From 970f32a2313fee96b986184a4399405f9c20ff82 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 7 Feb 2022 15:20:56 -0500
Subject: [PATCH 001/473] Add CUDA 11.5 to max compute and compute capability
 arrays (#3203)

* Add CUDA 11.5 to max compute and compute capability arrays

* Add CUDA 11.6 to max compute and compute capability arrays

Signed-off-by: Pradeep Garigipati <pradeep.garigipati@gmail.com>

Co-authored-by: Pradeep Garigipati <pradeep.garigipati@gmail.com>
---
 src/backend/cuda/device_manager.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 1a994424e6..ca46388484 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -95,6 +95,8 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {11060, 8, 6, 0},
+    {11050, 8, 6, 0},
     {11040, 8, 6, 0},
     {11030, 8, 6, 0},
     {11020, 8, 6, 0},
@@ -127,6 +129,8 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
+        {11060, 510.39f, 511.23f},
+        {11050, 495.29f, 496.13f},
         {11040, 470.42f, 471.11f},
         {11030, 465.19f, 465.89f},
         {11020, 460.27f, 460.82f},

From d2b09a6881054d22f586e440eef9e7c8252f2de0 Mon Sep 17 00:00:00 2001
From: Pradeep Garigipati <pradeep.garigipati@gmail.com>
Date: Sat, 12 Feb 2022 12:43:09 +0530
Subject: [PATCH 002/473] Fix mkl_version inclusion guard in CPU and OpenCL
 backends

Signed-off-by: Pradeep Garigipati <pradeep.garigipati@gmail.com>
---
 src/backend/cpu/solve.cpp            | 2 +-
 src/backend/opencl/cpu/cpu_solve.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/backend/cpu/solve.cpp b/src/backend/cpu/solve.cpp
index c5126275cb..52843d2fae 100644
--- a/src/backend/cpu/solve.cpp
+++ b/src/backend/cpu/solve.cpp
@@ -15,7 +15,7 @@
 #include <copy.hpp>
 #include <lapack_helper.hpp>
 #include <math.hpp>
-#if INTEL_MKL_VERSION >= 20210004
+#if USE_MKL
 #include <mkl_version.h>
 #endif
 #include <queue.hpp>
diff --git a/src/backend/opencl/cpu/cpu_solve.cpp b/src/backend/opencl/cpu/cpu_solve.cpp
index 3afdeca804..8b2cd79f64 100644
--- a/src/backend/opencl/cpu/cpu_solve.cpp
+++ b/src/backend/opencl/cpu/cpu_solve.cpp
@@ -12,7 +12,7 @@
 #include <cpu/cpu_helper.hpp>
 #include <cpu/cpu_solve.hpp>
 #include <math.hpp>
-#if INTEL_MKL_VERSION >= 20210004
+#if USE_MKL
 #include <mkl_version.h>
 #endif
 #include <algorithm>

From f58b849ca1d993030a807eccb1c653d325646857 Mon Sep 17 00:00:00 2001
From: Pradeep Garigipati <pradeep.garigipati@gmail.com>
Date: Sat, 12 Feb 2022 13:56:15 +0530
Subject: [PATCH 003/473] Update windows GA job to use new VS toolchain for
 respective GH image

Signed-off-by: Pradeep Garigipati <pradeep.garigipati@gmail.com>
---
 .github/workflows/win_cpu_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/win_cpu_build.yml b/.github/workflows/win_cpu_build.yml
index e265f6f877..72c6955238 100644
--- a/.github/workflows/win_cpu_build.yml
+++ b/.github/workflows/win_cpu_build.yml
@@ -48,7 +48,7 @@ jobs:
                   $dashboard = if($prnum -eq $null) { "Continuous" } else { "Experimental" }
                   $buildname = "$buildname-cpu-openblas"
                   mkdir build && cd build
-                  cmake .. -G "Visual Studio 16 2019" -A x64 `
+                  cmake .. -G "Visual Studio 17 2022" -A x64 `
                       -DVCPKG_ROOT:PATH="~/vcpkg" `
                       -DVCPKG_MANIFEST_MODE:BOOL=OFF `
                       -DAF_BUILD_CUDA:BOOL=OFF -DAF_BUILD_OPENCL:BOOL=OFF `

From 13cbff8a0a3cf0703f6223903d670f269293ead3 Mon Sep 17 00:00:00 2001
From: Pradeep Garigipati <pradeep.garigipati@gmail.com>
Date: Sat, 12 Feb 2022 23:10:47 +0530
Subject: [PATCH 004/473] intl/uintl versions of select/replace to fix accuracy
 issues

Without specific intl/uintl versions of these functions, when a
succifiently large 64 bit integer value is passed to select/replace
the output is incorrect or getting transformed to zero.

Signed-off-by: Pradeep Garigipati <pradeep.garigipati@gmail.com>
---
 include/af/data.h                    | 161 ++++++++++++++++++++++++++-
 src/api/c/deconvolution.cpp          |   3 +-
 src/api/c/replace.cpp                |  26 ++++-
 src/api/c/select.cpp                 | 148 ++++++++++++------------
 src/api/cpp/data.cpp                 |  32 ++++++
 src/api/unified/data.cpp             |  36 ++++++
 src/backend/cpu/kernel/select.hpp    |   7 +-
 src/backend/cpu/select.cpp           |  20 ++--
 src/backend/cpu/select.hpp           |   4 +-
 src/backend/cuda/kernel/select.hpp   |   4 +-
 src/backend/cuda/select.cpp          |  42 +++----
 src/backend/cuda/select.hpp          |   4 +-
 src/backend/opencl/kernel/select.hpp |   7 +-
 src/backend/opencl/select.cpp        |  42 +++----
 src/backend/opencl/select.hpp        |   4 +-
 test/CMakeLists.txt                  |   4 +-
 test/replace.cpp                     |  11 +-
 test/select.cpp                      |   9 +-
 18 files changed, 406 insertions(+), 158 deletions(-)

diff --git a/include/af/data.h b/include/af/data.h
index 05ef5f9f35..52ebb78ed7 100644
--- a/include/af/data.h
+++ b/include/af/data.h
@@ -409,7 +409,7 @@ namespace af
        \param[in]  cond is the conditional array.
        \param[in]  b is the replacement value.
 
-       \note Values of \p a are replaced with corresponding values of \p b, when \p cond is false.
+       \note Values of \p a are replaced with value \p b, when \p cond is false.
 
        \ingroup data_func_replace
     */
@@ -432,6 +432,81 @@ namespace af
     AFAPI array pad(const array &in, const dim4 &beginPadding,
                     const dim4 &endPadding, const borderType padFillType);
 #endif
+
+#if AF_API_VERSION >= 39
+    /**
+       \param[inout]  a is the input array
+       \param[in]  cond is the conditional array.
+       \param[in]  b is the replacement scalar value.
+
+       \note Values of \p a are replaced with value \p b, when \p cond is false.
+
+       \ingroup data_func_replace
+    */
+    AFAPI void replace(array &a, const array &cond, const long long b);
+
+    /**
+       \param[inout]  a is the input array
+       \param[in]  cond is the conditional array.
+       \param[in]  b is the replacement scalar value.
+
+       \note Values of \p a are replaced with value \p b, when \p cond is false.
+
+       \ingroup data_func_replace
+    */
+    AFAPI void replace(array &a, const array &cond,
+                       const unsigned long long b);
+
+    /**
+       \param[in]  cond is the conditional array
+       \param[in]  a is the array containing elements from the true part of the
+                   condition
+       \param[in]  b is a scalar assigned to \p out when \p cond is false
+       \return  the output containing elements of \p a when \p cond is true
+                else the value \p b
+
+       \ingroup data_func_select
+    */
+    AFAPI array select(const array &cond, const array &a, const long long b);
+
+    /**
+       \param[in]  cond is the conditional array
+       \param[in]  a is the array containing elements from the true part of the
+                   condition
+       \param[in]  b is a scalar assigned to \p out when \p cond is false
+       \return  the output containing elements of \p a when \p cond is true
+                else the value \p b
+
+       \ingroup data_func_select
+    */
+    AFAPI array select(const array &cond, const array &a,
+                       const unsigned long long b);
+
+    /**
+       \param[in]  cond is the conditional array
+       \param[in]  a is a scalar assigned to \p out when \p cond is true
+       \param[in]  b is the array containing elements from the false part of the
+                   condition
+       \return  the output containing the value \p a when \p cond is true else
+                elements from \p b
+
+       \ingroup data_func_select
+    */
+    AFAPI array select(const array &cond, const long long a, const array &b);
+
+    /**
+       \param[in]  cond is the conditional array
+       \param[in]  a is a scalar assigned to \p out when \p cond is true
+       \param[in]  b is the array containing elements from the false part of the
+                   condition
+       \return  the output containing the value \p a when \p cond is true else
+                elements from \p b
+
+       \ingroup data_func_select
+    */
+    AFAPI array select(const array &cond, const unsigned long long a,
+                       const array &b);
+#endif
 }
 #endif
 
@@ -735,6 +810,90 @@ extern "C" {
                         const af_border_type pad_fill_type);
 #endif
 
+#if AF_API_VERSION >= 39
+    /**
+       \param[inout]  a is the input array
+       \param[in]  cond is the conditional array.
+       \param[in]  b is the replacement array.
+
+       \note Values of \p a are replaced with corresponding values of \p b, when
+       \p cond is false.
+
+       \ingroup data_func_replace
+    */
+    AFAPI af_err af_replace_scalar_long(af_array a, const af_array cond,
+                                        const long long b);
+
+    /**
+       \param[inout]  a is the input array
+       \param[in]  cond is the conditional array.
+       \param[in]  b is the replacement array.
+
+       \note Values of \p a are replaced with corresponding values of \p b, when
+       \p cond is false.
+
+       \ingroup data_func_replace
+    */
+    AFAPI af_err af_replace_scalar_ulong(af_array a, const af_array cond,
+                                         const unsigned long long b);
+
+    /**
+       \param[out] out is the output containing elements of \p a when \p cond is
+       true else elements from \p b
+       \param[in]  cond is the conditional array
+       \param[in]  a is the array containing elements from the true part of the
+       condition
+       \param[in]  b is a scalar assigned to \p out when \p cond is
+       false
+
+       \ingroup data_func_select
+    */
+    AFAPI af_err af_select_scalar_r_long(af_array *out, const af_array cond,
+                                         const af_array a, const long long b);
+
+    /**
+       \param[out] out is the output containing elements of \p a when \p cond is
+       true else elements from \p b
+       \param[in]  cond is the conditional array
+       \param[in]  a is the array containing elements from the true part of the
+       condition
+       \param[in]  b is a scalar assigned to \p out when \p cond is
+       false
+
+       \ingroup data_func_select
+    */
+    AFAPI af_err af_select_scalar_r_ulong(af_array *out, const af_array cond,
+                                          const af_array a,
+                                          const unsigned long long b);
+
+    /**
+       \param[out] out is the output containing elements of \p a when \p cond is
+       true else elements from \p b
+       \param[in]  cond is the conditional array
+       \param[in]  a is a scalar assigned to \p out when \p cond is true
+       \param[in]  b is the array containing elements from the false part of the
+       condition
+
+       \ingroup data_func_select
+    */
+    AFAPI af_err af_select_scalar_l_long(af_array *out, const af_array cond,
+                                         const long long a, const af_array b);
+
+    /**
+       \param[out] out is the output containing elements of \p a when \p cond is
+       true else elements from \p b
+       \param[in]  cond is the conditional array
+       \param[in]  a is a scalar assigned to \p out when \p cond is true
+       \param[in]  b is the array containing elements from the false part of the
+       condition
+
+       \ingroup data_func_select
+    */
+    AFAPI af_err af_select_scalar_l_ulong(af_array *out, const af_array cond,
+                                          const unsigned long long a,
+                                          const af_array b);
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/api/c/deconvolution.cpp b/src/api/c/deconvolution.cpp
index 43c83965e3..21180b2d8b 100644
--- a/src/api/c/deconvolution.cpp
+++ b/src/api/c/deconvolution.cpp
@@ -19,6 +19,7 @@
 #include <fftconvolve.hpp>
 #include <handle.hpp>
 #include <logic.hpp>
+#include <math.hpp>
 #include <reduce.hpp>
 #include <select.hpp>
 #include <shift.hpp>
@@ -294,7 +295,7 @@ af_array invDeconv(const af_array in, const af_array ker, const float gamma,
     auto cond   = logicOp<T, af_ge_t>(absVal, THRESH, absVal.dims());
     auto val    = arithOp<CT, af_div_t>(numer, denom, numer.dims());
 
-    select_scalar<CT, false>(val, cond, val, 0);
+    select_scalar<CT, false>(val, cond, val, scalar<CT>(0.0));
 
     auto ival =
         fft_c2r<CT, T>(val, 1 / static_cast<double>(nElems), odims, BASE_DIM);
diff --git a/src/api/c/replace.cpp b/src/api/c/replace.cpp
index 27455982e9..bd4814157a 100644
--- a/src/api/c/replace.cpp
+++ b/src/api/c/replace.cpp
@@ -82,13 +82,15 @@ af_err af_replace(af_array a, const af_array cond, const af_array b) {
     return AF_SUCCESS;
 }
 
-template<typename T>
-void replace_scalar(af_array a, const af_array cond, const double b) {
-    select_scalar<T, false>(getCopyOnWriteArray<T>(a), getArray<char>(cond),
-                            getArray<T>(a), b);
+template<typename ArrayType, typename ScalarType>
+void replace_scalar(af_array a, const af_array cond, const ScalarType& b) {
+    select_scalar<ArrayType, false>(
+        getCopyOnWriteArray<ArrayType>(a), getArray<char>(cond),
+        getArray<ArrayType>(a), detail::scalar<ArrayType>(b));
 }
 
-af_err af_replace_scalar(af_array a, const af_array cond, const double b) {
+template<typename ScalarType>
+af_err replaceScalar(af_array a, const af_array cond, const ScalarType b) {
     try {
         const ArrayInfo& ainfo = getInfo(a);
         const ArrayInfo& cinfo = getInfo(cond);
@@ -121,3 +123,17 @@ af_err af_replace_scalar(af_array a, const af_array cond, const double b) {
     CATCHALL;
     return AF_SUCCESS;
 }
+
+af_err af_replace_scalar(af_array a, const af_array cond, const double b) {
+    return replaceScalar(a, cond, b);
+}
+
+af_err af_replace_scalar_long(af_array a, const af_array cond,
+                              const long long b) {
+    return replaceScalar(a, cond, b);
+}
+
+af_err af_replace_scalar_ulong(af_array a, const af_array cond,
+                               const unsigned long long b) {
+    return replaceScalar(a, cond, b);
+}
diff --git a/src/api/c/select.cpp b/src/api/c/select.cpp
index 952a8568fa..31d7facbcd 100644
--- a/src/api/c/select.cpp
+++ b/src/api/c/select.cpp
@@ -88,71 +88,90 @@ af_err af_select(af_array* out, const af_array cond, const af_array a,
     return AF_SUCCESS;
 }
 
-template<typename T, bool flip>
-af_array select_scalar(const af_array cond, const af_array a, const double b,
-                       const dim4& odims) {
-    Array<T> out = createSelectNode<T, flip>(getArray<char>(cond),
-                                             getArray<T>(a), b, odims);
-    return getHandle<T>(out);
+template<typename ArrayType, typename ScalarType, bool flip>
+af_array select_scalar(const af_array cond, const af_array a,
+                       const ScalarType b, const dim4& odims) {
+    auto scalar = detail::scalar<ArrayType>(b);
+    auto out    = createSelectNode<ArrayType, flip>(
+        getArray<char>(cond), getArray<ArrayType>(a), scalar, odims);
+    return getHandle(out);
 }
 
-af_err af_select_scalar_r(af_array* out, const af_array cond, const af_array a,
-                          const double b) {
+template<typename ScalarType, bool IsScalarTrueOutput>
+af_err selectScalar(af_array* out, const af_array cond, const af_array e,
+                    const ScalarType c) {
     try {
-        const ArrayInfo& ainfo = getInfo(a);
+        const ArrayInfo& einfo = getInfo(e);
         const ArrayInfo& cinfo = getInfo(cond);
 
         ARG_ASSERT(1, cinfo.getType() == b8);
 
-        dim4 adims     = ainfo.dims();
+        dim4 edims     = einfo.dims();
         dim4 cond_dims = cinfo.dims();
         dim4 odims(1);
 
         for (int i = 0; i < 4; i++) {
-            DIM_ASSERT(1, cond_dims[i] == adims[i] || cond_dims[i] == 1 ||
-                              adims[i] == 1);
-            odims[i] = std::max(cond_dims[i], adims[i]);
+            DIM_ASSERT(1, cond_dims[i] == edims[i] || cond_dims[i] == 1 ||
+                              edims[i] == 1);
+            odims[i] = std::max(cond_dims[i], edims[i]);
         }
 
         af_array res;
 
-        switch (ainfo.getType()) {
+        switch (einfo.getType()) {
             case f16:
-                res = select_scalar<half, false>(cond, a, b, odims);
+                res = select_scalar<half, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case f32:
-                res = select_scalar<float, false>(cond, a, b, odims);
+                res = select_scalar<float, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case f64:
-                res = select_scalar<double, false>(cond, a, b, odims);
+                res = select_scalar<double, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case c32:
-                res = select_scalar<cfloat, false>(cond, a, b, odims);
+                res = select_scalar<cfloat, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case c64:
-                res = select_scalar<cdouble, false>(cond, a, b, odims);
+                res = select_scalar<cdouble, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
+                break;
+            case s32:
+                res = select_scalar<int, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
-            case s32: res = select_scalar<int, false>(cond, a, b, odims); break;
             case u32:
-                res = select_scalar<uint, false>(cond, a, b, odims);
+                res = select_scalar<uint, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case s16:
-                res = select_scalar<short, false>(cond, a, b, odims);
+                res = select_scalar<short, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case u16:
-                res = select_scalar<ushort, false>(cond, a, b, odims);
+                res = select_scalar<ushort, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case s64:
-                res = select_scalar<intl, false>(cond, a, b, odims);
+                res = select_scalar<intl, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case u64:
-                res = select_scalar<uintl, false>(cond, a, b, odims);
+                res = select_scalar<uintl, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
             case u8:
-                res = select_scalar<uchar, false>(cond, a, b, odims);
+                res = select_scalar<uchar, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
                 break;
-            case b8: res = select_scalar<char, false>(cond, a, b, odims); break;
-            default: TYPE_ERROR(2, ainfo.getType());
+            case b8:
+                res = select_scalar<char, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
+                break;
+            default: TYPE_ERROR((IsScalarTrueOutput ? 3 : 2), einfo.getType());
         }
 
         std::swap(*out, res);
@@ -161,59 +180,32 @@ af_err af_select_scalar_r(af_array* out, const af_array cond, const af_array a,
     return AF_SUCCESS;
 }
 
-af_err af_select_scalar_l(af_array* out, const af_array cond, const double a,
-                          const af_array b) {
-    try {
-        const ArrayInfo& binfo = getInfo(b);
-        const ArrayInfo& cinfo = getInfo(cond);
-
-        ARG_ASSERT(1, cinfo.getType() == b8);
+af_err af_select_scalar_r(af_array* out, const af_array cond, const af_array a,
+                          const double b) {
+    return selectScalar<double, false>(out, cond, a, b);
+}
 
-        dim4 bdims     = binfo.dims();
-        dim4 cond_dims = cinfo.dims();
-        dim4 odims(1);
+af_err af_select_scalar_r_long(af_array* out, const af_array cond,
+                               const af_array a, const long long b) {
+    return selectScalar<long long, false>(out, cond, a, b);
+}
 
-        for (int i = 0; i < 4; i++) {
-            DIM_ASSERT(1, cond_dims[i] == bdims[i] || cond_dims[i] == 1 ||
-                              bdims[i] == 1);
-            odims[i] = std::max(cond_dims[i], bdims[i]);
-        }
+af_err af_select_scalar_r_ulong(af_array* out, const af_array cond,
+                                const af_array a, const unsigned long long b) {
+    return selectScalar<unsigned long long, false>(out, cond, a, b);
+}
 
-        af_array res;
+af_err af_select_scalar_l(af_array* out, const af_array cond, const double a,
+                          const af_array b) {
+    return selectScalar<double, true>(out, cond, b, a);
+}
 
-        switch (binfo.getType()) {
-            case f16: res = select_scalar<half, true>(cond, b, a, odims); break;
-            case f32:
-                res = select_scalar<float, true>(cond, b, a, odims);
-                break;
-            case f64:
-                res = select_scalar<double, true>(cond, b, a, odims);
-                break;
-            case c32:
-                res = select_scalar<cfloat, true>(cond, b, a, odims);
-                break;
-            case c64:
-                res = select_scalar<cdouble, true>(cond, b, a, odims);
-                break;
-            case s32: res = select_scalar<int, true>(cond, b, a, odims); break;
-            case u32: res = select_scalar<uint, true>(cond, b, a, odims); break;
-            case s16:
-                res = select_scalar<short, true>(cond, b, a, odims);
-                break;
-            case u16:
-                res = select_scalar<ushort, true>(cond, b, a, odims);
-                break;
-            case s64: res = select_scalar<intl, true>(cond, b, a, odims); break;
-            case u64:
-                res = select_scalar<uintl, true>(cond, b, a, odims);
-                break;
-            case u8: res = select_scalar<uchar, true>(cond, b, a, odims); break;
-            case b8: res = select_scalar<char, true>(cond, b, a, odims); break;
-            default: TYPE_ERROR(2, binfo.getType());
-        }
+af_err af_select_scalar_l_long(af_array* out, const af_array cond,
+                               const long long a, const af_array b) {
+    return selectScalar<long long, true>(out, cond, b, a);
+}
 
-        std::swap(*out, res);
-    }
-    CATCHALL;
-    return AF_SUCCESS;
+af_err af_select_scalar_l_ulong(af_array* out, const af_array cond,
+                                const unsigned long long a, const af_array b) {
+    return selectScalar<unsigned long long, true>(out, cond, b, a);
 }
diff --git a/src/api/cpp/data.cpp b/src/api/cpp/data.cpp
index 5ca5077b91..3f86520bd0 100644
--- a/src/api/cpp/data.cpp
+++ b/src/api/cpp/data.cpp
@@ -313,6 +313,38 @@ void replace(array &a, const array &cond, const double &b) {
     AF_THROW(af_replace_scalar(a.get(), cond.get(), b));
 }
 
+void replace(array &a, const array &cond, const long long b) {
+    AF_THROW(af_replace_scalar_long(a.get(), cond.get(), b));
+}
+
+void replace(array &a, const array &cond, const unsigned long long b) {
+    AF_THROW(af_replace_scalar_ulong(a.get(), cond.get(), b));
+}
+
+array select(const array &cond, const array &a, const long long b) {
+    af_array res;
+    AF_THROW(af_select_scalar_r_long(&res, cond.get(), a.get(), b));
+    return array(res);
+}
+
+array select(const array &cond, const array &a, const unsigned long long b) {
+    af_array res;
+    AF_THROW(af_select_scalar_r_ulong(&res, cond.get(), a.get(), b));
+    return array(res);
+}
+
+array select(const array &cond, const long long a, const array &b) {
+    af_array res;
+    AF_THROW(af_select_scalar_l_long(&res, cond.get(), a, b.get()));
+    return array(res);
+}
+
+array select(const array &cond, const unsigned long long a, const array &b) {
+    af_array res;
+    AF_THROW(af_select_scalar_l_ulong(&res, cond.get(), a, b.get()));
+    return array(res);
+}
+
 array pad(const array &in, const dim4 &beginPadding, const dim4 &endPadding,
           const borderType padFillType) {
     af_array out = 0;
diff --git a/src/api/unified/data.cpp b/src/api/unified/data.cpp
index b67868d181..3fb7312fdd 100644
--- a/src/api/unified/data.cpp
+++ b/src/api/unified/data.cpp
@@ -148,3 +148,39 @@ af_err af_pad(af_array *out, const af_array in, const unsigned b_ndims,
     CHECK_ARRAYS(in);
     CALL(af_pad, out, in, b_ndims, b_dims, e_ndims, e_dims, ptype);
 }
+
+af_err af_replace_scalar_long(af_array a, const af_array cond,
+                              const long long b) {
+    CHECK_ARRAYS(a, cond);
+    CALL(af_replace_scalar_long, a, cond, b);
+}
+
+af_err af_replace_scalar_ulong(af_array a, const af_array cond,
+                               const unsigned long long b) {
+    CHECK_ARRAYS(a, cond);
+    CALL(af_replace_scalar_ulong, a, cond, b);
+}
+
+af_err af_select_scalar_r_long(af_array *out, const af_array cond,
+                               const af_array a, const long long b) {
+    CHECK_ARRAYS(cond, a);
+    CALL(af_select_scalar_r_long, out, cond, a, b);
+}
+
+af_err af_select_scalar_r_ulong(af_array *out, const af_array cond,
+                                const af_array a, const unsigned long long b) {
+    CHECK_ARRAYS(cond, a);
+    CALL(af_select_scalar_r_ulong, out, cond, a, b);
+}
+
+af_err af_select_scalar_l_long(af_array *out, const af_array cond,
+                               const long long a, const af_array b) {
+    CHECK_ARRAYS(cond, b);
+    CALL(af_select_scalar_l_long, out, cond, a, b);
+}
+
+af_err af_select_scalar_l_ulong(af_array *out, const af_array cond,
+                                const unsigned long long a, const af_array b) {
+    CHECK_ARRAYS(cond, b);
+    CALL(af_select_scalar_l_ulong, out, cond, a, b);
+}
diff --git a/src/backend/cpu/kernel/select.hpp b/src/backend/cpu/kernel/select.hpp
index 6ab9e9ec5b..88a95fd5bc 100644
--- a/src/backend/cpu/kernel/select.hpp
+++ b/src/backend/cpu/kernel/select.hpp
@@ -71,8 +71,7 @@ void select(Param<T> out, CParam<char> cond, CParam<T> a, CParam<T> b) {
 }
 
 template<typename T, bool flip>
-void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a,
-                   const double b) {
+void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const T b) {
     af::dim4 astrides = a.strides();
     af::dim4 adims    = a.dims();
     af::dim4 cstrides = cond.strides();
@@ -85,6 +84,8 @@ void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a,
     data_t<T> *optr       = out.get();
     const char *cptr      = cond.get();
 
+    const compute_t<T> scalar = static_cast<compute_t<T>>(b);
+
     bool is_a_same[] = {adims[0] == odims[0], adims[1] == odims[1],
                         adims[2] == odims[2], adims[3] == odims[3]};
 
@@ -110,7 +111,7 @@ void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a,
                     bool cval = is_c_same[0] ? cptr[c_off1 + i] : cptr[c_off1];
                     compute_t<T> aval = static_cast<compute_t<T>>(
                         is_a_same[0] ? aptr[a_off1 + i] : aptr[a_off1]);
-                    optr[o_off1 + i] = (flip ^ cval) ? aval : b;
+                    optr[o_off1 + i] = (flip ^ cval) ? aval : scalar;
                 }
             }
         }
diff --git a/src/backend/cpu/select.cpp b/src/backend/cpu/select.cpp
index 31812949de..a801bb5e86 100644
--- a/src/backend/cpu/select.cpp
+++ b/src/backend/cpu/select.cpp
@@ -27,19 +27,19 @@ void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
-                   const double &b) {
+                   const T &b) {
     getQueue().enqueue(kernel::select_scalar<T, flip>, out, cond, a, b);
 }
 
-#define INSTANTIATE(T)                                                        \
-    template void select<T>(Array<T> & out, const Array<char> &cond,          \
-                            const Array<T> &a, const Array<T> &b);            \
-    template void select_scalar<T, true>(Array<T> & out,                      \
-                                         const Array<char> &cond,             \
-                                         const Array<T> &a, const double &b); \
-    template void select_scalar<T, false>(Array<T> & out,                     \
-                                          const Array<char> &cond,            \
-                                          const Array<T> &a, const double &b);
+#define INSTANTIATE(T)                                                   \
+    template void select<T>(Array<T> & out, const Array<char> &cond,     \
+                            const Array<T> &a, const Array<T> &b);       \
+    template void select_scalar<T, true>(Array<T> & out,                 \
+                                         const Array<char> &cond,        \
+                                         const Array<T> &a, const T &b); \
+    template void select_scalar<T, false>(Array<T> & out,                \
+                                          const Array<char> &cond,       \
+                                          const Array<T> &a, const T &b);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/cpu/select.hpp b/src/backend/cpu/select.hpp
index dfe13ae9ea..b92a8d36c5 100644
--- a/src/backend/cpu/select.hpp
+++ b/src/backend/cpu/select.hpp
@@ -16,7 +16,7 @@ void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
-                   const double &b);
+                   const T &b);
 
 template<typename T>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
@@ -28,7 +28,7 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
-                          const double &b, const af::dim4 &odims) {
+                          const T &b, const af::dim4 &odims) {
     Array<T> out = createEmptyArray<T>(odims);
     select_scalar<T, flip>(out, cond, a, b);
     return out;
diff --git a/src/backend/cuda/kernel/select.hpp b/src/backend/cuda/kernel/select.hpp
index 433875c009..6f8972e04f 100644
--- a/src/backend/cuda/kernel/select.hpp
+++ b/src/backend/cuda/kernel/select.hpp
@@ -57,7 +57,7 @@ void select(Param<T> out, CParam<char> cond, CParam<T> a, CParam<T> b,
 }
 
 template<typename T>
-void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const double b,
+void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const T b,
                    int ndims, bool flip) {
     auto selectScalar =
         common::getKernel("cuda::selectScalar", {select_cuh_src},
@@ -77,7 +77,7 @@ void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const double b,
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
-    selectScalar(qArgs, out, cond, a, scalar<T>(b), blk_x, blk_y);
+    selectScalar(qArgs, out, cond, a, b, blk_x, blk_y);
     POST_LAUNCH_CHECK();
 }
 
diff --git a/src/backend/cuda/select.cpp b/src/backend/cuda/select.cpp
index 0b554d1dbf..6f6f399960 100644
--- a/src/backend/cuda/select.cpp
+++ b/src/backend/cuda/select.cpp
@@ -34,7 +34,7 @@ void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
-                   const double &b) {
+                   const T &b) {
     kernel::select_scalar<T>(out, cond, a, b, out.ndims(), flip);
 }
 
@@ -68,10 +68,10 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
-                          const double &b_val, const af::dim4 &odims) {
+                          const T &b_val, const af::dim4 &odims) {
     auto cond_node   = cond.getNode();
     auto a_node      = a.getNode();
-    Array<T> b       = createScalarNode<T>(odims, scalar<T>(b_val));
+    Array<T> b       = createScalarNode<T>(odims, b_val);
     auto b_node      = b.getNode();
     auto a_height    = a_node->getHeight();
     auto b_height    = b_node->getHeight();
@@ -96,24 +96,24 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
     return createNodeArray<T>(odims, node);
 }
 
-#define INSTANTIATE(T)                                                        \
-    template Array<T> createSelectNode<T>(                                    \
-        const Array<char> &cond, const Array<T> &a, const Array<T> &b,        \
-        const af::dim4 &odims);                                               \
-    template Array<T> createSelectNode<T, true>(                              \
-        const Array<char> &cond, const Array<T> &a, const double &b_val,      \
-        const af::dim4 &odims);                                               \
-    template Array<T> createSelectNode<T, false>(                             \
-        const Array<char> &cond, const Array<T> &a, const double &b_val,      \
-        const af::dim4 &odims);                                               \
-    template void select<T>(Array<T> & out, const Array<char> &cond,          \
-                            const Array<T> &a, const Array<T> &b);            \
-    template void select_scalar<T, true>(Array<T> & out,                      \
-                                         const Array<char> &cond,             \
-                                         const Array<T> &a, const double &b); \
-    template void select_scalar<T, false>(Array<T> & out,                     \
-                                          const Array<char> &cond,            \
-                                          const Array<T> &a, const double &b)
+#define INSTANTIATE(T)                                                   \
+    template Array<T> createSelectNode<T>(                               \
+        const Array<char> &cond, const Array<T> &a, const Array<T> &b,   \
+        const af::dim4 &odims);                                          \
+    template Array<T> createSelectNode<T, true>(                         \
+        const Array<char> &cond, const Array<T> &a, const T &b_val,      \
+        const af::dim4 &odims);                                          \
+    template Array<T> createSelectNode<T, false>(                        \
+        const Array<char> &cond, const Array<T> &a, const T &b_val,      \
+        const af::dim4 &odims);                                          \
+    template void select<T>(Array<T> & out, const Array<char> &cond,     \
+                            const Array<T> &a, const Array<T> &b);       \
+    template void select_scalar<T, true>(Array<T> & out,                 \
+                                         const Array<char> &cond,        \
+                                         const Array<T> &a, const T &b); \
+    template void select_scalar<T, false>(Array<T> & out,                \
+                                          const Array<char> &cond,       \
+                                          const Array<T> &a, const T &b)
 
 INSTANTIATE(float);
 INSTANTIATE(double);
diff --git a/src/backend/cuda/select.hpp b/src/backend/cuda/select.hpp
index edd51a93bb..6552ca3ccd 100644
--- a/src/backend/cuda/select.hpp
+++ b/src/backend/cuda/select.hpp
@@ -17,7 +17,7 @@ void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
-                   const double &b);
+                   const T &b);
 
 template<typename T>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
@@ -25,5 +25,5 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
-                          const double &b_val, const af::dim4 &odims);
+                          const T &b_val, const af::dim4 &odims);
 }  // namespace cuda
diff --git a/src/backend/opencl/kernel/select.hpp b/src/backend/opencl/kernel/select.hpp
index cd98ac5662..743f200d5c 100644
--- a/src/backend/opencl/kernel/select.hpp
+++ b/src/backend/opencl/kernel/select.hpp
@@ -72,8 +72,8 @@ void select(Param out, Param cond, Param a, Param b, int ndims) {
 }
 
 template<typename T>
-void select_scalar(Param out, Param cond, Param a, const double b,
-                   const int ndims, const bool flip) {
+void select_scalar(Param out, Param cond, Param a, const T b, const int ndims,
+                   const bool flip) {
     std::vector<TemplateArg> targs = {
         TemplateTypename<T>(),
         TemplateArg(flip),
@@ -103,8 +103,7 @@ void select_scalar(Param out, Param cond, Param a, const double b,
                        groups_1 * out.info.dims[3] * local[1]);
 
     selectOp(cl::EnqueueArgs(getQueue(), global, local), *out.data, out.info,
-             *cond.data, cond.info, *a.data, a.info, scalar<T>(b), groups_0,
-             groups_1);
+             *cond.data, cond.info, *a.data, a.info, b, groups_0, groups_1);
 }
 }  // namespace kernel
 }  // namespace opencl
diff --git a/src/backend/opencl/select.cpp b/src/backend/opencl/select.cpp
index 9821e7ee89..32c2734f75 100644
--- a/src/backend/opencl/select.cpp
+++ b/src/backend/opencl/select.cpp
@@ -56,10 +56,10 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
-                          const double &b_val, const dim4 &odims) {
+                          const T &b_val, const dim4 &odims) {
     auto cond_node   = cond.getNode();
     auto a_node      = a.getNode();
-    Array<T> b       = createScalarNode<T>(odims, scalar<T>(b_val));
+    Array<T> b       = createScalarNode<T>(odims, b_val);
     auto b_node      = b.getNode();
     auto a_height    = a_node->getHeight();
     auto b_height    = b_node->getHeight();
@@ -92,28 +92,28 @@ void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
-                   const double &b) {
+                   const T &b) {
     kernel::select_scalar<T>(out, cond, a, b, out.ndims(), flip);
 }
 
-#define INSTANTIATE(T)                                                        \
-    template Array<T> createSelectNode<T>(                                    \
-        const Array<char> &cond, const Array<T> &a, const Array<T> &b,        \
-        const af::dim4 &odims);                                               \
-    template Array<T> createSelectNode<T, true>(                              \
-        const Array<char> &cond, const Array<T> &a, const double &b_val,      \
-        const af::dim4 &odims);                                               \
-    template Array<T> createSelectNode<T, false>(                             \
-        const Array<char> &cond, const Array<T> &a, const double &b_val,      \
-        const af::dim4 &odims);                                               \
-    template void select<T>(Array<T> & out, const Array<char> &cond,          \
-                            const Array<T> &a, const Array<T> &b);            \
-    template void select_scalar<T, true>(Array<T> & out,                      \
-                                         const Array<char> &cond,             \
-                                         const Array<T> &a, const double &b); \
-    template void select_scalar<T, false>(Array<T> & out,                     \
-                                          const Array<char> &cond,            \
-                                          const Array<T> &a, const double &b)
+#define INSTANTIATE(T)                                                   \
+    template Array<T> createSelectNode<T>(                               \
+        const Array<char> &cond, const Array<T> &a, const Array<T> &b,   \
+        const af::dim4 &odims);                                          \
+    template Array<T> createSelectNode<T, true>(                         \
+        const Array<char> &cond, const Array<T> &a, const T &b_val,      \
+        const af::dim4 &odims);                                          \
+    template Array<T> createSelectNode<T, false>(                        \
+        const Array<char> &cond, const Array<T> &a, const T &b_val,      \
+        const af::dim4 &odims);                                          \
+    template void select<T>(Array<T> & out, const Array<char> &cond,     \
+                            const Array<T> &a, const Array<T> &b);       \
+    template void select_scalar<T, true>(Array<T> & out,                 \
+                                         const Array<char> &cond,        \
+                                         const Array<T> &a, const T &b); \
+    template void select_scalar<T, false>(Array<T> & out,                \
+                                          const Array<char> &cond,       \
+                                          const Array<T> &a, const T &b)
 
 INSTANTIATE(float);
 INSTANTIATE(double);
diff --git a/src/backend/opencl/select.hpp b/src/backend/opencl/select.hpp
index 01b99ae554..4dbd0635da 100644
--- a/src/backend/opencl/select.hpp
+++ b/src/backend/opencl/select.hpp
@@ -17,7 +17,7 @@ void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
-                   const double &b);
+                   const T &b);
 
 template<typename T>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
@@ -25,5 +25,5 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
 
 template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
-                          const double &b_val, const af::dim4 &odims);
+                          const T &b_val, const af::dim4 &odims);
 }  // namespace opencl
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 57e0a307a8..a43cfb51d3 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -388,7 +388,7 @@ make_test(SRC rank_dense.cpp SERIAL)
 make_test(SRC reduce.cpp CXX11)
 make_test(SRC regions.cpp)
 make_test(SRC reorder.cpp)
-make_test(SRC replace.cpp)
+make_test(SRC replace.cpp CXX11)
 make_test(SRC resize.cpp)
 make_test(SRC rng_match.cpp CXX11 BACKENDS "unified")
 make_test(SRC rotate.cpp)
@@ -396,7 +396,7 @@ make_test(SRC rotate_linear.cpp)
 make_test(SRC sat.cpp)
 make_test(SRC scan.cpp)
 make_test(SRC scan_by_key.cpp)
-make_test(SRC select.cpp)
+make_test(SRC select.cpp CXX11)
 make_test(SRC set.cpp CXX11)
 make_test(SRC shift.cpp)
 make_test(SRC gloh.cpp)
diff --git a/test/replace.cpp b/test/replace.cpp
index c8787dc5ee..26baf63a9d 100644
--- a/test/replace.cpp
+++ b/test/replace.cpp
@@ -13,8 +13,10 @@
 #include <testHelpers.hpp>
 #include <af/dim4.hpp>
 #include <af/traits.hpp>
+
 #include <iostream>
 #include <string>
+#include <type_traits>
 #include <vector>
 
 using af::array;
@@ -77,6 +79,11 @@ void replaceTest(const dim4 &dims) {
 template<typename T>
 void replaceScalarTest(const dim4 &dims) {
     SUPPORTED_TYPE_CHECK(T);
+    using scalar_t =
+        typename std::conditional<std::is_same<T, intl>::value ||
+                                      std::is_same<T, uintl>::value,
+                                  T, double>::type;
+
     dtype ty = (dtype)dtype_traits<T>::af_type;
 
     array a = randu(dims, ty);
@@ -85,7 +92,7 @@ void replaceScalarTest(const dim4 &dims) {
 
     array c    = a.copy();
     array cond = randu(dims, ty) > a;
-    double b   = 3;
+    scalar_t b = static_cast<scalar_t>(3);
 
     replace(c, cond, b);
     int num = (int)a.elements();
@@ -170,7 +177,7 @@ TEST(Replace, ISSUE_1683) {
     A.host(ha1.data());
 
     array B = A(0, span);
-    replace(B, A(0, span) > 0.5, 0);
+    replace(B, A(0, span) > 0.5, 0.0);
 
     vector<float> ha2(A.elements());
     A.host(ha2.data());
diff --git a/test/select.cpp b/test/select.cpp
index 9ee331dff2..7df6b6a862 100644
--- a/test/select.cpp
+++ b/test/select.cpp
@@ -11,13 +11,13 @@
 #include <gtest/gtest.h>
 #include <half.hpp>
 #include <testHelpers.hpp>
-
 #include <af/dim4.hpp>
 #include <af/traits.hpp>
 
 #include <cstdio>
 #include <iostream>
 #include <string>
+#include <type_traits>
 #include <vector>
 
 using af::array;
@@ -83,11 +83,16 @@ void selectTest(const dim4& dims) {
 template<typename T, bool is_right>
 void selectScalarTest(const dim4& dims) {
     SUPPORTED_TYPE_CHECK(T);
+    using scalar_t =
+        typename std::conditional<std::is_same<T, intl>::value ||
+                                      std::is_same<T, uintl>::value,
+                                  T, double>::type;
+
     dtype ty = (dtype)dtype_traits<T>::af_type;
 
     array a    = randu(dims, ty);
     array cond = randu(dims, ty) > a;
-    double b   = 3;
+    scalar_t b = static_cast<scalar_t>(3);
 
     if (a.isinteger()) { a = (a % (1 << 30)).as(ty); }
 

From bb892342c1e7bef1f0e70fac16a4e2f005de2f69 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 7 Feb 2022 16:42:35 -0500
Subject: [PATCH 005/473] Use c++11 when building tests

---
 test/CMakeLists.txt |  2 +-
 test/dot.cpp        | 12 ++++++++++--
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index a43cfb51d3..66e87d9a67 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -61,7 +61,7 @@ if(NOT TARGET mmio)
 endif()
 
 # Reset the CXX flags for tests
-set(CMAKE_CXX_STANDARD 98)
+set(CMAKE_CXX_STANDARD 11)
 
 # TODO(pradeep) perhaps rename AF_USE_RELATIVE_TEST_DIR to AF_WITH_TEST_DATA_DIR
 #               with empty default value
diff --git a/test/dot.cpp b/test/dot.cpp
index 8a1905397c..37b84d2818 100644
--- a/test/dot.cpp
+++ b/test/dot.cpp
@@ -47,8 +47,14 @@ typedef ::testing::Types<cfloat, cdouble> TestTypesC;
 TYPED_TEST_CASE(DotF, TestTypesF);
 TYPED_TEST_CASE(DotC, TestTypesC);
 
-bool isinf(af::af_cfloat val) { return isinf(val.real) || isinf(val.imag); }
-bool isinf(af::af_cdouble val) { return isinf(val.real) || isinf(val.imag); }
+bool isinf(af::af_cfloat val) {
+    using std::isinf;
+    return isinf(val.real) || isinf(val.imag);
+}
+bool isinf(af::af_cdouble val) {
+    using std::isinf;
+    return isinf(val.real) || isinf(val.imag);
+}
 
 template<typename T>
 void dotTest(string pTestFile, const int resultIdx,
@@ -135,6 +141,8 @@ void dotAllTest(string pTestFile, const int resultIdx,
 
     vector<T> goldData = tests[resultIdx];
 
+    using ::isinf;
+    using std::isinf;
     if (false == (isinf(rval) && isinf(goldData[0]))) {
         compare<T>(rval, ival, goldData[0]);
     }

From 333009069be3ef6e8545b419c6a36bcb22cf25e7 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 7 Feb 2022 16:46:39 -0500
Subject: [PATCH 006/473] Use boost's epsilon difference when comparing
 floating point values

This commit changes the way we compare floating point values in the
tests to use the boost math's epsilon difference to compare two floating
point values for equality. This is a more accurate form of equality and
handles differences in half float values when the values reach a
certain threshold.
---
 test/CMakeLists.txt          |   1 +
 test/arrayfire_test.cpp      |  44 +++++++++++-
 test/join.cpp                |   2 +-
 test/relative_difference.hpp | 135 +++++++++++++++++++++++++++++++++++
 test/testHelpers.hpp         |  13 ----
 5 files changed, 179 insertions(+), 16 deletions(-)
 create mode 100644 test/relative_difference.hpp

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 66e87d9a67..af9afe4991 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -108,6 +108,7 @@ target_include_directories(arrayfire_test
     ${ArrayFire_BINARY_DIR}/include
     ${ArrayFire_SOURCE_DIR}/extern/half/include
     mmio
+    $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
     ${${gtest_prefix}_SOURCE_DIR}/googletest/include)
 
 if(WIN32)
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index de9b423fe5..63896a791a 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -16,6 +16,8 @@
 #include <af/internal.h>
 
 #include <gtest/gtest.h>
+#include <half.hpp>
+#include <relative_difference.hpp>
 
 #include <algorithm>
 #include <cfloat>
@@ -159,7 +161,7 @@ ::testing::AssertionResult assertArrayEq(std::string aName, std::string bName,
             return elemWiseEq<unsigned short>(aName, bName, a, b, maxAbsDiff);
             break;
         case f16:
-            return elemWiseEq<af::half>(aName, bName, a, b, maxAbsDiff);
+            return elemWiseEq<half_float::half>(aName, bName, a, b, maxAbsDiff);
             break;
         default:
             return ::testing::AssertionFailure()
@@ -1501,6 +1503,45 @@ ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
     }
 }
 
+struct absMatch {
+    float diff_;
+    absMatch(float diff) : diff_(diff) {}
+
+    template<typename T>
+    bool operator()(T lhs, T rhs) {
+        if (diff_ > 0) {
+            using half_float::abs;
+            using std::abs;
+            return abs(rhs - lhs) <= diff_;
+        } else {
+            return boost::math::epsilon_difference(lhs, rhs) < T(1.f);
+        }
+    }
+};
+
+template<>
+bool absMatch::operator()<af::af_cfloat>(af::af_cfloat lhs, af::af_cfloat rhs) {
+    return af::abs(rhs - lhs) <= diff_;
+}
+
+template<>
+bool absMatch::operator()<af::af_cdouble>(af::af_cdouble lhs,
+                                          af::af_cdouble rhs) {
+    return af::abs(rhs - lhs) <= diff_;
+}
+
+template<>
+bool absMatch::operator()<std::complex<float> >(std::complex<float> lhs,
+                                                std::complex<float> rhs) {
+    return std::abs(rhs - lhs) <= diff_;
+}
+
+template<>
+bool absMatch::operator()<std::complex<double> >(std::complex<double> lhs,
+                                                 std::complex<double> rhs) {
+    return std::abs(rhs - lhs) <= diff_;
+}
+
 template<typename T>
 ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
                                       const std::vector<T> &a, af::dim4 aDims,
@@ -1687,7 +1728,6 @@ INSTANTIATE(long long);
 INSTANTIATE(unsigned long long);
 INSTANTIATE(std::complex<float>);
 INSTANTIATE(std::complex<double>);
-INSTANTIATE(af_half);
 #undef INSTANTIATE
 
 int main(int argc, char **argv) {
diff --git a/test/join.cpp b/test/join.cpp
index 24120c2b3f..0024fe5542 100644
--- a/test/join.cpp
+++ b/test/join.cpp
@@ -48,7 +48,7 @@ class Join : public ::testing::Test {
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
                          intl, uintl, char, unsigned char, short, ushort,
-                         af_half>
+                         half_float::half>
     TestTypes;
 
 // register the type list
diff --git a/test/relative_difference.hpp b/test/relative_difference.hpp
new file mode 100644
index 0000000000..3fdfb28dc3
--- /dev/null
+++ b/test/relative_difference.hpp
@@ -0,0 +1,135 @@
+//  (C) Copyright John Maddock 2006, 2015
+//  Use, modification and distribution are subject to the
+//  Boost Software License, Version 1.0. (See accompanying file
+//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef BOOST_MATH_RELATIVE_ERROR
+#define BOOST_MATH_RELATIVE_ERROR
+
+#include <boost/math/special_functions/fpclassify.hpp>
+#include <boost/math/tools/precision.hpp>
+#include <boost/math/tools/promotion.hpp>
+
+namespace boost {
+namespace math {
+
+template<class T, class U>
+typename boost::math::tools::promote_args<T, U>::type relative_difference(
+    const T& arg_a, const U& arg_b) {
+    typedef typename boost::math::tools::promote_args<T, U>::type result_type;
+    result_type a = arg_a;
+    result_type b = arg_b;
+    BOOST_MATH_STD_USING
+#ifdef BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS
+    //
+    // If math.h has no long double support we can't rely
+    // on the math functions generating exponents outside
+    // the range of a double:
+    //
+    result_type min_val = (std::max)(
+        tools::min_value<result_type>(),
+        static_cast<result_type>((std::numeric_limits<double>::min)()));
+    result_type max_val = (std::min)(
+        tools::max_value<result_type>(),
+        static_cast<result_type>((std::numeric_limits<double>::max)()));
+#else
+    result_type min_val = tools::min_value<result_type>();
+    result_type max_val = tools::max_value<result_type>();
+#endif
+    // Screen out NaN's first, if either value is a NaN then the distance is
+    // "infinite":
+    if ((boost::math::isnan)(a) || (boost::math::isnan)(b)) return max_val;
+    // Screen out infinities:
+    if (fabs(b) > max_val) {
+        if (fabs(a) > max_val)
+            return (a < 0) == (b < 0)
+                       ? result_type(0)
+                       : max_val;  // one infinity is as good as another!
+        else
+            return max_val;  // one infinity and one finite value implies
+                             // infinite difference
+    } else if (fabs(a) > max_val)
+        return max_val;  // one infinity and one finite value implies infinite
+                         // difference
+
+    //
+    // If the values have different signs, treat as infinite difference:
+    //
+    if (((a < 0) != (b < 0)) && (a != 0) && (b != 0)) return max_val;
+    a = fabs(a);
+    b = fabs(b);
+    //
+    // Now deal with zero's, if one value is zero (or denorm) then treat it the
+    // same as min_val for the purposes of the calculation that follows:
+    //
+    if (a < min_val) a = min_val;
+    if (b < min_val) b = min_val;
+
+    return (std::max)(fabs((a - b) / a), fabs((a - b) / b));
+}
+
+#if (defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__)) && \
+    (LDBL_MAX_EXP <= DBL_MAX_EXP)
+template<>
+inline boost::math::tools::promote_args<double, double>::type
+relative_difference(const double& arg_a, const double& arg_b) {
+    BOOST_MATH_STD_USING
+    double a = arg_a;
+    double b = arg_b;
+    //
+    // On Mac OS X we evaluate "double" functions at "long double" precision,
+    // but "long double" actually has a very slightly narrower range than
+    // "double"! Therefore use the range of "long double" as our limits since
+    // results outside that range may have been truncated to 0 or INF:
+    //
+    double min_val = (std::max)((double)tools::min_value<long double>(),
+                                tools::min_value<double>());
+    double max_val = (std::min)((double)tools::max_value<long double>(),
+                                tools::max_value<double>());
+
+    // Screen out NaN's first, if either value is a NaN then the distance is
+    // "infinite":
+    if ((boost::math::isnan)(a) || (boost::math::isnan)(b)) return max_val;
+    // Screen out infinities:
+    if (fabs(b) > max_val) {
+        if (fabs(a) > max_val)
+            return 0;  // one infinity is as good as another!
+        else
+            return max_val;  // one infinity and one finite value implies
+                             // infinite difference
+    } else if (fabs(a) > max_val)
+        return max_val;  // one infinity and one finite value implies infinite
+                         // difference
+
+    //
+    // If the values have different signs, treat as infinite difference:
+    //
+    if (((a < 0) != (b < 0)) && (a != 0) && (b != 0)) return max_val;
+    a = fabs(a);
+    b = fabs(b);
+    //
+    // Now deal with zero's, if one value is zero (or denorm) then treat it the
+    // same as min_val for the purposes of the calculation that follows:
+    //
+    if (a < min_val) a = min_val;
+    if (b < min_val) b = min_val;
+
+    return (std::max)(fabs((a - b) / a), fabs((a - b) / b));
+}
+#endif
+
+template<class T, class U>
+inline typename boost::math::tools::promote_args<T, U>::type epsilon_difference(
+    const T& arg_a, const U& arg_b) {
+    typedef typename boost::math::tools::promote_args<T, U>::type result_type;
+    result_type r = relative_difference(arg_a, arg_b);
+    if (tools::max_value<result_type>() *
+            boost::math::tools::epsilon<result_type>() <
+        r)
+        return tools::max_value<result_type>();
+    return r / boost::math::tools::epsilon<result_type>();
+}
+}  // namespace math
+}  // namespace boost
+
+#endif
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 33b03db93b..024b46657f 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -273,19 +273,6 @@ ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
                                       const std::vector<T> &b, af::dim4 bDims,
                                       float maxAbsDiff, IntegerTag);
 
-struct absMatch {
-    float diff_;
-    absMatch(float diff) : diff_(diff) {}
-
-    template<typename T>
-    bool operator()(T lhs, T rhs) {
-        using af::abs;
-        using half_float::abs;
-        using std::abs;
-        return abs(rhs - lhs) <= diff_;
-    }
-};
-
 template<typename T>
 ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
                                       const std::vector<T> &a, af::dim4 aDims,

From 24665c7afb2d2f72cce34494c2ae2acce7ee6205 Mon Sep 17 00:00:00 2001
From: Pradeep Garigipati <pradeep.garigipati@gmail.com>
Date: Sat, 19 Feb 2022 11:23:55 +0530
Subject: [PATCH 007/473] Remove double underscore from identifiers

---
 src/backend/common/graphics_common.cpp | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/backend/common/graphics_common.cpp b/src/backend/common/graphics_common.cpp
index e8e24834b9..fc8256f999 100644
--- a/src/backend/common/graphics_common.cpp
+++ b/src/backend/common/graphics_common.cpp
@@ -182,10 +182,10 @@ void makeContextCurrent(fg_window window) {
 double step_round(const double in, const bool dir) {
     if (in == 0) { return 0; }
 
-    static const double __log2 = log10(2);
-    static const double __log4 = log10(4);
-    static const double __log6 = log10(6);
-    static const double __log8 = log10(8);
+    static const double LOG2 = log10(2);
+    static const double LOG4 = log10(4);
+    static const double LOG6 = log10(6);
+    static const double LOG8 = log10(8);
 
     // log_in is of the form "s abc.xyz", where
     // s is either + or -; + indicates abs(in) >= 1 and - indicates 0 < abs(in)
@@ -206,25 +206,25 @@ double step_round(const double in, const bool dir) {
 
     // Round up
     if (op_dir) {
-        if (dec <= __log2) {
+        if (dec <= LOG2) {
             mult = 2;
-        } else if (dec <= __log4) {
+        } else if (dec <= LOG4) {
             mult = 4;
-        } else if (dec <= __log6) {
+        } else if (dec <= LOG6) {
             mult = 6;
-        } else if (dec <= __log8) {
+        } else if (dec <= LOG8) {
             mult = 8;
         } else {
             mult = 10;
         }
     } else {  // Round down
-        if (dec < __log2) {
+        if (dec < LOG2) {
             mult = 1;
-        } else if (dec < __log4) {
+        } else if (dec < LOG4) {
             mult = 2;
-        } else if (dec < __log6) {
+        } else if (dec < LOG6) {
             mult = 4;
-        } else if (dec < __log8) {
+        } else if (dec < LOG8) {
             mult = 6;
         } else {
             mult = 8;

From 394466f234f61d0e2a7664236c822ba4c12bb08c Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 24 Feb 2022 13:44:58 -0500
Subject: [PATCH 008/473] Update docs to new doxygen version. Fix warnings

---
 docs/doxygen.mk                | 586 +++++++++++++++++++++------------
 docs/footer.htm                |  68 +---
 docs/header.htm                |  66 ++--
 docs/pages/install.md          |   8 +-
 docs/pages/using_on_linux.md   |   2 +-
 docs/pages/using_on_osx.md     |   4 +-
 docs/pages/using_on_windows.md |  33 +-
 7 files changed, 452 insertions(+), 315 deletions(-)

diff --git a/docs/doxygen.mk b/docs/doxygen.mk
index b9bfa4158e..b7eded0238 100644
--- a/docs/doxygen.mk
+++ b/docs/doxygen.mk
@@ -1,4 +1,4 @@
-# Doxyfile 1.8.14
+# Doxyfile 1.9.3
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -17,10 +17,10 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all text
-# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
-# built into libc) for the transcoding. See
+# This tag specifies the encoding used for all characters in the configuration
+# file that follow. The default is UTF-8 which is also the encoding used for all
+# text before the first occurrence of this tag. Doxygen uses libiconv (or the
+# iconv built into libc) for the transcoding. See
 # https://www.gnu.org/software/libiconv/ for the list of possible encodings.
 # The default value is: UTF-8.
 
@@ -32,13 +32,13 @@ DOXYFILE_ENCODING      = UTF-8
 # title of most generated pages and in a few other places.
 # The default value is: My Project.
 
-PROJECT_NAME           = "${PROJECT_NAME}"
+PROJECT_NAME           = ${PROJECT_NAME}
 
 # The PROJECT_NUMBER tag can be used to enter a project or revision number. This
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "${AF_VERSION}"
+PROJECT_NUMBER         = ${AF_VERSION}
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -180,6 +180,16 @@ SHORT_NAMES            = NO
 
 JAVADOC_AUTOBRIEF      = YES
 
+# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line
+# such as
+# /***************
+# as being the beginning of a Javadoc-style comment "banner". If set to NO, the
+# Javadoc-style will behave just like regular comments and it will not be
+# interpreted by doxygen.
+# The default value is: NO.
+
+JAVADOC_BANNER         = NO
+
 # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
 # line (until the first dot) of a Qt-style comment as the brief description. If
 # set to NO, the Qt-style will behave just like regular Qt-style comments (thus
@@ -200,6 +210,14 @@ QT_AUTOBRIEF           = NO
 
 MULTILINE_CPP_IS_BRIEF = NO
 
+# By default Python docstrings are displayed as preformatted text and doxygen's
+# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the
+# doxygen's special commands can be used and the contents of the docstring
+# documentation blocks is shown as doxygen documentation.
+# The default value is: YES.
+
+PYTHON_DOCSTRING       = YES
+
 # If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
 # documentation from any documented member that it re-implements.
 # The default value is: YES.
@@ -223,12 +241,16 @@ TAB_SIZE               = 4
 # the documentation. An alias has the form:
 # name=value
 # For example adding
-# "sideeffect=@par Side Effects:\n"
+# "sideeffect=@par Side Effects:^^"
 # will allow you to put the command \sideeffect (or @sideeffect) in the
 # documentation, which will result in a user-defined paragraph with heading
-# "Side Effects:". You can put \n's in the value part of an alias to insert
-# newlines (in the resulting output). You can put ^^ in the value part of an
-# alias to insert a newline as if a physical newline was in the original file.
+# "Side Effects:". Note that you cannot put \n's in the value part of an alias
+# to insert newlines (in the resulting output). You can put ^^ in the value part
+# of an alias to insert a newline as if a physical newline was in the original
+# file. When you need a literal { or } or , in the value part of an alias you
+# have to escape them by means of a backslash (\), this can lead to conflicts
+# with the commands \{ and \} for these it is advised to use the version @{ and
+# @} or use a double escape (\\{ and \\})
 
 ALIASES                = "support{1}=<DIV class=\"support\">\1</DIV>" \
                          "opencl=<IMG src=\"OpenCL.png\" alt=\"OpenCL Support\" />" \
@@ -246,17 +268,14 @@ ALIASES                = "support{1}=<DIV class=\"support\">\1</DIV>" \
                          "funcgroups{5}=\ingroup \3 \4 \5 \n @{ \n \defgroup \1 \2 \n @{ \n" \
                          "funcgroups{6}=\ingroup \3 \4 \5 \6 \n @{ \n \defgroup \1 \2 \n @{ \n" \
                          "endfuncgroups=@} \n @}" \
-                         "PR{1}=[[#\1](https://github.com/arrayfire/arrayfire/pull/\1)]"
-
-# Now add special commands for math equations. All of the following commands
-# are only expected to be used inside math mode
-ALIASES += "dims{4}=\f$ [\1 \ \2 \ \3 \ \4] \f$"
-ALIASES += "shape_eq{5}=\f$ \underset{[\2 \ \3 \ \4 \ \5]}{\1} \f$"
-ALIASES += "shape_t{5}=\underset{[\2 \ \3 \ \4 \ \5]}{\1}"
-ALIASES += "convolve_eq{2}=\f$ \1 \ast \2 \f$"
-ALIASES += "convolve_t{2}=\1 \ast \2"
-ALIASES += "set_eq{2}=\f$ \left\\{ \1 \ \Bigg\vert \ \2 \right\\} \f$"
-ALIASES += "set_t{2}=\left\\\{ \1 \ \Bigg\vert \ \2 \right\\\}"
+                         "PR{1}=[[#\1](https://github.com/arrayfire/arrayfire/pull/\1)]" \
+                         "dims{4}=\f$ [\1 \ \2 \ \3 \ \4] \f$" \
+                         "shape_eq{5}=\f$ \underset{[\2 \ \3 \ \4 \ \5]}{\1} \f$" \
+                         "shape_t{5}=\underset{[\2 \ \3 \ \4 \ \5]}{\1}" \
+                         "convolve_eq{2}=\f$ \1 \ast \2 \f$" \
+                         "convolve_t{2}=\1 \ast \2" \
+                         "set_eq{2}=\f$ \left\\{ \1 \ \Bigg\vert \ \2 \right\\} \f$" \
+                         "set_t{2}=\left\\\{ \1 \ \Bigg\vert \ \2 \right\\\}"
 
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
@@ -286,28 +305,40 @@ OPTIMIZE_FOR_FORTRAN   = NO
 
 OPTIMIZE_OUTPUT_VHDL   = NO
 
+# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice
+# sources only. Doxygen will then generate output that is more tailored for that
+# language. For instance, namespaces will be presented as modules, types will be
+# separated into more groups, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_SLICE  = NO
+
 # Doxygen selects the parser to use depending on the extension of the files it
 # parses. With this tag you can assign which parser to use for a given
 # extension. Doxygen has a built-in mapping, but you can override or extend it
 # using this tag. The format is ext=language, where ext is a file extension, and
-# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
-# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
-# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
-# Fortran. In the later case the parser tries to guess whether the code is fixed
-# or free formatted code, this is the default for Fortran type files), VHDL. For
-# instance to make doxygen treat .inc files as Fortran files (default is PHP),
-# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+# language is one of the parsers supported by doxygen: IDL, Java, JavaScript,
+# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice,
+# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
+# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
+# tries to guess whether the code is fixed or free formatted code, this is the
+# default for Fortran type files). For instance to make doxygen treat .inc files
+# as Fortran files (default is PHP), and .f files as C (default is Fortran),
+# use: inc=Fortran f=C.
 #
 # Note: For files without extension you can use no_extension as a placeholder.
 #
 # Note that for custom extensions you also need to set FILE_PATTERNS otherwise
-# the files are not read by doxygen.
+# the files are not read by doxygen. When specifying no_extension you should add
+# * to the FILE_PATTERNS.
+#
+# Note see also the list of default file extension mappings.
 
 EXTENSION_MAPPING      =
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
-# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# documentation. See https://daringfireball.net/projects/markdown/ for details.
 # The output of markdown processing is further processed by doxygen, so you can
 # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
 # case of backward compatibilities issues.
@@ -319,7 +350,7 @@ MARKDOWN_SUPPORT       = YES
 # to that level are automatically included in the table of contents, even if
 # they do not have an id attribute.
 # Note: This feature currently applies only to Markdown headings.
-# Minimum value: 0, maximum value: 99, default value: 0.
+# Minimum value: 0, maximum value: 99, default value: 5.
 # This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
 
 TOC_INCLUDE_HEADINGS   = 0
@@ -435,6 +466,19 @@ TYPEDEF_HIDES_STRUCT   = NO
 
 LOOKUP_CACHE_SIZE      = 0
 
+# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use
+# during processing. When set to 0 doxygen will based this on the number of
+# cores available in the system. You can set it explicitly to a value larger
+# than 0 to get more control over the balance between CPU load and processing
+# speed. At this moment only the input processing can be done using multiple
+# threads. Since this is still an experimental feature the default is set to 1,
+# which effectively disables parallel processing. Please report any issues you
+# encounter. Generating dot graphs in parallel is controlled by the
+# DOT_NUM_THREADS setting.
+# Minimum value: 0, maximum value: 32, default value: 1.
+
+NUM_PROC_THREADS       = 0
+
 #---------------------------------------------------------------------------
 # Build related configuration options
 #---------------------------------------------------------------------------
@@ -455,6 +499,12 @@ EXTRACT_ALL            = YES
 
 EXTRACT_PRIVATE        = NO
 
+# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual
+# methods of a class will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIV_VIRTUAL   = NO
+
 # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
 # scope will be included in the documentation.
 # The default value is: NO.
@@ -492,6 +542,13 @@ EXTRACT_LOCAL_METHODS  = NO
 
 EXTRACT_ANON_NSPACES   = NO
 
+# If this flag is set to YES, the name of an unnamed parameter in a declaration
+# will be determined by the corresponding definition. By default unnamed
+# parameters remain unnamed in the output.
+# The default value is: YES.
+
+RESOLVE_UNNAMED_PARAMS = YES
+
 # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
 # undocumented members inside documented classes or files. If set to NO these
 # members will be included in the various overviews, but no documentation
@@ -509,8 +566,8 @@ HIDE_UNDOC_MEMBERS     = NO
 HIDE_UNDOC_CLASSES     = NO
 
 # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
-# (class|struct|union) declarations. If set to NO, these declarations will be
-# included in the documentation.
+# declarations. If set to NO, these declarations will be included in the
+# documentation.
 # The default value is: NO.
 
 HIDE_FRIEND_COMPOUNDS  = NO
@@ -529,11 +586,18 @@ HIDE_IN_BODY_DOCS      = NO
 
 INTERNAL_DOCS          = NO
 
-# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
-# names in lower-case letters. If set to YES, upper-case letters are also
-# allowed. This is useful if you have classes or files whose names only differ
-# in case and if your file system supports case sensitive file names. Windows
-# and Mac users are advised to set this option to NO.
+# With the correct setting of option CASE_SENSE_NAMES doxygen will better be
+# able to match the capabilities of the underlying filesystem. In case the
+# filesystem is case sensitive (i.e. it supports files in the same directory
+# whose names only differ in casing), the option must be set to YES to properly
+# deal with such files in case they appear in the input. For filesystems that
+# are not case sensitive the option should be be set to NO to properly deal with
+# output files written for symbols that only differ in casing, such as for two
+# classes, one named CLASS and the other named Class, and to also support
+# references to files without having to specify the exact matching casing. On
+# Windows (including Cygwin) and MacOS, users should typically set this option
+# to NO, whereas on Linux or other Unix flavors it should typically be set to
+# YES.
 # The default value is: system dependent.
 
 CASE_SENSE_NAMES       = YES
@@ -552,6 +616,12 @@ HIDE_SCOPE_NAMES       = YES
 
 HIDE_COMPOUND_REFERENCE= NO
 
+# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class
+# will show which file needs to be included to use the class.
+# The default value is: YES.
+
+SHOW_HEADERFILE        = YES
+
 # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
 # the files that are included by a file in the documentation of that file.
 # The default value is: YES.
@@ -709,7 +779,8 @@ FILE_VERSION_FILTER    = "/bin/sh -c 'git log --pretty=\"format:%ci, (build %h)\
 # output files in an output format independent way. To create the layout file
 # that represents doxygen's defaults, run doxygen with the -l option. You can
 # optionally specify a file name after the option, if omitted DoxygenLayout.xml
-# will be used as the name of the layout file.
+# will be used as the name of the layout file. See also section "Changing the
+# layout of pages" for information.
 #
 # Note that if you run doxygen from a directory containing a file called
 # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
@@ -755,23 +826,35 @@ WARNINGS               = YES
 WARN_IF_UNDOCUMENTED   = YES
 
 # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
-# potential errors in the documentation, such as not documenting some parameters
-# in a documented function, or documenting parameters that don't exist or using
-# markup commands wrongly.
+# potential errors in the documentation, such as documenting some parameters in
+# a documented function twice, or documenting parameters that don't exist or
+# using markup commands wrongly.
 # The default value is: YES.
 
 WARN_IF_DOC_ERROR      = YES
 
+# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete
+# function parameter documentation. If set to NO, doxygen will accept that some
+# parameters have no documentation without warning.
+# The default value is: YES.
+
+WARN_IF_INCOMPLETE_DOC = YES
+
 # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
 # are documented, but have no documentation for their parameters or return
-# value. If set to NO, doxygen will only warn about wrong or incomplete
-# parameter documentation, but not about the absence of documentation.
+# value. If set to NO, doxygen will only warn about wrong parameter
+# documentation, but not about the absence of documentation. If EXTRACT_ALL is
+# set to YES then this flag will automatically be disabled. See also
+# WARN_IF_INCOMPLETE_DOC
 # The default value is: NO.
 
 WARN_NO_PARAMDOC       = YES
 
 # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
-# a warning is encountered.
+# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS
+# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but
+# at the end of the doxygen process doxygen will return with a non-zero status.
+# Possible values are: NO, YES and FAIL_ON_WARNINGS.
 # The default value is: NO.
 
 WARN_AS_ERROR          = NO
@@ -788,7 +871,10 @@ WARN_FORMAT            = "$file:$line: $text"
 
 # The WARN_LOGFILE tag can be used to specify a file to which warning and error
 # messages should be written. If left blank the output is written to standard
-# error (stderr).
+# error (stderr). In case the file specified cannot be opened for writing the
+# warning and error messages are written to standard error. When as file - is
+# specified the warning and error messages are written to standard output
+# (stdout).
 
 WARN_LOGFILE           =
 
@@ -810,8 +896,8 @@ INPUT                  = ${DOCS_DIR}/pages \
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
 # libiconv (or the iconv built into libc) for the transcoding. See the libiconv
-# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
-# possible encodings.
+# documentation (see:
+# https://www.gnu.org/software/libiconv/) for the list of possible encodings.
 # The default value is: UTF-8.
 
 INPUT_ENCODING         = UTF-8
@@ -824,11 +910,15 @@ INPUT_ENCODING         = UTF-8
 # need to set EXTENSION_MAPPING for the extension otherwise the files are not
 # read by doxygen.
 #
+# Note the list of default checked file patterns might differ from the list of
+# default file extension mappings.
+#
 # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
 # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
-# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
-# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08,
-# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf.
+# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml,
+# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C
+# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd,
+# *.vhdl, *.ucf, *.qsf and *.ice.
 
 FILE_PATTERNS          =
 
@@ -867,7 +957,7 @@ EXCLUDE_PATTERNS       = *.cpp
 # (namespaces, classes, functions, etc.) that should be excluded from the
 # output. The symbol name can be a fully qualified name, a word, or if the
 # wildcard * is used, a substring. Examples: ANamespace, AClass,
-# AClass::ANamespace, ANamespace::*Test
+# ANamespace::AClass, ANamespace::*Test
 #
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
@@ -987,7 +1077,7 @@ INLINE_SOURCES         = YES
 STRIP_CODE_COMMENTS    = YES
 
 # If the REFERENCED_BY_RELATION tag is set to YES then for each documented
-# function all documented functions referencing it will be listed.
+# entity all documented functions referencing it will be listed.
 # The default value is: NO.
 
 REFERENCED_BY_RELATION = NO
@@ -1024,7 +1114,7 @@ SOURCE_TOOLTIPS        = YES
 #
 # To use it do the following:
 # - Install the latest version of global
-# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file
 # - Make sure the INPUT points to the root of the source tree
 # - Run doxygen as normal
 #
@@ -1046,36 +1136,6 @@ USE_HTAGS              = NO
 
 VERBATIM_HEADERS       = YES
 
-# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the
-# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the
-# cost of reduced performance. This can be particularly helpful with template
-# rich C++ code for which doxygen's built-in parser lacks the necessary type
-# information.
-# Note: The availability of this option depends on whether or not doxygen was
-# generated with the -Duse-libclang=ON option for CMake.
-# The default value is: NO.
-
-#CLANG_ASSISTED_PARSING = NO
-
-# If clang assisted parsing is enabled you can provide the compiler with command
-# line options that you would normally use when invoking the compiler. Note that
-# the include paths will already be set by doxygen for the files and directories
-# specified with INPUT and INCLUDE_PATH.
-# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
-
-#CLANG_OPTIONS          = -Wno-pragma-once-outside-header
-
-# If clang assisted parsing is enabled you can provide the clang parser with the
-# path to the compilation database (see:
-# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files
-# were built. This is equivalent to specifying the "-p" option to a clang tool,
-# such as clang-check. These options will then be passed to the parser.
-# Note: The availability of this option depends on whether or not doxygen was
-# generated with the -Duse-libclang=ON option for CMake.
-# The default value is: 0.
-
-#CLANG_COMPILATION_DATABASE_PATH  = ${ArrayFire_BINARY_DIR}
-
 #---------------------------------------------------------------------------
 # Configuration options related to the alphabetical class index
 #---------------------------------------------------------------------------
@@ -1186,7 +1246,7 @@ HTML_EXTRA_FILES       =
 
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
-# this color. Hue is specified as an angle on a colorwheel, see
+# this color. Hue is specified as an angle on a color-wheel, see
 # https://en.wikipedia.org/wiki/Hue for more information. For instance the value
 # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
 # purple, and 360 is red again.
@@ -1196,7 +1256,7 @@ HTML_EXTRA_FILES       =
 HTML_COLORSTYLE_HUE    = 19
 
 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
-# in the HTML output. For a value of 0 the output will use grayscales only. A
+# in the HTML output. For a value of 0 the output will use gray-scales only. A
 # value of 255 will produce the most vivid colors.
 # Minimum value: 0, maximum value: 255, default value: 100.
 # This tag requires that the tag GENERATE_HTML is set to YES.
@@ -1225,9 +1285,9 @@ HTML_TIMESTAMP         = YES
 
 # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
 # documentation will contain a main index with vertical navigation menus that
-# are dynamically created via Javascript. If disabled, the navigation index will
+# are dynamically created via JavaScript. If disabled, the navigation index will
 # consists of multiple levels of tabs that are statically embedded in every HTML
-# page. Disable this option to support browsers that do not have Javascript,
+# page. Disable this option to support browsers that do not have JavaScript,
 # like the Qt help browser.
 # The default value is: YES.
 # This tag requires that the tag GENERATE_HTML is set to YES.
@@ -1257,13 +1317,14 @@ HTML_INDEX_NUM_ENTRIES = 100
 
 # If the GENERATE_DOCSET tag is set to YES, additional index files will be
 # generated that can be used as input for Apple's Xcode 3 integrated development
-# environment (see: https://developer.apple.com/tools/xcode/), introduced with
-# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
-# Makefile in the HTML output directory. Running make will produce the docset in
-# that directory and running make install will install the docset in
+# environment (see:
+# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To
+# create a documentation set, doxygen will generate a Makefile in the HTML
+# output directory. Running make will produce the docset in that directory and
+# running make install will install the docset in
 # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
-# startup. See https://developer.apple.com/tools/creatingdocsetswithdoxygen.html
-# for more information.
+# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy
+# genXcode/_index.html for more information.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
@@ -1277,6 +1338,13 @@ GENERATE_DOCSET        = NO
 
 DOCSET_FEEDNAME        = "Doxygen generated docs"
 
+# This tag determines the URL of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDURL         =
+
 # This tag specifies a string that should uniquely identify the documentation
 # set bundle. This should be a reverse domain-name style string, e.g.
 # com.mycompany.MyDocSet. Doxygen will append .docset to the name.
@@ -1302,8 +1370,12 @@ DOCSET_PUBLISHER_NAME  = Publisher
 # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
 # additional HTML index files: index.hhp, index.hhc, and index.hhk. The
 # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
-# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
-# Windows.
+# on Windows. In the beginning of 2021 Microsoft took the original page, with
+# a.o. the download links, offline the HTML help workshop was already many years
+# in maintenance mode). You can download the HTML help workshop from the web
+# archives at Installation executable (see:
+# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo
+# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe).
 #
 # The HTML Help Workshop contains a compiler that can convert all HTML output
 # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
@@ -1333,7 +1405,7 @@ CHM_FILE               =
 HHC_LOCATION           =
 
 # The GENERATE_CHI flag controls if a separate .chi index file is generated
-# (YES) or that it should be included in the master .chm file (NO).
+# (YES) or that it should be included in the main .chm file (NO).
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTMLHELP is set to YES.
 
@@ -1378,7 +1450,8 @@ QCH_FILE               =
 
 # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
 # Project output. For more information please see Qt Help Project / Namespace
-# (see: http://doc.qt.io/qt-4.8/qthelpproject.html#namespace).
+# (see:
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace).
 # The default value is: org.doxygen.Project.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1386,7 +1459,8 @@ QHP_NAMESPACE          = org.doxygen.Project
 
 # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
 # Help Project output. For more information please see Qt Help Project / Virtual
-# Folders (see: http://doc.qt.io/qt-4.8/qthelpproject.html#virtual-folders).
+# Folders (see:
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders).
 # The default value is: doc.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1394,28 +1468,30 @@ QHP_VIRTUAL_FOLDER     = doc
 
 # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
 # filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://doc.qt.io/qt-4.8/qthelpproject.html#custom-filters).
+# Filters (see:
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_CUST_FILTER_NAME   =
 
 # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
 # custom filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://doc.qt.io/qt-4.8/qthelpproject.html#custom-filters).
+# Filters (see:
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_CUST_FILTER_ATTRS  =
 
 # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
 # project's filter section matches. Qt Help Project / Filter Attributes (see:
-# http://doc.qt.io/qt-4.8/qthelpproject.html#filter-attributes).
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_SECT_FILTER_ATTRS  =
 
-# The QHG_LOCATION tag can be used to specify the location of Qt's
-# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
-# generated .qhp file.
+# The QHG_LOCATION tag can be used to specify the location (absolute path
+# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to
+# run qhelpgenerator on the generated .qhp file.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHG_LOCATION           =
@@ -1458,16 +1534,28 @@ DISABLE_INDEX          = NO
 # to work a browser that supports JavaScript, DHTML, CSS and frames is required
 # (i.e. any modern browser). Windows users are probably better off using the
 # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
-# further fine-tune the look of the index. As an example, the default style
-# sheet generated by doxygen has an example that shows how to put an image at
-# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
-# the same information as the tab index, you could consider setting
-# DISABLE_INDEX to YES when enabling this option.
+# further fine tune the look of the index (see "Fine-tuning the output"). As an
+# example, the default style sheet generated by doxygen has an example that
+# shows how to put an image at the root of the tree instead of the PROJECT_NAME.
+# Since the tree basically has the same information as the tab index, you could
+# consider setting DISABLE_INDEX to YES when enabling this option.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
 GENERATE_TREEVIEW      = YES
 
+# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the
+# FULL_SIDEBAR option determines if the side bar is limited to only the treeview
+# area (value NO) or if it should extend to the full height of the window (value
+# YES). Setting this to YES gives a layout similar to
+# https://docs.readthedocs.io with more room for contents, but less room for the
+# project logo, title, and description. If either GENERATE_TREEVIEW or
+# DISABLE_INDEX is set to NO, this option has no effect.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FULL_SIDEBAR           = NO
+
 # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
 # doxygen will group on one line in the generated HTML documentation.
 #
@@ -1492,6 +1580,24 @@ TREEVIEW_WIDTH         = 250
 
 EXT_LINKS_IN_WINDOW    = NO
 
+# If the OBFUSCATE_EMAILS tag is set to YES, doxygen will obfuscate email
+# addresses.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+OBFUSCATE_EMAILS       = YES
+
+# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg
+# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see
+# https://inkscape.org) to generate formulas as SVG images instead of PNGs for
+# the HTML output. These images will generally look nicer at scaled resolutions.
+# Possible values are: png (the default) and svg (looks nicer but requires the
+# pdf2svg or inkscape tool).
+# The default value is: png.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FORMULA_FORMAT    = png
+
 # Use this tag to change the font size of LaTeX formulas included as images in
 # the HTML documentation. When you change the font size after a successful
 # doxygen run you need to manually remove any form_*.png images from the HTML
@@ -1512,8 +1618,14 @@ FORMULA_FONTSIZE       = 12
 
 FORMULA_TRANSPARENT    = YES
 
+# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands
+# to create new LaTeX commands to be used in formulas as building blocks. See
+# the section "Including formulas" for details.
+
+FORMULA_MACROFILE      =
+
 # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
-# https://www.mathjax.org) which uses client side Javascript for the rendering
+# https://www.mathjax.org) which uses client side JavaScript for the rendering
 # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
 # installed or if you want to formulas look prettier in the HTML output. When
 # enabled you may also need to install MathJax separately and configure the path
@@ -1523,11 +1635,29 @@ FORMULA_TRANSPARENT    = YES
 
 USE_MATHJAX            = YES
 
+# With MATHJAX_VERSION it is possible to specify the MathJax version to be used.
+# Note that the different versions of MathJax have different requirements with
+# regards to the different settings, so it is possible that also other MathJax
+# settings have to be changed when switching between the different MathJax
+# versions.
+# Possible values are: MathJax_2 and MathJax_3.
+# The default value is: MathJax_2.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_VERSION        = MathJax_2
+
 # When MathJax is enabled you can set the default output format to be used for
-# the MathJax output. See the MathJax site (see:
-# http://docs.mathjax.org/en/latest/output.html) for more details.
+# the MathJax output. For more details about the output format see MathJax
+# version 2 (see:
+# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3
+# (see:
+# http://docs.mathjax.org/en/latest/web/components/output.html).
 # Possible values are: HTML-CSS (which is slower, but has the best
-# compatibility), NativeMML (i.e. MathML) and SVG.
+# compatibility. This is the name for Mathjax version 2, for MathJax version 3
+# this will be translated into chtml), NativeMML (i.e. MathML. Only supported
+# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This
+# is the name for Mathjax version 3, for MathJax version 2 this will be
+# translated into HTML-CSS) and SVG.
 # The default value is: HTML-CSS.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
@@ -1540,22 +1670,29 @@ MATHJAX_FORMAT         = HTML-CSS
 # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
 # Content Delivery Network so you can quickly see the result without installing
 # MathJax. However, it is strongly recommended to install a local copy of
-# MathJax from https://www.mathjax.org before deployment.
-# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/.
+# MathJax from https://www.mathjax.org before deployment. The default value is:
+# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2
+# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1
 
 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
 # extension names that should be enabled during MathJax rendering. For example
+# for MathJax version 2 (see
+# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions):
 # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# For example for MathJax version 3 (see
+# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html):
+# MATHJAX_EXTENSIONS = ams
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
 MATHJAX_EXTENSIONS     =
 
 # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
 # of code that will be used on startup of the MathJax code. See the MathJax site
-# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# (see:
+# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an
 # example see the documentation.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
@@ -1583,7 +1720,7 @@ MATHJAX_CODEFILE       =
 SEARCHENGINE           = NO
 
 # When the SERVER_BASED_SEARCH tag is enabled the search engine will be
-# implemented using a web server instead of a web client using Javascript. There
+# implemented using a web server instead of a web client using JavaScript. There
 # are two flavors of web server based searching depending on the EXTERNAL_SEARCH
 # setting. When disabled, doxygen will generate a PHP script for searching and
 # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
@@ -1602,7 +1739,8 @@ SERVER_BASED_SEARCH    = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: https://xapian.org/).
+# Xapian (see:
+# https://xapian.org/).
 #
 # See the section "External Indexing and Searching" for details.
 # The default value is: NO.
@@ -1615,8 +1753,9 @@ EXTERNAL_SEARCH        = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: https://xapian.org/). See the section "External Indexing and
-# Searching" for details.
+# Xapian (see:
+# https://xapian.org/). See the section "External Indexing and Searching" for
+# details.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
 SEARCHENGINE_URL       =
@@ -1667,21 +1806,35 @@ LATEX_OUTPUT           = latex
 # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
 # invoked.
 #
-# Note that when enabling USE_PDFLATEX this option is only used for generating
-# bitmaps for formulas in the HTML output, but not in the Makefile that is
-# written to the output directory.
-# The default file is: latex.
+# Note that when not enabling USE_PDFLATEX the default is latex when enabling
+# USE_PDFLATEX the default is pdflatex and when in the later case latex is
+# chosen this is overwritten by pdflatex. For specific output languages the
+# default can have been set differently, this depends on the implementation of
+# the output language.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_CMD_NAME         = latex
 
 # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
 # index for LaTeX.
+# Note: This tag is used in the Makefile / make.bat.
+# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file
+# (.tex).
 # The default file is: makeindex.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 MAKEINDEX_CMD_NAME     = makeindex
 
+# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to
+# generate index for LaTeX. In case there is no backslash (\) as first character
+# it will be automatically added in the LaTeX code.
+# Note: This tag is used in the generated output file (.tex).
+# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat.
+# The default value is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_MAKEINDEX_CMD    = makeindex
+
 # If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
 # documents. This may be useful for small projects and may help to save some
 # trees in general.
@@ -1711,29 +1864,31 @@ PAPER_TYPE             = a4
 
 EXTRA_PACKAGES         =
 
-# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
-# generated LaTeX document. The header should contain everything until the first
-# chapter. If it is left blank doxygen will generate a standard header. See
-# section "Doxygen usage" for information on how to let doxygen write the
-# default header to a separate file.
+# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for
+# the generated LaTeX document. The header should contain everything until the
+# first chapter. If it is left blank doxygen will generate a standard header. It
+# is highly recommended to start with a default header using
+# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty
+# and then modify the file new_header.tex. See also section "Doxygen usage" for
+# information on how to generate the default header that doxygen normally uses.
 #
-# Note: Only use a user-defined header if you know what you are doing! The
-# following commands have a special meaning inside the header: $title,
-# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
-# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
-# string, for the replacement values of the other commands the user is referred
-# to HTML_HEADER.
+# Note: Only use a user-defined header if you know what you are doing!
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. The following
+# commands have a special meaning inside the header (and footer): For a
+# description of the possible markers and block names see the documentation.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_HEADER           =
 
-# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
-# generated LaTeX document. The footer should contain everything after the last
-# chapter. If it is left blank doxygen will generate a standard footer. See
+# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for
+# the generated LaTeX document. The footer should contain everything after the
+# last chapter. If it is left blank doxygen will generate a standard footer. See
 # LATEX_HEADER for more information on how to generate a default footer and what
-# special commands can be used inside the footer.
-#
-# Note: Only use a user-defined footer if you know what you are doing!
+# special commands can be used inside the footer. See also section "Doxygen
+# usage" for information on how to generate the default footer that doxygen
+# normally uses. Note: Only use a user-defined footer if you know what you are
+# doing!
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_FOOTER           =
@@ -1766,9 +1921,11 @@ LATEX_EXTRA_FILES      =
 
 PDF_HYPERLINKS         = YES
 
-# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
-# the PDF file directly from the LaTeX files. Set this option to YES, to get a
-# higher quality PDF documentation.
+# If the USE_PDFLATEX tag is set to YES, doxygen will use the engine as
+# specified with LATEX_CMD_NAME to generate the PDF file directly from the LaTeX
+# files. Set this option to YES, to get a higher quality PDF documentation.
+#
+# See also section LATEX_CMD_NAME for selecting the engine.
 # The default value is: YES.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
@@ -1776,8 +1933,7 @@ USE_PDFLATEX           = YES
 
 # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
 # command to the generated LaTeX files. This will instruct LaTeX to keep running
-# if errors occur, instead of asking the user for help. This option is also used
-# when generating formulas in HTML.
+# if errors occur, instead of asking the user for help.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
@@ -1790,16 +1946,6 @@ LATEX_BATCHMODE        = NO
 
 LATEX_HIDE_INDICES     = NO
 
-# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
-# code with syntax highlighting in the LaTeX output.
-#
-# Note that which sources are shown also depends on other settings such as
-# SOURCE_BROWSER.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_SOURCE_CODE      = NO
-
 # The LATEX_BIB_STYLE tag can be used to specify the style to use for the
 # bibliography, e.g. plainnat, or ieeetr. See
 # https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
@@ -1816,6 +1962,14 @@ LATEX_BIB_STYLE        = plain
 
 LATEX_TIMESTAMP        = NO
 
+# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
+# path from which the emoji images will be read. If a relative path is entered,
+# it will be relative to the LATEX_OUTPUT directory. If left blank the
+# LATEX_OUTPUT directory will be used.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EMOJI_DIRECTORY  =
+
 #---------------------------------------------------------------------------
 # Configuration options related to the RTF output
 #---------------------------------------------------------------------------
@@ -1855,9 +2009,9 @@ COMPACT_RTF            = NO
 
 RTF_HYPERLINKS         = NO
 
-# Load stylesheet definitions from file. Syntax is similar to doxygen's config
-# file, i.e. a series of assignments. You only have to provide replacements,
-# missing definitions are set to their default value.
+# Load stylesheet definitions from file. Syntax is similar to doxygen's
+# configuration file, i.e. a series of assignments. You only have to provide
+# replacements, missing definitions are set to their default value.
 #
 # See also section "Doxygen usage" for information on how to generate the
 # default style sheet that doxygen normally uses.
@@ -1866,22 +2020,12 @@ RTF_HYPERLINKS         = NO
 RTF_STYLESHEET_FILE    =
 
 # Set optional variables used in the generation of an RTF document. Syntax is
-# similar to doxygen's config file. A template extensions file can be generated
-# using doxygen -e rtf extensionFile.
+# similar to doxygen's configuration file. A template extensions file can be
+# generated using doxygen -e rtf extensionFile.
 # This tag requires that the tag GENERATE_RTF is set to YES.
 
 RTF_EXTENSIONS_FILE    =
 
-# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
-# with syntax highlighting in the RTF output.
-#
-# Note that which sources are shown also depends on other settings such as
-# SOURCE_BROWSER.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_RTF is set to YES.
-
-RTF_SOURCE_CODE        = NO
-
 #---------------------------------------------------------------------------
 # Configuration options related to the man page output
 #---------------------------------------------------------------------------
@@ -1953,6 +2097,13 @@ XML_OUTPUT             = xml
 
 XML_PROGRAMLISTING     = YES
 
+# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
+# namespace members in file scope as well, matching the HTML output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_NS_MEMB_FILE_SCOPE = NO
+
 #---------------------------------------------------------------------------
 # Configuration options related to the DOCBOOK output
 #---------------------------------------------------------------------------
@@ -1971,15 +2122,6 @@ GENERATE_DOCBOOK       = NO
 
 DOCBOOK_OUTPUT         = docbook
 
-# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
-# program listings (including syntax highlighting and cross-referencing
-# information) to the DOCBOOK output. Note that enabling this will significantly
-# increase the size of the DOCBOOK output.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
-
-DOCBOOK_PROGRAMLISTING = NO
-
 #---------------------------------------------------------------------------
 # Configuration options for the AutoGen Definitions output
 #---------------------------------------------------------------------------
@@ -2158,30 +2300,10 @@ EXTERNAL_GROUPS        = YES
 
 EXTERNAL_PAGES         = YES
 
-# The PERL_PATH should be the absolute path and name of the perl script
-# interpreter (i.e. the result of 'which perl').
-# The default file (with absolute path) is: /usr/bin/perl.
-
 #---------------------------------------------------------------------------
 # Configuration options related to the dot tool
 #---------------------------------------------------------------------------
 
-# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
-# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
-# NO turns the diagrams off. Note that this option also works with HAVE_DOT
-# disabled, but it is recommended to install and use dot, since it yields more
-# powerful graphs.
-# The default value is: YES.
-
-CLASS_DIAGRAMS         = YES
-
-# You can define message sequence charts within doxygen comments using the \msc
-# command. Doxygen will then run the mscgen tool (see:
-# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
-# documentation. The MSCGEN_PATH tag allows you to specify the directory where
-# the mscgen tool resides. If left empty the tool is assumed to be found in the
-# default search path.
-
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The
 # DIA_PATH tag allows you to specify the directory where the dia binary resides.
@@ -2238,11 +2360,14 @@ DOT_FONTSIZE           = 10
 
 DOT_FONTPATH           =
 
-# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
-# each documented class showing the direct and indirect inheritance relations.
-# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# If the CLASS_GRAPH tag is set to YES (or GRAPH) then doxygen will generate a
+# graph for each documented class showing the direct and indirect inheritance
+# relations. In case HAVE_DOT is set as well dot will be used to draw the graph,
+# otherwise the built-in generator will be used. If the CLASS_GRAPH tag is set
+# to TEXT the direct and indirect inheritance relations will be shown as texts /
+# links.
+# Possible values are: NO, YES, TEXT and GRAPH.
 # The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
 
 CLASS_GRAPH            = YES
 
@@ -2279,10 +2404,32 @@ UML_LOOK               = NO
 # but if the number exceeds 15, the total amount of fields shown is limited to
 # 10.
 # Minimum value: 0, maximum value: 100, default value: 10.
-# This tag requires that the tag HAVE_DOT is set to YES.
+# This tag requires that the tag UML_LOOK is set to YES.
 
 UML_LIMIT_NUM_FIELDS   = 10
 
+# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and
+# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS
+# tag is set to YES, doxygen will add type and arguments for attributes and
+# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen
+# will not generate fields with class member information in the UML graphs. The
+# class diagrams will look similar to the default class diagrams but using UML
+# notation for the relationships.
+# Possible values are: NO, YES and NONE.
+# The default value is: NO.
+# This tag requires that the tag UML_LOOK is set to YES.
+
+DOT_UML_DETAILS        = NO
+
+# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters
+# to display on a single line. If the actual line length exceeds this threshold
+# significantly it will wrapped across multiple lines. Some heuristics are apply
+# to avoid ugly line breaks.
+# Minimum value: 0, maximum value: 1000, default value: 17.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_WRAP_THRESHOLD     = 17
+
 # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
 # collaboration graphs will show the relations between templates and their
 # instances.
@@ -2349,6 +2496,13 @@ GRAPHICAL_HIERARCHY    = YES
 
 DIRECTORY_GRAPH        = YES
 
+# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels
+# of child directories generated in directory dependency graphs by dot.
+# Minimum value: 1, maximum value: 25, default value: 1.
+# This tag requires that the tag DIRECTORY_GRAPH is set to YES.
+
+DIR_GRAPH_MAX_DEPTH    = 1
+
 # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
 # generated by dot. For an explanation of the image formats see the section
 # output formats in the documentation of the dot tool (Graphviz (see:
@@ -2402,10 +2556,10 @@ MSCFILE_DIRS           =
 DIAFILE_DIRS           =
 
 # When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
-# path where java can find the plantuml.jar file. If left blank, it is assumed
-# PlantUML is not used or called during a preprocessing step. Doxygen will
-# generate a warning when it encounters a \startuml command in this case and
-# will not generate output for the diagram.
+# path where java can find the plantuml.jar file or to the filename of jar file
+# to be used. If left blank, it is assumed PlantUML is not used or called during
+# a preprocessing step. Doxygen will generate a warning when it encounters a
+# \startuml command in this case and will not generate output for the diagram.
 
 PLANTUML_JAR_PATH      =
 
@@ -2467,14 +2621,18 @@ DOT_MULTI_TARGETS      = NO
 # If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
 # explaining the meaning of the various boxes and arrows in the dot generated
 # graphs.
+# Note: This tag requires that UML_LOOK isn't set, i.e. the doxygen internal
+# graphical representation for inheritance and collaboration diagrams is used.
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
 GENERATE_LEGEND        = YES
 
-# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
+# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate
 # files that are used to generate the various graphs.
+#
+# Note: This setting is not only used for dot files but also for msc temporary
+# files.
 # The default value is: YES.
-# This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_CLEANUP            = YES
diff --git a/docs/footer.htm b/docs/footer.htm
index 2ca612336a..ca355c3af8 100644
--- a/docs/footer.htm
+++ b/docs/footer.htm
@@ -1,57 +1,17 @@
+<!-- HTML footer for doxygen 1.9.3-->
+<!-- start footer part -->
+<!--BEGIN GENERATE_TREEVIEW-->
+<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
+    <ul>
+        $navpath
+        <li class="footer">$generatedby <a href="https://www.doxygen.org/index.html"><img class="footer" src="$relpath^doxygen.svg" width="104" height="31" alt="doxygen"/></a> $doxygenversion </li>
+    </ul>
 </div>
-</div>
-</div>
-</div>
-</div>
-
-<!--Google Analytics-->
-<script type="text/javascript">
-  var _gaq = _gaq || [];
-  _gaq.push(['_setAccount', 'UA-130950618-1']);
-  _gaq.push(['_setDomainName', '.arrayfire.com']);
-  _gaq.push(['_trackPageview']);
-
-  (function() {
-    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
-    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'https://www') + '.google-analytics.com/ga.js';
-    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
-  })();
-</script>
-
-<!--Spectate-->
-<script type="text/javascript">
-  sAId = "151";
-  sCId = "688";
-
-  (function() {
-    function async_load(){
-      var s = document.createElement('script'); s.type = 'text/javascript';
-      s.src = (('https:' == document.location.protocol) ? "https://ssl" : "https://cdn") + ".spectate.com/s.js";
-      var c = document.getElementsByTagName('script')[0]; c.parentNode.insertBefore(s, c);
-    }
-    if(window.attachEvent) { window.attachEvent('onload', async_load); }
-    else { window.addEventListener('load', async_load, false); }
-  })();
-</script>
-
-<!--Adroll-->
-<script type="text/javascript">
-adroll_adv_id = "ZRWI4W4RTRHENOWGXZY5JQ";
-adroll_pix_id = "QLXGBK3MSFB6LOL6PES2MT";
-(function () {
-var oldonload = window.onload;
-window.onload = function(){
-   __adroll_loaded=true;
-   var scr = document.createElement("script");
-   var host = (("https:" == document.location.protocol) ? "https://s.adroll.com" : "https://a.adroll.com");
-   scr.setAttribute('async', 'true');
-   scr.type = "text/javascript";
-   scr.src = host + "/j/roundtrip.js";
-   ((document.getElementsByTagName('head') || [null])[0] ||
-    document.getElementsByTagName('script')[0].parentNode).appendChild(scr);
-   if(oldonload){oldonload()}};
-}());
-</script>
-
+<!--END GENERATE_TREEVIEW-->
+<!--BEGIN !GENERATE_TREEVIEW-->
+<hr class="footer"/><address class="footer"><small>
+    $generatedby&#160;<a href="https://www.doxygen.org/index.html"><img class="footer" src="$relpath^doxygen.svg" width="104" height="31" alt="doxygen"/></a> $doxygenversion
+</small></address>
+<!--END !GENERATE_TREEVIEW-->
 </body>
 </html>
diff --git a/docs/header.htm b/docs/header.htm
index cc7a161d56..5704d89dfb 100644
--- a/docs/header.htm
+++ b/docs/header.htm
@@ -1,14 +1,28 @@
-<!-- HTML header for doxygen 1.8.17-->
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="https://www.w3.org/1999/xhtml">
+<!-- HTML header for doxygen 1.9.3-->
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml">
 <head>
+<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=UA-130950618-1"></script>
+<script>
+    window.dataLayer = window.dataLayer || [];
+    function gtag(){dataLayer.push(arguments);}
+    gtag('js', new Date());
+
+    gtag('config', 'UA-130950618-1');
+</script>
 <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
-<meta http-equiv="X-UA-Compatible" content="IE=9"/>
+<meta http-equiv="X-UA-Compatible" content="IE=11"/>
 <meta name="generator" content="Doxygen $doxygenversion"/>
 <meta name="viewport" content="width=device-width, initial-scale=1"/>
 <!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
 <!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
 <link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<script type="text/javascript">var page_layout=1;</script>
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
 <script type="text/javascript" src="$relpath^jquery.js"></script>
 <script type="text/javascript" src="$relpath^dynsections.js"></script>
 $treeview
@@ -18,47 +32,53 @@
 $extrastylesheet
 </head>
 <body>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<div id="side-nav" class="ui-resizable side-nav-resizable"><!-- do not remove this div, it is closed by doxygen! -->
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+
 <div id="top"><!-- do not remove this div, it is closed by doxygen! -->
 
 <!--BEGIN TITLEAREA-->
 <div id="titlearea">
 <table cellspacing="0" cellpadding="0" width="100%">
  <tbody>
- <tr style="height: 56px;">
+ <tr id="projectrow">
   <!--BEGIN PROJECT_LOGO-->
   <td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td>
   <!--END PROJECT_LOGO-->
   <!--BEGIN PROJECT_NAME-->
+  <!--<td id="projectalign">
+   <div id="projectname">$projectname<!--BEGIN PROJECT_NUMBER--><span id="projectnumber">&#160;$projectnumber</span><!--END PROJECT_NUMBER-->
+   </div>
+   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF-->
+  </td>-->
   <!--END PROJECT_NAME-->
   <!--BEGIN !PROJECT_NAME-->
    <!--BEGIN PROJECT_BRIEF-->
-    <td style="padding-left: 0.5em;">
+    <td>
     <div id="projectbrief">$projectbrief</div>
     </td>
    <!--END PROJECT_BRIEF-->
   <!--END !PROJECT_NAME-->
   <!--BEGIN DISABLE_INDEX-->
    <!--BEGIN SEARCHENGINE-->
-   <td>$searchbox</td>
+     <!--BEGIN !FULL_SIDEBAR-->
+    <td>$searchbox</td>
+     <!--END !FULL_SIDEBAR-->
    <!--END SEARCHENGINE-->
   <!--END DISABLE_INDEX-->
-   <td id="gsearch">
-       <div><script>
-             (function() {
-                 var cx = '004356362924927882526:zup3ehe-7bs';
-                 var gcse = document.createElement('script');
-                 gcse.type = 'text/javascript';
-                 gcse.async = true;
-                 gcse.src = (document.location.protocol == 'https:' ? 'https:' : 'http:') +
-                            '//www.google.com/cse/cse.js?cx=' + cx;
-                 var s = document.getElementsByTagName('script')[0];
-                 s.parentNode.insertBefore(gcse, s);
-             })();
-       </script>
-       <gcse:search></gcse:search>
-       <div>
-   </td>
+    <td id="gsearch">
+        <script async src="https://cse.google.com/cse.js?cx=004356362924927882526:zup3ehe-7bs"></script>
+        <div class="gcse-search"></div>
+    </td>
  </tr>
+  <!--BEGIN SEARCHENGINE-->
+   <!--BEGIN FULL_SIDEBAR-->
+ <tr><td colspan="2">$searchbox</td></tr>
+   <!--END FULL_SIDEBAR-->
+  <!--END SEARCHENGINE-->
  </tbody>
 </table>
 </div>
diff --git a/docs/pages/install.md b/docs/pages/install.md
index 2cbabab9b9..7a78b95f71 100644
--- a/docs/pages/install.md
+++ b/docs/pages/install.md
@@ -20,13 +20,13 @@ OpenCL backend, you will need to have the OpenCL **runtime** installed on your
 system. Drivers and runtimes should be downloaded and installed from your device
 vendor’s website.
 
-# Install Instructions
+# Install Instructions {#InstallInstructions}
 
 * [Windows](#Windows)
 * [Linux](#Linux)
 * [macOS](#macOS)
 
-## <a name="Windows"></a> Windows
+## Windows {#Windows}
 
 Prior to installing ArrayFire on Windows,
 [download](https://www.microsoft.com/en-in/download/details.aspx?id=48145)
@@ -41,7 +41,7 @@ can find ArrayFire DLLs.
 For more information on using ArrayFire on Windows, visit the following
 [page](http://arrayfire.org/docs/using_on_windows.htm).
 
-## <a name="Linux"></a> Linux
+## Linux {#Linux}
 
 There are two ways to install ArrayFire on Linux.
 1. Package Manager
@@ -90,7 +90,7 @@ __Fedora, Redhat, CentOS__
     yum install freeimage fontconfig mesa-libGLU
 
 
-## <a name="macOS"></a> macOS
+## macOS {#macOS}
 
 Once you have downloaded the ArrayFire installer, execute the installer by
 either double clicking on the ArrayFire `pkg` file or running the following
diff --git a/docs/pages/using_on_linux.md b/docs/pages/using_on_linux.md
index 87cab953bc..4948763d77 100644
--- a/docs/pages/using_on_linux.md
+++ b/docs/pages/using_on_linux.md
@@ -8,7 +8,7 @@ requirements are that you include the ArrayFire header directories and link with
 the ArrayFire library you intend to use i.e. CUDA, OpenCL, CPU, or Unified
 backends.
 
-## <a name="big-picture"/> The big picture
+## The big picture  {#big-picture}
 
 On Linux, we recommend installing ArrayFire to `/opt/arrayfire` directory. The
 installer will populate files in the following sub-directories:
diff --git a/docs/pages/using_on_osx.md b/docs/pages/using_on_osx.md
index f5643e3f93..272898ec5e 100644
--- a/docs/pages/using_on_osx.md
+++ b/docs/pages/using_on_osx.md
@@ -30,7 +30,7 @@ CMake or Makefiles with CMake being our preferred build system.
 * [CMake](#CMake)
 * [Makefiles](#Makefiles)
 
-## <a name="CMake"></a>CMake
+## CMake {#CMake}
 
 The CMake build system can be used to create ArrayFire projects. As [discussed
 above](#big-picture), ArrayFire ships with a series of CMake scripts to make
@@ -80,7 +80,7 @@ you would modify the `cmake` command above to contain the following definition:
 
 You can also specify this information in the `ccmake` command-line interface.
 
-## <a name="Makefiles"></a> Makefiles
+## Makefiles {#Makefiles}
 
 Building ArrayFire projects with Makefiles is fairly similar to CMake except you
 must specify all paths and libraries manually.
diff --git a/docs/pages/using_on_windows.md b/docs/pages/using_on_windows.md
index 99d321b886..924fca2794 100644
--- a/docs/pages/using_on_windows.md
+++ b/docs/pages/using_on_windows.md
@@ -2,10 +2,9 @@ Using ArrayFire with Microsoft Windows and Visual Studio {#using_on_windows}
 ============================================================================
 
 If you have not already done so, please make sure you have installed,
-configured, and tested ArrayFire following the [installation instructions](\ref
-installing).
+configured, and tested ArrayFire following the [installation instructions](#installing).
 
-## The big picture
+# The big picture
 
 The ArrayFire Windows installer creates the following:
 1. **AF_PATH** environment variable to point to the installation location. The
@@ -26,12 +25,12 @@ If you chose not to modify PATH during installation please make sure to do so
 manually so that all applications using ArrayFire libraries will be able to find
 the required DLLs.
 
-## <a name="section1" />Build and Run Helloworld
+# Build and Run Helloworld {#section1}
 
 This can be done in two ways either by using CMake build tool or using Visual
 Studio directly.
 
-### <a name="section1part1"/> Using CMake
+##  Using CMake {#section1part1}
 1. Download and install [CMake](https://cmake.org/download/), preferrably the
    latest version.
 2. Open CMake-GUI and set the field __Where is the source code__ to the root
@@ -59,7 +58,7 @@ Studio directly.
 10. Once the helloworld example builds, you will see a console window with the
     output from helloworld program.
 
-### <a name="section1part2"/> Using Visual Studio
+## Using Visual Studio {#section1part2}
 
 1. Open Visual Studio of your choice and create an empty C++ project.
 2. Right click the project and add an existing source file
@@ -76,16 +75,16 @@ Studio directly.
 7. Build and run the project. You will see a console window with the output from
    helloworld program.
 
-## <a name="section2" />Using ArrayFire within Existing Visual Studio Projects
+# Using ArrayFire within Existing Visual Studio Projects {#section2}
 This is divided into three parts:
-* [Part A: Adding ArrayFire to an existing solution (Single
-  Backend)](#section3partA)
-* [Part B: Adding ArrayFire CUDA to a new/existing CUDA project](#section3partB)
-* [Part C: Project with all ArrayFire backends](#section3partC)
+* [Part A: Adding ArrayFire to an existing solution (Single Backend)](#section2partA)
+* [Part B: Adding ArrayFire CUDA to a new/existing CUDA project](#section2partB)
+* [Part C: Project with all ArrayFire backends](#section2partC)
+
+## Part A: Adding ArrayFire to an existing solution (Single Backend) {#section2partA}
 
-### <a name="section3partA" />Part A: Adding ArrayFire to an existing solution (Single Backend)
 Note: If you plan on using Native CUDA code in the project, use the steps under
-[Part B](#section3partB).
+[Part B](#section2partB).
 
 Adding a single backend to an existing project is quite simple.
 
@@ -97,7 +96,7 @@ Adding a single backend to an existing project is quite simple.
    Properties -> Linker -> Input -> Additional Dependencies_. based on your
    preferred backend.
 
-### <a name="section3partB" />Part B: Adding ArrayFire CUDA to a new/existing CUDA project
+## Part B: Adding ArrayFire CUDA to a new/existing CUDA project {#section2partB}
 Lastly, if your project contains custom CUDA code, the instructions are slightly
 different as it requires using a CUDA NVCC Project:
 
@@ -109,15 +108,15 @@ different as it requires using a CUDA NVCC Project:
 4. Add `afcpu.lib`, `afcuda.lib`, `afopencl.lib`, or `af.lib` to _Project Properties ->
    Linker -> Input -> Additional Dependencies_. based on your preferred backend.
 
-### <a name="section3partC" />Part C: Project with all ArrayFire backends
+### Part C: Project with all ArrayFire backends {#section2partC}
 If you wish to create a project that allows you to use all the ArrayFire
 backends with ease, you should use `af.lib` in step 3 from [Part
-A](#section3partA).
+A](#section2partA).
 
 You can alternately download the template project from [ArrayFire Template
 Projects](https://github.com/arrayfire/arrayfire-project-templates)
 
-## <a name="section4" />Using ArrayFire with CMake
+# <a name="section3" />Using ArrayFire with CMake
 ArrayFire ships with a series of CMake scripts to make finding and using our
 library easy.
 

From bcda3cdbc245b4c24e78efdb5285c8241f6516c5 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 24 Feb 2022 13:56:52 -0500
Subject: [PATCH 009/473] Move 3.8.1 release notes to master branch.

This should have been included in the master before being backported
---
 docs/pages/release_notes.md | 64 +++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 571f37801f..259b927772 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -1,6 +1,70 @@
 Release Notes {#releasenotes}
 ==============
 
+v3.8.1
+======
+
+## Improvements
+
+- moddims now uses JIT approach for certain special cases - \PR{3177}
+- Embed Version Info in Windows DLLs - \PR{3025} 
+- OpenCL device max parameter is now queries from device properties - \PR{3032} 
+- JIT Performance Optimization: Unique funcName generation sped up - \PR{3040} 
+- Improved readability of log traces  - \PR{3050} 
+- Use short function name in non-debug build error messages - \PR{3060} 
+- SIFT/GLOH are now available as part of website binaries - \PR{3071} 
+- Short-circuit zero elements case in detail::copyArray backend function - \PR{3059} 
+- Speedup of kernel caching mechanism - \PR{3043} 
+- Add short-circuit check for empty Arrays in JIT evalNodes - \PR{3072} 
+- Performance optimization of indexing using dynamic thread block sizes - \PR{3111} 
+- ArrayFire starting with this release will use Intel MKL single dynamic library which resolves lot of linking issues unified library had when user applications used MKL themselves - \PR{3120} 
+- Add shortcut check for zero elements in af_write_array - \PR{3130} 
+- Speedup join by eliminating temp buffers for cascading joins - \PR{3145} 
+- Added batch support for solve - \PR{1705} 
+- Use pinned memory to copy device pointers in CUDA solve - \PR{1705} 
+- Added package manager instructions to docs - \PR{3076} 
+- CMake Build Improvements - \PR{3027} , \PR{3089} , \PR{3037} , \PR{3072} , \PR{3095} , \PR{3096} , \PR{3097} , \PR{3102} , \PR{3106} , \PR{3105} , \PR{3120} , \PR{3136} , \PR{3135} , \PR{3137} , \PR{3119} , \PR{3150} , \PR{3138} , \PR{3156} , \PR{3139} , \PR{1705} , \PR{3162} 
+- CPU backend improvements - \PR{3010} , \PR{3138} , \PR{3161} 
+- CUDA backend improvements - \PR{3066} , \PR{3091} , \PR{3093} , \PR{3125} , \PR{3143} , \PR{3161} 
+- OpenCL backend improvements - \PR{3091} , \PR{3068} , \PR{3127} , \PR{3010} , \PR{3039} , \PR{3138} , \PR{3161} 
+- General(including JIT) performance improvements across backends - \PR{3167} 
+- Testing improvements - \PR{3072} , \PR{3131} , \PR{3151} , \PR{3141} , \PR{3153} , \PR{3152} , \PR{3157} , \PR{1705} , \PR{3170} , \PR{3167} 
+- Update CLBlast to latest version - \PR{3135} , \PR{3179} 
+- Improved Otsu threshold computation helper in canny algorithm - \PR{3169} 
+- Modified default parameters for fftR2C and fftC2R C++ API from 0 to 1.0 - \PR{3178} 
+- Use appropriate MKL getrs_batch_strided API based on MKL Versions - \PR{3181} 
+
+## Fixes
+
+- Fixed a bug JIT kernel disk caching - \PR{3182} 
+- Fixed stream used by thrust(CUDA backend) functions - \PR{3029}  
+- Added workaround for new cuSparse API that was added by CUDA amid fix releases - \PR{3057} 
+- Fixed `const` array indexing inside `gfor` - \PR{3078} 
+- Handle zero elements in copyData to host - \PR{3059} 
+- Fixed double free regression in OpenCL backend - \PR{3091} 
+- Fixed an infinite recursion bug in NaryNode JIT Node - \PR{3072} 
+- Added missing input validation check in sparse-dense arithmetic operations - \PR{3129} 
+- Fixed bug in `getMappedPtr` in OpenCL due to invalid lambda capture - \PR{3163} 
+- Fixed bug in `getMappedPtr` on Arrays that are not ready - \PR{3163} 
+- Fixed edgeTraceKernel for CPU devices on OpenCL backend - \PR{3164} 
+- Fixed windows build issue(s) with VS2019 - \PR{3048}
+- API documentation fixes - \PR{3075} , \PR{3076} , \PR{3143} , \PR{3161} 
+- CMake Build Fixes - \PR{3088} 
+- Fixed the tutorial link in README - \PR{3033} 
+- Fixed function name typo in timing tutorial - \PR{3028} 
+- Fixed couple of bugs in CPU backend canny implementation - \PR{3169} 
+- Fixed reference count of array(s) used in JIT operations. It is related to arrayfire's internal memory book keeping. The behavior/accuracy of arrayfire code wasn't broken earlier. It corrected the reference count to be of optimal value in the said scenarios. This may potentially reduce memory usage in some narrow cases - \PR{3167} 
+- Added assert that checks if topk is called with a negative value for k - \PR{3176} 
+- Fixed an Issue where countByKey would give incorrect results for any n > 128 - \PR{3175} 
+
+## Contributions
+
+Special thanks to our contributors:
+[HO-COOH][https://github.com/HO-COOH]
+[Willy Born][https://github.com/willyborn]
+[Gilad Avidov][https://github.com/avidov]
+[Pavan Yalamanchili][https://github.com/pavanky]
+
 v3.8.0
 ======
 

From 259c2ffcc58684b47dda3d93d5be2eda11e00394 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 3 Feb 2022 19:20:28 -0500
Subject: [PATCH 010/473] handles empty arrays in join_many

---
 include/af/data.h           | 10 +++++++
 src/api/c/join.cpp          | 56 ++++++++++++++++++++++++++++++++-----
 src/api/c/rgb_gray.cpp      |  6 +++-
 src/api/c/surface.cpp       |  7 ++++-
 src/api/c/vector_field.cpp  | 10 +++++--
 src/api/c/ycbcr_rgb.cpp     | 11 ++++++--
 src/backend/cpu/join.cpp    | 29 +++----------------
 src/backend/cpu/join.hpp    |  2 +-
 src/backend/cuda/join.cpp   | 30 +++-----------------
 src/backend/cuda/join.hpp   |  2 +-
 src/backend/opencl/join.cpp | 29 +++----------------
 src/backend/opencl/join.hpp |  2 +-
 test/join.cpp               | 20 +++++++++++++
 13 files changed, 122 insertions(+), 92 deletions(-)

diff --git a/include/af/data.h b/include/af/data.h
index 52ebb78ed7..6da90fe801 100644
--- a/include/af/data.h
+++ b/include/af/data.h
@@ -200,6 +200,8 @@ namespace af
         \param[in] second is the second input array
         \return the array that joins input arrays along the given dimension
 
+        \note empty arrays will be ignored
+
         \ingroup manip_func_join
     */
     AFAPI array join(const int dim, const array &first, const array &second);
@@ -213,6 +215,8 @@ namespace af
         \param[in] third is the third input array
         \return the array that joins input arrays along the given dimension
 
+        \note empty arrays will be ignored
+
         \ingroup manip_func_join
     */
     AFAPI array join(const int dim, const array &first, const array &second, const array &third);
@@ -227,6 +231,8 @@ namespace af
         \param[in] fourth is the fourth input array
         \return the array that joins input arrays along the given dimension
 
+        \note empty arrays will be ignored
+
         \ingroup manip_func_join
     */
     AFAPI array join(const int dim, const array &first, const array &second,
@@ -622,6 +628,8 @@ extern "C" {
         \param[in] first is the first input array
         \param[in] second is the second input array
 
+        \note empty arrays will be ignored
+
         \ingroup manip_func_join
     */
     AFAPI af_err af_join(af_array *out, const int dim, const af_array first, const af_array second);
@@ -636,6 +644,8 @@ extern "C" {
         \param[in] n_arrays number of arrays to join
         \param[in] inputs is an array of af_arrays containing handles to the arrays to be joined
 
+        \note empty arrays will be ignored
+
         \ingroup manip_func_join
     */
     AFAPI af_err af_join_many(af_array *out, const int dim, const unsigned n_arrays, const af_array *inputs);
diff --git a/src/api/c/join.cpp b/src/api/c/join.cpp
index 79e45d3f9f..dad2bc1ffd 100644
--- a/src/api/c/join.cpp
+++ b/src/api/c/join.cpp
@@ -14,6 +14,7 @@
 #include <handle.hpp>
 #include <join.hpp>
 #include <af/data.h>
+#include <algorithm>
 #include <vector>
 
 using af::dim4;
@@ -21,6 +22,7 @@ using common::half;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
+using detail::createEmptyArray;
 using detail::intl;
 using detail::uchar;
 using detail::uint;
@@ -43,8 +45,30 @@ static inline af_array join_many(const int dim, const unsigned n_arrays,
 
     for (unsigned i = 0; i < n_arrays; i++) {
         inputs_.push_back(getArray<T>(inputs[i]));
+        if (inputs_.back().isEmpty()) { inputs_.pop_back(); }
     }
-    return getHandle(join<T>(dim, inputs_));
+
+    // All dimensions except join dimension must be equal
+    // calculate odims size
+    std::vector<af::dim4> idims(inputs_.size());
+    dim_t dim_size = 0;
+    for (unsigned i = 0; i < idims.size(); i++) {
+        idims[i] = inputs_[i].dims();
+        dim_size += idims[i][dim];
+    }
+
+    af::dim4 odims;
+    for (int i = 0; i < 4; i++) {
+        if (i == dim) {
+            odims[i] = dim_size;
+        } else {
+            odims[i] = idims[0][i];
+        }
+    }
+
+    Array<T> out = createEmptyArray<T>(odims);
+    join<T>(out, dim, inputs_);
+    return getHandle(out);
 }
 
 af_err af_join(af_array *out, const int dim, const af_array first,
@@ -117,24 +141,42 @@ af_err af_join_many(af_array *out, const int dim, const unsigned n_arrays,
 
         ARG_ASSERT(1, dim >= 0 && dim < 4);
 
+        bool allEmpty = std::all_of(
+            info.begin(), info.end(),
+            [](const ArrayInfo &i) -> bool { return i.elements() <= 0; });
+        if (allEmpty) {
+            af_array ret = nullptr;
+            AF_CHECK(af_retain_array(&ret, inputs[0]));
+            std::swap(*out, ret);
+            return AF_SUCCESS;
+        }
+
+        auto first_valid_afinfo = std::find_if(
+            info.begin(), info.end(),
+            [](const ArrayInfo &i) -> bool { return i.elements() > 0; });
+
+        af_dtype assertType = first_valid_afinfo->getType();
         for (unsigned i = 1; i < n_arrays; i++) {
-            ARG_ASSERT(3, info[0].getType() == info[i].getType());
-            DIM_ASSERT(3, info[i].elements() > 0);
+            if (info[i].elements() > 0) {
+                ARG_ASSERT(3, assertType == info[i].getType());
+            }
         }
 
         // All dimensions except join dimension must be equal
-        // Compute output dims
+        af::dim4 assertDims = first_valid_afinfo->dims();
         for (int i = 0; i < 4; i++) {
             if (i != dim) {
-                for (unsigned j = 1; j < n_arrays; j++) {
-                    DIM_ASSERT(3, dims[0][i] == dims[j][i]);
+                for (unsigned j = 0; j < n_arrays; j++) {
+                    if (info[j].elements() > 0) {
+                        DIM_ASSERT(3, assertDims[i] == dims[j][i]);
+                    }
                 }
             }
         }
 
         af_array output;
 
-        switch (info[0].getType()) {
+        switch (assertType) {
             case f32: output = join_many<float>(dim, n_arrays, inputs); break;
             case c32: output = join_many<cfloat>(dim, n_arrays, inputs); break;
             case f64: output = join_many<double>(dim, n_arrays, inputs); break;
diff --git a/src/api/c/rgb_gray.cpp b/src/api/c/rgb_gray.cpp
index e801881447..635474e846 100644
--- a/src/api/c/rgb_gray.cpp
+++ b/src/api/c/rgb_gray.cpp
@@ -26,6 +26,7 @@ using af::dim4;
 using common::cast;
 using detail::arithOp;
 using detail::Array;
+using detail::createEmptyArray;
 using detail::createValueArray;
 using detail::join;
 using detail::scalar;
@@ -96,7 +97,10 @@ static af_array gray2rgb(const af_array& in, const float r, const float g,
     AF_CHECK(af_release_array(mod_input));
 
     // join channels
-    return getHandle(join<cType>(2, {expr3, expr1, expr2}));
+    dim4 odims(expr1.dims()[0], expr1.dims()[1], 3);
+    Array<cType> out = createEmptyArray<cType>(odims);
+    join<cType>(out, 2, {expr3, expr1, expr2});
+    return getHandle(out);
 }
 
 template<typename T, typename cType, bool isRGB2GRAY>
diff --git a/src/api/c/surface.cpp b/src/api/c/surface.cpp
index 92e916e2f4..986cedae09 100644
--- a/src/api/c/surface.cpp
+++ b/src/api/c/surface.cpp
@@ -26,6 +26,7 @@ using af::dim4;
 using common::modDims;
 using detail::Array;
 using detail::copy_surface;
+using detail::createEmptyArray;
 using detail::forgeManager;
 using detail::reduce_all;
 using detail::uchar;
@@ -72,7 +73,11 @@ fg_chart setup_surface(fg_window window, const af_array xVals,
 
     // Now join along first dimension, skip reorder
     std::vector<Array<T>> inputs{xIn, yIn, zIn};
-    Array<T> Z = join(0, inputs);
+
+    dim4 odims(3, rowDims[1]);
+    Array<T> out = createEmptyArray<T>(odims);
+    join(out, 0, inputs);
+    Array<T> Z = out;
 
     ForgeManager& fgMngr = forgeManager();
 
diff --git a/src/api/c/vector_field.cpp b/src/api/c/vector_field.cpp
index c2f764c5c7..fa48328462 100644
--- a/src/api/c/vector_field.cpp
+++ b/src/api/c/vector_field.cpp
@@ -25,6 +25,7 @@
 using af::dim4;
 using detail::Array;
 using detail::copy_vector_field;
+using detail::createEmptyArray;
 using detail::forgeManager;
 using detail::reduce;
 using detail::transpose;
@@ -50,8 +51,13 @@ fg_chart setup_vector_field(fg_window window, const vector<af_array>& points,
     }
 
     // Join for set up vector
-    Array<T> pIn = detail::join(1, pnts);
-    Array<T> dIn = detail::join(1, dirs);
+    dim4 odims(3, points.size());
+    Array<T> out_pnts = createEmptyArray<T>(odims);
+    Array<T> out_dirs = createEmptyArray<T>(odims);
+    detail::join(out_pnts, 1, pnts);
+    detail::join(out_dirs, 1, dirs);
+    Array<T> pIn = out_pnts;
+    Array<T> dIn = out_dirs;
 
     // do transpose if required
     if (transpose_) {
diff --git a/src/api/c/ycbcr_rgb.cpp b/src/api/c/ycbcr_rgb.cpp
index b5beee4fae..d3c56a7117 100644
--- a/src/api/c/ycbcr_rgb.cpp
+++ b/src/api/c/ycbcr_rgb.cpp
@@ -20,6 +20,7 @@
 using af::dim4;
 using detail::arithOp;
 using detail::Array;
+using detail::createEmptyArray;
 using detail::createValueArray;
 using detail::join;
 using detail::scalar;
@@ -108,7 +109,10 @@ static af_array convert(const af_array& in, const af_ycc_std standard) {
                    INV_112 * (kb - 1) * kb * invKl);
         Array<T> B = mix<T>(Y_, Cb_, INV_219, INV_112 * (1 - kb));
         // join channels
-        return getHandle(join<T>(2, {R, G, B}));
+        dim4 odims(R.dims()[0], R.dims()[1], 3);
+        Array<T> rgbout = createEmptyArray<T>(odims);
+        join<T>(rgbout, 2, {R, G, B});
+        return getHandle(rgbout);
     }
     Array<T> Ey = mix<T>(X, Y, Z, kr, kl, kb);
     Array<T> Ecr =
@@ -119,7 +123,10 @@ static af_array convert(const af_array& in, const af_ycc_std standard) {
     Array<T> Cr = digitize<T>(Ecr, 224.0, 128.0);
     Array<T> Cb = digitize<T>(Ecb, 224.0, 128.0);
     // join channels
-    return getHandle(join<T>(2, {Y_, Cb, Cr}));
+    dim4 odims(Y_.dims()[0], Y_.dims()[1], 3);
+    Array<T> ycbcrout = createEmptyArray<T>(odims);
+    join<T>(ycbcrout, 2, {Y_, Cb, Cr});
+    return getHandle(ycbcrout);
 }
 
 template<bool isYCbCr2RGB>
diff --git a/src/backend/cpu/join.cpp b/src/backend/cpu/join.cpp
index 5b9382ee25..52f73747e2 100644
--- a/src/backend/cpu/join.cpp
+++ b/src/backend/cpu/join.cpp
@@ -44,26 +44,8 @@ Array<T> join(const int dim, const Array<T> &first, const Array<T> &second) {
 }
 
 template<typename T>
-Array<T> join(const int dim, const std::vector<Array<T>> &inputs) {
-    // All dimensions except join dimension must be equal
-    // Compute output dims
-    af::dim4 odims;
+void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs) {
     const dim_t n_arrays = inputs.size();
-    std::vector<af::dim4> idims(n_arrays);
-
-    dim_t dim_size = 0;
-    for (unsigned i = 0; i < idims.size(); i++) {
-        idims[i] = inputs[i].dims();
-        dim_size += idims[i][dim];
-    }
-
-    for (int i = 0; i < 4; i++) {
-        if (i == dim) {
-            odims[i] = dim_size;
-        } else {
-            odims[i] = idims[0][i];
-        }
-    }
 
     std::vector<Array<T> *> input_ptrs(inputs.size());
     std::transform(
@@ -71,11 +53,8 @@ Array<T> join(const int dim, const std::vector<Array<T>> &inputs) {
         [](const Array<T> &input) { return const_cast<Array<T> *>(&input); });
     evalMultiple(input_ptrs);
     std::vector<CParam<T>> inputParams(inputs.begin(), inputs.end());
-    Array<T> out = createEmptyArray<T>(odims);
 
     getQueue().enqueue(kernel::join<T>, dim, out, inputParams, n_arrays);
-
-    return out;
 }
 
 #define INSTANTIATE(T)                                              \
@@ -98,9 +77,9 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 
-#define INSTANTIATE(T)                       \
-    template Array<T> join<T>(const int dim, \
-                              const std::vector<Array<T>> &inputs);
+#define INSTANTIATE(T)                                   \
+    template void join<T>(Array<T> & out, const int dim, \
+                          const std::vector<Array<T>> &inputs);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/cpu/join.hpp b/src/backend/cpu/join.hpp
index 622e70c742..efabe9c8a5 100644
--- a/src/backend/cpu/join.hpp
+++ b/src/backend/cpu/join.hpp
@@ -15,5 +15,5 @@ template<typename T>
 Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
 
 template<typename T>
-Array<T> join(const int dim, const std::vector<Array<T>> &inputs);
+void join(Array<T> &output, const int dim, const std::vector<Array<T>> &inputs);
 }  // namespace cpu
diff --git a/src/backend/cuda/join.cpp b/src/backend/cuda/join.cpp
index 47f5a56205..880716e22b 100644
--- a/src/backend/cuda/join.cpp
+++ b/src/backend/cuda/join.cpp
@@ -69,36 +69,14 @@ void join_wrapper(const int dim, Array<T> &out,
 }
 
 template<typename T>
-Array<T> join(const int dim, const std::vector<Array<T>> &inputs) {
-    // All dimensions except join dimension must be equal
-    // Compute output dims
-    af::dim4 odims;
-    const dim_t n_arrays = inputs.size();
-    std::vector<af::dim4> idims(n_arrays);
-
-    dim_t dim_size = 0;
-    for (size_t i = 0; i < idims.size(); i++) {
-        idims[i] = inputs[i].dims();
-        dim_size += idims[i][dim];
-    }
-
-    for (int i = 0; i < 4; i++) {
-        if (i == dim) {
-            odims[i] = dim_size;
-        } else {
-            odims[i] = idims[0][i];
-        }
-    }
-
+void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs) {
     std::vector<Array<T> *> input_ptrs(inputs.size());
     std::transform(
         begin(inputs), end(inputs), begin(input_ptrs),
         [](const Array<T> &input) { return const_cast<Array<T> *>(&input); });
     evalMultiple(input_ptrs);
-    Array<T> out = createEmptyArray<T>(odims);
 
     join_wrapper<T>(dim, out, inputs);
-    return out;
 }
 
 #define INSTANTIATE(T)                                              \
@@ -121,9 +99,9 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 
-#define INSTANTIATE(T)                       \
-    template Array<T> join<T>(const int dim, \
-                              const std::vector<Array<T>> &inputs);
+#define INSTANTIATE(T)                                   \
+    template void join<T>(Array<T> & out, const int dim, \
+                          const std::vector<Array<T>> &inputs);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/cuda/join.hpp b/src/backend/cuda/join.hpp
index 7f88e5cad1..cf74076b8a 100644
--- a/src/backend/cuda/join.hpp
+++ b/src/backend/cuda/join.hpp
@@ -14,5 +14,5 @@ template<typename T>
 Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
 
 template<typename T>
-Array<T> join(const int dim, const std::vector<Array<T>> &inputs);
+void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs);
 }  // namespace cuda
diff --git a/src/backend/opencl/join.cpp b/src/backend/opencl/join.cpp
index 162229af7f..0c7109a895 100644
--- a/src/backend/opencl/join.cpp
+++ b/src/backend/opencl/join.cpp
@@ -72,37 +72,15 @@ void join_wrapper(const int dim, Array<T> &out,
 }
 
 template<typename T>
-Array<T> join(const int dim, const vector<Array<T>> &inputs) {
-    // All dimensions except join dimension must be equal
-    // Compute output dims
-    dim4 odims;
-    const dim_t n_arrays = inputs.size();
-    vector<dim4> idims(n_arrays);
-
-    dim_t dim_size = 0;
-    for (size_t i = 0; i < idims.size(); i++) {
-        idims[i] = inputs[i].dims();
-        dim_size += idims[i][dim];
-    }
-
-    for (int i = 0; i < 4; i++) {
-        if (i == dim) {
-            odims[i] = dim_size;
-        } else {
-            odims[i] = idims[0][i];
-        }
-    }
-
+void join(Array<T> &out, const int dim, const vector<Array<T>> &inputs) {
     vector<Array<T> *> input_ptrs(inputs.size());
     transform(
         begin(inputs), end(inputs), begin(input_ptrs),
         [](const Array<T> &input) { return const_cast<Array<T> *>(&input); });
     evalMultiple(input_ptrs);
     vector<Param> inputParams(inputs.begin(), inputs.end());
-    Array<T> out = createEmptyArray<T>(odims);
 
     join_wrapper<T>(dim, out, inputs);
-    return out;
 }
 
 #define INSTANTIATE(T)                                              \
@@ -125,8 +103,9 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 
-#define INSTANTIATE(T) \
-    template Array<T> join<T>(const int dim, const vector<Array<T>> &inputs);
+#define INSTANTIATE(T)                                   \
+    template void join<T>(Array<T> & out, const int dim, \
+                          const vector<Array<T>> &inputs);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/opencl/join.hpp b/src/backend/opencl/join.hpp
index 2f05a4fcf9..ea101d03f2 100644
--- a/src/backend/opencl/join.hpp
+++ b/src/backend/opencl/join.hpp
@@ -14,5 +14,5 @@ template<typename T>
 Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
 
 template<typename T>
-Array<T> join(const int dim, const std::vector<Array<T>> &inputs);
+void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs);
 }  // namespace opencl
diff --git a/test/join.cpp b/test/join.cpp
index 0024fe5542..4a98763b9b 100644
--- a/test/join.cpp
+++ b/test/join.cpp
@@ -246,3 +246,23 @@ TEST(Join, SameSize) {
 
     ASSERT_VEC_ARRAY_EQ(hgold, dim4(10 + 10 + 10), d);
 }
+
+TEST(Join, ManyEmpty) {
+    array gold = af::constant(0, 15, 5);
+    array a    = af::randn(5, 5);
+    array e;
+    array c  = af::randn(10, 5);
+    array ee = af::join(0, e, e);
+    ASSERT_EQ(ee.elements(), 0);
+    array eee = af::join(0, e, e, e);
+    ASSERT_EQ(eee.elements(), 0);
+
+    array eeac                     = af::join(0, e, e, a, c);
+    array eace                     = af::join(0, e, a, c, e);
+    array acee                     = af::join(0, a, c, e, e);
+    gold(af::seq(0, 4), af::span)  = a;
+    gold(af::seq(5, 14), af::span) = c;
+    ASSERT_ARRAYS_EQ(gold, eeac);
+    ASSERT_ARRAYS_EQ(gold, eace);
+    ASSERT_ARRAYS_EQ(gold, acee);
+}

From 60277cf173881e3a7aa8530db5b68649078bdce7 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 9 Feb 2022 22:25:53 -0500
Subject: [PATCH 011/473] fixes missing glfw with AF_BUILD_FORGE

---
 CMakeModules/AFconfigure_forge_dep.cmake | 66 ++++++++++++------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/CMakeModules/AFconfigure_forge_dep.cmake b/CMakeModules/AFconfigure_forge_dep.cmake
index 162e26c3ee..f15014e72b 100644
--- a/CMakeModules/AFconfigure_forge_dep.cmake
+++ b/CMakeModules/AFconfigure_forge_dep.cmake
@@ -8,34 +8,16 @@
 set(FG_VERSION_MAJOR 1)
 set(FG_VERSION_MINOR 0)
 set(FG_VERSION_PATCH 8)
+set(FG_VERSION "${FG_VERSION_MAJOR}.${FG_VERSION_MINOR}.${FG_VERSION_PATCH}")
+set(FG_API_VERSION_CURRENT ${FG_VERSION_MAJOR}${FG_VERSION_MINOR})
 
-find_package(Forge
-  ${FG_VERSION_MAJOR}.${FG_VERSION_MINOR}.${FG_VERSION_PATCH}
-  QUIET
-)
 
-if(TARGET Forge::forge)
-  get_target_property(fg_lib_type Forge::forge TYPE)
-  if(NOT ${fg_lib_type} STREQUAL "STATIC_LIBRARY")
-      install(FILES
-          $<TARGET_FILE:Forge::forge>
-          $<$<PLATFORM_ID:Linux>:$<TARGET_SONAME_FILE:Forge::forge>>
-          $<$<PLATFORM_ID:Darwin>:$<TARGET_SONAME_FILE:Forge::forge>>
-          $<$<PLATFORM_ID:Linux>:$<TARGET_LINKER_FILE:Forge::forge>>
-          $<$<PLATFORM_ID:Darwin>:$<TARGET_LINKER_FILE:Forge::forge>>
-          DESTINATION "${AF_INSTALL_LIB_DIR}"
-          COMPONENT common_backend_dependencies)
-  endif()
-else()
-  set(FG_VERSION "${FG_VERSION_MAJOR}.${FG_VERSION_MINOR}.${FG_VERSION_PATCH}")
-  set(FG_API_VERSION_CURRENT ${FG_VERSION_MAJOR}${FG_VERSION_MINOR})
+if(AF_BUILD_FORGE)
+    af_dep_check_and_populate(${forge_prefix}
+        URI https://github.com/arrayfire/forge.git
+        REF "v${FG_VERSION}"
+    )
 
-  af_dep_check_and_populate(${forge_prefix}
-    URI https://github.com/arrayfire/forge.git
-    REF "v${FG_VERSION}"
-  )
-
-  if(AF_BUILD_FORGE)
     set(af_FETCHCONTENT_BASE_DIR ${FETCHCONTENT_BASE_DIR})
     set(af_FETCHCONTENT_QUIET ${FETCHCONTENT_QUIET})
     set(af_FETCHCONTENT_FULLY_DISCONNECTED ${FETCHCONTENT_FULLY_DISCONNECTED})
@@ -67,9 +49,9 @@ else()
     set(FETCHCONTENT_QUIET ${af_FETCHCONTENT_QUIET})
     set(FETCHCONTENT_FULLY_DISCONNECTED ${af_FETCHCONTENT_FULLY_DISCONNECTED})
     set(FETCHCONTENT_UPDATES_DISCONNECTED ${af_FETCHCONTENT_UPDATES_DISCONNECTED})
-
     install(FILES
         $<TARGET_FILE:forge>
+        $<TARGET_RUNTIME_DLLS:forge>
         $<$<PLATFORM_ID:Linux>:$<TARGET_SONAME_FILE:forge>>
         $<$<PLATFORM_ID:Darwin>:$<TARGET_SONAME_FILE:forge>>
         $<$<PLATFORM_ID:Linux>:$<TARGET_LINKER_FILE:forge>>
@@ -77,10 +59,28 @@ else()
         DESTINATION "${AF_INSTALL_LIB_DIR}"
         COMPONENT common_backend_dependencies)
     set_property(TARGET forge APPEND_STRING PROPERTY COMPILE_FLAGS " -w")
-  else(AF_BUILD_FORGE)
-    configure_file(
-      ${${forge_prefix}_SOURCE_DIR}/CMakeModules/version.h.in
-      ${${forge_prefix}_BINARY_DIR}/include/fg/version.h
-      )
-  endif(AF_BUILD_FORGE)
-endif()
+else(AF_BUILD_FORGE)
+    find_package(Forge
+	    ${FG_VERSION_MAJOR}.${FG_VERSION_MINOR}.${FG_VERSION_PATCH}
+	    QUIET
+    )
+
+    if(TARGET Forge::forge)
+        get_target_property(fg_lib_type Forge::forge TYPE)
+        if(NOT ${fg_lib_type} STREQUAL "STATIC_LIBRARY")
+            install(FILES
+                    $<TARGET_FILE:Forge::forge>
+                    $<$<PLATFORM_ID:Linux>:$<TARGET_SONAME_FILE:Forge::forge>>
+                    $<$<PLATFORM_ID:Darwin>:$<TARGET_SONAME_FILE:Forge::forge>>
+                    $<$<PLATFORM_ID:Linux>:$<TARGET_LINKER_FILE:Forge::forge>>
+                    $<$<PLATFORM_ID:Darwin>:$<TARGET_LINKER_FILE:Forge::forge>>
+                    DESTINATION "${AF_INSTALL_LIB_DIR}"
+                    COMPONENT common_backend_dependencies)
+        endif()
+    else()
+        configure_file(
+		  ${${forge_prefix}_SOURCE_DIR}/CMakeModules/version.h.in
+		  ${${forge_prefix}_BINARY_DIR}/include/fg/version.h
+		  )
+    endif()
+endif(AF_BUILD_FORGE)

From d596bf79cdb26246f5ed36f3d34c5a895674c464 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 11 Feb 2022 16:31:11 -0500
Subject: [PATCH 012/473] fix intel defaults in ci workflows, fix
 configure_file for non-building forge

---
 .github/workflows/release_src_artifact.yml |  2 +-
 CMakeModules/AFconfigure_forge_dep.cmake   | 15 ++++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/release_src_artifact.yml b/.github/workflows/release_src_artifact.yml
index 273c7a9249..c616c8db5b 100644
--- a/.github/workflows/release_src_artifact.yml
+++ b/.github/workflows/release_src_artifact.yml
@@ -46,7 +46,7 @@ jobs:
               run: |
                   cd ${GITHUB_WORKSPACE}/arrayfire-full-${AF_VER}
                   mkdir build && cd build
-                  cmake .. -DAF_BUILD_FORGE:BOOL=ON
+                  cmake .. -DAF_BUILD_FORGE:BOOL=ON -DAF_COMPUTE_LIBRARY="FFTW/LAPACK/BLAS"
 
             - name: Create source tarball
               id: create-src-tarball
diff --git a/CMakeModules/AFconfigure_forge_dep.cmake b/CMakeModules/AFconfigure_forge_dep.cmake
index f15014e72b..0b3352cf12 100644
--- a/CMakeModules/AFconfigure_forge_dep.cmake
+++ b/CMakeModules/AFconfigure_forge_dep.cmake
@@ -61,8 +61,8 @@ if(AF_BUILD_FORGE)
     set_property(TARGET forge APPEND_STRING PROPERTY COMPILE_FLAGS " -w")
 else(AF_BUILD_FORGE)
     find_package(Forge
-	    ${FG_VERSION_MAJOR}.${FG_VERSION_MINOR}.${FG_VERSION_PATCH}
-	    QUIET
+        ${FG_VERSION_MAJOR}.${FG_VERSION_MINOR}.${FG_VERSION_PATCH}
+        QUIET
     )
 
     if(TARGET Forge::forge)
@@ -78,9 +78,14 @@ else(AF_BUILD_FORGE)
                     COMPONENT common_backend_dependencies)
         endif()
     else()
+        af_dep_check_and_populate(${forge_prefix}
+            URI https://github.com/arrayfire/forge.git
+            REF "v${FG_VERSION}"
+        )
+
         configure_file(
-		  ${${forge_prefix}_SOURCE_DIR}/CMakeModules/version.h.in
-		  ${${forge_prefix}_BINARY_DIR}/include/fg/version.h
-		  )
+            ${${forge_prefix}_SOURCE_DIR}/CMakeModules/version.h.in
+            ${${forge_prefix}_BINARY_DIR}/include/fg/version.h
+        )
     endif()
 endif(AF_BUILD_FORGE)

From f4dc55c18b092209d05acfe5ba24537a2b0c4095 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 3 Mar 2022 15:18:48 -0500
Subject: [PATCH 013/473] check cmake version for TARGET_RUNETIME_DLLS
 generator

---
 CMakeModules/AFconfigure_forge_dep.cmake | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/CMakeModules/AFconfigure_forge_dep.cmake b/CMakeModules/AFconfigure_forge_dep.cmake
index 0b3352cf12..6944d9e9f1 100644
--- a/CMakeModules/AFconfigure_forge_dep.cmake
+++ b/CMakeModules/AFconfigure_forge_dep.cmake
@@ -51,13 +51,21 @@ if(AF_BUILD_FORGE)
     set(FETCHCONTENT_UPDATES_DISCONNECTED ${af_FETCHCONTENT_UPDATES_DISCONNECTED})
     install(FILES
         $<TARGET_FILE:forge>
-        $<TARGET_RUNTIME_DLLS:forge>
         $<$<PLATFORM_ID:Linux>:$<TARGET_SONAME_FILE:forge>>
         $<$<PLATFORM_ID:Darwin>:$<TARGET_SONAME_FILE:forge>>
         $<$<PLATFORM_ID:Linux>:$<TARGET_LINKER_FILE:forge>>
         $<$<PLATFORM_ID:Darwin>:$<TARGET_LINKER_FILE:forge>>
         DESTINATION "${AF_INSTALL_LIB_DIR}"
         COMPONENT common_backend_dependencies)
+
+    if(AF_INSTALL_STANDALONE)
+        cmake_minimum_required(VERSION 3.21)
+        install(FILES
+            $<TARGET_RUNTIME_DLLS:forge>
+            DESTINATION "${AF_INSTALL_LIB_DIR}"
+            COMPONENT common_backend_dependencies)
+    endif(AF_INSTALL_STANDALONE)
+
     set_property(TARGET forge APPEND_STRING PROPERTY COMPILE_FLAGS " -w")
 else(AF_BUILD_FORGE)
     find_package(Forge

From fcaa40caa9b98b49cf71f7b678ca0dc0914568fe Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 8 Mar 2022 17:51:58 -0500
Subject: [PATCH 014/473] Set AF_COMPUTE_LIBRARY to MKL only if found.

---
 CMakeLists.txt | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 129927c0d2..db3b8978d8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -75,7 +75,12 @@ option(AF_WITH_STACKTRACE  "Add stacktraces to the error messages." ON)
 option(AF_CACHE_KERNELS_TO_DISK "Enable caching kernels to disk" ON)
 option(AF_WITH_STATIC_MKL "Link against static Intel MKL libraries" OFF)
 
-set(AF_COMPUTE_LIBRARY "Intel-MKL"
+set(default_compute_library "FFTW/LAPACK/BLAS")
+if(MKL_FOUND)
+  set(default_compute_library "Intel-MKL")
+endif()
+
+set(AF_COMPUTE_LIBRARY ${default_compute_library}
     CACHE STRING "Compute library for signal processing and linear algebra routines")
 set_property(CACHE AF_COMPUTE_LIBRARY
     PROPERTY STRINGS "Intel-MKL" "FFTW/LAPACK/BLAS")

From 27d424532bd2d06e1f10656f5826a3a9c55bc452 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 10 Mar 2022 10:59:21 -0500
Subject: [PATCH 015/473] fix multiprocess filename collisions in imageio tests
 (#3204)

* fix multiprocess filename collisions in imageio
* change imageio names to include backend to avoid collisions
---
 test/arrayfire_test.cpp | 16 ++++++++++++++++
 test/imageio.cpp        | 27 +++++++++++++++++++--------
 test/testHelpers.hpp    |  3 +++
 3 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index 63896a791a..a7d823e040 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -100,6 +100,22 @@ std::string readNextNonEmptyLine(std::ifstream &file) {
     return result;
 }
 
+std::string getBackendName() {
+    af::Backend backend = af::getActiveBackend();
+    if (backend == AF_BACKEND_OPENCL)
+        return std::string("opencl");
+    else if (backend == AF_BACKEND_CUDA)
+        return std::string("cuda");
+
+    return std::string("cpu");
+}
+
+std::string getTestName() {
+    std::string testname =
+        ::testing::UnitTest::GetInstance()->current_test_info()->name();
+    return testname;
+}
+
 namespace half_float {
 std::ostream &operator<<(std::ostream &os, half_float::half val) {
     os << (float)val;
diff --git a/test/imageio.cpp b/test/imageio.cpp
index cd66348b9f..9dc85a5865 100644
--- a/test/imageio.cpp
+++ b/test/imageio.cpp
@@ -160,8 +160,11 @@ TEST(ImageIO, SavePNGCPP) {
     input(9, 0, 2)          = 255;
     input(9, 9, span)       = 255;
 
-    saveImage("SaveCPP.png", input);
-    array out = loadImage("SaveCPP.png", true);
+    std::string testname  = getTestName() + "_" + getBackendName();
+    std::string imagename = "SaveCPP_" + testname + ".png";
+
+    saveImage(imagename.c_str(), input);
+    array out = loadImage(imagename.c_str(), true);
 
     ASSERT_FALSE(anyTrue<bool>(out - input));
 }
@@ -177,8 +180,11 @@ TEST(ImageIO, SaveBMPCPP) {
     input(9, 0, 2)          = 255;
     input(9, 9, span)       = 255;
 
-    saveImage("SaveCPP.bmp", input);
-    array out = loadImage("SaveCPP.bmp", true);
+    std::string testname  = getTestName() + "_" + getBackendName();
+    std::string imagename = "SaveCPP_" + testname + ".bmp";
+
+    saveImage(imagename.c_str(), input);
+    array out = loadImage(imagename.c_str(), true);
 
     ASSERT_FALSE(anyTrue<bool>(out - input));
 }
@@ -285,9 +291,12 @@ TEST(ImageIO, SaveImage16CPP) {
     array input     = randu(dims, u16);
     array input_255 = (input / 257).as(u16);
 
-    saveImage("saveImage16CPP.png", input);
+    std::string testname  = getTestName() + "_" + getBackendName();
+    std::string imagename = "saveImage16CPP_" + testname + ".png";
 
-    array img = loadImage("saveImage16CPP.png", true);
+    saveImage(imagename.c_str(), input);
+
+    array img = loadImage(imagename.c_str(), true);
     ASSERT_EQ(img.type(), f32);  // loadImage should always return float
 
     ASSERT_FALSE(anyTrue<bool>(abs(img - input_255)));
@@ -357,9 +366,11 @@ void saveLoadImageNativeCPPTest(dim4 dims) {
 
     array input = randu(dims, (af_dtype)dtype_traits<T>::af_type);
 
-    saveImageNative("saveImageNative.png", input);
+    std::string imagename = getTestName() + "_" + getBackendName() + ".png";
+
+    saveImageNative(imagename.c_str(), input);
 
-    array loaded = loadImageNative("saveImageNative.png");
+    array loaded = loadImageNative(imagename.c_str());
     ASSERT_EQ(loaded.type(), input.type());
 
     ASSERT_FALSE(anyTrue<bool>(input - loaded));
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 024b46657f..2e13ff9bbf 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -69,6 +69,9 @@ typedef unsigned char uchar;
 typedef unsigned int uint;
 typedef unsigned short ushort;
 
+std::string getBackendName();
+std::string getTestName();
+
 std::string readNextNonEmptyLine(std::ifstream &file);
 
 namespace half_float {

From e7625d1eee9a7f916dba9716937c00c5a4576d0d Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Fri, 5 Nov 2021 19:20:39 +0100
Subject: [PATCH 016/473] Improved precision of timeit

---
 src/api/cpp/timing.cpp | 66 ++++++++++++++++++++----------------------
 1 file changed, 31 insertions(+), 35 deletions(-)

diff --git a/src/api/cpp/timing.cpp b/src/api/cpp/timing.cpp
index 847c8d7873..285cb0cdb9 100644
--- a/src/api/cpp/timing.cpp
+++ b/src/api/cpp/timing.cpp
@@ -10,6 +10,7 @@
 #include <af/device.h>
 #include <af/timing.h>
 #include <algorithm>
+#include <array>
 #include <cmath>
 #include <vector>
 
@@ -71,43 +72,38 @@ double timer::stop(timer start) { return time_seconds(start, time_now()); }
 double timer::stop() { return time_seconds(_timer_, time_now()); }
 
 double timeit(void (*fn)()) {
-    // parameters
-    static const int trials      = 10;  // trial runs
-    static const int s_trials    = 5;   // trial runs
-    static const double min_time = 1;   // seconds
+    // Minimum target duration to limit impact of clock precision
+    constexpr double targetDurationPerTest = 0.050;
+    // samples during which the nr of cycles are determined to obtain target
+    // duration
+    constexpr int testSamples = 2;
+    // cycles needed to include CPU-GPU overlapping (if present)
+    constexpr int minCycles = 3;
+    // initial cycles used for the test samples
+    int cycles = minCycles;
+    // total number of real samples taken, of which the median is returned
+    constexpr int nrSamples = 10;
 
-    std::vector<double> sample_times(s_trials);
-
-    // estimate time for a few samples
-    for (int i = 0; i < s_trials; ++i) {
-        sync();
-        timer start = timer::start();
-        fn();
-        sync();
-        sample_times[i] = timer::stop(start);
-    }
-
-    // Sort sample times and select the median time
-    std::sort(sample_times.begin(), sample_times.end());
-
-    double median_time = sample_times[s_trials / 2];
-
-    // Run a bunch of batches of fn
-    // Each batch runs trial runs before sync
-    // If trials * median_time < min time,
-    //   then run (min time / (trials * median_time)) batches
-    // else
-    //   run 1 batch
-    int batches = static_cast<int>(ceilf(min_time / (trials * median_time)));
-    double run_time = 0;
-
-    for (int b = 0; b < batches; b++) {
-        timer start = timer::start();
-        for (int i = 0; i < trials; ++i) { fn(); }
-        sync();
-        run_time += timer::stop(start) / trials;
+    std::array<double, nrSamples> X;
+    for (int s = -testSamples; s < nrSamples; ++s) {
+        af::sync();
+        af::timer start = af::timer::start();
+        for (int i = cycles; i > 0; --i) { fn(); }
+        af::sync();
+        const double time = af::timer::stop(start);
+        if (s >= 0) {
+            // real sample, so store it for later processing
+            X[s] = time;
+        } else {
+            // test sample, so improve nr cycles
+            cycles = std::max(
+                minCycles,
+                static_cast<int>(trunc(targetDurationPerTest / time * cycles)));
+        };
     }
-    return run_time / batches;
+    std::sort(X.begin(), X.end());
+    // returns the median (iso of mean), to limit impact of outliers
+    return X[nrSamples / 2] / cycles;
 }
 
 }  // namespace af

From cb507b1d386f3db0e6af2f9af4c4c55fd032f5ba Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 10 Mar 2022 15:08:25 -0500
Subject: [PATCH 017/473] Add span-lite span header to the project

---
 CMakeLists.txt                                       | 5 +++++
 src/backend/common/CMakeLists.txt                    | 1 +
 src/backend/opencl/kernel/sort_by_key/CMakeLists.txt | 1 +
 3 files changed, 7 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index db3b8978d8..5ccfff22bb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -220,6 +220,11 @@ if(NOT TARGET glad::glad)
     )
 endif()
 
+af_dep_check_and_populate(span-lite
+  URI https://github.com/martinmoene/span-lite
+  REF "ccf2351"
+  )
+
 af_dep_check_and_populate(${assets_prefix}
   URI https://github.com/arrayfire/assets.git
   REF master
diff --git a/src/backend/common/CMakeLists.txt b/src/backend/common/CMakeLists.txt
index 9805b42ae4..9ac53b8454 100644
--- a/src/backend/common/CMakeLists.txt
+++ b/src/backend/common/CMakeLists.txt
@@ -102,6 +102,7 @@ endif()
 target_include_directories(afcommon_interface
   INTERFACE
     ${ArrayFire_SOURCE_DIR}/src/backend
+    ${span-lite_SOURCE_DIR}/include
     ${ArrayFire_BINARY_DIR}
   SYSTEM INTERFACE
     $<$<PLATFORM_ID:Darwin>:${OPENGL_INCLUDE_DIR}>
diff --git a/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt b/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
index 32d078faa2..e7a7ca27f3 100644
--- a/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
+++ b/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
@@ -30,6 +30,7 @@ foreach(SBK_TYPE ${SBK_TYPES})
         ../../api/c
         ../common
         ../../../include
+        ${span-lite_SOURCE_DIR}/include
         ${CMAKE_CURRENT_BINARY_DIR})
 
     target_include_directories(opencl_sort_by_key_${SBK_TYPE}

From 9f04bd4fbaf004703d0481606bbea1962b46ee43 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 10 Mar 2022 15:13:48 -0500
Subject: [PATCH 018/473] Allow passesJitHeuristics to accept multiple nodes

This commit will change passesJitHeuristics function to accept
multiple root nodes to determine if the resulting kernel is passing
in too many parameters. This change will allow us to use this function
in eval multiple functions.
---
 src/backend/common/jit/NaryNode.hpp |  5 ++-
 src/backend/cpu/Array.cpp           | 43 +++++++++++---------
 src/backend/cpu/Array.hpp           |  3 +-
 src/backend/cuda/Array.cpp          | 63 +++++++++++++++++------------
 src/backend/cuda/Array.hpp          |  3 +-
 src/backend/cuda/select.cpp         |  6 ++-
 src/backend/opencl/Array.cpp        | 62 ++++++++++++++++------------
 src/backend/opencl/Array.hpp        |  3 +-
 src/backend/opencl/select.cpp       |  8 ++--
 9 files changed, 117 insertions(+), 79 deletions(-)

diff --git a/src/backend/common/jit/NaryNode.hpp b/src/backend/common/jit/NaryNode.hpp
index 885edb277d..c03af9c2a5 100644
--- a/src/backend/common/jit/NaryNode.hpp
+++ b/src/backend/common/jit/NaryNode.hpp
@@ -14,6 +14,7 @@
 #include <common/defines.hpp>
 #include <common/jit/Node.hpp>
 
+#include <nonstd/span.hpp>
 #include <array>
 #include <iomanip>
 #include <sstream>
@@ -100,13 +101,15 @@ common::Node_ptr createNaryNode(
     const af::dim4 &odims, FUNC createNode,
     std::array<const detail::Array<Ti> *, N> &&children) {
     std::array<common::Node_ptr, N> childNodes;
+    std::array<common::Node *, N> nodes;
     for (int i = 0; i < N; i++) {
         childNodes[i] = move(children[i]->getNode());
+        nodes[i]      = childNodes[i].get();
     }
 
     common::Node_ptr ptr = createNode(childNodes);
 
-    switch (detail::passesJitHeuristics<Ti>(ptr.get())) {
+    switch (detail::passesJitHeuristics<Ti>(nodes)) {
         case kJITHeuristics::Pass: {
             return ptr;
         }
diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index 5b2385866c..dcd79dd9ed 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -30,6 +30,7 @@
 #include <af/seq.h>
 #include <af/traits.hpp>
 
+#include <nonstd/span.hpp>
 #include <algorithm>  // IWYU pragma: keep
 #include <cstddef>
 #include <cstring>
@@ -44,6 +45,7 @@ using common::Node_ptr;
 using common::NodeIterator;
 using cpu::jit::BufferNode;
 
+using nonstd::span;
 using std::adjacent_find;
 using std::copy;
 using std::is_standard_layout;
@@ -227,28 +229,31 @@ Array<T> createEmptyArray(const dim4 &dims) {
 }
 
 template<typename T>
-kJITHeuristics passesJitHeuristics(Node *root_node) {
+kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     if (!evalFlag()) { return kJITHeuristics::Pass; }
-    if (root_node->getHeight() > static_cast<int>(getMaxJitSize())) {
-        return kJITHeuristics::TreeHeight;
+    size_t bytes = 0;
+    for (Node *n : root_nodes) {
+        if (n->getHeight() > static_cast<int>(getMaxJitSize())) {
+            return kJITHeuristics::TreeHeight;
+        }
+        // Check if approaching the memory limit
+        if (getMemoryPressure() >= getMemoryPressureThreshold()) {
+            NodeIterator<Node> it(n);
+            NodeIterator<Node> end_node;
+            bytes = accumulate(it, end_node, bytes,
+                               [=](const size_t prev, const Node &n) {
+                                   // getBytes returns the size of the data
+                                   // Array. Sub arrays will be represented
+                                   // by their parent size.
+                                   return prev + n.getBytes();
+                               });
+        }
     }
 
-    // Check if approaching the memory limit
-    if (getMemoryPressure() >= getMemoryPressureThreshold()) {
-        NodeIterator<Node> it(root_node);
-        NodeIterator<Node> end_node;
-        size_t bytes = accumulate(it, end_node, size_t(0),
-                                  [=](const size_t prev, const Node &n) {
-                                      // getBytes returns the size of the data
-                                      // Array. Sub arrays will be represented
-                                      // by their parent size.
-                                      return prev + n.getBytes();
-                                  });
-
-        if (jitTreeExceedsMemoryPressure(bytes)) {
-            return kJITHeuristics::MemoryPressure;
-        }
+    if (jitTreeExceedsMemoryPressure(bytes)) {
+        return kJITHeuristics::MemoryPressure;
     }
+
     return kJITHeuristics::Pass;
 }
 
@@ -343,7 +348,7 @@ void Array<T>::setDataDims(const dim4 &new_dims) {
     template void writeDeviceDataArray<T>(                                    \
         Array<T> & arr, const void *const data, const size_t bytes);          \
     template void evalMultiple<T>(vector<Array<T> *> arrays);                 \
-    template kJITHeuristics passesJitHeuristics<T>(Node * n);                 \
+    template kJITHeuristics passesJitHeuristics<T>(span<Node *> n);           \
     template void Array<T>::setDataDims(const dim4 &new_dims);
 
 INSTANTIATE(float)
diff --git a/src/backend/cpu/Array.hpp b/src/backend/cpu/Array.hpp
index 792b582de2..8db2ee7e44 100644
--- a/src/backend/cpu/Array.hpp
+++ b/src/backend/cpu/Array.hpp
@@ -22,6 +22,7 @@
 #include <af/dim4.hpp>
 #include <af/seq.h>
 
+#include <nonstd/span.hpp>
 #include <algorithm>
 #include <cstddef>
 #include <memory>
@@ -100,7 +101,7 @@ template<typename T>
 void destroyArray(Array<T> *A);
 
 template<typename T>
-kJITHeuristics passesJitHeuristics(common::Node *node);
+kJITHeuristics passesJitHeuristics(nonstd::span<common::Node *> node);
 
 template<typename T>
 void *getDevicePtr(const Array<T> &arr) {
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index 44169eccbd..134645f496 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -30,6 +30,7 @@ using common::Node_ptr;
 using common::NodeIterator;
 using cuda::jit::BufferNode;
 
+using nonstd::span;
 using std::accumulate;
 using std::move;
 using std::shared_ptr;
@@ -245,27 +246,33 @@ Node_ptr Array<T>::getNode() const {
 /// 2. The number of parameters we are passing into the kernel exceeds the
 ///    limitation on the platform. For NVIDIA this is 4096 bytes. The
 template<typename T>
-kJITHeuristics passesJitHeuristics(Node *root_node) {
+kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     if (!evalFlag()) { return kJITHeuristics::Pass; }
-    if (root_node->getHeight() > static_cast<int>(getMaxJitSize())) {
-        return kJITHeuristics::TreeHeight;
+    for (Node *n : root_nodes) {
+        if (n->getHeight() > static_cast<int>(getMaxJitSize())) {
+            return kJITHeuristics::TreeHeight;
+        }
     }
 
     // A lightweight check based on the height of the node. This is an
     // inexpensive operation and does not traverse the JIT tree.
-    if (root_node->getHeight() > 6 ||
-        getMemoryPressure() >= getMemoryPressureThreshold()) {
+    int heightCheckLimit = 6;
+    bool atHeightLimit =
+        std::any_of(std::begin(root_nodes), std::end(root_nodes),
+                    [heightCheckLimit](Node *n) {
+                        return (n->getHeight() + 1 >= heightCheckLimit);
+                    });
+    if (atHeightLimit || getMemoryPressure() >= getMemoryPressureThreshold()) {
         // The size of the parameters without any extra arguments from the
         // JIT tree. This includes one output Param object and 4 integers.
-        constexpr size_t base_param_size =
-            sizeof(Param<T>) + (4 * sizeof(uint));
+        size_t base_param_size =
+            sizeof(Param<T>) * root_nodes.size() + (4 * sizeof(uint));
 
         // extra padding for safety to avoid failure during compilation
         constexpr size_t jit_padding_size = 256;  //@umar dontfix!
         // This is the maximum size of the params that can be allowed by the
         // CUDA platform.
-        constexpr size_t max_param_size =
-            4096 - base_param_size - jit_padding_size;
+        size_t max_param_size = 4096 - base_param_size - jit_padding_size;
 
         struct tree_info {
             size_t total_buffer_size;
@@ -273,22 +280,26 @@ kJITHeuristics passesJitHeuristics(Node *root_node) {
             size_t param_scalar_size;
         };
         NodeIterator<> end_node;
-        tree_info info =
-            accumulate(NodeIterator<>(root_node), end_node, tree_info{0, 0, 0},
-                       [](tree_info &prev, const Node &node) {
-                           if (node.isBuffer()) {
-                               const auto &buf_node =
-                                   static_cast<const BufferNode<T> &>(node);
-                               // getBytes returns the size of the data Array.
-                               // Sub arrays will be represented by their parent
-                               // size.
-                               prev.total_buffer_size += buf_node.getBytes();
-                               prev.num_buffers++;
-                           } else {
-                               prev.param_scalar_size += node.getParamBytes();
-                           }
-                           return prev;
-                       });
+        tree_info info = tree_info{0, 0, 0};
+
+        for (Node *n : root_nodes) {
+            info = accumulate(
+                NodeIterator<>(n), end_node, info,
+                [](tree_info &prev, const Node &node) {
+                    if (node.isBuffer()) {
+                        const auto &buf_node =
+                            static_cast<const BufferNode<T> &>(node);
+                        // getBytes returns the size of the data Array.
+                        // Sub arrays will be represented by their
+                        // parent size.
+                        prev.total_buffer_size += buf_node.getBytes();
+                        prev.num_buffers++;
+                    } else {
+                        prev.param_scalar_size += node.getParamBytes();
+                    }
+                    return prev;
+                });
+        }
         size_t param_size =
             info.num_buffers * sizeof(Param<T>) + info.param_scalar_size;
 
@@ -440,7 +451,7 @@ void Array<T>::setDataDims(const dim4 &new_dims) {
     template void writeDeviceDataArray<T>(                                    \
         Array<T> & arr, const void *const data, const size_t bytes);          \
     template void evalMultiple<T>(std::vector<Array<T> *> arrays);            \
-    template kJITHeuristics passesJitHeuristics<T>(Node * n);                 \
+    template kJITHeuristics passesJitHeuristics<T>(span<Node *> n);           \
     template void Array<T>::setDataDims(const dim4 &new_dims);
 
 INSTANTIATE(float)
diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp
index b279ffcab4..52dbed7aeb 100644
--- a/src/backend/cuda/Array.hpp
+++ b/src/backend/cuda/Array.hpp
@@ -22,6 +22,7 @@
 #include <af/dim4.hpp>
 #include "traits.hpp"
 
+#include <nonstd/span.hpp>
 #include <vector>
 
 namespace cuda {
@@ -103,7 +104,7 @@ void destroyArray(Array<T> *A);
 /// \returns false if the kernel generated by this node will fail to compile
 ///          or its nodes are consuming too much memory.
 template<typename T>
-kJITHeuristics passesJitHeuristics(common::Node *node);
+kJITHeuristics passesJitHeuristics(nonstd::span<common::Node *> node);
 
 template<typename T>
 void *getDevicePtr(const Array<T> &arr) {
diff --git a/src/backend/cuda/select.cpp b/src/backend/cuda/select.cpp
index 6f6f399960..739e150c05 100644
--- a/src/backend/cuda/select.cpp
+++ b/src/backend/cuda/select.cpp
@@ -53,7 +53,8 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
         NaryNode(static_cast<af::dtype>(dtype_traits<T>::af_type), "__select",
                  3, {{cond_node, a_node, b_node}}, af_select_t, height));
 
-    if (detail::passesJitHeuristics<T>(node.get()) != kJITHeuristics::Pass) {
+    std::array<common::Node *, 1> nodes{node.get()};
+    if (detail::passesJitHeuristics<T>(nodes) != kJITHeuristics::Pass) {
         if (a_height > max(b_height, cond_height)) {
             a.eval();
         } else if (b_height > cond_height) {
@@ -83,7 +84,8 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
         (flip ? "__not_select" : "__select"), 3, {{cond_node, a_node, b_node}},
         flip ? af_not_select_t : af_select_t, height));
 
-    if (detail::passesJitHeuristics<T>(node.get()) != kJITHeuristics::Pass) {
+    std::array<common::Node *, 1> nodes{node.get()};
+    if (detail::passesJitHeuristics<T>(nodes) != kJITHeuristics::Pass) {
         if (a_height > max(b_height, cond_height)) {
             a.eval();
         } else if (b_height > cond_height) {
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 3aa63b40d4..6e490f82a8 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -45,6 +45,7 @@ using common::Node_ptr;
 using common::NodeIterator;
 using opencl::jit::BufferNode;
 
+using nonstd::span;
 using std::accumulate;
 using std::is_standard_layout;
 using std::make_shared;
@@ -293,10 +294,12 @@ Node_ptr Array<T>::getNode() const {
 /// 2. The number of parameters we are passing into the kernel exceeds the
 ///    limitation on the platform. For NVIDIA this is 4096 bytes. The
 template<typename T>
-kJITHeuristics passesJitHeuristics(Node *root_node) {
+kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     if (!evalFlag()) { return kJITHeuristics::Pass; }
-    if (root_node->getHeight() > static_cast<int>(getMaxJitSize())) {
-        return kJITHeuristics::TreeHeight;
+    for (const Node *n : root_nodes) {
+        if (n->getHeight() > static_cast<int>(getMaxJitSize())) {
+            return kJITHeuristics::TreeHeight;
+        }
     }
 
     bool isBufferLimit = getMemoryPressure() >= getMemoryPressureThreshold();
@@ -312,12 +315,18 @@ kJITHeuristics passesJitHeuristics(Node *root_node) {
 
     // A lightweight check based on the height of the node. This is
     // an inexpensive operation and does not traverse the JIT tree.
-    bool isParamLimit = (root_node->getHeight() >= heightCheckLimit);
-    if (isParamLimit || isBufferLimit) {
+    bool atHeightLimit =
+        std::any_of(std::begin(root_nodes), std::end(root_nodes),
+                    [heightCheckLimit](Node *n) {
+                        return (n->getHeight() + 1 >= heightCheckLimit);
+                    });
+
+    if (atHeightLimit || isBufferLimit) {
         // This is the base parameter size if the kernel had no
         // arguments
-        constexpr size_t base_param_size =
-            sizeof(T *) + sizeof(KParam) + (3 * sizeof(uint));
+        size_t base_param_size =
+            (sizeof(T *) + sizeof(KParam)) * root_nodes.size() +
+            (3 * sizeof(uint));
 
         const cl::Device &device = getDevice();
         size_t max_param_size = device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>();
@@ -332,28 +341,31 @@ kJITHeuristics passesJitHeuristics(Node *root_node) {
             size_t num_buffers;
             size_t param_scalar_size;
         };
-        NodeIterator<> it(root_node);
-        tree_info info =
-            accumulate(it, NodeIterator<>(), tree_info{0, 0, 0},
-                       [](tree_info &prev, Node &n) {
-                           if (n.isBuffer()) {
-                               auto &buf_node = static_cast<BufferNode &>(n);
-                               // getBytes returns the size of the data Array.
-                               // Sub arrays will be represented by their parent
-                               // size.
-                               prev.total_buffer_size += buf_node.getBytes();
-                               prev.num_buffers++;
-                           } else {
-                               prev.param_scalar_size += n.getParamBytes();
-                           }
-                           return prev;
-                       });
+
+        tree_info info{0, 0, 0};
+        for (Node *n : root_nodes) {
+            NodeIterator<> it(n);
+            info = accumulate(
+                it, NodeIterator<>(), info, [](tree_info &prev, Node &n) {
+                    if (n.isBuffer()) {
+                        auto &buf_node = static_cast<BufferNode &>(n);
+                        // getBytes returns the size of the data Array.
+                        // Sub arrays will be represented by their parent
+                        // size.
+                        prev.total_buffer_size += buf_node.getBytes();
+                        prev.num_buffers++;
+                    } else {
+                        prev.param_scalar_size += n.getParamBytes();
+                    }
+                    return prev;
+                });
+        }
         isBufferLimit = jitTreeExceedsMemoryPressure(info.total_buffer_size);
 
         size_t param_size = (info.num_buffers * (sizeof(KParam) + sizeof(T *)) +
                              info.param_scalar_size);
 
-        isParamLimit = param_size >= max_param_size;
+        bool isParamLimit = param_size >= max_param_size;
 
         if (isParamLimit) { return kJITHeuristics::KernelParameterSize; }
         if (isBufferLimit) { return kJITHeuristics::MemoryPressure; }
@@ -513,7 +525,7 @@ size_t Array<T>::getAllocatedBytes() const {
     template void writeDeviceDataArray<T>(                                    \
         Array<T> & arr, const void *const data, const size_t bytes);          \
     template void evalMultiple<T>(vector<Array<T> *> arrays);                 \
-    template kJITHeuristics passesJitHeuristics<T>(Node * node);              \
+    template kJITHeuristics passesJitHeuristics<T>(span<Node *> node);        \
     template void *getDevicePtr<T>(const Array<T> &arr);                      \
     template void Array<T>::setDataDims(const dim4 &new_dims);                \
     template size_t Array<T>::getAllocatedBytes() const;
diff --git a/src/backend/opencl/Array.hpp b/src/backend/opencl/Array.hpp
index 67290207df..d3362cfa9a 100644
--- a/src/backend/opencl/Array.hpp
+++ b/src/backend/opencl/Array.hpp
@@ -23,6 +23,7 @@
 
 #include <af/dim4.hpp>
 
+#include <nonstd/span.hpp>
 #include <algorithm>
 #include <cstdlib>
 #include <memory>
@@ -108,7 +109,7 @@ void destroyArray(Array<T> *A);
 /// \returns false if the kernel generated by this node will fail to compile
 ///          or its nodes are consuming too much memory.
 template<typename T>
-kJITHeuristics passesJitHeuristics(common::Node *node);
+kJITHeuristics passesJitHeuristics(nonstd::span<common::Node *> node);
 
 template<typename T>
 void *getDevicePtr(const Array<T> &arr);
diff --git a/src/backend/opencl/select.cpp b/src/backend/opencl/select.cpp
index 32c2734f75..d652df25c6 100644
--- a/src/backend/opencl/select.cpp
+++ b/src/backend/opencl/select.cpp
@@ -15,6 +15,7 @@
 #include <err_opencl.hpp>
 #include <scalar.hpp>
 
+#include <nonstd/span.hpp>
 #include <memory>
 
 using af::dim4;
@@ -40,8 +41,8 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
     auto node = make_shared<NaryNode>(
         NaryNode(static_cast<af::dtype>(dtype_traits<T>::af_type), "__select",
                  3, {{cond_node, a_node, b_node}}, af_select_t, height));
-
-    if (detail::passesJitHeuristics<T>(node.get()) != kJITHeuristics::Pass) {
+    std::array<common::Node *, 1> nodes{node.get()};
+    if (detail::passesJitHeuristics<T>(nodes) != kJITHeuristics::Pass) {
         if (a_height > max(b_height, cond_height)) {
             a.eval();
         } else if (b_height > cond_height) {
@@ -71,7 +72,8 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
         (flip ? "__not_select" : "__select"), 3, {{cond_node, a_node, b_node}},
         (flip ? af_not_select_t : af_select_t), height));
 
-    if (detail::passesJitHeuristics<T>(node.get()) != kJITHeuristics::Pass) {
+    std::array<common::Node *, 1> nodes{node.get()};
+    if (detail::passesJitHeuristics<T>(nodes) != kJITHeuristics::Pass) {
         if (a_height > max(b_height, cond_height)) {
             a.eval();
         } else if (b_height > cond_height) {

From ddb55c40c095252311cbb7d23707eba295595b0a Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 11 Mar 2022 18:34:45 -0500
Subject: [PATCH 019/473] Add some debugging macros

---
 src/backend/common/ArrayFireTypesIO.hpp | 37 ++++++++++
 src/backend/common/CMakeLists.txt       |  2 +
 src/backend/common/debug.hpp            | 62 +++++++++++++++++
 src/backend/common/jit/NodeIO.hpp       | 93 +++++++++++++++++++++++++
 4 files changed, 194 insertions(+)
 create mode 100644 src/backend/common/ArrayFireTypesIO.hpp
 create mode 100644 src/backend/common/debug.hpp
 create mode 100644 src/backend/common/jit/NodeIO.hpp

diff --git a/src/backend/common/ArrayFireTypesIO.hpp b/src/backend/common/ArrayFireTypesIO.hpp
new file mode 100644
index 0000000000..234df93b43
--- /dev/null
+++ b/src/backend/common/ArrayFireTypesIO.hpp
@@ -0,0 +1,37 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <spdlog/fmt/bundled/ranges.h>
+#include <spdlog/fmt/ostr.h>
+#include <af/seq.h>
+
+template<>
+struct fmt::formatter<af_seq> {
+    // Parses format specifications of the form ['f' | 'e'].
+    constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) {
+        return ctx.begin();
+    }
+
+    // Formats the point p using the parsed format specification (presentation)
+    // stored in this formatter.
+    template<typename FormatContext>
+    auto format(const af_seq& p, FormatContext& ctx) -> decltype(ctx.out()) {
+        // ctx.out() is an output iterator to write to.
+        if (p.begin == af_span.begin && p.end == af_span.end &&
+            p.step == af_span.step) {
+            return format_to(ctx.out(), "span");
+        }
+        if (p.begin == p.end) { return format_to(ctx.out(), "{}", p.begin); }
+        if (p.step == 1) {
+            return format_to(ctx.out(), "({} -> {})", p.begin, p.end);
+        }
+        return format_to(ctx.out(), "({} -({})-> {})", p.begin, p.step, p.end);
+    }
+};
diff --git a/src/backend/common/CMakeLists.txt b/src/backend/common/CMakeLists.txt
index 9ac53b8454..125c620754 100644
--- a/src/backend/common/CMakeLists.txt
+++ b/src/backend/common/CMakeLists.txt
@@ -15,6 +15,7 @@ target_sources(afcommon_interface
     ${CMAKE_CURRENT_SOURCE_DIR}/jit/NaryNode.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/jit/Node.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/jit/Node.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/jit/NodeIO.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/jit/NodeIterator.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/jit/ScalarNode.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/jit/UnaryNode.hpp
@@ -25,6 +26,7 @@ target_sources(afcommon_interface
     ${CMAKE_CURRENT_SOURCE_DIR}/AllocatorInterface.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ArrayInfo.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ArrayInfo.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ArrayFireTypesIO.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/DefaultMemoryManager.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/DefaultMemoryManager.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/DependencyModule.cpp
diff --git a/src/backend/common/debug.hpp b/src/backend/common/debug.hpp
new file mode 100644
index 0000000000..6c2c6cbfb8
--- /dev/null
+++ b/src/backend/common/debug.hpp
@@ -0,0 +1,62 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#define FMT_HEADER_ONLY
+#include <boost/stacktrace.hpp>
+#include <common/ArrayFireTypesIO.hpp>
+#include <common/jit/NodeIO.hpp>
+#include <spdlog/fmt/bundled/format.h>
+#include <iostream>
+
+#define DBGTRACE(msg)                                              \
+    fmt::print(std::cout, __FILE__ ":{}:{}\n{}\n", __LINE__, #msg, \
+               boost::stacktrace::stacktrace())
+
+namespace debugging {
+
+template<typename first>
+void print(const char *F, const first &FF) {
+    fmt::print(std::cout, "{} = {}", F, FF);
+}
+
+template<typename first, typename... ARGS>
+void print(const char *F, const first &FF, ARGS... args) {
+    fmt::print(std::cout, "{} = {} | ", F, FF);
+    print(args...);
+}
+}  // namespace debugging
+
+#define SHOW1(val1) debugging::print(#val1, val1)
+#define SHOW2(val1, val2) debugging::print(#val1, val1, #val2, val2)
+#define SHOW3(val1, val2, val3) \
+    debugging::print(#val1, val1, #val2, val2, #val3, val3)
+
+#define SHOW4(val1, val2, val3, val4) \
+    debugging::print(#val1, val1, #val2, val2, #val3, val3, #val4, val4)
+#define SHOW5(val1, val2, val3, val4, val5)                              \
+    debugging::print(#val1, val1, #val2, val2, #val3, val3, #val4, val4, \
+                     #val5, val5)
+
+#define GET_MACRO(_1, _2, _3, _4, _5, NAME, ...) NAME
+
+#define SHOW(...)                                                 \
+    do {                                                          \
+        fmt::print(std::cout, "{}:({}): ", __FILE__, __LINE__);   \
+        GET_MACRO(__VA_ARGS__, SHOW5, SHOW4, SHOW3, SHOW2, SHOW1) \
+        (__VA_ARGS__);                                            \
+        fmt::print(std::cout, "\n");                              \
+    } while (0)
+
+#define PRINTVEC(val)                                                        \
+    do {                                                                     \
+        fmt::print(std::cout, "{}:({}):{} [{}]\n", __FILE__, __LINE__, #val, \
+                   fmt::join(val, ", "));                                    \
+    } while (0)
diff --git a/src/backend/common/jit/NodeIO.hpp b/src/backend/common/jit/NodeIO.hpp
new file mode 100644
index 0000000000..55d40c2b2d
--- /dev/null
+++ b/src/backend/common/jit/NodeIO.hpp
@@ -0,0 +1,93 @@
+/*******************************************************
+ * Copyright (c) 2021, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <common/jit/Node.hpp>
+#include <common/util.hpp>
+#include <spdlog/fmt/bundled/format.h>
+
+#include <common/TemplateArg.hpp>
+
+template<>
+struct fmt::formatter<af::dtype> : fmt::formatter<char> {
+    template<typename FormatContext>
+    auto format(const af::dtype& p, FormatContext& ctx) -> decltype(ctx.out()) {
+        format_to(ctx.out(), "{}", getName(p));
+        return ctx.out();
+    }
+};
+
+template<>
+struct fmt::formatter<common::Node> {
+    // Presentation format: 'p' - pointer, 't' - type.
+    // char presentation;
+    bool pointer;
+    bool type;
+    bool children;
+    bool op;
+
+    // Parses format specifications of the form ['f' | 'e'].
+    constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) {
+        auto it = ctx.begin(), end = ctx.end();
+
+        if (it == end || *it == '}') {
+            pointer = type = children = op = true;
+            return it;
+        }
+
+        while (it != end && *it != '}') {
+            switch (*it) {
+                case 'p': pointer = true; break;
+                case 't': type = true; break;
+                case 'c': children = true; break;
+                case 'o': op = true; break;
+                default: throw format_error("invalid format");
+            }
+            ++it;
+        }
+
+        // Return an iterator past the end of the parsed range:
+        return it;
+    }
+
+    // Formats the point p using the parsed format specification (presentation)
+    // stored in this formatter.
+    template<typename FormatContext>
+    auto format(const common::Node& node, FormatContext& ctx)
+        -> decltype(ctx.out()) {
+        // ctx.out() is an output iterator to write to.
+
+        format_to(ctx.out(), "{{");
+        if (pointer) format_to(ctx.out(), "{} ", (void*)&node);
+        if (op) {
+            if (node.isBuffer()) {
+                format_to(ctx.out(), "buffer ");
+            } else {
+                format_to(ctx.out(), "{} ", getOpEnumStr(node.getOp()));
+            }
+        }
+        if (type) format_to(ctx.out(), "{} ", node.getType());
+        if (children) {
+            int count;
+            for (count = 0; count < common::Node::kMaxChildren &&
+                            node.m_children[count].get() != nullptr;
+                 count++) {}
+            if (count > 0) {
+                format_to(ctx.out(), "children: {{ ");
+                for (int i = 0; i < count; i++) {
+                    format_to(ctx.out(), "{} ", *(node.m_children[i].get()));
+                }
+                format_to(ctx.out(), "\b}} ");
+            }
+        }
+        format_to(ctx.out(), "\b}}");
+
+        return ctx.out();
+    }
+};

From c5af6ef031096f6f1227ed3d831c95aaffeb5906 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 21 Mar 2022 13:39:34 -0400
Subject: [PATCH 020/473] Add a function to check if Node is a scalar object

---
 src/backend/common/jit/Node.cpp       | 2 ++
 src/backend/common/jit/Node.hpp       | 5 +++++
 src/backend/common/jit/NodeIO.hpp     | 4 +++-
 src/backend/common/jit/ScalarNode.hpp | 3 +++
 src/backend/cpu/jit/ScalarNode.hpp    | 2 ++
 5 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/backend/common/jit/Node.cpp b/src/backend/common/jit/Node.cpp
index b59222de86..83767f502f 100644
--- a/src/backend/common/jit/Node.cpp
+++ b/src/backend/common/jit/Node.cpp
@@ -63,6 +63,8 @@ bool NodePtr_equalto::operator()(const Node *l, const Node *r) const noexcept {
 
 auto isBuffer(const Node &ptr) -> bool { return ptr.isBuffer(); }
 
+auto isScalar(const Node &ptr) -> bool { return ptr.isScalar(); }
+
 /// Returns true if the buffer is linear
 bool Node::isLinear(const dim_t dims[4]) const { return true; }
 
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 3cad47f03e..0b284c072e 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -241,6 +241,9 @@ class Node {
     // Returns true if this node is a Buffer
     virtual bool isBuffer() const { return false; }
 
+    // Returns true if this node is a Buffer
+    virtual bool isScalar() const { return false; }
+
     /// Returns true if the buffer is linear
     virtual bool isLinear(const dim_t dims[4]) const;
 
@@ -300,4 +303,6 @@ std::string getFuncName(const std::vector<Node *> &output_nodes,
 
 auto isBuffer(const Node &ptr) -> bool;
 
+auto isScalar(const Node &ptr) -> bool;
+
 }  // namespace common
diff --git a/src/backend/common/jit/NodeIO.hpp b/src/backend/common/jit/NodeIO.hpp
index 55d40c2b2d..050c8e3a7c 100644
--- a/src/backend/common/jit/NodeIO.hpp
+++ b/src/backend/common/jit/NodeIO.hpp
@@ -66,8 +66,10 @@ struct fmt::formatter<common::Node> {
         format_to(ctx.out(), "{{");
         if (pointer) format_to(ctx.out(), "{} ", (void*)&node);
         if (op) {
-            if (node.isBuffer()) {
+            if (isBuffer(node)) {
                 format_to(ctx.out(), "buffer ");
+            } else if (isScalar(node)) {
+                format_to(ctx.out(), "scalar ", getOpEnumStr(node.getOp()));
             } else {
                 format_to(ctx.out(), "{} ", getOpEnumStr(node.getOp()));
             }
diff --git a/src/backend/common/jit/ScalarNode.hpp b/src/backend/common/jit/ScalarNode.hpp
index bf0978359f..126e8860f7 100644
--- a/src/backend/common/jit/ScalarNode.hpp
+++ b/src/backend/common/jit/ScalarNode.hpp
@@ -84,6 +84,9 @@ class ScalarNode : public common::Node {
                   << ";\n";
     }
 
+    // Returns true if this node is a Buffer
+    virtual bool isScalar() const { return false; }
+
     std::string getNameStr() const final { return detail::shortname<T>(false); }
 
     // Return the info for the params and the size of the buffers
diff --git a/src/backend/cpu/jit/ScalarNode.hpp b/src/backend/cpu/jit/ScalarNode.hpp
index 657cbbf355..79a9f40f22 100644
--- a/src/backend/cpu/jit/ScalarNode.hpp
+++ b/src/backend/cpu/jit/ScalarNode.hpp
@@ -58,6 +58,8 @@ class ScalarNode : public TNode<T> {
         UNUSED(kerStream);
         UNUSED(ids);
     }
+
+    bool isScalar() const final { return true; }
 };
 }  // namespace jit
 

From 6543bee5dbc74931c986b753fc77649837cd2745 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 21 Mar 2022 16:00:56 -0400
Subject: [PATCH 021/473] Download only mkl instead of basekit when building
 the CI env

---
 .github/workflows/unix_cpu_build.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 9fcb37b87e..47dff97a42 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -83,7 +83,8 @@ jobs:
                   sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
                   sudo sh -c 'echo deb https://apt.repos.intel.com/oneapi all main > /etc/apt/sources.list.d/oneAPI.list'
                   sudo apt-get -qq update
-                  sudo apt-get install -y intel-basekit
+                  sudo apt-get install -y intel-oneapi-mkl-devel
+                  echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> ${GITHUB_ENV}
 
             - name: Install OpenBLAS for Ubuntu
               if: matrix.os != 'macos-latest' && matrix.blas_backend == 'OpenBLAS'
@@ -107,7 +108,7 @@ jobs:
                       -DAF_BUILD_CUDA:BOOL=OFF -DAF_BUILD_OPENCL:BOOL=OFF \
                       -DAF_BUILD_UNIFIED:BOOL=OFF -DAF_BUILD_EXAMPLES:BOOL=ON \
                       -DAF_BUILD_FORGE:BOOL=ON \
-                      -DAF_COMPUTE_LIBRARY:STRING=$backend \
+                      -DAF_COMPUTE_LIBRARY:STRING=${backend} \
                       -DBUILDNAME:STRING=${buildname} ..
                   echo "CTEST_DASHBOARD=${dashboard}" >> $GITHUB_ENV
 

From 86cdffd219bdb4ab13d5810ab950176f55506234 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 19 Mar 2022 00:51:16 -0400
Subject: [PATCH 022/473] Fix nested and duplicate moddims jit issue with the
 CPU backend

Fix an issue that caused errors with nested moddims caused errors with the CPU
backend. this was caused when the moddims function was called back to back on
the same array.

Another issue that this fixes is when you have the same node which are composed
of moddims arrays in the same jit tree
---
 src/backend/cpu/kernel/Array.hpp | 104 +++++++++++++++++++------------
 test/moddims.cpp                 |  67 ++++++++++++++++++++
 2 files changed, 130 insertions(+), 41 deletions(-)

diff --git a/src/backend/cpu/kernel/Array.hpp b/src/backend/cpu/kernel/Array.hpp
index 32ef5f6634..48987a5d4d 100644
--- a/src/backend/cpu/kernel/Array.hpp
+++ b/src/backend/cpu/kernel/Array.hpp
@@ -21,16 +21,16 @@
 namespace cpu {
 namespace kernel {
 
-/// Clones nodes and update the child pointers
+/// Clones node_index_map and update the child pointers
 std::vector<std::shared_ptr<common::Node>> cloneNodes(
-    const std::vector<common::Node *> &nodes,
+    const std::vector<common::Node *> &node_index_map,
     const std::vector<common::Node_ids> &ids) {
     using common::Node;
     // find all moddims in the tree
     std::vector<std::shared_ptr<Node>> node_clones;
-    node_clones.reserve(nodes.size());
-    transform(begin(nodes), end(nodes), back_inserter(node_clones),
-              [](Node *n) { return n->clone(); });
+    node_clones.reserve(node_index_map.size());
+    transform(begin(node_index_map), end(node_index_map),
+              back_inserter(node_clones), [](Node *n) { return n->clone(); });
 
     for (common::Node_ids id : ids) {
         auto &children = node_clones[id.id]->m_children;
@@ -41,7 +41,8 @@ std::vector<std::shared_ptr<common::Node>> cloneNodes(
     return node_clones;
 }
 
-/// Sets the shape of the buffer nodes under the moddims node to the new shape
+/// Sets the shape of the buffer node_index_map under the moddims node to the
+/// new shape
 void propagateModdimsShape(
     std::vector<std::shared_ptr<common::Node>> &node_clones) {
     using common::NodeIterator;
@@ -63,14 +64,13 @@ void propagateModdimsShape(
     }
 }
 
-/// Removes nodes whos operation matchs a unary operation \p op.
-void removeNodeOfOperation(std::vector<std::shared_ptr<common::Node>> &nodes,
-                           std::vector<common::Node_ids> &ids, af_op_t op) {
+/// Removes node_index_map whos operation matchs a unary operation \p op.
+void removeNodeOfOperation(
+    std::vector<std::shared_ptr<common::Node>> &node_index_map, af_op_t op) {
     using common::Node;
 
-    std::vector<std::vector<std::shared_ptr<Node>>::iterator> moddims_loc;
-    for (size_t nid = 0; nid < nodes.size(); nid++) {
-        auto &node = nodes[nid];
+    for (size_t nid = 0; nid < node_index_map.size(); nid++) {
+        auto &node = node_index_map[nid];
 
         for (int i = 0;
              i < Node::kMaxChildren && node->m_children[i] != nullptr; i++) {
@@ -78,15 +78,47 @@ void removeNodeOfOperation(std::vector<std::shared_ptr<common::Node>> &nodes,
                 // replace moddims
                 auto moddim_node    = node->m_children[i];
                 node->m_children[i] = moddim_node->m_children[0];
-
-                int parent_id = ids[nid].id;
-                int moddim_id = ids[parent_id].child_ids[i];
-                moddims_loc.emplace_back(begin(nodes) + moddim_id);
             }
         }
     }
 
-    for (auto &loc : moddims_loc) { nodes.erase(loc); }
+    node_index_map.erase(remove_if(begin(node_index_map), end(node_index_map),
+                                   [op](std::shared_ptr<Node> &node) {
+                                       return node->getOp() == op;
+                                   }),
+                         end(node_index_map));
+}
+
+/// Returns the cloned output_nodes located in the node_clones array
+///
+/// This function returns the new cloned version of the output_nodes_ from
+/// the node_clones array. If the output node is a moddim node, then it will
+/// set the output node to be its first non-moddim node child
+template<typename T>
+std::vector<TNode<T> *> getClonedOutputNodes(
+    common::Node_map_t &node_index_map,
+    const std::vector<std::shared_ptr<common::Node>> &node_clones,
+    const std::vector<common::Node_ptr> &output_nodes_) {
+    std::vector<TNode<T> *> cloned_output_nodes;
+    cloned_output_nodes.reserve(output_nodes_.size());
+    for (auto &n : output_nodes_) {
+        TNode<T> *ptr;
+        if (n->getOp() == af_moddims_t) {
+            // if the output node is a moddims node, then set the output node
+            // to be the child of the moddims node. This is necessary because
+            // we remove the moddim node_index_map from the tree later
+            int child_index = node_index_map[n->m_children[0].get()];
+            ptr = static_cast<TNode<T> *>(node_clones[child_index].get());
+            while (ptr->getOp() == af_moddims_t) {
+                ptr = static_cast<TNode<T> *>(ptr->m_children[0].get());
+            }
+        } else {
+            int node_index = node_index_map[n.get()];
+            ptr = static_cast<TNode<T> *>(node_clones[node_index].get());
+        }
+        cloned_output_nodes.push_back(ptr);
+    }
+    return cloned_output_nodes;
 }
 
 template<typename T>
@@ -100,41 +132,29 @@ void evalMultiple(std::vector<Param<T>> arrays,
     af::dim4 odims = arrays[0].dims();
     af::dim4 ostrs = arrays[0].strides();
 
-    Node_map_t nodes;
+    Node_map_t node_index_map;
     std::vector<T *> ptrs;
-    std::vector<TNode<T> *> output_nodes;
     std::vector<common::Node *> full_nodes;
     std::vector<common::Node_ids> ids;
 
     int narrays = static_cast<int>(arrays.size());
+    ptrs.reserve(narrays);
     for (int i = 0; i < narrays; i++) {
         ptrs.push_back(arrays[i].get());
-        output_nodes_[i]->getNodesMap(nodes, full_nodes, ids);
+        output_nodes_[i]->getNodesMap(node_index_map, full_nodes, ids);
     }
-
     auto node_clones = cloneNodes(full_nodes, ids);
 
-    for (auto &n : output_nodes_) {
-        if (n->getOp() == af_moddims_t) {
-            // if the output node is a moddims node, then set the output node to
-            // be the child of the moddims node. This is necessary because we
-            // remove the moddim nodes from the tree later
-            output_nodes.push_back(static_cast<TNode<T> *>(
-                node_clones[nodes[n->m_children[0].get()]].get()));
-        } else {
-            output_nodes.push_back(
-                static_cast<TNode<T> *>(node_clones[nodes[n.get()]].get()));
-        }
-    }
-
+    std::vector<TNode<T> *> cloned_output_nodes =
+        getClonedOutputNodes<T>(node_index_map, node_clones, output_nodes_);
     propagateModdimsShape(node_clones);
-    removeNodeOfOperation(node_clones, ids, af_moddims_t);
+    removeNodeOfOperation(node_clones, af_moddims_t);
 
     bool is_linear = true;
     for (auto &node : node_clones) { is_linear &= node->isLinear(odims.get()); }
 
     int num_nodes        = node_clones.size();
-    int num_output_nodes = output_nodes.size();
+    int num_output_nodes = cloned_output_nodes.size();
     if (is_linear) {
         int num = arrays[0].dims().elements();
         int cnum =
@@ -145,8 +165,9 @@ void evalMultiple(std::vector<Param<T>> arrays,
                 node_clones[n]->calc(i, lim);
             }
             for (int n = 0; n < num_output_nodes; n++) {
-                std::copy(output_nodes[n]->m_val.begin(),
-                          output_nodes[n]->m_val.begin() + lim, ptrs[n] + i);
+                std::copy(cloned_output_nodes[n]->m_val.begin(),
+                          cloned_output_nodes[n]->m_val.begin() + lim,
+                          ptrs[n] + i);
             }
         }
     } else {
@@ -170,9 +191,10 @@ void evalMultiple(std::vector<Param<T>> arrays,
                             node_clones[n]->calc(x, y, z, w, lim);
                         }
                         for (int n = 0; n < num_output_nodes; n++) {
-                            std::copy(output_nodes[n]->m_val.begin(),
-                                      output_nodes[n]->m_val.begin() + lim,
-                                      ptrs[n] + id);
+                            std::copy(
+                                cloned_output_nodes[n]->m_val.begin(),
+                                cloned_output_nodes[n]->m_val.begin() + lim,
+                                ptrs[n] + id);
                         }
                     }
                 }
diff --git a/test/moddims.cpp b/test/moddims.cpp
index 6794e4c90e..630e4e6783 100644
--- a/test/moddims.cpp
+++ b/test/moddims.cpp
@@ -279,3 +279,70 @@ TEST(Moddims, jit) {
     gold = moddims(gold, 5, 10);
     ASSERT_ARRAYS_EQ(gold, a);
 }
+
+TEST(Moddims, JitNested) {
+    array a    = af::constant(1, 5, 5);
+    array b    = moddims(moddims(moddims(a, 25), 1, 5, 5), 5, 5);
+    array gold = af::constant(1, 5, 5);
+    gold.eval();
+    ASSERT_ARRAYS_EQ(gold, b);
+}
+
+TEST(Moddims, JitDuplicate) {
+    array a = af::constant(1, 5, 5);
+    array b = af::moddims(a, 25);
+    array c = b + b;
+
+    array gold = af::constant(2, 25);
+    gold.eval();
+    ASSERT_ARRAYS_EQ(gold, c);
+}
+
+TEST(Moddims, JitNestedAndDuplicate) {
+    array a = af::constant(1, 10, 10);
+    array b = af::constant(1, 10, 10);
+    array c = af::constant(2, 100) + moddims(a + b, 100);
+    array d = moddims(
+        moddims(af::constant(2, 1, 10, 10) + moddims(c, 1, 10, 10), 100), 10,
+        10);
+    array e    = d + d;
+    array gold = af::constant(12, 10, 10);
+    gold.eval();
+    ASSERT_ARRAYS_EQ(gold, e);
+}
+
+TEST(Moddims, JitTileThenModdims) {
+    array a    = af::constant(1, 10);
+    array b    = tile(a, 1, 10);
+    array c    = moddims(b, 100);
+    array gold = af::constant(1, 100);
+    gold.eval();
+    ASSERT_ARRAYS_EQ(gold, c);
+}
+
+TEST(Moddims, JitModdimsThenTiled) {
+    array a    = af::constant(1, 10);
+    array b    = moddims(a, 1, 10);
+    array c    = tile(b, 10);
+    array gold = af::constant(1, 10, 10);
+    gold.eval();
+    ASSERT_ARRAYS_EQ(gold, c);
+}
+
+TEST(Moddims, JitTileThenMultipleModdims) {
+    array a    = af::constant(1, 10);
+    array b    = tile(a, 1, 10);
+    array c    = moddims(moddims(b, 100), 10, 10);
+    array gold = af::constant(1, 10, 10);
+    gold.eval();
+    ASSERT_ARRAYS_EQ(gold, c);
+}
+
+TEST(Moddims, JitMultipleModdimsThenTiled) {
+    array a    = af::constant(1, 10);
+    array b    = moddims(moddims(a, 1, 10), 1, 1, 10);
+    array c    = tile(b, 10);
+    array gold = af::constant(1, 10, 1, 10);
+    gold.eval();
+    ASSERT_ARRAYS_EQ(gold, c);
+}

From 699df329363072375273f69a2ff3ed19a92f7b7c Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 13 Jan 2022 13:40:34 -0500
Subject: [PATCH 023/473] Link afcuda with static numeric libs by default

ArrayFire's CUDA backend linked against the CUDA numeric libraries
staticly before this change. This caused the libafcuda library to be
in the 1.1GB range for CUDA 11.5 even if you were targeting one compute
capability. This is partially due to the fact that the linker does not
remove the compute capabilities of older architectures when linking.

One way around this would be to use nvprune to remove the architectures
that are not being used by the compute cability when building. This
approach is not yet implemented.

This commit will revert back to dynamically linking the CUDA numeric
libraries by default. You can still select the old behavior by setting
the AF_WITH_STATIC_CUDA_NUMERIC_LIBS option in CMake
---
 CMakeLists.txt                  |  1 +
 src/backend/cuda/CMakeLists.txt | 41 ++++++++++++++++-----------------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5ccfff22bb..dce9076c8c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -74,6 +74,7 @@ option(AF_WITH_LOGGING  "Build ArrayFire with logging support" ON)
 option(AF_WITH_STACKTRACE  "Add stacktraces to the error messages." ON)
 option(AF_CACHE_KERNELS_TO_DISK "Enable caching kernels to disk" ON)
 option(AF_WITH_STATIC_MKL "Link against static Intel MKL libraries" OFF)
+option(AF_WITH_STATIC_CUDA_NUMERIC_LIBS "Link libafcuda with static numeric libraries(cublas, cufft, etc.)" OFF)
 
 set(default_compute_library "FFTW/LAPACK/BLAS")
 if(MKL_FOUND)
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 218878e163..f10ae0dc0c 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -47,7 +47,7 @@ endif()
 
 find_cuda_helper_libs(nvrtc)
 find_cuda_helper_libs(nvrtc-builtins)
-if(UNIX)
+if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   af_find_static_cuda_libs(culibos)
   af_find_static_cuda_libs(cublas_static)
   af_find_static_cuda_libs(cublasLt_static)
@@ -312,8 +312,7 @@ if(CUDA_VERSION_MAJOR VERSION_GREATER 10 OR
   target_compile_definitions(af_cuda_static_cuda_library PRIVATE AF_USE_NEW_CUSPARSE_API)
 endif()
 
-if(UNIX)
-
+if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   check_cxx_compiler_flag("-Wl,--start-group -Werror" group_flags)
   if(group_flags)
     set(START_GROUP -Wl,--start-group)
@@ -349,7 +348,7 @@ if(UNIX)
   set(CUDA_SEPARABLE_COMPILATION ${pior_val_CUDA_SEPARABLE_COMPILATION})
 else()
   target_link_libraries(af_cuda_static_cuda_library
-    PRIVATE
+    PUBLIC
       Boost::boost
       ${CUDA_CUBLAS_LIBRARIES}
       ${CUDA_CUFFT_LIBRARIES}
@@ -771,10 +770,10 @@ function(afcu_collect_libs libname)
 
   if(cuda_args_LIB_MAJOR AND cuda_args_LIB_MINOR)
     set(lib_major ${cuda_args_LIB_MAJOR})
-	set(lib_minor ${cuda_args_LIB_MINOR})
+	  set(lib_minor ${cuda_args_LIB_MINOR})
   else()
     set(lib_major ${CUDA_VERSION_MAJOR})
-	set(lib_minor ${CUDA_VERSION_MINOR})
+	  set(lib_minor ${CUDA_VERSION_MINOR})
   endif()
   set(lib_version "${lib_major}.${lib_minor}")
 
@@ -832,24 +831,24 @@ endfunction()
 if(AF_INSTALL_STANDALONE)
   if(AF_WITH_CUDNN)
     afcu_collect_cudnn_libs("")
-	if(cuDNN_VERSION_MAJOR VERSION_GREATER 8 OR cuDNN_VERSION_MAJOR VERSION_EQUAL 8)
-	  # cudnn changed how dlls are shipped starting major version 8
+    if(cuDNN_VERSION_MAJOR VERSION_GREATER 8 OR cuDNN_VERSION_MAJOR VERSION_EQUAL 8)
+      # cudnn changed how dlls are shipped starting major version 8
       # except the main dll a lot of the other DLLs are loaded upon demand
-	  afcu_collect_cudnn_libs(adv_infer)
-	  afcu_collect_cudnn_libs(adv_train)
-	  afcu_collect_cudnn_libs(cnn_infer)
-	  afcu_collect_cudnn_libs(cnn_train)
-	  afcu_collect_cudnn_libs(ops_infer)
-	  afcu_collect_cudnn_libs(ops_train)
-	endif()
+      afcu_collect_cudnn_libs(adv_infer)
+      afcu_collect_cudnn_libs(adv_train)
+      afcu_collect_cudnn_libs(cnn_infer)
+      afcu_collect_cudnn_libs(cnn_train)
+      afcu_collect_cudnn_libs(ops_infer)
+      afcu_collect_cudnn_libs(ops_train)
+    endif()
   endif()
 
-  if(WIN32)
-	if(CUDA_VERSION_MAJOR VERSION_EQUAL 11)
-      afcu_collect_libs(cufft LIB_MAJOR 10 LIB_MINOR 4)
-	else()
-      afcu_collect_libs(cufft)
-	endif()
+  if(WIN32 OR NOT AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
+    if(CUDA_VERSION_MAJOR VERSION_EQUAL 11)
+        afcu_collect_libs(cufft LIB_MAJOR 10 LIB_MINOR 4)
+    else()
+        afcu_collect_libs(cufft)
+    endif()
     afcu_collect_libs(cublas)
     if(CUDA_VERSION VERSION_GREATER 10.0)
       afcu_collect_libs(cublasLt)

From e3f9559375bb3ac42ba0202bc77f45a4dd0a40a4 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 13 Jan 2022 17:04:35 -0500
Subject: [PATCH 024/473] Fix find_library call when searching for CUDA
 libraries

---
 src/backend/cuda/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index f10ae0dc0c..fd81ebd3eb 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -797,8 +797,8 @@ function(afcu_collect_libs libname)
             COMPONENT   cuda_dependencies)
   else () #UNIX
     find_library(CUDA_${libname}_LIBRARY
-      NAME ${libname}
-      PATH
+      NAMES ${libname}
+      PATHS
         ${dlib_path_prefix})
 
     get_filename_component(outpath "${CUDA_${libname}_LIBRARY}" REALPATH)

From 453cdc3f520a7e4f179ef344d261847c18f77e34 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 22 Mar 2022 17:26:17 -0400
Subject: [PATCH 025/473] Prune CUDA static numerical libraries for specifice
 compute capability

Prune CUDA static libraries so that the binary size of the final executable
is smaller. This commit will run the nvprune utility on some static libraries
(cublasLt, cublas, cusolver, and cusparse) to remove unused architectures
from the binary. The resulting binary is significantly smaller when targeting
a single compute capability.
---
 CMakeModules/AFcuda_helpers.cmake | 25 +++++++++++++++-
 src/backend/cuda/CMakeLists.txt   | 49 ++++++++++++++++---------------
 2 files changed, 50 insertions(+), 24 deletions(-)

diff --git a/CMakeModules/AFcuda_helpers.cmake b/CMakeModules/AFcuda_helpers.cmake
index 4fde494df8..578c49956b 100644
--- a/CMakeModules/AFcuda_helpers.cmake
+++ b/CMakeModules/AFcuda_helpers.cmake
@@ -5,14 +5,37 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-
+find_program(NVPRUNE NAMES nvprune)
 # The following macro uses a macro defined by
 # FindCUDA module from cmake.
 function(af_find_static_cuda_libs libname)
+  cmake_parse_arguments(fscl "PRUNE" "" "" ${ARGN})
+
   set(search_name
     "${CMAKE_STATIC_LIBRARY_PREFIX}${libname}${CMAKE_STATIC_LIBRARY_SUFFIX}")
   cuda_find_library_local_first(CUDA_${libname}_LIBRARY
     ${search_name} "${libname} static library")
+
+  if(fscl_PRUNE)
+    get_filename_component(af_${libname} ${CUDA_${libname}_LIBRARY} NAME)
+
+    set(liboutput ${CMAKE_CURRENT_BINARY_DIR}/${af_${libname}})
+    add_custom_command(OUTPUT ${liboutput}.depend
+      COMMAND ${NVPRUNE} ${cuda_architecture_flags} ${CUDA_${libname}_LIBRARY} -o ${liboutput}
+      COMMAND ${CMAKE_COMMAND} -E touch ${liboutput}.depend
+      BYPRODUCTS ${liboutput}
+      MAIN_DEPENDENCY ${CUDA_${libname}_LIBRARY}
+      COMMENT "Pruning ${CUDA_${libname}_LIBRARY} for ${cuda_build_targets}"
+      VERBATIM)
+    add_custom_target(AF_CUDA_${libname}_LIBRARY_TARGET
+      DEPENDS ${liboutput}.depend)
+    list(APPEND cuda_pruned_libraries AF_CUDA_${libname}_LIBRARY_TARGET PARENT_SCOPE)
+
+    set(AF_CUDA_${libname}_LIBRARY ${liboutput} PARENT_SCOPE)
+    mark_as_advanced(AF_CUDA_${libname}_LIBRARY)
+  else()
+    set(AF_CUDA_${libname}_LIBRARY ${CUDA_${libname}_LIBRARY} PARENT_SCOPE)
+  endif()
   mark_as_advanced(CUDA_${libname}_LIBRARY)
 endfunction()
 
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index fd81ebd3eb..7694170aa5 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -45,14 +45,27 @@ else()
   set(use_static_cuda_lapack OFF)
 endif()
 
+set(CUDA_architecture_build_targets "Auto" CACHE
+  STRING "The compute architectures targeted by this build. (Options: Auto;3.0;Maxwell;All;Common)")
+
+cuda_select_nvcc_arch_flags(cuda_architecture_flags ${CUDA_architecture_build_targets})
+
+string(REGEX REPLACE "-gencodearch=compute_[0-9]+,code=sm_([0-9]+)" "\\1|" cuda_build_targets ${cuda_architecture_flags})
+string(REGEX REPLACE "-gencodearch=compute_[0-9]+,code=compute_([0-9]+)" "\\1+PTX|" cuda_build_targets ${cuda_build_targets})
+string(REGEX REPLACE "([0-9]+)([0-9])\\|" "\\1.\\2 " cuda_build_targets ${cuda_build_targets})
+string(REGEX REPLACE "([0-9]+)([0-9]\\+PTX)\\|" "\\1.\\2 " cuda_build_targets ${cuda_build_targets})
+message(STATUS "CUDA_architecture_build_targets: ${CUDA_architecture_build_targets} ( ${cuda_build_targets} )")
+
+set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};${cuda_architecture_flags})
+
 find_cuda_helper_libs(nvrtc)
 find_cuda_helper_libs(nvrtc-builtins)
 if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   af_find_static_cuda_libs(culibos)
-  af_find_static_cuda_libs(cublas_static)
-  af_find_static_cuda_libs(cublasLt_static)
+  af_find_static_cuda_libs(cublas_static PRUNE)
+  af_find_static_cuda_libs(cublasLt_static PRUNE)
   af_find_static_cuda_libs(cufft_static)
-  af_find_static_cuda_libs(cusparse_static)
+  af_find_static_cuda_libs(cusparse_static PRUNE)
 
   # FIXME When NVCC resolves this particular issue.
   # NVCC doesn't like -l<full_path_static_lib>, hence we cannot
@@ -67,8 +80,8 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   set(af_cuda_static_flags "${af_cuda_static_flags};-lcusparse_static")
 
   if(${use_static_cuda_lapack})
-    af_find_static_cuda_libs(cusolver_static)
-    set(cusolver_static_lib "${CUDA_cusolver_static_LIBRARY}")
+    af_find_static_cuda_libs(cusolver_static PRUNE)
+    set(cusolver_static_lib "${AF_CUDA_cusolver_static_LIBRARY}")
 
     # NVIDIA LAPACK library liblapack_static.a is a subset of LAPACK and only
     # contains GPU accelerated stedc and bdsqr. The user has to link
@@ -84,19 +97,6 @@ endif()
 
 get_filename_component(CUDA_LIBRARIES_PATH ${CUDA_cudart_static_LIBRARY} DIRECTORY CACHE)
 
-set(CUDA_architecture_build_targets "Auto" CACHE
-    STRING "The compute architectures targeted by this build. (Options: Auto;3.0;Maxwell;All;Common)")
-
-cuda_select_nvcc_arch_flags(cuda_architecture_flags ${CUDA_architecture_build_targets})
-
-string(REGEX REPLACE "-gencodearch=compute_[0-9]+,code=sm_([0-9]+)" "\\1|" cuda_build_targets ${cuda_architecture_flags})
-string(REGEX REPLACE "-gencodearch=compute_[0-9]+,code=compute_([0-9]+)" "\\1+PTX|" cuda_build_targets ${cuda_build_targets})
-string(REGEX REPLACE "([0-9]+)([0-9])\\|" "\\1.\\2 " cuda_build_targets ${cuda_build_targets})
-string(REGEX REPLACE "([0-9]+)([0-9]\\+PTX)\\|" "\\1.\\2 " cuda_build_targets ${cuda_build_targets})
-message(STATUS "CUDA_architecture_build_targets: ${CUDA_architecture_build_targets} ( ${cuda_build_targets} )")
-
-set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};${cuda_architecture_flags})
-
 mark_as_advanced(
     CUDA_LIBRARIES_PATH
     CUDA_architecture_build_targets)
@@ -327,9 +327,10 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
       ${cusolver_lib}
       ${START_GROUP}
       ${CUDA_culibos_LIBRARY} #also a static libary
-      ${CUDA_cublas_static_LIBRARY}
-      ${CUDA_cufft_static_LIBRARY}
-      ${CUDA_cusparse_static_LIBRARY}
+      ${AF_CUDA_cublas_static_LIBRARY}
+      ${AF_CUDA_cufft_static_LIBRARY}
+      ${AF_CUDA_cusparse_static_LIBRARY}
+      ${AF_CUDA_cublasLt_static_LIBRARY}
       ${cusolver_static_lib}
       ${END_GROUP}
   )
@@ -337,7 +338,7 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   if(CUDA_VERSION VERSION_GREATER 10.0)
     target_link_libraries(af_cuda_static_cuda_library
       PRIVATE
-        ${CUDA_cublasLt_static_LIBRARY})
+        ${AF_CUDA_cublasLt_static_LIBRARY})
   endif()
   if(CUDA_VERSION VERSION_GREATER 9.5)
     target_link_libraries(af_cuda_static_cuda_library
@@ -687,7 +688,9 @@ add_library(ArrayFire::afcuda ALIAS afcuda)
 
 add_dependencies(afcuda ${jit_kernel_targets} ${nvrtc_kernel_targets})
 add_dependencies(af_cuda_static_cuda_library ${nvrtc_kernel_targets})
-add_dependencies(afcuda af_cuda_static_cuda_library)
+if(cuda_pruned_libraries)
+  add_dependencies(afcuda ${cuda_pruned_libraries})
+endif()
 
 target_include_directories (afcuda
   PUBLIC

From 83aad432c50732f0645d911bbec2ed62c7459ddb Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 22 Mar 2022 18:31:32 -0400
Subject: [PATCH 026/473] Remove adv_infer and adv_train cudnn libs from
 install step

---
 src/backend/cuda/CMakeLists.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 7694170aa5..d75b96296a 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -837,8 +837,6 @@ if(AF_INSTALL_STANDALONE)
     if(cuDNN_VERSION_MAJOR VERSION_GREATER 8 OR cuDNN_VERSION_MAJOR VERSION_EQUAL 8)
       # cudnn changed how dlls are shipped starting major version 8
       # except the main dll a lot of the other DLLs are loaded upon demand
-      afcu_collect_cudnn_libs(adv_infer)
-      afcu_collect_cudnn_libs(adv_train)
       afcu_collect_cudnn_libs(cnn_infer)
       afcu_collect_cudnn_libs(cnn_train)
       afcu_collect_cudnn_libs(ops_infer)

From b76b12711f7296bc544cd25b6f72111addb290cb Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 23 Mar 2022 20:16:57 -0400
Subject: [PATCH 027/473] Deterministic topK (#3210)

Add the ability to perform topk and maintain a stable order of the indices
---
 include/af/defines.h                          |   7 +-
 include/af/statistics.h                       |   6 +-
 src/backend/cpu/topk.cpp                      |  54 ++++++--
 src/backend/cuda/kernel/topk.hpp              |  56 ++++++--
 .../opencl/kernel/sort_by_key_impl.hpp        |   5 +-
 src/backend/opencl/topk.cpp                   |  56 ++++++--
 test/topk.cpp                                 | 128 +++++++++++++++++-
 7 files changed, 268 insertions(+), 44 deletions(-)

diff --git a/include/af/defines.h b/include/af/defines.h
index a346a14e24..611a025375 100644
--- a/include/af/defines.h
+++ b/include/af/defines.h
@@ -508,8 +508,11 @@ typedef enum {
 } af_diffusion_eq;
 
 typedef enum {
-    AF_TOPK_MIN     = 1,  ///< Top k min values
-    AF_TOPK_MAX     = 2,  ///< Top k max values
+    AF_TOPK_MIN         = 1,  ///< Top k min values
+    AF_TOPK_MAX         = 2,  ///< Top k max values
+    AF_TOPK_STABLE      = 4,  ///< Preserve order of indices for equal values
+    AF_TOPK_STABLE_MIN  = AF_TOPK_STABLE | AF_TOPK_MIN, ///< Top k min with stable indices
+    AF_TOPK_STABLE_MAX  = AF_TOPK_STABLE | AF_TOPK_MAX, ///< Top k max with stable indices
     AF_TOPK_DEFAULT = 0   ///< Default option (max)
 } af_topk_function;
 #endif
diff --git a/include/af/statistics.h b/include/af/statistics.h
index 9f7adf455a..86851a3a7b 100644
--- a/include/af/statistics.h
+++ b/include/af/statistics.h
@@ -320,7 +320,8 @@ AFAPI T corrcoef(const array& X, const array& Y);
 
    \note{This function is optimized for small values of k.}
    \note{The order of the returned keys may not be in the same order as the
-   appear in the input array}
+   appear in the input array, for a stable topk, set the AF_TOPK_STABLE flag
+   in the order param. These are equivalent to AF_TOPK_STABLE_MAX and AF_TOPK_STABLE_MIN}
    \ingroup stat_func_topk
 */
 AFAPI void topk(array &values, array &indices, const array& in, const int k,
@@ -673,7 +674,8 @@ AFAPI af_err af_corrcoef(double *realVal, double *imagVal, const af_array X, con
 
    \note{This function is optimized for small values of k.}
    \note{The order of the returned keys may not be in the same order as the
-         appear in the input array}
+   appear in the input array, for a stable topk, set the AF_TOPK_STABLE flag
+   in the order param. These are equivalent to AF_TOPK_STABLE_MAX and AF_TOPK_STABLE_MIN}
    \ingroup stat_func_topk
 */
 AFAPI af_err af_topk(af_array *values, af_array *indices, const af_array in,
diff --git a/src/backend/cpu/topk.cpp b/src/backend/cpu/topk.cpp
index 645e48d2e2..a87d257a8c 100644
--- a/src/backend/cpu/topk.cpp
+++ b/src/backend/cpu/topk.cpp
@@ -57,19 +57,49 @@ void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
             auto idx_itr = begin(idx) + i * in.strides()[1];
             auto* kiptr  = iptr + k * i;
 
-            if (order == AF_TOPK_MIN) {
-                // Sort the top k values in each column
-                partial_sort_copy(
-                    idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
-                    [ptr](const uint lhs, const uint rhs) -> bool {
-                        return compute_t<T>(ptr[lhs]) < compute_t<T>(ptr[rhs]);
-                    });
+            if (order & AF_TOPK_MIN) {
+                if (order & AF_TOPK_STABLE) {
+                    partial_sort_copy(
+                        idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+                        [ptr](const uint lhs, const uint rhs) -> bool {
+                            return compute_t<T>(ptr[lhs]) <
+                                           compute_t<T>(ptr[rhs])
+                                       ? true
+                                   : compute_t<T>(ptr[lhs]) ==
+                                           compute_t<T>(ptr[rhs])
+                                       ? (lhs < rhs)
+                                       : false;
+                        });
+                } else {
+                    partial_sort_copy(
+                        idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+                        [ptr](const uint lhs, const uint rhs) -> bool {
+                            return compute_t<T>(ptr[lhs]) <
+                                   compute_t<T>(ptr[rhs]);
+                        });
+                    // Sort the top k values in each column
+                }
             } else {
-                partial_sort_copy(
-                    idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
-                    [ptr](const uint lhs, const uint rhs) -> bool {
-                        return compute_t<T>(ptr[lhs]) >= compute_t<T>(ptr[rhs]);
-                    });
+                if (order & AF_TOPK_STABLE) {
+                    partial_sort_copy(
+                        idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+                        [ptr](const uint lhs, const uint rhs) -> bool {
+                            return compute_t<T>(ptr[lhs]) >
+                                           compute_t<T>(ptr[rhs])
+                                       ? true
+                                   : compute_t<T>(ptr[lhs]) ==
+                                           compute_t<T>(ptr[rhs])
+                                       ? (lhs < rhs)
+                                       : false;
+                        });
+                } else {
+                    partial_sort_copy(
+                        idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+                        [ptr](const uint lhs, const uint rhs) -> bool {
+                            return compute_t<T>(ptr[lhs]) >
+                                   compute_t<T>(ptr[rhs]);
+                        });
+                }
             }
 
             auto* kvptr = vptr + k * i;
diff --git a/src/backend/cuda/kernel/topk.hpp b/src/backend/cuda/kernel/topk.hpp
index 4552ab0b97..0d71d4949c 100644
--- a/src/backend/cuda/kernel/topk.hpp
+++ b/src/backend/cuda/kernel/topk.hpp
@@ -36,14 +36,26 @@ static __global__ void kerTopkDim0(Param<T> ovals, Param<uint> oidxs,
     using BlockRadixSortT = BlockRadixSort<compute_t<T>, TOPK_THRDS_PER_BLK,
                                            TOPK_IDX_THRD_LOAD, ValueType>;
 
-    __shared__ typename BlockRadixSortT::TempStorage smem;
+    struct keyValBlocks {
+        // used for rearranging each granule's data items
+        // we want each thread(granule) to own TOPK_IDX_THRD_LOAD=4 consecutive
+        // datum for both coalesced memory reads and this blocked layout we need
+        // this SMEM to rearrange
+        compute_t<T> keys[TOPK_IDX_THRD_LOAD * TOPK_THRDS_PER_BLK];
+        ValueType vals[TOPK_IDX_THRD_LOAD * TOPK_THRDS_PER_BLK];
+    };
+
+    union smemUnion {
+        // used for cub radix sort
+        typename BlockRadixSortT::TempStorage sortmem;
+        // used for rearranging
+        keyValBlocks blkt;
+    } __shared__ smem;
 
     const int bw = blockIdx.y / numLaunchBlocksY;
     const int bz = blockIdx.z;
     const int by = (blockIdx.y - bw * numLaunchBlocksY);
 
-    const uint gx       = blockIdx.x * blockDim.x + threadIdx.x;
-    const uint gxStride = blockDim.x * gridDim.x;
     const uint elements = ivals.dims[0];
 
     const data_t<T>* kdata = ivals.ptr + by * ivals.strides[1] +
@@ -60,21 +72,41 @@ static __global__ void kerTopkDim0(Param<T> ovals, Param<uint> oidxs,
     compute_t<T> keys[TOPK_IDX_THRD_LOAD];
     ValueType vals[TOPK_IDX_THRD_LOAD];
 
-    for (uint li = 0, i = gx; li < TOPK_IDX_THRD_LOAD; i += gxStride, li++) {
+    const int blockOffset =
+        blockDim.x * blockIdx.x * TOPK_IDX_THRD_LOAD + threadIdx.x;
+// each block will load consecutive data items while iterating a block-width at
+// a time [B0][][]...[][B1][][]...[] ... [BN][][]...[]
+#pragma unroll
+    for (uint li = 0, i = blockOffset; li < TOPK_IDX_THRD_LOAD;
+         i += blockDim.x, li++) {
         if (i < elements) {
-            keys[li] = static_cast<compute_t<T>>(kdata[i]);
-            vals[li] = (READ_INDEX) ? idata[i] : i;
+            smem.blkt.keys[li * TOPK_THRDS_PER_BLK + threadIdx.x] =
+                static_cast<compute_t<T>>(kdata[i]);
+            smem.blkt.vals[li * TOPK_THRDS_PER_BLK + threadIdx.x] =
+                (READ_INDEX) ? idata[i] : i;
         } else {
-            keys[li] = (order == AF_TOPK_MAX) ? minval<compute_t<T>>()
-                                              : maxval<compute_t<T>>();
-            vals[li] = maxval<ValueType>();
+            smem.blkt.keys[li * TOPK_THRDS_PER_BLK + threadIdx.x] =
+                (order & AF_TOPK_MAX) ? minval<compute_t<T>>()
+                                      : maxval<compute_t<T>>();
+            smem.blkt.vals[li * TOPK_THRDS_PER_BLK + threadIdx.x] =
+                maxval<ValueType>();
         }
     }
+    __syncthreads();
 
-    if (order == AF_TOPK_MAX) {
-        BlockRadixSortT(smem).SortDescendingBlockedToStriped(keys, vals);
+#pragma unroll
+    for (uint li = 0; li < TOPK_IDX_THRD_LOAD; li++) {
+        // transposed read into registers for cub radix sort
+        keys[li] = smem.blkt.keys[li + (threadIdx.x * TOPK_IDX_THRD_LOAD)];
+        vals[li] = smem.blkt.vals[li + (threadIdx.x * TOPK_IDX_THRD_LOAD)];
+    }
+    __syncthreads();
+
+    if (order & AF_TOPK_MAX) {
+        BlockRadixSortT(smem.sortmem)
+            .SortDescendingBlockedToStriped(keys, vals);
     } else {
-        BlockRadixSortT(smem).SortBlockedToStriped(keys, vals);
+        BlockRadixSortT(smem.sortmem).SortBlockedToStriped(keys, vals);
     }
 
     if (threadIdx.x < k) {
diff --git a/src/backend/opencl/kernel/sort_by_key_impl.hpp b/src/backend/opencl/kernel/sort_by_key_impl.hpp
index 02f23cfa67..2d6f84493b 100644
--- a/src/backend/opencl/kernel/sort_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/sort_by_key_impl.hpp
@@ -222,10 +222,11 @@ void sort0ByKey(Param pKey, Param pVal, bool isAscending) {
     // But this is only useful before GPU is saturated
     // The GPU is saturated at around 1000,000 integers
     // Call batched sort only if both conditions are met
-    if (higherDims > 4 && pKey.info.dims[0] < 1000000)
+    if (higherDims > 4 && pKey.info.dims[0] < 1000000) {
         kernel::sortByKeyBatched<Tk, Tv>(pKey, pVal, 0, isAscending);
-    else
+    } else {
         kernel::sort0ByKeyIterative<Tk, Tv>(pKey, pVal, isAscending);
+    }
 }
 
 #define INSTANTIATE(Tk, Tv)                                           \
diff --git a/src/backend/opencl/topk.cpp b/src/backend/opencl/topk.cpp
index 5795ddd380..08155b9d8a 100644
--- a/src/backend/opencl/topk.cpp
+++ b/src/backend/opencl/topk.cpp
@@ -94,19 +94,49 @@ void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
             auto idx_itr = begin(idx) + i * in.strides()[1];
             auto kiptr   = iptr + k * i;
 
-            if (order == AF_TOPK_MIN) {
-                // Sort the top k values in each column
-                partial_sort_copy(
-                    idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
-                    [ptr](const uint lhs, const uint rhs) -> bool {
-                        return compute_t<T>(ptr[lhs]) < compute_t<T>(ptr[rhs]);
-                    });
+            if (order & AF_TOPK_MIN) {
+                if (order & AF_TOPK_STABLE) {
+                    partial_sort_copy(
+                        idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+                        [ptr](const uint lhs, const uint rhs) -> bool {
+                            return (compute_t<T>(ptr[lhs]) <
+                                    compute_t<T>(ptr[rhs]))
+                                       ? true
+                                   : compute_t<T>(ptr[lhs]) ==
+                                           compute_t<T>(ptr[rhs])
+                                       ? (lhs < rhs)
+                                       : false;
+                        });
+                } else {
+                    // Sort the top k values in each column
+                    partial_sort_copy(
+                        idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+                        [ptr](const uint lhs, const uint rhs) -> bool {
+                            return compute_t<T>(ptr[lhs]) <
+                                   compute_t<T>(ptr[rhs]);
+                        });
+                }
             } else {
-                partial_sort_copy(
-                    idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
-                    [ptr](const uint lhs, const uint rhs) -> bool {
-                        return compute_t<T>(ptr[lhs]) >= compute_t<T>(ptr[rhs]);
-                    });
+                if (order & AF_TOPK_STABLE) {
+                    partial_sort_copy(
+                        idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+                        [ptr](const uint lhs, const uint rhs) -> bool {
+                            return (compute_t<T>(ptr[lhs]) >
+                                    compute_t<T>(ptr[rhs]))
+                                       ? true
+                                   : compute_t<T>(ptr[lhs]) ==
+                                           compute_t<T>(ptr[rhs])
+                                       ? (lhs < rhs)
+                                       : false;
+                        });
+                } else {
+                    partial_sort_copy(
+                        idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+                        [ptr](const uint lhs, const uint rhs) -> bool {
+                            return compute_t<T>(ptr[lhs]) >
+                                   compute_t<T>(ptr[rhs]);
+                        });
+                }
             }
             ev_val.wait();
 
@@ -128,7 +158,7 @@ void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
     } else {
         auto values  = createEmptyArray<T>(in.dims());
         auto indices = createEmptyArray<unsigned>(in.dims());
-        sort_index(values, indices, in, dim, order == AF_TOPK_MIN);
+        sort_index(values, indices, in, dim, order & AF_TOPK_MIN);
         auto indVec = indexForTopK(k);
         vals        = index<T>(values, indVec.data());
         idxs        = index<unsigned>(indices, indVec.data());
diff --git a/test/topk.cpp b/test/topk.cpp
index 46eba3f159..46c4355d6a 100644
--- a/test/topk.cpp
+++ b/test/topk.cpp
@@ -113,7 +113,7 @@ void topkTest(const int ndims, const dim_t* dims, const unsigned k,
         for (size_t i = b * bSize; i < ((b + 1) * bSize); ++i)
             kvPairs.push_back(make_pair(inData[i], (i - b * bSize)));
 
-        if (order == AF_TOPK_MIN) {
+        if (order & AF_TOPK_MIN) {
             stable_sort(kvPairs.begin(), kvPairs.end(),
                         [](const KeyValuePair& lhs, const KeyValuePair& rhs) {
                             return lhs.first < rhs.first;
@@ -233,6 +233,74 @@ TEST(TopK, ValidationCheck_DefaultDim) {
     ASSERT_SUCCESS(af_release_array(idx));
 }
 
+// stable variants
+TYPED_TEST(TopK, Max1D0_Stable) {
+    af_dtype t    = (af_dtype)dtype_traits<TypeParam>::af_type;
+    dim_t dims[4] = {type_max(t), 1, 1, 1};
+    topkTest<TypeParam>(1, dims, 5, 0, AF_TOPK_STABLE_MAX);
+}
+
+TYPED_TEST(TopK, Max2D0_Stable) {
+    af_dtype t    = (af_dtype)dtype_traits<TypeParam>::af_type;
+    dim_t dims[4] = {type_max(t) / 10, 10, 1, 1};
+    topkTest<TypeParam>(2, dims, 3, 0, AF_TOPK_STABLE_MAX);
+}
+
+TYPED_TEST(TopK, Max3D0_Stable) {
+    af_dtype t    = (af_dtype)dtype_traits<TypeParam>::af_type;
+    dim_t dims[4] = {type_max(t) / 100, 10, 10, 1};
+    topkTest<TypeParam>(2, dims, 5, 0, AF_TOPK_STABLE_MAX);
+}
+
+TYPED_TEST(TopK, Max4D0_Stable) {
+    af_dtype t    = (af_dtype)dtype_traits<TypeParam>::af_type;
+    dim_t dims[4] = {type_max(t) / 1000, 10, 10, 10};
+    topkTest<TypeParam>(2, dims, 5, 0, AF_TOPK_STABLE_MAX);
+}
+
+TYPED_TEST(TopK, Min1D0_Stable) {
+    af_dtype t    = (af_dtype)dtype_traits<TypeParam>::af_type;
+    dim_t dims[4] = {type_max(t), 1, 1, 1};
+    topkTest<TypeParam>(1, dims, 5, 0, AF_TOPK_STABLE_MIN);
+}
+
+TYPED_TEST(TopK, Min2D0_Stable) {
+    af_dtype t    = (af_dtype)dtype_traits<TypeParam>::af_type;
+    dim_t dims[4] = {type_max(t) / 10, 10, 1, 1};
+    topkTest<TypeParam>(2, dims, 3, 0, AF_TOPK_STABLE_MIN);
+}
+
+TYPED_TEST(TopK, Min3D0_Stable) {
+    af_dtype t    = (af_dtype)dtype_traits<TypeParam>::af_type;
+    dim_t dims[4] = {type_max(t) / 100, 10, 10, 1};
+    topkTest<TypeParam>(2, dims, 5, 0, AF_TOPK_STABLE_MIN);
+}
+
+TYPED_TEST(TopK, Min4D0_Stable) {
+    af_dtype t    = (af_dtype)dtype_traits<TypeParam>::af_type;
+    dim_t dims[4] = {type_max(t) / 1000, 10, 10, 10};
+    topkTest<TypeParam>(2, dims, 5, 0, AF_TOPK_STABLE_MIN);
+}
+
+TEST(TopK, ValidationCheck_DimN_Stable) {
+    dim_t dims[4] = {10, 10, 1, 1};
+    af_array out, idx, in;
+    ASSERT_SUCCESS(af_randu(&in, 2, dims, f32));
+    ASSERT_EQ(AF_ERR_NOT_SUPPORTED,
+              af_topk(&out, &idx, in, 10, 1, AF_TOPK_STABLE_MAX));
+    ASSERT_SUCCESS(af_release_array(in));
+}
+
+TEST(TopK, ValidationCheck_DefaultDim_Stable) {
+    dim_t dims[4] = {10, 10, 1, 1};
+    af_array out, idx, in;
+    ASSERT_SUCCESS(af_randu(&in, 4, dims, f32));
+    ASSERT_SUCCESS(af_topk(&out, &idx, in, 10, -1, AF_TOPK_STABLE_MAX));
+    ASSERT_SUCCESS(af_release_array(in));
+    ASSERT_SUCCESS(af_release_array(out));
+    ASSERT_SUCCESS(af_release_array(idx));
+}
+
 struct topk_params {
     int d0;
     int d1;
@@ -367,3 +435,61 @@ TEST(TopK, KLessThan0) {
     EXPECT_THROW(topk(vals, idx, a, k), af::exception)
         << "K cannot be less than 0";
 }
+
+TEST(TopK, DeterministicTiesMin) {
+    af::array a           = af::constant(1, 500);
+    a(af::seq(0, 499, 2)) = 7;
+    af::array vals_min, idx_min;
+
+    int k = 6;
+    topk(vals_min, idx_min, a, k, 0, AF_TOPK_STABLE_MIN);
+
+    af::array expected_idx_min   = af::seq(1, 499, 2);
+    af::array k_expected_idx_min = expected_idx_min(af::seq(0, k - 1));
+    ASSERT_ARRAYS_EQ(idx_min, k_expected_idx_min.as(u32));
+}
+
+TEST(TopK, DeterministicTiesMax) {
+    af::array a           = af::constant(1, 500);
+    a(af::seq(0, 499, 2)) = 7;
+    af::array vals_max, idx_max;
+
+    int k = 6;
+    topk(vals_max, idx_max, a, k, 0, AF_TOPK_STABLE_MAX);
+
+    af::array expected_idx_max   = af::seq(0, 499, 2);
+    af::array k_expected_idx_max = expected_idx_max(af::seq(0, k - 1));
+    ASSERT_ARRAYS_EQ(idx_max, k_expected_idx_max.as(u32));
+}
+
+TEST(TopK, DeterministicTiesBatchedMin) {
+    const int nbatch = 10;
+    af::array a      = af::constant(1, 500, nbatch, nbatch, nbatch);
+    a(af::seq(0, 499, 2), af::span, af::span, af::span) = 7;
+    af::array vals_min, idx_min;
+
+    int k = 6;
+    topk(vals_min, idx_min, a, k, 0, AF_TOPK_STABLE_MIN);
+
+    af::array expected_idx_min = af::seq(1, 499, 2);
+    af::array k_expected_idx_min =
+        af::tile(expected_idx_min(af::seq(0, k - 1)),
+                 af::dim4(1, nbatch, nbatch, nbatch));
+    ASSERT_ARRAYS_EQ(idx_min, k_expected_idx_min.as(u32));
+}
+
+TEST(TopK, DeterministicTiesBatchedMax) {
+    const int nbatch = 10;
+    af::array a      = af::constant(1, 500, nbatch, nbatch, nbatch);
+    a(af::seq(0, 499, 2), af::span, af::span, af::span) = 7;
+    af::array vals_max, idx_max;
+
+    int k = 6;
+    topk(vals_max, idx_max, a, k, 0, AF_TOPK_STABLE_MAX);
+
+    af::array expected_idx_max = af::seq(0, 499, 2);
+    af::array k_expected_idx_max =
+        af::tile(expected_idx_max(af::seq(0, k - 1)),
+                 af::dim4(1, nbatch, nbatch, nbatch));
+    ASSERT_ARRAYS_EQ(idx_max, k_expected_idx_max.as(u32));
+}

From 35bd3f88c8cde9ca99b2ff0626e22f54feb8fc0e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 25 Mar 2022 22:07:14 -0400
Subject: [PATCH 028/473] Add support for staticly linking nvrtc starting CUDA
 11.5

---
 src/backend/cuda/CMakeLists.txt | 57 +++++++++++++++++++++------------
 1 file changed, 36 insertions(+), 21 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index d75b96296a..8bd6a18391 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -60,6 +60,8 @@ set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};${cuda_architecture_flags})
 
 find_cuda_helper_libs(nvrtc)
 find_cuda_helper_libs(nvrtc-builtins)
+list(APPEND nvrtc_libs ${CUDA_nvrtc_LIBRARY})
+
 if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   af_find_static_cuda_libs(culibos)
   af_find_static_cuda_libs(cublas_static PRUNE)
@@ -67,6 +69,15 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   af_find_static_cuda_libs(cufft_static)
   af_find_static_cuda_libs(cusparse_static PRUNE)
 
+  if(CUDA_VERSION VERSION_GREATER 11.4)
+    af_find_static_cuda_libs(nvrtc_static)
+    af_find_static_cuda_libs(nvrtc-builtins_static)
+    af_find_static_cuda_libs(nvptxcompiler_static)
+    set(nvrtc_libs ${AF_CUDA_nvrtc_static_LIBRARY}
+                   ${AF_CUDA_nvrtc-builtins_static_LIBRARY}
+                   ${AF_CUDA_nvptxcompiler_static_LIBRARY})
+  endif()
+
   # FIXME When NVCC resolves this particular issue.
   # NVCC doesn't like -l<full_path_static_lib>, hence we cannot
   # use ${CMAKE_*_LIBRARY} variables in the following flags.
@@ -328,9 +339,10 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
       ${START_GROUP}
       ${CUDA_culibos_LIBRARY} #also a static libary
       ${AF_CUDA_cublas_static_LIBRARY}
+      ${AF_CUDA_cublasLt_static_LIBRARY}
       ${AF_CUDA_cufft_static_LIBRARY}
       ${AF_CUDA_cusparse_static_LIBRARY}
-      ${AF_CUDA_cublasLt_static_LIBRARY}
+      ${nvrtc_libs}
       ${cusolver_static_lib}
       ${END_GROUP}
   )
@@ -340,6 +352,7 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
       PRIVATE
         ${AF_CUDA_cublasLt_static_LIBRARY})
   endif()
+
   if(CUDA_VERSION VERSION_GREATER 9.5)
     target_link_libraries(af_cuda_static_cuda_library
       PRIVATE
@@ -355,6 +368,7 @@ else()
       ${CUDA_CUFFT_LIBRARIES}
       ${CUDA_cusolver_LIBRARY}
       ${CUDA_cusparse_LIBRARY}
+      ${nvrtc_libs}
   )
 endif()
 
@@ -712,7 +726,6 @@ target_link_libraries(afcuda
     cpp_api_interface
     afcommon_interface
     ${CMAKE_DL_LIBS}
-    ${CUDA_nvrtc_LIBRARY}
     af_cuda_static_cuda_library
   )
 
@@ -860,26 +873,28 @@ if(AF_INSTALL_STANDALONE)
     afcu_collect_libs(cusolver)
   endif()
 
-  afcu_collect_libs(nvrtc FULL_VERSION)
-  if(CUDA_VERSION VERSION_GREATER 10.0)
-    afcu_collect_libs(nvrtc-builtins FULL_VERSION)
-  else()
-    if(APPLE)
-      afcu_collect_libs(cudart)
-
-      get_filename_component(nvrtc_outpath "${dlib_path_prefix}/${PX}nvrtc-builtins.${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}${SX}" REALPATH)
-      install(FILES       ${nvrtc_outpath}
-              DESTINATION ${AF_INSTALL_BIN_DIR}
-              RENAME      "${PX}nvrtc-builtins${SX}"
-              COMPONENT   cuda_dependencies)
-    elseif(UNIX)
-      get_filename_component(nvrtc_outpath "${dlib_path_prefix}/${PX}nvrtc-builtins${SX}" REALPATH)
-      install(FILES       ${nvrtc_outpath}
-              DESTINATION ${AF_INSTALL_LIB_DIR}
-              RENAME      "${PX}nvrtc-builtins${SX}"
-              COMPONENT   cuda_dependencies)
+  if(WIN32 OR CUDA_VERSION VERSION_LESS 11.5 OR NOT AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
+    afcu_collect_libs(nvrtc FULL_VERSION)
+    if(CUDA_VERSION VERSION_GREATER 10.0)
+      afcu_collect_libs(nvrtc-builtins FULL_VERSION)
     else()
-      afcu_collect_libs(nvrtc-builtins)
+      if(APPLE)
+        afcu_collect_libs(cudart)
+
+        get_filename_component(nvrtc_outpath "${dlib_path_prefix}/${PX}nvrtc-builtins.${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}${SX}" REALPATH)
+        install(FILES       ${nvrtc_outpath}
+                DESTINATION ${AF_INSTALL_BIN_DIR}
+                RENAME      "${PX}nvrtc-builtins${SX}"
+                COMPONENT   cuda_dependencies)
+      elseif(UNIX)
+        get_filename_component(nvrtc_outpath "${dlib_path_prefix}/${PX}nvrtc-builtins${SX}" REALPATH)
+        install(FILES       ${nvrtc_outpath}
+                DESTINATION ${AF_INSTALL_LIB_DIR}
+                RENAME      "${PX}nvrtc-builtins${SX}"
+                COMPONENT   cuda_dependencies)
+      else()
+        afcu_collect_libs(nvrtc-builtins)
+      endif()
     endif()
   endif()
 endif()

From ff774fe48b0e639307218b5116cae8a9985f0986 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sun, 27 Mar 2022 11:52:17 -0400
Subject: [PATCH 029/473] Fix prune by making the cuda_prune_library_targets a
 set with parent_scope

The cuda_prune_library_targets was not being exposed in the parent
scope because it was used in a list. This commit changes the list
to a set to append the targets to that CMake variable which allows
us to use PARENT_SCOPE in the command.
---
 CMakeModules/AFcuda_helpers.cmake | 6 +++---
 src/backend/cuda/CMakeLists.txt   | 5 +++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/CMakeModules/AFcuda_helpers.cmake b/CMakeModules/AFcuda_helpers.cmake
index 578c49956b..598c6cd233 100644
--- a/CMakeModules/AFcuda_helpers.cmake
+++ b/CMakeModules/AFcuda_helpers.cmake
@@ -27,11 +27,11 @@ function(af_find_static_cuda_libs libname)
       MAIN_DEPENDENCY ${CUDA_${libname}_LIBRARY}
       COMMENT "Pruning ${CUDA_${libname}_LIBRARY} for ${cuda_build_targets}"
       VERBATIM)
-    add_custom_target(AF_CUDA_${libname}_LIBRARY_TARGET
+    add_custom_target(prune_${libname}
       DEPENDS ${liboutput}.depend)
-    list(APPEND cuda_pruned_libraries AF_CUDA_${libname}_LIBRARY_TARGET PARENT_SCOPE)
+    set(cuda_pruned_library_targets ${cuda_pruned_library_targets};prune_${libname} PARENT_SCOPE)
 
-    set(AF_CUDA_${libname}_LIBRARY ${liboutput} PARENT_SCOPE)
+    set(AF_CUDA_${libname}_LIBRARY "${liboutput}" PARENT_SCOPE)
     mark_as_advanced(AF_CUDA_${libname}_LIBRARY)
   else()
     set(AF_CUDA_${libname}_LIBRARY ${CUDA_${libname}_LIBRARY} PARENT_SCOPE)
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 8bd6a18391..fe794bfb61 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -702,8 +702,9 @@ add_library(ArrayFire::afcuda ALIAS afcuda)
 
 add_dependencies(afcuda ${jit_kernel_targets} ${nvrtc_kernel_targets})
 add_dependencies(af_cuda_static_cuda_library ${nvrtc_kernel_targets})
-if(cuda_pruned_libraries)
-  add_dependencies(afcuda ${cuda_pruned_libraries})
+
+if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
+  add_dependencies(afcuda ${cuda_pruned_library_targets})
 endif()
 
 target_include_directories (afcuda

From c696425aaadb605a873d4933ef90a0ef2e87cf63 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sun, 27 Mar 2022 15:28:19 -0400
Subject: [PATCH 030/473] Make pruning static CUDA libs optional with flag

Make pruning CUDA static libraries optional for static CUDA libraries
because nvprune seems to fail for some combination of CUDA toolkits
and compute capabilities.
---
 CMakeLists.txt                    | 4 ++++
 CMakeModules/AFcuda_helpers.cmake | 4 ++--
 src/backend/cuda/CMakeLists.txt   | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dce9076c8c..1ef063ac52 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -76,6 +76,10 @@ option(AF_CACHE_KERNELS_TO_DISK "Enable caching kernels to disk" ON)
 option(AF_WITH_STATIC_MKL "Link against static Intel MKL libraries" OFF)
 option(AF_WITH_STATIC_CUDA_NUMERIC_LIBS "Link libafcuda with static numeric libraries(cublas, cufft, etc.)" OFF)
 
+if(AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
+  option(AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS "Prune CUDA static libraries to reduce binary size.(WARNING: May break some libs on older CUDA toolkits for some compute arch)" OFF)
+endif()
+
 set(default_compute_library "FFTW/LAPACK/BLAS")
 if(MKL_FOUND)
   set(default_compute_library "Intel-MKL")
diff --git a/CMakeModules/AFcuda_helpers.cmake b/CMakeModules/AFcuda_helpers.cmake
index 598c6cd233..59cfb2002a 100644
--- a/CMakeModules/AFcuda_helpers.cmake
+++ b/CMakeModules/AFcuda_helpers.cmake
@@ -6,6 +6,7 @@
 # http://arrayfire.com/licenses/BSD-3-Clause
 
 find_program(NVPRUNE NAMES nvprune)
+
 # The following macro uses a macro defined by
 # FindCUDA module from cmake.
 function(af_find_static_cuda_libs libname)
@@ -16,7 +17,7 @@ function(af_find_static_cuda_libs libname)
   cuda_find_library_local_first(CUDA_${libname}_LIBRARY
     ${search_name} "${libname} static library")
 
-  if(fscl_PRUNE)
+  if(fscl_PRUNE AND AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS)
     get_filename_component(af_${libname} ${CUDA_${libname}_LIBRARY} NAME)
 
     set(liboutput ${CMAKE_CURRENT_BINARY_DIR}/${af_${libname}})
@@ -32,7 +33,6 @@ function(af_find_static_cuda_libs libname)
     set(cuda_pruned_library_targets ${cuda_pruned_library_targets};prune_${libname} PARENT_SCOPE)
 
     set(AF_CUDA_${libname}_LIBRARY "${liboutput}" PARENT_SCOPE)
-    mark_as_advanced(AF_CUDA_${libname}_LIBRARY)
   else()
     set(AF_CUDA_${libname}_LIBRARY ${CUDA_${libname}_LIBRARY} PARENT_SCOPE)
   endif()
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index fe794bfb61..ee20e453ac 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -703,7 +703,7 @@ add_library(ArrayFire::afcuda ALIAS afcuda)
 add_dependencies(afcuda ${jit_kernel_targets} ${nvrtc_kernel_targets})
 add_dependencies(af_cuda_static_cuda_library ${nvrtc_kernel_targets})
 
-if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
+if(UNIX AND AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS)
   add_dependencies(afcuda ${cuda_pruned_library_targets})
 endif()
 

From 35861bf4cc42444158f7a210eda5ac7e46082e76 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sun, 27 Mar 2022 15:48:35 -0400
Subject: [PATCH 031/473] Add support for ccache to the CUDA backend

---
 CMakeModules/config_ccache.cmake |  4 ++++
 CMakeModules/launch-nvcc.in      | 10 ++++++++++
 2 files changed, 14 insertions(+)
 create mode 100644 CMakeModules/launch-nvcc.in

diff --git a/CMakeModules/config_ccache.cmake b/CMakeModules/config_ccache.cmake
index b112787d76..1bf3adaef6 100644
--- a/CMakeModules/config_ccache.cmake
+++ b/CMakeModules/config_ccache.cmake
@@ -14,11 +14,14 @@ if (UNIX)
     # Set up wrapper scripts
     set(C_LAUNCHER   "${CCACHE_PROGRAM}")
     set(CXX_LAUNCHER "${CCACHE_PROGRAM}")
+    set(NVCC_LAUNCHER "${CCACHE_PROGRAM}")
     configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-c.in   launch-c)
     configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-cxx.in launch-cxx)
+    configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-nvcc.in launch-nvcc)
     execute_process(COMMAND chmod a+rx
         "${ArrayFire_BINARY_DIR}/launch-c"
         "${ArrayFire_BINARY_DIR}/launch-cxx"
+        "${ArrayFire_BINARY_DIR}/launch-nvcc"
       )
     if(CMAKE_GENERATOR STREQUAL "Xcode")
       # Set Xcode project attributes to route compilation and linking
@@ -31,6 +34,7 @@ if (UNIX)
       # Support Unix Makefiles and Ninja
       set(CMAKE_C_COMPILER_LAUNCHER   "${ArrayFire_BINARY_DIR}/launch-c")
       set(CMAKE_CXX_COMPILER_LAUNCHER "${ArrayFire_BINARY_DIR}/launch-cxx")
+      set(CUDA_NVCC_EXECUTABLE "${ArrayFire_BINARY_DIR}/launch-nvcc")
     endif()
   endif()
   mark_as_advanced(CCACHE_PROGRAM)
diff --git a/CMakeModules/launch-nvcc.in b/CMakeModules/launch-nvcc.in
new file mode 100644
index 0000000000..47a4591850
--- /dev/null
+++ b/CMakeModules/launch-nvcc.in
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Xcode generator doesn't include the compiler as the
+# first argument, Ninja and Makefiles do. Handle both cases.
+if [ "$1" = "${CUDA_NVCC_EXECUTABLE}" ] ; then
+    shift
+fi
+
+export CCACHE_CPP2=true
+exec "${NVCC_LAUNCHER}" "${CUDA_NVCC_EXECUTABLE}" "$@"

From 5b2e8ea34ff6d7f35bd68768be886d50463dd6e4 Mon Sep 17 00:00:00 2001
From: Jacob Kahn <jacobkahn1@gmail.com>
Date: Wed, 6 Apr 2022 12:00:42 -0500
Subject: [PATCH 032/473] JIT optimization for sequential casts that are
 idempotent (#3031)

Adds a JIT optimization which removes sequential casts in cases that don't
result in a differently-typed result.

This commit removes the following casts:

* Casts for conversions between any floating point types.
* Casts from smaller integer types to larger integer type and back

Following casts are NOT removed

* Floating point to integer types and back
* Integer types from larger types to smaller types and back

Casts can be forced by calling eval on the casted intermediate array
---
 include/af/arith.h                        |  29 ++++++
 include/af/array.h                        |  29 +++++-
 src/backend/common/ArrayInfo.cpp          |  24 ++---
 src/backend/common/cast.hpp               | 121 +++++++++++++++++++++-
 src/backend/common/jit/BufferNodeBase.hpp |   2 +
 src/backend/common/jit/NaryNode.hpp       |  10 +-
 src/backend/common/jit/Node.hpp           |   5 +
 src/backend/common/traits.hpp             |  53 +++++++++-
 src/backend/cuda/Array.cpp                |   8 +-
 src/backend/cuda/cast.hpp                 |   1 -
 src/backend/opencl/Array.cpp              |   6 +-
 test/cast.cpp                             |  91 ++++++++++++++++
 12 files changed, 350 insertions(+), 29 deletions(-)

diff --git a/include/af/arith.h b/include/af/arith.h
index 6b0c08dea5..83240ffc6d 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -822,6 +822,35 @@ extern "C" {
     /**
        C Interface for casting an array from one type to another
 
+       This function casts an af_array object from one type to another. If the
+       type of the original array is the same as \p type then the same array is
+       returned.
+
+       \note Consecitive casting operations may be may be optimized out if the
+       original type of the af_array is the same as the final type. For example
+       if the original type is f64 which is then cast to f32 and then back to
+       f64, then the cast to f32 will be skipped and that operation will *NOT*
+       be performed by ArrayFire. The following table shows which casts will
+       be optimized out. outer -> inner -> outer
+       | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | u8 | b8 | s64 | u64 | s16 | u16 | f16 |
+       |---------|-----|-----|-----|-----|-----|-----|----|----|-----|-----|-----|-----|-----|
+       | f32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+       | f64     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+       | c32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+       | c64     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+       | s32     | x   | x   | x   | x   | x   | x   |    |    | x   | x   |     |     | x   |
+       | u32     | x   | x   | x   | x   | x   | x   |    |    | x   | x   |     |     | x   |
+       | u8      | x   | x   | x   | x   | x   | x   | x  | x  | x   | x   | x   | x   | x   |
+       | b8      | x   | x   | x   | x   | x   | x   | x  | x  | x   | x   | x   | x   | x   |
+       | s64     | x   | x   | x   | x   |     |     |    |    | x   | x   |     |     | x   |
+       | u64     | x   | x   | x   | x   |     |     |    |    | x   | x   |     |     | x   |
+       | s16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
+       | u16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
+       | f16     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+       If you want to avoid this behavior use af_eval after the first cast
+       operation. This will ensure that the cast operation is performed on the
+       af_array
+
        \param[out] out will contain the values in the specified type
        \param[in] in is the input
        \param[in] type is the target data type \ref af_dtype
diff --git a/include/af/array.h b/include/af/array.h
index b30d5694fc..bdd9ac4e9c 100644
--- a/include/af/array.h
+++ b/include/af/array.h
@@ -933,9 +933,34 @@ namespace af
         const array::array_proxy slices(int first, int last) const; ///< \copydoc slices
         /// @}
 
-        /// \brief Converts the array into another type
+        /// \brief Casts the array into another data type
         ///
-        ///  \param[in] type is the desired type(f32, s64, etc.)
+        /// \note Consecitive casting operations may be may be optimized out if
+        /// the original type of the af::array is the same as the final type.
+        /// For example if the original type is f64 which is then cast to f32
+        /// and then back to f64, then the cast to f32 will be skipped and that
+        /// operation will *NOT* be performed by ArrayFire. The following table
+        /// shows which casts will be optimized out. outer -> inner -> outer
+        /// | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | u8 | b8 | s64 | u64 | s16 | u16 | f16 |
+        /// |---------|-----|-----|-----|-----|-----|-----|----|----|-----|-----|-----|-----|-----|
+        /// | f32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+        /// | f64     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+        /// | c32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+        /// | c64     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+        /// | s32     | x   | x   | x   | x   | x   | x   |    |    | x   | x   |     |     | x   |
+        /// | u32     | x   | x   | x   | x   | x   | x   |    |    | x   | x   |     |     | x   |
+        /// | u8      | x   | x   | x   | x   | x   | x   | x  | x  | x   | x   | x   | x   | x   |
+        /// | b8      | x   | x   | x   | x   | x   | x   | x  | x  | x   | x   | x   | x   | x   |
+        /// | s64     | x   | x   | x   | x   |     |     |    |    | x   | x   |     |     | x   |
+        /// | u64     | x   | x   | x   | x   |     |     |    |    | x   | x   |     |     | x   |
+        /// | s16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
+        /// | u16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
+        /// | f16     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+        /// If you want to avoid this behavior use af_eval after the first cast
+        /// operation. This will ensure that the cast operation is performed on
+        /// the af::array
+        ///
+        /// \param[in] type is the desired type(f32, s64, etc.)
         /// \returns an array with the type specified by \p type
         const array as(dtype type) const;
 
diff --git a/src/backend/common/ArrayInfo.cpp b/src/backend/common/ArrayInfo.cpp
index 6cf55d20ea..585b48d403 100644
--- a/src/backend/common/ArrayInfo.cpp
+++ b/src/backend/common/ArrayInfo.cpp
@@ -9,6 +9,7 @@
 
 #include <common/ArrayInfo.hpp>
 #include <common/err_common.hpp>
+#include <common/traits.hpp>
 #include <algorithm>
 #include <functional>
 #include <numeric>
@@ -93,28 +94,23 @@ bool ArrayInfo::isVector() const {
     return singular_dims == AF_MAX_DIMS - 1 && non_singular_dims == 1;
 }
 
-bool ArrayInfo::isComplex() const { return ((type == c32) || (type == c64)); }
+bool ArrayInfo::isComplex() const { return common::isComplex(type); }
 
-bool ArrayInfo::isReal() const { return !isComplex(); }
+bool ArrayInfo::isReal() const { return common::isReal(type); }
 
-bool ArrayInfo::isDouble() const { return (type == f64 || type == c64); }
+bool ArrayInfo::isDouble() const { return common::isDouble(type); }
 
-bool ArrayInfo::isSingle() const { return (type == f32 || type == c32); }
+bool ArrayInfo::isSingle() const { return common::isSingle(type); }
 
-bool ArrayInfo::isHalf() const { return (type == f16); }
+bool ArrayInfo::isHalf() const { return common::isHalf(type); }
 
-bool ArrayInfo::isRealFloating() const {
-    return (type == f64 || type == f32 || type == f16);
-}
+bool ArrayInfo::isRealFloating() const { return common::isRealFloating(type); }
 
-bool ArrayInfo::isFloating() const { return (!isInteger() && !isBool()); }
+bool ArrayInfo::isFloating() const { return common::isFloating(type); }
 
-bool ArrayInfo::isInteger() const {
-    return (type == s32 || type == u32 || type == s64 || type == u64 ||
-            type == s16 || type == u16 || type == u8);
-}
+bool ArrayInfo::isInteger() const { return common::isInteger(type); }
 
-bool ArrayInfo::isBool() const { return (type == b8); }
+bool ArrayInfo::isBool() const { return common::isBool(type); }
 
 bool ArrayInfo::isLinear() const {
     if (ndims() == 1) { return dim_strides[0] == 1; }
diff --git a/src/backend/common/cast.hpp b/src/backend/common/cast.hpp
index b266d8517a..d80caacfe6 100644
--- a/src/backend/common/cast.hpp
+++ b/src/backend/common/cast.hpp
@@ -10,37 +10,150 @@
 #pragma once
 #include <Array.hpp>
 #include <cast.hpp>
+#include <common/Logger.hpp>
+#include <memory>
 
 #ifdef AF_CPU
 #include <jit/UnaryNode.hpp>
 #endif
 
 namespace common {
+/// This function determines if consecutive cast operations should be
+/// removed from a JIT AST.
+///
+/// This function returns true if consecutive cast operations in the JIT AST
+/// should be removed. Multiple cast operations are removed when going from
+/// a smaller type to a larger type and back again OR if the conversion is
+/// between two floating point types including complex types.
+///
+///                  Cast operations that will be removed
+///                        outer -> inner -> outer
+///
+///                                inner cast
+///           f32  f64  c32  c64  s32  u32   u8   b8  s64  u64  s16  u16  f16
+///     f32    x    x    x    x                                            x
+///     f64    x    x    x    x                                            x
+///  o  c32    x    x    x    x                                            x
+///  u  c64    x    x    x    x                                            x
+///  t  s32    x    x    x    x    x    x              x    x              x
+///  e  u32    x    x    x    x    x    x              x    x              x
+///  r   u8    x    x    x    x    x    x    x    x    x    x    x    x    x
+///      b8    x    x    x    x    x    x    x    x    x    x    x    x    x
+///  c  s64    x    x    x    x                        x    x              x
+///  a  u64    x    x    x    x                        x    x              x
+///  s  s16    x    x    x    x    x    x              x    x    x    x    x
+///  t  u16    x    x    x    x    x    x              x    x    x    x    x
+///     f16    x    x    x    x                                            x
+///
+/// \param[in] outer The type of the second cast and the child of the
+///            previous cast
+/// \param[in] inner  The type of the first cast
+///
+/// \returns True if the inner cast operation should be removed
+constexpr bool canOptimizeCast(af::dtype outer, af::dtype inner) {
+    if (isFloating(outer)) {
+        if (isFloating(inner)) { return true; }
+    } else {
+        if (isFloating(inner)) { return true; }
+        if (dtypeSize(inner) >= dtypeSize(outer)) { return true; }
+    }
+
+    return false;
+}
 
 #ifdef AF_CPU
 template<typename To, typename Ti>
 struct CastWrapper {
+    static spdlog::logger *getLogger() noexcept {
+        static std::shared_ptr<spdlog::logger> logger =
+            common::loggerFactory("ast");
+        return logger.get();
+    }
+
     detail::Array<To> operator()(const detail::Array<Ti> &in) {
         using cpu::jit::UnaryNode;
 
-        Node_ptr in_node = in.getNode();
+        common::Node_ptr in_node = in.getNode();
+        constexpr af::dtype to_dtype =
+            static_cast<af::dtype>(af::dtype_traits<To>::af_type);
+        constexpr af::dtype in_dtype =
+            static_cast<af::dtype>(af::dtype_traits<Ti>::af_type);
+
+        if (canOptimizeCast(to_dtype, in_dtype)) {
+            // JIT optimization in the cast of multiple sequential casts that
+            // become idempotent - check to see if the previous operation was
+            // also a cast
+            // TODO: handle arbitrarily long chains of casts
+            auto in_node_unary =
+                std::dynamic_pointer_cast<UnaryNode<To, Ti, af_cast_t>>(
+                    in_node);
+
+            if (in_node_unary && in_node_unary->getOp() == af_cast_t) {
+                // child child's output type is the input type of the child
+                AF_TRACE("Cast optimiztion performed by removing cast to {}",
+                         af::dtype_traits<Ti>::getName());
+                auto in_child_node = in_node_unary->getChildren()[0];
+                if (in_child_node->getType() == to_dtype) {
+                    // ignore the input node and simply connect a noop node from
+                    // the child's child to produce this op's output
+                    return detail::createNodeArray<To>(in.dims(),
+                                                       in_child_node);
+                }
+            }
+        }
+
         auto node = std::make_shared<UnaryNode<To, Ti, af_cast_t>>(in_node);
 
         return detail::createNodeArray<To>(in.dims(), move(node));
     }
 };
 #else
+
 template<typename To, typename Ti>
 struct CastWrapper {
+    static spdlog::logger *getLogger() noexcept {
+        static std::shared_ptr<spdlog::logger> logger =
+            common::loggerFactory("ast");
+        return logger.get();
+    }
+
     detail::Array<To> operator()(const detail::Array<Ti> &in) {
+        using common::UnaryNode;
         detail::CastOp<To, Ti> cop;
         common::Node_ptr in_node = in.getNode();
-        common::UnaryNode *node  = new common::UnaryNode(
-            static_cast<af::dtype>(dtype_traits<To>::af_type), cop.name(),
-            in_node, af_cast_t);
+        constexpr af::dtype to_dtype =
+            static_cast<af::dtype>(dtype_traits<To>::af_type);
+        constexpr af::dtype in_dtype =
+            static_cast<af::dtype>(af::dtype_traits<Ti>::af_type);
+
+        if (canOptimizeCast(to_dtype, in_dtype)) {
+            // JIT optimization in the cast of multiple sequential casts that
+            // become idempotent - check to see if the previous operation was
+            // also a cast
+            // TODO: handle arbitrarily long chains of casts
+            auto in_node_unary =
+                std::dynamic_pointer_cast<common::UnaryNode>(in_node);
+
+            if (in_node_unary && in_node_unary->getOp() == af_cast_t) {
+                // child child's output type is the input type of the child
+                AF_TRACE("Cast optimiztion performed by removing cast to {}",
+                         dtype_traits<Ti>::getName());
+                auto in_child_node = in_node_unary->getChildren()[0];
+                if (in_child_node->getType() == to_dtype) {
+                    // ignore the input node and simply connect a noop node from
+                    // the child's child to produce this op's output
+                    return detail::createNodeArray<To>(in.dims(),
+                                                       in_child_node);
+                }
+            }
+        }
+
+        common::UnaryNode *node =
+            new common::UnaryNode(to_dtype, cop.name(), in_node, af_cast_t);
         return detail::createNodeArray<To>(in.dims(), common::Node_ptr(node));
     }
 };
+
 #endif
 
 template<typename T>
diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index 5027cd5671..8bb8185378 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -34,6 +34,8 @@ class BufferNodeBase : public common::Node {
         return std::make_unique<BufferNodeBase>(*this);
     }
 
+    DataType getDataPointer() const { return m_data; }
+
     void setData(ParamType param, DataType data, const unsigned bytes,
                  bool is_linear) {
         m_param         = param;
diff --git a/src/backend/common/jit/NaryNode.hpp b/src/backend/common/jit/NaryNode.hpp
index c03af9c2a5..5e97e249dd 100644
--- a/src/backend/common/jit/NaryNode.hpp
+++ b/src/backend/common/jit/NaryNode.hpp
@@ -26,9 +26,11 @@ namespace common {
 class NaryNode : public Node {
    private:
     int m_num_children;
-    af_op_t m_op;
     const char *m_op_str;
 
+   protected:
+    af_op_t m_op;
+
    public:
     NaryNode(const af::dtype type, const char *op_str, const int num_children,
              const std::array<common::Node_ptr, Node::kMaxChildren> &&children,
@@ -39,8 +41,8 @@ class NaryNode : public Node {
                   const std::array<common::Node_ptr, Node::kMaxChildren>>(
                   children))
         , m_num_children(num_children)
-        , m_op(op)
-        , m_op_str(op_str) {
+        , m_op_str(op_str)
+        , m_op(op) {
         static_assert(std::is_nothrow_move_assignable<NaryNode>::value,
                       "NaryNode is not move assignable");
         static_assert(std::is_nothrow_move_constructible<NaryNode>::value,
@@ -61,8 +63,8 @@ class NaryNode : public Node {
         using std::swap;
         Node::swap(other);
         swap(m_num_children, other.m_num_children);
-        swap(m_op, other.m_op);
         swap(m_op_str, other.m_op_str);
+        swap(m_op, other.m_op);
     }
 
     af_op_t getOp() const noexcept final { return m_op; }
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 0b284c072e..ca557a50d6 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -181,6 +181,10 @@ class Node {
         UNUSED(lim);
     }
 
+    const std::array<Node_ptr, kMaxChildren> &getChildren() const {
+        return m_children;
+    }
+
     /// Generates the variable that stores the thread's/work-item's offset into
     /// the memory.
     ///
@@ -247,6 +251,7 @@ class Node {
     /// Returns true if the buffer is linear
     virtual bool isLinear(const dim_t dims[4]) const;
 
+    /// Returns the type
     af::dtype getType() const { return m_type; }
 
     /// Returns the string representation of the type
diff --git a/src/backend/common/traits.hpp b/src/backend/common/traits.hpp
index 8f27ce952f..cfd07b8a0e 100644
--- a/src/backend/common/traits.hpp
+++ b/src/backend/common/traits.hpp
@@ -8,6 +8,7 @@
  ********************************************************/
 #pragma once
 
+#include <common/err_common.hpp>
 #include <af/defines.h>
 
 namespace af {
@@ -17,13 +18,63 @@ struct dtype_traits;
 
 namespace common {
 class half;
+
+namespace {
+
+inline size_t dtypeSize(af::dtype type) {
+    switch (type) {
+        case u8:
+        case b8: return 1;
+        case s16:
+        case u16:
+        case f16: return 2;
+        case s32:
+        case u32:
+        case f32: return 4;
+        case u64:
+        case s64:
+        case c32:
+        case f64: return 8;
+        case c64: return 16;
+        default: AF_RETURN_ERROR("Unsupported type", AF_ERR_INTERNAL);
+    }
+}
+
+constexpr bool isComplex(af::dtype type) {
+    return ((type == c32) || (type == c64));
+}
+
+constexpr bool isReal(af::dtype type) { return !isComplex(type); }
+
+constexpr bool isDouble(af::dtype type) { return (type == f64 || type == c64); }
+
+constexpr bool isSingle(af::dtype type) { return (type == f32 || type == c32); }
+
+constexpr bool isHalf(af::dtype type) { return (type == f16); }
+
+constexpr bool isRealFloating(af::dtype type) {
+    return (type == f64 || type == f32 || type == f16);
+}
+
+constexpr bool isInteger(af::dtype type) {
+    return (type == s32 || type == u32 || type == s64 || type == u64 ||
+            type == s16 || type == u16 || type == u8);
 }
 
+constexpr bool isBool(af::dtype type) { return (type == b8); }
+
+constexpr bool isFloating(af::dtype type) {
+    return (!isInteger(type) && !isBool(type));
+}
+
+}  // namespace
+}  // namespace common
+
 namespace af {
 template<>
 struct dtype_traits<common::half> {
     enum { af_type = f16, ctype = f16 };
     typedef common::half base_type;
-    static const char* getName() { return "half"; }
+    static const char *getName() { return "half"; }
 };
 }  // namespace af
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index 134645f496..c6347d1bbe 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -21,7 +21,7 @@
 #include <cstddef>
 #include <memory>
 #include <numeric>
-#include <utility>
+#include <vector>
 
 using af::dim4;
 using common::half;
@@ -129,7 +129,11 @@ Array<T>::Array(const af::dim4 &dims, common::Node_ptr n)
     , data()
     , data_dims(dims)
     , node(move(n))
-    , owner(true) {}
+    , owner(true) {
+    if (node->isBuffer()) {
+        data = std::static_pointer_cast<BufferNode<T>>(node)->getDataPointer();
+    }
+}
 
 template<typename T>
 Array<T>::Array(const af::dim4 &dims, const af::dim4 &strides, dim_t offset_,
diff --git a/src/backend/cuda/cast.hpp b/src/backend/cuda/cast.hpp
index bae9b3cbb6..cfcc9a8042 100644
--- a/src/backend/cuda/cast.hpp
+++ b/src/backend/cuda/cast.hpp
@@ -16,7 +16,6 @@
 #include <optypes.hpp>
 #include <types.hpp>
 #include <af/dim4.hpp>
-#include <complex>
 
 namespace cuda {
 
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 6e490f82a8..f3dd8d97ed 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -100,7 +100,11 @@ Array<T>::Array(const dim4 &dims, Node_ptr n)
            static_cast<af_dtype>(dtype_traits<T>::af_type))
     , data_dims(dims)
     , node(std::move(n))
-    , owner(true) {}
+    , owner(true) {
+    if (node->isBuffer()) {
+        data = std::static_pointer_cast<BufferNode>(node)->getDataPointer();
+    }
+}
 
 template<typename T>
 Array<T>::Array(const dim4 &dims, const T *const in_data)
diff --git a/test/cast.cpp b/test/cast.cpp
index 75ff9aca42..96178a470c 100644
--- a/test/cast.cpp
+++ b/test/cast.cpp
@@ -14,6 +14,9 @@
 #include <af/array.h>
 #include <af/data.h>
 #include <af/random.h>
+#include <algorithm>
+#include <cstdlib>
+#include <vector>
 
 using af::cdouble;
 using af::cfloat;
@@ -99,3 +102,91 @@ COMPLEX_REAL_TESTS(cfloat, float)
 COMPLEX_REAL_TESTS(cfloat, double)
 COMPLEX_REAL_TESTS(cdouble, float)
 COMPLEX_REAL_TESTS(cdouble, double)
+
+TEST(CAST_TEST, Test_JIT_DuplicateCastNoop) {
+    // Does a trivial cast - check JIT kernel trace to ensure a __noop is
+    // generated since we don't have a way to test it directly
+    af_dtype ta = (af_dtype)dtype_traits<float>::af_type;
+    af_dtype tb = (af_dtype)dtype_traits<double>::af_type;
+    dim4 dims(num, 1, 1, 1);
+    af_array a, b, c;
+    af_randu(&a, dims.ndims(), dims.get(), ta);
+
+    af_cast(&b, a, tb);
+    af_cast(&c, b, ta);
+
+    std::vector<float> a_vals(num);
+    std::vector<float> c_vals(num);
+    ASSERT_SUCCESS(af_get_data_ptr((void **)&a_vals[0], a));
+    ASSERT_SUCCESS(af_get_data_ptr((void **)&c_vals[0], c));
+
+    for (size_t i = 0; i < num; ++i) { ASSERT_FLOAT_EQ(a_vals[i], c_vals[i]); }
+
+    af_release_array(a);
+    af_release_array(b);
+    af_release_array(c);
+}
+
+TEST(Cast, ImplicitCast) {
+    using namespace af;
+    array a = randu(100, 100, f64);
+    array b = a.as(f32);
+
+    array c = max(abs(a - b));
+    ASSERT_ARRAYS_NEAR(constant(0, 1, 100, f64), c, 1e-7);
+}
+
+TEST(Cast, ConstantCast) {
+    using namespace af;
+    array a = constant(1, 100, f64);
+    array b = a.as(f32);
+
+    array c = max(abs(a - b));
+    ASSERT_ARRAYS_NEAR(c, constant(0, 1, f64), 1e-7);
+}
+
+TEST(Cast, OpCast) {
+    using namespace af;
+    array a = constant(1, 100, f64);
+    a       = a + a;
+    array b = a.as(f32);
+
+    array c = max(abs(a - b));
+    ASSERT_ARRAYS_NEAR(c, constant(0, 1, f64), 1e-7);
+}
+TEST(Cast, ImplicitCastIndexed) {
+    using namespace af;
+    array a = randu(100, 100, f64);
+    array b = a(span, 1).as(f32);
+    array c = max(abs(a(span, 1) - b));
+    ASSERT_ARRAYS_NEAR(constant(0, 1, 1, f64), c, 1e-7);
+}
+
+TEST(Cast, ImplicitCastIndexedNonLinear) {
+    using namespace af;
+    array a = randu(100, 100, f64);
+    array b = a(seq(10, 20, 2), 1).as(f32);
+    array c = max(abs(a(seq(10, 20, 2), 1) - b));
+    ASSERT_ARRAYS_NEAR(constant(0, 1, 1, f64), c, 1e-7);
+}
+
+TEST(Cast, ImplicitCastIndexedNonLinearArray) {
+    using namespace af;
+    array a   = randu(100, 100, f64);
+    array idx = seq(10, 20, 2);
+    array b   = a(idx, 1).as(f32);
+    array c   = max(abs(a(idx, 1) - b));
+    ASSERT_ARRAYS_NEAR(constant(0, 1, 1, f64), c, 1e-7);
+}
+
+TEST(Cast, ImplicitCastIndexedAndScoped) {
+    using namespace af;
+    array c;
+    {
+        array a = randu(100, 100, f64);
+        array b = a(span, 1).as(f32);
+        c       = abs(a(span, 1) - b);
+    }
+    c = max(c);
+    ASSERT_ARRAYS_NEAR(constant(0, 1, 1, f64), c, 1e-7);
+}

From 8c232900aa0e448b3e226ab00d2656b05b3d8edf Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 30 Mar 2022 13:21:49 -0400
Subject: [PATCH 033/473] Create the af_multiple_option CMake macro

This commit adds the af_multiple_option macro which allows you to
create a CMake variable that has limited set of optional string
values assigned to it.
---
 CMakeLists.txt                   | 20 ++++++++++++--------
 CMakeModules/InternalUtils.cmake | 19 +++++++++++++++++++
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1ef063ac52..784ed20144 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -85,17 +85,21 @@ if(MKL_FOUND)
   set(default_compute_library "Intel-MKL")
 endif()
 
-set(AF_COMPUTE_LIBRARY ${default_compute_library}
-    CACHE STRING "Compute library for signal processing and linear algebra routines")
-set_property(CACHE AF_COMPUTE_LIBRARY
-    PROPERTY STRINGS "Intel-MKL" "FFTW/LAPACK/BLAS")
+af_multiple_option(NAME        AF_COMPUTE_LIBRARY
+                   DEFAULT     ${default_compute_library}
+                   DESCRIPTION "Compute library for signal processing and linear algebra routines"
+                   OPTIONS     "Intel-MKL" "FFTW/LAPACK/BLAS")
 
 if(WIN32)
-  set(AF_STACKTRACE_TYPE "Windbg" CACHE STRING "The type of backtrace features. Windbg(simple), None")
-  set_property(CACHE AF_STACKTRACE_TYPE PROPERTY STRINGS "Windbg" "None")
+  af_multiple_option(NAME         AF_STACKTRACE_TYPE
+                     DEFAULT      "Windbg"
+                     DESCRIPTION  "The type of backtrace features. Windbg(simple), None"
+                     OPTIONS       "Windbg" "None")
 else()
-  set(AF_STACKTRACE_TYPE "Basic" CACHE STRING "The type of backtrace features. Basic(simple), libbacktrace(fancy), addr2line(fancy), None")
-  set_property(CACHE AF_STACKTRACE_TYPE PROPERTY STRINGS "Basic" "libbacktrace" "addr2line" "None")
+  af_multiple_option(NAME         AF_STACKTRACE_TYPE
+                     DEFAULT      "Basic"
+                     DESCRIPTION  "The type of backtrace features. Basic(simple), libbacktrace(fancy), addr2line(fancy), None"
+                     OPTIONS       "Basic" "libbacktrace" "addr2line" "None")
 endif()
 
 option(AF_INSTALL_STANDALONE "Build installers that include all dependencies" OFF)
diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index 1c1a8e5f5f..8fd21e7447 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -223,6 +223,25 @@ macro(af_mkl_batch_check)
   check_symbol_exists(sgetrf_batch_strided "mkl_lapack.h" MKL_BATCH)
 endmacro()
 
+# Creates a CACHEd CMake variable which has limited set of possible string values
+# Argumehts:
+#   NAME: The name of the variable
+#   DEFAULT: The default value of the variable
+#   DESCRIPTION: The description of the variable
+#   OPTIONS: The possible set of values for the option
+#
+# Example:
+#
+# af_multiple_option(NAME        AF_COMPUTE_LIBRARY
+#                    DEFAULT     "Intel-MKL"
+#                    DESCRIPTION "Compute library for signal processing and linear algebra routines"
+#                    OPTIONS     "Intel-MKL" "FFTW/LAPACK/BLAS")
+macro(af_multiple_option)
+  cmake_parse_arguments(opt "" "NAME;DEFAULT;DESCRIPTION" "OPTIONS" ${ARGN})
+  set(${opt_NAME} ${opt_DEFAULT} CACHE STRING ${opt_DESCRIPTION})
+  set_property(CACHE ${opt_NAME} PROPERTY STRINGS ${opt_OPTIONS})
+endmacro()
+
 mark_as_advanced(
     pkgcfg_lib_PC_CBLAS_cblas
     pkgcfg_lib_PC_LAPACKE_lapacke

From d7905f7299ed4cfbfaba7a77ed049372635d007d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 28 Mar 2022 18:13:07 -0400
Subject: [PATCH 034/473] Make cuSparse a runtime dependency. Optionally allow
 static linking

This PR adds the ability to load cuSparse at runtime and not at link
time. This allows us to not have cuSparse on the system at startup if
you don't need to use the sparse functionallity in CUDA. Optionally
it also allows you to staticly link against the cuSparse library
if you want to package the library with ArrayFire.
---
 src/backend/common/DependencyModule.hpp |   5 +
 src/backend/cuda/CMakeLists.txt         |  30 ++++--
 src/backend/cuda/cusparse.hpp           |  11 +-
 src/backend/cuda/cusparseModule.cpp     | 135 ++++++++++++++++++++++++
 src/backend/cuda/cusparseModule.hpp     |  96 +++++++++++++++++
 src/backend/cuda/platform.cpp           |   6 +-
 src/backend/cuda/sparse.cu              |  43 ++++----
 src/backend/cuda/sparse_arith.cu        |  21 ++--
 src/backend/cuda/sparse_blas.cu         |  37 ++++---
 9 files changed, 328 insertions(+), 56 deletions(-)
 create mode 100644 src/backend/cuda/cusparseModule.cpp
 create mode 100644 src/backend/cuda/cusparseModule.hpp

diff --git a/src/backend/common/DependencyModule.hpp b/src/backend/common/DependencyModule.hpp
index d4f456dbe8..923ba96a47 100644
--- a/src/backend/common/DependencyModule.hpp
+++ b/src/backend/common/DependencyModule.hpp
@@ -38,6 +38,11 @@ class DependencyModule {
     std::vector<void*> functions;
 
    public:
+    /// Loads the library \p plugin_file_name from the \p paths locations
+    /// \param plugin_file_name  The name of the library without any prefix or
+    ///                          extensions
+    /// \param paths             The locations to search for the libraries if
+    ///                          not found in standard locations
     DependencyModule(const char* plugin_file_name,
                      const char** paths = nullptr);
 
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index ee20e453ac..8f25f1bea1 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -63,11 +63,23 @@ find_cuda_helper_libs(nvrtc-builtins)
 list(APPEND nvrtc_libs ${CUDA_nvrtc_LIBRARY})
 
 if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
+  # The libraries that may be staticly linked or may be loaded at runtime
+  set(AF_CUDA_optionally_static_libraries)
+
+  af_multiple_option(NAME        AF_cusparse_LINK_LOADING
+    DEFAULT     "Module"
+    DESCRIPTION "The approach to load the cusparse library. Static linking(Static) or Dynamic runtime loading(Module) of the module"
+    OPTIONS     "Module" "Static")
+
+  if(AF_cusparse_LINK_LOADING STREQUAL "Static")
+    af_find_static_cuda_libs(cusparse_static PRUNE)
+    list(APPEND AF_CUDA_optionally_static_libraries ${AF_CUDA_cusparse_static_LIBRARY})
+  endif()
+
   af_find_static_cuda_libs(culibos)
   af_find_static_cuda_libs(cublas_static PRUNE)
   af_find_static_cuda_libs(cublasLt_static PRUNE)
   af_find_static_cuda_libs(cufft_static)
-  af_find_static_cuda_libs(cusparse_static PRUNE)
 
   if(CUDA_VERSION VERSION_GREATER 11.4)
     af_find_static_cuda_libs(nvrtc_static)
@@ -88,7 +100,6 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
     set(af_cuda_static_flags "${af_cuda_static_flags};-lcublasLt_static")
   endif()
   set(af_cuda_static_flags "${af_cuda_static_flags};-lcufft_static")
-  set(af_cuda_static_flags "${af_cuda_static_flags};-lcusparse_static")
 
   if(${use_static_cuda_lapack})
     af_find_static_cuda_libs(cusolver_static PRUNE)
@@ -341,11 +352,10 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
       ${AF_CUDA_cublas_static_LIBRARY}
       ${AF_CUDA_cublasLt_static_LIBRARY}
       ${AF_CUDA_cufft_static_LIBRARY}
-      ${AF_CUDA_cusparse_static_LIBRARY}
+      ${AF_CUDA_optionally_static_libraries}
       ${nvrtc_libs}
       ${cusolver_static_lib}
-      ${END_GROUP}
-  )
+      ${END_GROUP})
 
   if(CUDA_VERSION VERSION_GREATER 10.0)
     target_link_libraries(af_cuda_static_cuda_library
@@ -367,7 +377,6 @@ else()
       ${CUDA_CUBLAS_LIBRARIES}
       ${CUDA_CUFFT_LIBRARIES}
       ${CUDA_cusolver_LIBRARY}
-      ${CUDA_cusparse_LIBRARY}
       ${nvrtc_libs}
   )
 endif()
@@ -536,6 +545,8 @@ cuda_add_library(afcuda
     cusolverDn.hpp
     cusparse.cpp
     cusparse.hpp
+    cusparseModule.cpp
+    cusparseModule.hpp
     device_manager.cpp
     device_manager.hpp
     debug_cuda.hpp
@@ -690,6 +701,13 @@ if(AF_WITH_CUDNN)
     )
 endif()
 
+if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS AND AF_cusparse_LINK_LOADING STREQUAL "Static")
+  target_compile_definitions(afcuda
+    PRIVATE
+      AF_cusparse_STATIC_LINKING)
+endif()
+
+
 arrayfire_set_default_cxx_flags(afcuda)
 
 # NOTE: Do not add additional CUDA specific definitions here. Add it to the
diff --git a/src/backend/cuda/cusparse.hpp b/src/backend/cuda/cusparse.hpp
index 7eb54900b4..b7a332a856 100644
--- a/src/backend/cuda/cusparse.hpp
+++ b/src/backend/cuda/cusparse.hpp
@@ -12,15 +12,16 @@
 #include <common/defines.hpp>
 #include <common/err_common.hpp>
 #include <common/unique_handle.hpp>
+#include <cusparseModule.hpp>
 #include <cusparse_v2.h>
 
 // clang-format off
-DEFINE_HANDLER(cusparseHandle_t, cusparseCreate, cusparseDestroy);
-DEFINE_HANDLER(cusparseMatDescr_t, cusparseCreateMatDescr, cusparseDestroyMatDescr);
+DEFINE_HANDLER(cusparseHandle_t, cuda::getCusparsePlugin().cusparseCreate, cuda::getCusparsePlugin().cusparseDestroy);
+DEFINE_HANDLER(cusparseMatDescr_t, cuda::getCusparsePlugin().cusparseCreateMatDescr, cuda::getCusparsePlugin().cusparseDestroyMatDescr);
 #if defined(AF_USE_NEW_CUSPARSE_API)
-DEFINE_HANDLER(cusparseSpMatDescr_t, cusparseCreateCsr, cusparseDestroySpMat);
-DEFINE_HANDLER(cusparseDnVecDescr_t, cusparseCreateDnVec, cusparseDestroyDnVec);
-DEFINE_HANDLER(cusparseDnMatDescr_t, cusparseCreateDnMat, cusparseDestroyDnMat);
+DEFINE_HANDLER(cusparseSpMatDescr_t, cuda::getCusparsePlugin().cusparseCreateCsr, cuda::getCusparsePlugin().cusparseDestroySpMat);
+DEFINE_HANDLER(cusparseDnVecDescr_t, cuda::getCusparsePlugin().cusparseCreateDnVec, cuda::getCusparsePlugin().cusparseDestroyDnVec);
+DEFINE_HANDLER(cusparseDnMatDescr_t, cuda::getCusparsePlugin().cusparseCreateDnMat, cuda::getCusparsePlugin().cusparseDestroyDnMat);
 #endif
 // clang-format on
 
diff --git a/src/backend/cuda/cusparseModule.cpp b/src/backend/cuda/cusparseModule.cpp
new file mode 100644
index 0000000000..f229372b43
--- /dev/null
+++ b/src/backend/cuda/cusparseModule.cpp
@@ -0,0 +1,135 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <cusparseModule.hpp>
+
+#include <common/err_common.hpp>
+#include <af/defines.h>
+
+#include <cuda.h>
+#include <string>
+
+namespace cuda {
+
+cusparseModule::cusparseModule()
+    :
+#ifdef AF_cusparse_STATIC_LINKING
+    module(nullptr, nullptr)
+#else
+    module("cusparse", nullptr)
+#endif
+{
+#ifdef AF_cusparse_STATIC_LINKING
+    AF_TRACE("CuSparse linked staticly.");
+#undef MODULE_FUNCTION_INIT
+#define MODULE_FUNCTION_INIT(NAME) NAME = &::NAME
+#else
+    if (!module.isLoaded()) {
+        AF_TRACE(
+            "WARNING: Unable to load cuSparse: {}\n"
+            "cuSparse failed to load. Try installing cuSparse or check if\n"
+            "cuSparse is in the search path. On Linux, you can set the\n"
+            "LD_DEBUG=libs environment variable to debug loading issues.\n"
+            "Falling back to matmul based implementation",
+            module.getErrorMessage());
+
+        return;
+    }
+#endif
+
+    MODULE_FUNCTION_INIT(cusparseCcsc2dense);
+    MODULE_FUNCTION_INIT(cusparseCcsr2dense);
+    MODULE_FUNCTION_INIT(cusparseCdense2csc);
+    MODULE_FUNCTION_INIT(cusparseCdense2csr);
+    MODULE_FUNCTION_INIT(cusparseCgthr);
+    MODULE_FUNCTION_INIT(cusparseCnnz);
+    MODULE_FUNCTION_INIT(cusparseCreateCsr);
+    MODULE_FUNCTION_INIT(cusparseCreateDnMat);
+    MODULE_FUNCTION_INIT(cusparseCreateDnVec);
+    MODULE_FUNCTION_INIT(cusparseCreateIdentityPermutation);
+    MODULE_FUNCTION_INIT(cusparseCreate);
+    MODULE_FUNCTION_INIT(cusparseCreateMatDescr);
+    MODULE_FUNCTION_INIT(cusparseDcsc2dense);
+    MODULE_FUNCTION_INIT(cusparseDcsr2dense);
+    MODULE_FUNCTION_INIT(cusparseDdense2csc);
+    MODULE_FUNCTION_INIT(cusparseDdense2csr);
+    MODULE_FUNCTION_INIT(cusparseDestroyDnMat);
+    MODULE_FUNCTION_INIT(cusparseDestroyDnVec);
+    MODULE_FUNCTION_INIT(cusparseDestroy);
+    MODULE_FUNCTION_INIT(cusparseDestroyMatDescr);
+    MODULE_FUNCTION_INIT(cusparseDestroySpMat);
+    MODULE_FUNCTION_INIT(cusparseDgthr);
+    MODULE_FUNCTION_INIT(cusparseDnnz);
+    MODULE_FUNCTION_INIT(cusparseScsc2dense);
+    MODULE_FUNCTION_INIT(cusparseScsr2dense);
+    MODULE_FUNCTION_INIT(cusparseSdense2csc);
+    MODULE_FUNCTION_INIT(cusparseSdense2csr);
+    MODULE_FUNCTION_INIT(cusparseSetMatIndexBase);
+    MODULE_FUNCTION_INIT(cusparseSetMatType);
+    MODULE_FUNCTION_INIT(cusparseSetStream);
+    MODULE_FUNCTION_INIT(cusparseSgthr);
+    MODULE_FUNCTION_INIT(cusparseSnnz);
+    MODULE_FUNCTION_INIT(cusparseSpMM_bufferSize);
+    MODULE_FUNCTION_INIT(cusparseSpMM);
+    MODULE_FUNCTION_INIT(cusparseSpMV_bufferSize);
+    MODULE_FUNCTION_INIT(cusparseSpMV);
+    MODULE_FUNCTION_INIT(cusparseXcoo2csr);
+    MODULE_FUNCTION_INIT(cusparseXcoosort_bufferSizeExt);
+    MODULE_FUNCTION_INIT(cusparseXcoosortByColumn);
+    MODULE_FUNCTION_INIT(cusparseXcoosortByRow);
+    MODULE_FUNCTION_INIT(cusparseXcsr2coo);
+#if CUDA_VERSION >= 11000
+    MODULE_FUNCTION_INIT(cusparseXcsrgeam2Nnz);
+#else
+    MODULE_FUNCTION_INIT(cusparseXcsrgeamNnz);
+#endif
+    MODULE_FUNCTION_INIT(cusparseZcsc2dense);
+    MODULE_FUNCTION_INIT(cusparseZcsr2dense);
+#if CUDA_VERSION >= 11000
+    MODULE_FUNCTION_INIT(cusparseScsrgeam2_bufferSizeExt);
+    MODULE_FUNCTION_INIT(cusparseScsrgeam2);
+    MODULE_FUNCTION_INIT(cusparseDcsrgeam2_bufferSizeExt);
+    MODULE_FUNCTION_INIT(cusparseDcsrgeam2);
+    MODULE_FUNCTION_INIT(cusparseCcsrgeam2_bufferSizeExt);
+    MODULE_FUNCTION_INIT(cusparseCcsrgeam2);
+    MODULE_FUNCTION_INIT(cusparseZcsrgeam2_bufferSizeExt);
+    MODULE_FUNCTION_INIT(cusparseZcsrgeam2);
+#else
+    MODULE_FUNCTION_INIT(cusparseScsrgeam);
+    MODULE_FUNCTION_INIT(cusparseDcsrgeam);
+    MODULE_FUNCTION_INIT(cusparseCcsrgeam);
+    MODULE_FUNCTION_INIT(cusparseZcsrgeam);
+#endif
+    MODULE_FUNCTION_INIT(cusparseZdense2csc);
+    MODULE_FUNCTION_INIT(cusparseZdense2csr);
+    MODULE_FUNCTION_INIT(cusparseZgthr);
+    MODULE_FUNCTION_INIT(cusparseZnnz);
+
+#ifndef AF_cusparse_STATIC_LINKING
+    if (!module.symbolsLoaded()) {
+        std::string error_message =
+            "Error loading cuSparse symbols. ArrayFire was unable to load some "
+            "symbols from the cuSparse library. Please create an issue on the "
+            "ArrayFire repository with information about the installed "
+            "cuSparse and ArrayFire on your system.";
+        AF_ERROR(error_message, AF_ERR_LOAD_LIB);
+    }
+#endif
+}
+
+spdlog::logger* cusparseModule::getLogger() const noexcept {
+    return module.getLogger();
+}
+
+cusparseModule& getCusparsePlugin() noexcept {
+    static auto* plugin = new cusparseModule();
+    return *plugin;
+}
+
+}  // namespace cuda
diff --git a/src/backend/cuda/cusparseModule.hpp b/src/backend/cuda/cusparseModule.hpp
new file mode 100644
index 0000000000..57878c2cf8
--- /dev/null
+++ b/src/backend/cuda/cusparseModule.hpp
@@ -0,0 +1,96 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/DependencyModule.hpp>
+#include <cuda.h>
+#include <cusparse_v2.h>
+
+namespace cuda {
+class cusparseModule {
+    common::DependencyModule module;
+
+   public:
+    cusparseModule();
+    ~cusparseModule() = default;
+
+    MODULE_MEMBER(cusparseCcsc2dense);
+    MODULE_MEMBER(cusparseCcsr2dense);
+    MODULE_MEMBER(cusparseCdense2csc);
+    MODULE_MEMBER(cusparseCdense2csr);
+    MODULE_MEMBER(cusparseCgthr);
+    MODULE_MEMBER(cusparseCnnz);
+    MODULE_MEMBER(cusparseCreateCsr);
+    MODULE_MEMBER(cusparseCreateDnMat);
+    MODULE_MEMBER(cusparseCreateDnVec);
+    MODULE_MEMBER(cusparseCreateIdentityPermutation);
+    MODULE_MEMBER(cusparseCreate);
+    MODULE_MEMBER(cusparseCreateMatDescr);
+    MODULE_MEMBER(cusparseDcsc2dense);
+    MODULE_MEMBER(cusparseDcsr2dense);
+    MODULE_MEMBER(cusparseDdense2csc);
+    MODULE_MEMBER(cusparseDdense2csr);
+    MODULE_MEMBER(cusparseDestroyDnMat);
+    MODULE_MEMBER(cusparseDestroyDnVec);
+    MODULE_MEMBER(cusparseDestroy);
+    MODULE_MEMBER(cusparseDestroyMatDescr);
+    MODULE_MEMBER(cusparseDestroySpMat);
+    MODULE_MEMBER(cusparseDgthr);
+    MODULE_MEMBER(cusparseDnnz);
+    MODULE_MEMBER(cusparseScsc2dense);
+    MODULE_MEMBER(cusparseScsr2dense);
+    MODULE_MEMBER(cusparseSdense2csc);
+    MODULE_MEMBER(cusparseSdense2csr);
+    MODULE_MEMBER(cusparseSetMatIndexBase);
+    MODULE_MEMBER(cusparseSetMatType);
+    MODULE_MEMBER(cusparseSetStream);
+    MODULE_MEMBER(cusparseSgthr);
+    MODULE_MEMBER(cusparseSnnz);
+    MODULE_MEMBER(cusparseSpMM_bufferSize);
+    MODULE_MEMBER(cusparseSpMM);
+    MODULE_MEMBER(cusparseSpMV_bufferSize);
+    MODULE_MEMBER(cusparseSpMV);
+    MODULE_MEMBER(cusparseXcoo2csr);
+    MODULE_MEMBER(cusparseXcoosort_bufferSizeExt);
+    MODULE_MEMBER(cusparseXcoosortByColumn);
+    MODULE_MEMBER(cusparseXcoosortByRow);
+    MODULE_MEMBER(cusparseXcsr2coo);
+    MODULE_MEMBER(cusparseZcsc2dense);
+    MODULE_MEMBER(cusparseZcsr2dense);
+
+#if CUDA_VERSION >= 11000
+    MODULE_MEMBER(cusparseXcsrgeam2Nnz);
+    MODULE_MEMBER(cusparseCcsrgeam2_bufferSizeExt);
+    MODULE_MEMBER(cusparseCcsrgeam2);
+    MODULE_MEMBER(cusparseDcsrgeam2_bufferSizeExt);
+    MODULE_MEMBER(cusparseDcsrgeam2);
+    MODULE_MEMBER(cusparseScsrgeam2_bufferSizeExt);
+    MODULE_MEMBER(cusparseScsrgeam2);
+    MODULE_MEMBER(cusparseZcsrgeam2_bufferSizeExt);
+    MODULE_MEMBER(cusparseZcsrgeam2);
+#else
+    MODULE_MEMBER(cusparseXcsrgeamNnz);
+    MODULE_MEMBER(cusparseCcsrgeam);
+    MODULE_MEMBER(cusparseDcsrgeam);
+    MODULE_MEMBER(cusparseScsrgeam);
+    MODULE_MEMBER(cusparseZcsrgeam);
+#endif
+
+    MODULE_MEMBER(cusparseZdense2csc);
+    MODULE_MEMBER(cusparseZdense2csr);
+    MODULE_MEMBER(cusparseZgthr);
+    MODULE_MEMBER(cusparseZnnz);
+
+    spdlog::logger* getLogger() const noexcept;
+};
+
+cusparseModule& getCusparsePlugin() noexcept;
+
+}  // namespace cuda
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index dd715e4691..ab94cf298f 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -29,6 +29,7 @@
 #include <cufft.hpp>
 #include <cusolverDn.hpp>
 #include <cusparse.hpp>
+#include <cusparseModule.hpp>
 #include <device_manager.hpp>
 #include <driver.h>
 #include <err_cuda.hpp>
@@ -84,7 +85,7 @@ unique_handle<cublasHandle_t> *cublasManager(const int deviceId) {
     thread_local once_flag initFlags[DeviceManager::MAX_DEVICES];
 
     call_once(initFlags[deviceId], [&] {
-        handles[deviceId].create();
+        CUBLAS_CHECK((cublasStatus_t)handles[deviceId].create());
         // TODO(pradeep) When multiple streams per device
         // is added to CUDA backend, move the cublasSetStream
         // call outside of call_once scope.
@@ -159,12 +160,13 @@ unique_handle<cusparseHandle_t> *cusparseManager(const int deviceId) {
         handles[DeviceManager::MAX_DEVICES];
     thread_local once_flag initFlags[DeviceManager::MAX_DEVICES];
     call_once(initFlags[deviceId], [&] {
+        auto &_ = getCusparsePlugin();
         handles[deviceId].create();
         // TODO(pradeep) When multiple streams per device
         // is added to CUDA backend, move the cublasSetStream
         // call outside of call_once scope.
         CUSPARSE_CHECK(
-            cusparseSetStream(handles[deviceId], cuda::getStream(deviceId)));
+            _.cusparseSetStream(handles[deviceId], cuda::getStream(deviceId)));
     });
     return &handles[deviceId];
 }
diff --git a/src/backend/cuda/sparse.cu b/src/backend/cuda/sparse.cu
index 47dad93e07..27b805e9ea 100644
--- a/src/backend/cuda/sparse.cu
+++ b/src/backend/cuda/sparse.cu
@@ -15,6 +15,7 @@
 #include <complex.hpp>
 #include <copy.hpp>
 #include <cusparse.hpp>
+#include <cusparseModule.hpp>
 #include <kernel/sparse.hpp>
 #include <lookup.hpp>
 #include <math.hpp>
@@ -122,8 +123,9 @@ struct gthr_func_def_t {
 #define SPARSE_FUNC(FUNC, TYPE, PREFIX)                                     \
     template<>                                                              \
     typename FUNC##_func_def_t<TYPE>::FUNC##_func_def FUNC##_func<TYPE>() { \
-        return (FUNC##_func_def_t<TYPE>::FUNC##_func_def) &                 \
-               cusparse##PREFIX##FUNC;                                      \
+        cusparseModule &_ = getCusparsePlugin();                            \
+        return (FUNC##_func_def_t<TYPE>::FUNC##_func_def)(                  \
+            _.cusparse##PREFIX##FUNC);                                      \
     }
 
 SPARSE_FUNC_DEF(dense2csr)
@@ -194,11 +196,12 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
     const int M = in.dims()[0];
     const int N = in.dims()[1];
 
+    cusparseModule &_ = getCusparsePlugin();
     // Create Sparse Matrix Descriptor
     cusparseMatDescr_t descr = 0;
-    CUSPARSE_CHECK(cusparseCreateMatDescr(&descr));
-    cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
-    cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
+    CUSPARSE_CHECK(_.cusparseCreateMatDescr(&descr));
+    _.cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
+    _.cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
 
     int d                   = -1;
     cusparseDirection_t dir = CUSPARSE_DIRECTION_ROW;
@@ -238,7 +241,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
             nnzPerDir.get(), values.get(), rowIdx.get(), colIdx.get()));
 
     // Destory Sparse Matrix Descriptor
-    CUSPARSE_CHECK(cusparseDestroyMatDescr(descr));
+    CUSPARSE_CHECK(_.cusparseDestroyMatDescr(descr));
 
     return createArrayDataSparseArray<T>(in.dims(), values, rowIdx, colIdx,
                                          stype);
@@ -262,10 +265,11 @@ Array<T> sparseConvertCOOToDense(const SparseArray<T> &in) {
 template<typename T, af_storage stype>
 Array<T> sparseConvertStorageToDense(const SparseArray<T> &in) {
     // Create Sparse Matrix Descriptor
+    cusparseModule &_        = getCusparsePlugin();
     cusparseMatDescr_t descr = 0;
-    CUSPARSE_CHECK(cusparseCreateMatDescr(&descr));
-    cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
-    cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
+    CUSPARSE_CHECK(_.cusparseCreateMatDescr(&descr));
+    _.cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
+    _.cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO);
 
     int M          = in.dims()[0];
     int N          = in.dims()[1];
@@ -284,7 +288,7 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in) {
                                 in.getColIdx().get(), dense.get(), d_strides1));
 
     // Destory Sparse Matrix Descriptor
-    CUSPARSE_CHECK(cusparseDestroyMatDescr(descr));
+    CUSPARSE_CHECK(_.cusparseDestroyMatDescr(descr));
 
     return dense;
 }
@@ -297,6 +301,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
     int nNZ                  = in.getNNZ();
     SparseArray<T> converted = createEmptySparseArray<T>(in.dims(), nNZ, dest);
 
+    cusparseModule &_ = getCusparsePlugin();
     if (src == AF_STORAGE_CSR && dest == AF_STORAGE_COO) {
         // Copy colIdx as is
         CUDA_CHECK(
@@ -305,13 +310,13 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
                             cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
         // cusparse function to expand compressed row into coordinate
-        CUSPARSE_CHECK(cusparseXcsr2coo(
+        CUSPARSE_CHECK(_.cusparseXcsr2coo(
             sparseHandle(), in.getRowIdx().get(), nNZ, in.dims()[0],
             converted.getRowIdx().get(), CUSPARSE_INDEX_BASE_ZERO));
 
         // Call sort
         size_t pBufferSizeInBytes = 0;
-        CUSPARSE_CHECK(cusparseXcoosort_bufferSizeExt(
+        CUSPARSE_CHECK(_.cusparseXcoosort_bufferSizeExt(
             sparseHandle(), in.dims()[0], in.dims()[1], nNZ,
             converted.getRowIdx().get(), converted.getColIdx().get(),
             &pBufferSizeInBytes));
@@ -320,9 +325,9 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
 
         shared_ptr<int> P(memAlloc<int>(nNZ).release(), memFree<int>);
         CUSPARSE_CHECK(
-            cusparseCreateIdentityPermutation(sparseHandle(), nNZ, P.get()));
+            _.cusparseCreateIdentityPermutation(sparseHandle(), nNZ, P.get()));
 
-        CUSPARSE_CHECK(cusparseXcoosortByColumn(
+        CUSPARSE_CHECK(_.cusparseXcoosortByColumn(
             sparseHandle(), in.dims()[0], in.dims()[1], nNZ,
             converted.getRowIdx().get(), converted.getColIdx().get(), P.get(),
             (void *)pBuffer.get()));
@@ -344,7 +349,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
         // Call sort to convert column major to row major
         {
             size_t pBufferSizeInBytes = 0;
-            CUSPARSE_CHECK(cusparseXcoosort_bufferSizeExt(
+            CUSPARSE_CHECK(_.cusparseXcoosort_bufferSizeExt(
                 sparseHandle(), cooT.dims()[0], cooT.dims()[1], nNZ,
                 cooT.getRowIdx().get(), cooT.getColIdx().get(),
                 &pBufferSizeInBytes));
@@ -352,10 +357,10 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
                 memAlloc<char>(pBufferSizeInBytes).release(), memFree<char>);
 
             shared_ptr<int> P(memAlloc<int>(nNZ).release(), memFree<int>);
-            CUSPARSE_CHECK(cusparseCreateIdentityPermutation(sparseHandle(),
-                                                             nNZ, P.get()));
+            CUSPARSE_CHECK(_.cusparseCreateIdentityPermutation(sparseHandle(),
+                                                               nNZ, P.get()));
 
-            CUSPARSE_CHECK(cusparseXcoosortByRow(
+            CUSPARSE_CHECK(_.cusparseXcoosortByRow(
                 sparseHandle(), cooT.dims()[0], cooT.dims()[1], nNZ,
                 cooT.getRowIdx().get(), cooT.getColIdx().get(), P.get(),
                 (void *)pBuffer.get()));
@@ -376,7 +381,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
                             cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
 
         // cusparse function to compress row from coordinate
-        CUSPARSE_CHECK(cusparseXcoo2csr(
+        CUSPARSE_CHECK(_.cusparseXcoo2csr(
             sparseHandle(), cooT.getRowIdx().get(), nNZ, cooT.dims()[0],
             converted.getRowIdx().get(), CUSPARSE_INDEX_BASE_ZERO));
 
diff --git a/src/backend/cuda/sparse_arith.cu b/src/backend/cuda/sparse_arith.cu
index 11a38c58e1..a41c356397 100644
--- a/src/backend/cuda/sparse_arith.cu
+++ b/src/backend/cuda/sparse_arith.cu
@@ -115,10 +115,11 @@ SparseArray<T> arithOp(const SparseArray<T> &lhs, const Array<T> &rhs,
     template<typename T>               \
     FUNC##_def<T> FUNC##_func();
 
-#define SPARSE_ARITH_OP_FUNC(FUNC, TYPE, INFIX) \
-    template<>                                  \
-    FUNC##_def<TYPE> FUNC##_func<TYPE>() {      \
-        return cusparse##INFIX##FUNC;           \
+#define SPARSE_ARITH_OP_FUNC(FUNC, TYPE, INFIX)  \
+    template<>                                   \
+    FUNC##_def<TYPE> FUNC##_func<TYPE>() {       \
+        cusparseModule &_ = getCusparsePlugin(); \
+        return _.cusparse##INFIX##FUNC;          \
     }
 
 #if CUDA_VERSION >= 11000
@@ -139,7 +140,8 @@ SPARSE_ARITH_OP_BUFFER_SIZE_FUNC_DEF(csrgeam2);
 #define SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(FUNC, TYPE, INFIX)        \
     template<>                                                     \
     FUNC##_buffer_size_def<TYPE> FUNC##_buffer_size_func<TYPE>() { \
-        return cusparse##INFIX##FUNC##_bufferSizeExt;              \
+        cusparseModule &_ = getCusparsePlugin();                   \
+        return _.cusparse##INFIX##FUNC##_bufferSizeExt;            \
     }
 
 SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2, float, S);
@@ -206,8 +208,9 @@ SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
     int baseC, nnzC;
     int *nnzcDevHostPtr = &nnzC;
 
-    T alpha = scalar<T>(1);
-    T beta  = op == af_sub_t ? scalar<T>(-1) : alpha;
+    T alpha           = scalar<T>(1);
+    T beta            = op == af_sub_t ? scalar<T>(-1) : alpha;
+    cusparseModule &_ = getCusparsePlugin();
 
 #if CUDA_VERSION >= 11000
     size_t pBufferSize = 0;
@@ -219,12 +222,12 @@ SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
 
     auto tmpBuffer = createEmptyArray<char>(dim4(pBufferSize));
 
-    CUSPARSE_CHECK(cusparseXcsrgeam2Nnz(
+    CUSPARSE_CHECK(_.cusparseXcsrgeam2Nnz(
         sparseHandle(), M, N, desc, nnzA, csrRowPtrA, csrColPtrA, desc, nnzB,
         csrRowPtrB, csrColPtrB, desc, csrRowPtrC, nnzcDevHostPtr,
         tmpBuffer.get()));
 #else
-    CUSPARSE_CHECK(cusparseXcsrgeamNnz(
+    CUSPARSE_CHECK(_.cusparseXcsrgeamNnz(
         sparseHandle(), M, N, desc, nnzA, csrRowPtrA, csrColPtrA, desc, nnzB,
         csrRowPtrB, csrColPtrB, desc, csrRowPtrC, nnzcDevHostPtr));
 #endif
diff --git a/src/backend/cuda/sparse_blas.cu b/src/backend/cuda/sparse_blas.cu
index 179c17615d..33a2957a62 100644
--- a/src/backend/cuda/sparse_blas.cu
+++ b/src/backend/cuda/sparse_blas.cu
@@ -14,6 +14,7 @@
 #include <cudaDataType.hpp>
 #include <cuda_runtime.h>
 #include <cusparse.hpp>
+#include <cusparseModule.hpp>
 #include <cusparse_descriptor_helpers.hpp>
 #include <math.hpp>
 #include <platform.hpp>
@@ -41,8 +42,9 @@ size_t spmvBufferSize(cusparseOperation_t opA, const T *alpha,
                       const cusparseSpMatDescr_t matA,
                       const cusparseDnVecDescr_t vecX, const T *beta,
                       const cusparseDnVecDescr_t vecY) {
-    size_t retVal = 0;
-    CUSPARSE_CHECK(cusparseSpMV_bufferSize(
+    size_t retVal     = 0;
+    cusparseModule &_ = getCusparsePlugin();
+    CUSPARSE_CHECK(_.cusparseSpMV_bufferSize(
         sparseHandle(), opA, alpha, matA, vecX, beta, vecY, getComputeType<T>(),
         CUSPARSE_CSRMV_ALG1, &retVal));
     return retVal;
@@ -52,9 +54,10 @@ template<typename T>
 void spmv(cusparseOperation_t opA, const T *alpha,
           const cusparseSpMatDescr_t matA, const cusparseDnVecDescr_t vecX,
           const T *beta, const cusparseDnVecDescr_t vecY, void *buffer) {
-    CUSPARSE_CHECK(cusparseSpMV(sparseHandle(), opA, alpha, matA, vecX, beta,
-                                vecY, getComputeType<T>(),
-                                CUSPARSE_MV_ALG_DEFAULT, buffer));
+    cusparseModule &_ = getCusparsePlugin();
+    CUSPARSE_CHECK(_.cusparseSpMV(sparseHandle(), opA, alpha, matA, vecX, beta,
+                                  vecY, getComputeType<T>(),
+                                  CUSPARSE_MV_ALG_DEFAULT, buffer));
 }
 
 template<typename T>
@@ -62,8 +65,9 @@ size_t spmmBufferSize(cusparseOperation_t opA, cusparseOperation_t opB,
                       const T *alpha, const cusparseSpMatDescr_t matA,
                       const cusparseDnMatDescr_t matB, const T *beta,
                       const cusparseDnMatDescr_t matC) {
-    size_t retVal = 0;
-    CUSPARSE_CHECK(cusparseSpMM_bufferSize(
+    size_t retVal     = 0;
+    cusparseModule &_ = getCusparsePlugin();
+    CUSPARSE_CHECK(_.cusparseSpMM_bufferSize(
         sparseHandle(), opA, opB, alpha, matA, matB, beta, matC,
         getComputeType<T>(), CUSPARSE_CSRMM_ALG1, &retVal));
     return retVal;
@@ -73,9 +77,10 @@ template<typename T>
 void spmm(cusparseOperation_t opA, cusparseOperation_t opB, const T *alpha,
           const cusparseSpMatDescr_t matA, const cusparseDnMatDescr_t matB,
           const T *beta, const cusparseDnMatDescr_t matC, void *buffer) {
-    CUSPARSE_CHECK(cusparseSpMM(sparseHandle(), opA, opB, alpha, matA, matB,
-                                beta, matC, getComputeType<T>(),
-                                CUSPARSE_CSRMM_ALG1, buffer));
+    cusparseModule &_ = getCusparsePlugin();
+    CUSPARSE_CHECK(_.cusparseSpMM(sparseHandle(), opA, opB, alpha, matA, matB,
+                                  beta, matC, getComputeType<T>(),
+                                  CUSPARSE_CSRMM_ALG1, buffer));
 }
 
 #else
@@ -105,8 +110,9 @@ struct csrmm_func_def_t {
 #define SPARSE_FUNC(FUNC, TYPE, PREFIX)                                     \
     template<>                                                              \
     typename FUNC##_func_def_t<TYPE>::FUNC##_func_def FUNC##_func<TYPE>() { \
+        cusparseModule &_ = getCusparsePlugin();                            \
         return (FUNC##_func_def_t<TYPE>::FUNC##_func_def) &                 \
-               cusparse##PREFIX##FUNC;                                      \
+               _.cusparse##PREFIX##FUNC;                                    \
     }
 
 SPARSE_FUNC_DEF(csrmm)
@@ -174,11 +180,12 @@ Array<T> matmul(const common::SparseArray<T> &lhs, const Array<T> &rhs,
 
 #else
 
+    cusparseModule &_ = getCusparsePlugin();
     // Create Sparse Matrix Descriptor
     cusparseMatDescr_t descr = 0;
-    CUSPARSE_CHECK(cusparseCreateMatDescr(&descr));
-    CUSPARSE_CHECK(cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL));
-    CUSPARSE_CHECK(cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO));
+    CUSPARSE_CHECK(_.cusparseCreateMatDescr(&descr));
+    CUSPARSE_CHECK(_.cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL));
+    CUSPARSE_CHECK(_.cusparseSetMatIndexBase(descr, CUSPARSE_INDEX_BASE_ZERO));
 
     // Call Matrix-Vector or Matrix-Matrix
     // Note:
@@ -197,7 +204,7 @@ Array<T> matmul(const common::SparseArray<T> &lhs, const Array<T> &rhs,
             lhs.getRowIdx().get(), lhs.getColIdx().get(), rhs.get(),
             rStrides[1], &beta, out.get(), out.dims()[0]));
     }
-    CUSPARSE_CHECK(cusparseDestroyMatDescr(descr));
+    CUSPARSE_CHECK(_.cusparseDestroyMatDescr(descr));
 
 #endif
 

From 8bdcc77e1b648d36b686667bebc2139b6d186341 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 11 Apr 2022 15:12:24 -0400
Subject: [PATCH 035/473] reduce all -> array (#3199)

* change return type for reduce_all
adds single cuda kernel for reduce_all_array
adds cpu reduce_all
adds opencl reduce_all functions, kernel: todo
remove old versions of reduce_all
update missing reduction functions
adds missing reduce tests, other reduce functions
update test precision, fix kernel shared ptrs
fixes failing tests, clang format, fix init assignment
update api, minor unified error handling

* Fix reduce_all on Intel's OpenCL. Removed unused variables

Co-authored-by: Umar Arshad <umar@arrayfire.com>
---
 include/af/algorithm.h                  | 130 +++++++++++++
 src/api/c/anisotropic_diffusion.cpp     |   4 +-
 src/api/c/canny.cpp                     |  14 +-
 src/api/c/confidence_connected.cpp      |  19 +-
 src/api/c/corrcoef.cpp                  |  12 +-
 src/api/c/gaussian_kernel.cpp           |   4 +-
 src/api/c/hist.cpp                      |   5 +-
 src/api/c/histeq.cpp                    |   6 +-
 src/api/c/imgproc_common.hpp            |   8 +-
 src/api/c/norm.cpp                      |  19 +-
 src/api/c/rank.cpp                      |   4 +-
 src/api/c/reduce.cpp                    | 189 +++++++++++++++++-
 src/api/c/stdev.cpp                     |   8 +-
 src/api/c/surface.cpp                   |  14 +-
 src/api/c/var.cpp                       |  16 +-
 src/api/cpp/reduce.cpp                  |  28 +++
 src/api/unified/algorithm.cpp           |  27 +++
 src/api/unified/symbol_manager.hpp      |   5 +
 src/backend/cpu/kernel/reduce.hpp       |  41 ++++
 src/backend/cpu/reduce.cpp              |  52 ++---
 src/backend/cpu/reduce.hpp              |   3 +-
 src/backend/cpu/sparse.cpp              |   2 +-
 src/backend/cuda/kernel/reduce.hpp      | 249 ++++++++++++++++++------
 src/backend/cuda/reduce.hpp             |   3 +-
 src/backend/cuda/reduce_impl.hpp        |  11 +-
 src/backend/opencl/kernel/reduce.hpp    | 114 ++++++-----
 src/backend/opencl/kernel/reduce_all.cl | 160 +++++++++++++++
 src/backend/opencl/reduce.hpp           |   3 +-
 src/backend/opencl/reduce_impl.hpp      |  11 +-
 src/backend/opencl/sparse.cpp           |   2 +-
 src/backend/opencl/svd.cpp              |   2 +-
 test/mean.cpp                           |   5 +-
 test/reduce.cpp                         | 204 ++++++++++++++++++-
 33 files changed, 1150 insertions(+), 224 deletions(-)
 create mode 100644 src/backend/opencl/kernel/reduce_all.cl

diff --git a/include/af/algorithm.h b/include/af/algorithm.h
index 7c8cfdd393..801792a32a 100644
--- a/include/af/algorithm.h
+++ b/include/af/algorithm.h
@@ -674,6 +674,19 @@ extern "C" {
     */
     AFAPI af_err af_sum(af_array *out, const af_array in, const int dim);
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for sum of all elements in an array, resulting in an array
+
+       \param[out] out will contain the sum of all values in \p in
+       \param[in] in is the input array
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \ingroup reduce_func_sum
+    */
+    AFAPI af_err af_sum_all_array(af_array *out, const af_array in);
+#endif
+
 #if AF_API_VERSION >= 31
     /**
        C Interface for sum of elements in an array while replacing nans
@@ -690,6 +703,21 @@ extern "C" {
                             const int dim, const double nanval);
 #endif
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for sum of all elements in an array, resulting in an array with
+       nan substitution
+
+       \param[out] out will contain the sum of all values in \p in
+       \param[in] in is the input array
+       \param[in] nanval  The value that will replace the NaNs in \p in
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \ingroup reduce_func_sum
+    */
+    AFAPI af_err af_sum_nan_all_array(af_array *out, const af_array in, const double nanval);
+#endif
+
 #if AF_API_VERSION >= 37
     /**
        C Interface for sum of elements in an array according to key
@@ -741,6 +769,19 @@ extern "C" {
     */
     AFAPI af_err af_product(af_array *out, const af_array in, const int dim);
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for product of elements in an array, resulting in an array
+
+       \param[out] out will contain the product of all values in \p in
+       \param[in] in is the input array
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \ingroup reduce_func_product
+    */
+    AFAPI af_err af_product_all_array(af_array *out, const af_array in);
+#endif
+
 #if AF_API_VERSION >= 31
     /**
        C Interface for product of elements in an array while replacing nans
@@ -757,6 +798,21 @@ extern "C" {
     AFAPI af_err af_product_nan(af_array *out, const af_array in, const int dim, const double nanval);
 #endif
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for product of elements in an array, resulting in an array
+       while replacing nans
+
+       \param[out] out will contain the product of all values in \p in
+       \param[in] in   is the input array
+       \param[in] nanval  The value that will replace the NaNs in \p in
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \ingroup reduce_func_product
+    */
+    AFAPI af_err af_product_nan_all_array(af_array *out, const af_array in, const double nanval);
+#endif
+
 #if AF_API_VERSION >= 37
     /**
        C Interface for product of elements in an array according to key
@@ -1052,6 +1108,19 @@ extern "C" {
     */
     AFAPI af_err af_min_all(double *real, double *imag, const af_array in);
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for minimum values in an array, returning an array
+
+       \param[out] out will contain the minimum of all values in \p in
+       \param[in] in is the input array
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \ingroup reduce_func_min
+    */
+    AFAPI af_err af_min_all_array(af_array *out, const af_array in);
+#endif
+
     /**
        C Interface for getting maximum value of an array
 
@@ -1066,6 +1135,21 @@ extern "C" {
     */
     AFAPI af_err af_max_all(double *real, double *imag, const af_array in);
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for getting maximum value of an array, returning an array
+
+       \param[out] out will contain the maximum of all values in \p in
+       \param[in] in is the input array
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \note \p imag is always set to 0 when \p in is real.
+
+       \ingroup reduce_func_max
+    */
+    AFAPI af_err af_max_all_array(af_array *out, const af_array in);
+#endif
+
     /**
        C Interface for checking if all values in an array are true
 
@@ -1080,6 +1164,22 @@ extern "C" {
     */
     AFAPI af_err af_all_true_all(double *real, double *imag, const af_array in);
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for checking if all values in an array are true,
+       while returning an af_array
+
+       \param[out] out will contain 1 if all values of input \p in are true, 0 otherwise
+       \param[in] in is the input array
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \note \p imag is always set to 0.
+
+       \ingroup reduce_func_all_true
+    */
+    AFAPI af_err af_all_true_all_array(af_array *out, const af_array in);
+#endif
+
     /**
        C Interface for checking if any values in an array are true
 
@@ -1094,6 +1194,22 @@ extern "C" {
     */
     AFAPI af_err af_any_true_all(double *real, double *imag, const af_array in);
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for checking if any values in an array are true,
+       while returning an af_array
+
+       \param[out] out will contain 1 if any value of input \p in is true, 0 otherwise
+       \param[in] in is the input array
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \note \p imag is always set to 0.
+
+       \ingroup reduce_func_any_true
+    */
+    AFAPI af_err af_any_true_all_array(af_array *out, const af_array in);
+#endif
+
     /**
        C Interface for counting total number of non-zero values in an array
 
@@ -1108,6 +1224,20 @@ extern "C" {
     */
     AFAPI af_err af_count_all(double *real, double *imag, const af_array in);
 
+#if AF_API_VERSION >= 39
+    /**
+       C Interface for counting total number of non-zero values in an array,
+       while returning an af_array
+
+       \param[out] out contain the number of non-zero values in \p in.
+       \param[in] in is the input array
+       \return \ref AF_SUCCESS if the execution completes properly
+
+       \ingroup reduce_func_count
+    */
+    AFAPI af_err af_count_all_array(af_array *out, const af_array in);
+#endif
+
     /**
        C Interface for getting minimum values and their locations in an array
 
diff --git a/src/api/c/anisotropic_diffusion.cpp b/src/api/c/anisotropic_diffusion.cpp
index 24335a406e..fd2f83c5c1 100644
--- a/src/api/c/anisotropic_diffusion.cpp
+++ b/src/api/c/anisotropic_diffusion.cpp
@@ -28,6 +28,7 @@ using common::cast;
 using detail::arithOp;
 using detail::Array;
 using detail::createEmptyArray;
+using detail::getScalar;
 using detail::gradient;
 using detail::reduce_all;
 
@@ -48,7 +49,8 @@ af_array diffusion(const Array<float>& in, const float dt, const float K,
         auto g0Sqr = arithOp<float, af_mul_t>(g0, g0, dims);
         auto g1Sqr = arithOp<float, af_mul_t>(g1, g1, dims);
         auto sumd  = arithOp<float, af_add_t>(g0Sqr, g1Sqr, dims);
-        float avg  = reduce_all<af_add_t, float, float>(sumd, true, 0);
+        float avg =
+            getScalar<float>(reduce_all<af_add_t, float, float>(sumd, true, 0));
 
         anisotropicDiffusion(out, dt, 1.0f / (cnst * avg), fftype, eq);
     }
diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp
index 0c67ddb03d..d9d74da7d9 100644
--- a/src/api/c/canny.cpp
+++ b/src/api/c/canny.cpp
@@ -44,6 +44,7 @@ using detail::createEmptyArray;
 using detail::createHostDataArray;
 using detail::createSubArray;
 using detail::createValueArray;
+using detail::getScalar;
 using detail::histogram;
 using detail::iota;
 using detail::ireduce;
@@ -151,7 +152,9 @@ pair<Array<char>, Array<char>> computeCandidates(const Array<float>& supEdges,
                                                  const float t1,
                                                  const af_canny_threshold ct,
                                                  const float t2) {
-    float maxVal  = reduce_all<af_max_t, float, float>(supEdges);
+    float maxVal =
+        getScalar<float>(reduce_all<af_max_t, float, float>(supEdges));
+    ;
     auto NUM_BINS = static_cast<unsigned>(maxVal);
 
     auto lowRatio = createValueArray<float>(supEdges.dims(), t1);
@@ -171,10 +174,11 @@ pair<Array<char>, Array<char>> computeCandidates(const Array<float>& supEdges,
             return make_pair(strong, weak);
         };
         default: {
-            float minVal = reduce_all<af_min_t, float, float>(supEdges);
-            auto normG   = normalize(supEdges, minVal, maxVal);
-            auto T2      = createValueArray<float>(supEdges.dims(), t2);
-            auto T1      = createValueArray<float>(supEdges.dims(), t1);
+            float minVal =
+                getScalar<float>(reduce_all<af_min_t, float, float>(supEdges));
+            auto normG = normalize(supEdges, minVal, maxVal);
+            auto T2    = createValueArray<float>(supEdges.dims(), t2);
+            auto T1    = createValueArray<float>(supEdges.dims(), t1);
             Array<char> weak1 =
                 logicOp<float, af_ge_t>(normG, T1, normG.dims());
             Array<char> weak2 =
diff --git a/src/api/c/confidence_connected.cpp b/src/api/c/confidence_connected.cpp
index 174ed3c688..b42decc227 100644
--- a/src/api/c/confidence_connected.cpp
+++ b/src/api/c/confidence_connected.cpp
@@ -29,6 +29,7 @@ using common::createSpanIndex;
 using detail::arithOp;
 using detail::Array;
 using detail::createValueArray;
+using detail::getScalar;
 using detail::reduce_all;
 using detail::uchar;
 using detail::uint;
@@ -127,8 +128,8 @@ af_array ccHelper(const Array<T>& img, const Array<uint>& seedx,
     Array<CT> I2       = common::integralImage<CT>(in_2);
     Array<CT> S1       = sum(I1, _x, x_, _y, y_);
     Array<CT> S2       = sum(I2, _x, x_, _y, y_);
-    CT totSum          = reduce_all<af_add_t, CT, CT>(S1);
-    CT totSumSq        = reduce_all<af_add_t, CT, CT>(S2);
+    CT totSum          = getScalar<CT>(reduce_all<af_add_t, CT, CT>(S1));
+    CT totSumSq        = getScalar<CT>(reduce_all<af_add_t, CT, CT>(S2));
     CT totalNum        = numSeeds * nhoodSize;
     CT s1mean          = totSum / totalNum;
     CT s1var           = calcVar(totSumSq, totSum, totalNum);
@@ -137,8 +138,10 @@ af_array ccHelper(const Array<T>& img, const Array<uint>& seedx,
     CT upper           = s1mean + mult * s1stddev;
 
     Array<CT> seedIntensities = pointList(in, seedx, seedy);
-    CT maxSeedIntensity       = reduce_all<af_max_t, CT, CT>(seedIntensities);
-    CT minSeedIntensity       = reduce_all<af_min_t, CT, CT>(seedIntensities);
+    CT maxSeedIntensity =
+        getScalar<CT>(reduce_all<af_max_t, CT, CT>(seedIntensities));
+    CT minSeedIntensity =
+        getScalar<CT>(reduce_all<af_min_t, CT, CT>(seedIntensities));
 
     if (lower > minSeedIntensity) { lower = minSeedIntensity; }
     if (upper < maxSeedIntensity) { upper = maxSeedIntensity; }
@@ -155,7 +158,8 @@ af_array ccHelper(const Array<T>& img, const Array<uint>& seedx,
         // Segmented images are set with 1's and 0's thus essentially
         // making them into mask arrays for each iteration's input image
 
-        uint sampleCount = reduce_all<af_notzero_t, CT, uint>(segmented, true);
+        uint sampleCount = getScalar<uint>(
+            reduce_all<af_notzero_t, CT, uint>(segmented, true));
         if (sampleCount == 0) {
             // If no valid pixels are found, skip iterations
             break;
@@ -163,8 +167,9 @@ af_array ccHelper(const Array<T>& img, const Array<uint>& seedx,
         Array<CT> valids = arithOp<CT, af_mul_t>(segmented, in, inDims);
         Array<CT> vsqrd  = arithOp<CT, af_mul_t>(valids, valids, inDims);
 
-        CT validsSum  = reduce_all<af_add_t, CT, CT>(valids, true);
-        CT sumOfSqs   = reduce_all<af_add_t, CT, CT>(vsqrd, true);
+        CT validsSum =
+            getScalar<CT>(reduce_all<af_add_t, CT, CT>(valids, true));
+        CT sumOfSqs = getScalar<CT>(reduce_all<af_add_t, CT, CT>(vsqrd, true));
         CT validsMean = validsSum / sampleCount;
         CT validsVar  = calcVar(sumOfSqs, validsSum, CT(sampleCount));
         CT stddev     = sqrt(validsVar);
diff --git a/src/api/c/corrcoef.cpp b/src/api/c/corrcoef.cpp
index 2ee5e45d6a..0efc503cd4 100644
--- a/src/api/c/corrcoef.cpp
+++ b/src/api/c/corrcoef.cpp
@@ -11,6 +11,7 @@
 #include <backend.hpp>
 #include <common/cast.hpp>
 #include <common/err_common.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <math.hpp>
 #include <reduce.hpp>
@@ -26,6 +27,7 @@ using af::dim4;
 using common::cast;
 using detail::arithOp;
 using detail::Array;
+using detail::getScalar;
 using detail::intl;
 using detail::reduce_all;
 using detail::uchar;
@@ -41,16 +43,16 @@ static To corrcoef(const af_array& X, const af_array& Y) {
     const dim4& dims = xIn.dims();
     dim_t n          = xIn.elements();
 
-    To xSum = reduce_all<af_add_t, To, To>(xIn);
-    To ySum = reduce_all<af_add_t, To, To>(yIn);
+    To xSum = getScalar<To>(reduce_all<af_add_t, To, To>(xIn));
+    To ySum = getScalar<To>(reduce_all<af_add_t, To, To>(yIn));
 
     Array<To> xSq = arithOp<To, af_mul_t>(xIn, xIn, dims);
     Array<To> ySq = arithOp<To, af_mul_t>(yIn, yIn, dims);
     Array<To> xy  = arithOp<To, af_mul_t>(xIn, yIn, dims);
 
-    To xSqSum = reduce_all<af_add_t, To, To>(xSq);
-    To ySqSum = reduce_all<af_add_t, To, To>(ySq);
-    To xySum  = reduce_all<af_add_t, To, To>(xy);
+    To xSqSum = getScalar<To>(reduce_all<af_add_t, To, To>(xSq));
+    To ySqSum = getScalar<To>(reduce_all<af_add_t, To, To>(ySq));
+    To xySum  = getScalar<To>(reduce_all<af_add_t, To, To>(xy));
 
     To result =
         (n * xySum - xSum * ySum) / (std::sqrt(n * xSqSum - xSum * xSum) *
diff --git a/src/api/c/gaussian_kernel.cpp b/src/api/c/gaussian_kernel.cpp
index 79492f87ea..529aa378e9 100644
--- a/src/api/c/gaussian_kernel.cpp
+++ b/src/api/c/gaussian_kernel.cpp
@@ -10,6 +10,7 @@
 #include <arith.hpp>
 #include <backend.hpp>
 #include <common/err_common.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <math.hpp>
 #include <range.hpp>
@@ -24,6 +25,7 @@ using af::dim4;
 using detail::arithOp;
 using detail::Array;
 using detail::createValueArray;
+using detail::getScalar;
 using detail::range;
 using detail::reduce_all;
 using detail::scalar;
@@ -77,7 +79,7 @@ Array<T> gaussianKernel(const int rows, const int cols, const double sigma_r,
 
     // Use this instead of (2 * pi * sig^2);
     // This ensures the window adds up to 1
-    T norm_factor = reduce_all<af_add_t, T, T>(tmp);
+    T norm_factor = getScalar<T>(reduce_all<af_add_t, T, T>(tmp));
 
     Array<T> norm = createValueArray(odims, norm_factor);
     Array<T> res  = arithOp<T, af_div_t>(tmp, norm, odims);
diff --git a/src/api/c/hist.cpp b/src/api/c/hist.cpp
index 0fad162819..4b74e33cdf 100644
--- a/src/api/c/hist.cpp
+++ b/src/api/c/hist.cpp
@@ -12,6 +12,7 @@
 #include <common/cast.hpp>
 #include <common/err_common.hpp>
 #include <common/graphics_common.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <hist_graphics.hpp>
 #include <reduce.hpp>
@@ -20,6 +21,7 @@
 using detail::Array;
 using detail::copy_histogram;
 using detail::forgeManager;
+using detail::getScalar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -57,7 +59,8 @@ fg_chart setup_histogram(fg_window const window, const af_array in,
         float xMin, xMax, yMin, yMax, zMin, zMax;
         FG_CHECK(_.fg_get_chart_axes_limits(&xMin, &xMax, &yMin, &yMax, &zMin,
                                             &zMax, chart));
-        T freqMax = detail::reduce_all<af_max_t, T, T>(histogramInput);
+        T freqMax =
+            getScalar<T>(detail::reduce_all<af_max_t, T, T>(histogramInput));
 
         if (xMin == 0 && xMax == 0 && yMin == 0 && yMax == 0) {
             // No previous limits. Set without checking
diff --git a/src/api/c/histeq.cpp b/src/api/c/histeq.cpp
index 0c2ce6f8ca..8fef8a2684 100644
--- a/src/api/c/histeq.cpp
+++ b/src/api/c/histeq.cpp
@@ -12,6 +12,7 @@
 #include <common/cast.hpp>
 #include <common/err_common.hpp>
 #include <common/moddims.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <lookup.hpp>
 #include <reduce.hpp>
@@ -27,6 +28,7 @@ using common::modDims;
 using detail::arithOp;
 using detail::Array;
 using detail::createValueArray;
+using detail::getScalar;
 using detail::intl;
 using detail::lookup;
 using detail::reduce_all;
@@ -50,8 +52,8 @@ static af_array hist_equal(const af_array& in, const af_array& hist) {
 
     Array<float> cdf = scan<af_add_t, float, float>(fHist, 0);
 
-    float minCdf = reduce_all<af_min_t, float, float>(cdf);
-    float maxCdf = reduce_all<af_max_t, float, float>(cdf);
+    float minCdf = getScalar<float>(reduce_all<af_min_t, float, float>(cdf));
+    float maxCdf = getScalar<float>(reduce_all<af_max_t, float, float>(cdf));
     float factor = static_cast<float>(grayLevels - 1) / (maxCdf - minCdf);
 
     // constant array of min value from cdf
diff --git a/src/api/c/imgproc_common.hpp b/src/api/c/imgproc_common.hpp
index bf16be980a..214fbe6c7a 100644
--- a/src/api/c/imgproc_common.hpp
+++ b/src/api/c/imgproc_common.hpp
@@ -12,6 +12,7 @@
 #include <arith.hpp>
 #include <backend.hpp>
 #include <common/cast.hpp>
+#include <copy.hpp>
 #include <logic.hpp>
 #include <reduce.hpp>
 #include <scan.hpp>
@@ -46,9 +47,10 @@ detail::Array<To> convRange(const detail::Array<Ti>& in,
                             const To newLow = To(0), const To newHigh = To(1)) {
     auto dims  = in.dims();
     auto input = common::cast<To, Ti>(in);
-    To high    = detail::reduce_all<af_max_t, To, To>(input);
-    To low     = detail::reduce_all<af_min_t, To, To>(input);
-    To range   = high - low;
+    To high =
+        detail::getScalar<To>(detail::reduce_all<af_max_t, To, To>(input));
+    To low = detail::getScalar<To>(detail::reduce_all<af_min_t, To, To>(input));
+    To range = high - low;
 
     if (std::abs(range) < 1.0e-6) {
         if (low == To(0) && newLow == To(0)) {
diff --git a/src/api/c/norm.cpp b/src/api/c/norm.cpp
index 79f064ebb7..84444eed58 100644
--- a/src/api/c/norm.cpp
+++ b/src/api/c/norm.cpp
@@ -12,6 +12,7 @@
 #include <common/ArrayInfo.hpp>
 #include <common/err_common.hpp>
 #include <complex.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <lu.hpp>
 #include <math.hpp>
@@ -29,6 +30,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createEmptyArray;
 using detail::createValueArray;
+using detail::getScalar;
 using detail::reduce;
 using detail::reduce_all;
 using detail::scalar;
@@ -37,11 +39,11 @@ template<typename T>
 double matrixNorm(const Array<T> &A, double p) {
     if (p == 1) {
         Array<T> colSum = reduce<af_add_t, T, T>(A, 0);
-        return reduce_all<af_max_t, T, T>(colSum);
+        return getScalar<T>(reduce_all<af_max_t, T, T>(colSum));
     }
     if (p == af::Inf) {
         Array<T> rowSum = reduce<af_add_t, T, T>(A, 1);
-        return reduce_all<af_max_t, T, T>(rowSum);
+        return getScalar<T>(reduce_all<af_max_t, T, T>(rowSum));
     }
 
     AF_ERROR("This type of norm is not supported in ArrayFire\n",
@@ -50,17 +52,17 @@ double matrixNorm(const Array<T> &A, double p) {
 
 template<typename T>
 double vectorNorm(const Array<T> &A, double p) {
-    if (p == 1) { return reduce_all<af_add_t, T, T>(A); }
+    if (p == 1) { return getScalar<T>(reduce_all<af_add_t, T, T>(A)); }
     if (p == af::Inf) {
-        return reduce_all<af_max_t, T, T>(A);
+        return getScalar<T>(reduce_all<af_max_t, T, T>(A));
     } else if (p == 2) {
         Array<T> A_sq = arithOp<T, af_mul_t>(A, A, A.dims());
-        return std::sqrt(reduce_all<af_add_t, T, T>(A_sq));
+        return std::sqrt(getScalar<T>(reduce_all<af_add_t, T, T>(A_sq)));
     }
 
     Array<T> P   = createValueArray<T>(A.dims(), scalar<T>(p));
     Array<T> A_p = arithOp<T, af_pow_t>(A, P, A.dims());
-    return std::pow(reduce_all<af_add_t, T, T>(A_p), T(1.0 / p));
+    return std::pow(getScalar<T>(reduce_all<af_add_t, T, T>(A_p)), T(1.0 / p));
 }
 
 template<typename T>
@@ -78,12 +80,13 @@ double LPQNorm(const Array<T> &A, double p, double q) {
         A_p_norm         = arithOp<T, af_pow_t>(A_p_sum, invP, invP.dims());
     }
 
-    if (q == 1) { return reduce_all<af_add_t, T, T>(A_p_norm); }
+    if (q == 1) { return getScalar<T>(reduce_all<af_add_t, T, T>(A_p_norm)); }
 
     Array<T> Q          = createValueArray<T>(A_p_norm.dims(), scalar<T>(q));
     Array<T> A_p_norm_q = arithOp<T, af_pow_t>(A_p_norm, Q, Q.dims());
 
-    return std::pow(reduce_all<af_add_t, T, T>(A_p_norm_q), T(1.0 / q));
+    return std::pow(getScalar<T>(reduce_all<af_add_t, T, T>(A_p_norm_q)),
+                    T(1.0 / q));
 }
 
 template<typename T>
diff --git a/src/api/c/rank.cpp b/src/api/c/rank.cpp
index 8880814a82..770c331a7a 100644
--- a/src/api/c/rank.cpp
+++ b/src/api/c/rank.cpp
@@ -11,6 +11,7 @@
 #include <common/ArrayInfo.hpp>
 #include <common/err_common.hpp>
 #include <complex.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <logic.hpp>
 #include <qr.hpp>
@@ -25,6 +26,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createEmptyArray;
 using detail::createValueArray;
+using detail::getScalar;
 using detail::logicOp;
 using detail::reduce;
 using detail::reduce_all;
@@ -52,7 +54,7 @@ static inline uint rank(const af_array in, double tol) {
     Array<BT> val  = createValueArray<BT>(R.dims(), scalar<BT>(tol));
     Array<char> gt = logicOp<BT, af_gt_t>(R, val, val.dims());
     Array<char> at = reduce<af_or_t, char, char>(gt, 1);
-    return reduce_all<af_notzero_t, char, uint>(at);
+    return getScalar<uint>(reduce_all<af_notzero_t, char, uint>(at));
 }
 
 af_err af_rank(uint* out, const af_array in, const double tol) {
diff --git a/src/api/c/reduce.cpp b/src/api/c/reduce.cpp
index 544ced2368..1849255257 100644
--- a/src/api/c/reduce.cpp
+++ b/src/api/c/reduce.cpp
@@ -10,6 +10,7 @@
 #include <backend.hpp>
 #include <common/err_common.hpp>
 #include <common/half.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <ireduce.hpp>
 #include <math.hpp>
@@ -25,6 +26,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::createEmptyArray;
+using detail::getScalar;
 using detail::imag;
 using detail::intl;
 using detail::real;
@@ -533,11 +535,19 @@ af_err af_any_true_by_key(af_array *keys_out, af_array *vals_out,
                                              dim);
 }
 
+template<af_op_t op, typename Ti, typename To>
+static inline af_array reduce_all_array(const af_array in,
+                                        bool change_nan = false,
+                                        double nanval   = 0) {
+    return getHandle(
+        detail::reduce_all<op, Ti, To>(getArray<Ti>(in), change_nan, nanval));
+}
+
 template<af_op_t op, typename Ti, typename Tacc, typename Tret = double>
 static inline Tret reduce_all(const af_array in, bool change_nan = false,
                               double nanval = 0) {
-    return static_cast<Tret>(
-        reduce_all<op, Ti, Tacc>(getArray<Ti>(in), change_nan, nanval));
+    return static_cast<Tret>(getScalar<Tacc>(
+        reduce_all<op, Ti, Tacc>(getArray<Ti>(in), change_nan, nanval)));
 }
 
 template<af_op_t op, typename To>
@@ -574,6 +584,38 @@ static af_err reduce_all_type(double *real, double *imag, const af_array in) {
     return AF_SUCCESS;
 }
 
+template<af_op_t op, typename To>
+static af_err reduce_all_type_array(af_array *out, const af_array in) {
+    try {
+        const ArrayInfo &in_info = getInfo(in);
+        af_dtype type            = in_info.getType();
+
+        af_array res;
+        switch (type) {
+            // clang-format off
+            case f32: res = reduce_all_array<op, float,   To>(in); break;
+            case f64: res = reduce_all_array<op, double,  To>(in); break;
+            case c32: res = reduce_all_array<op, cfloat,  To>(in); break;
+            case c64: res = reduce_all_array<op, cdouble, To>(in); break;
+            case u32: res = reduce_all_array<op, uint,    To>(in); break;
+            case s32: res = reduce_all_array<op, int,     To>(in); break;
+            case u64: res = reduce_all_array<op, uintl,   To>(in); break;
+            case s64: res = reduce_all_array<op, intl,    To>(in); break;
+            case u16: res = reduce_all_array<op, ushort,  To>(in); break;
+            case s16: res = reduce_all_array<op, short,   To>(in); break;
+            case b8:  res = reduce_all_array<op, char,    To>(in); break;
+            case u8:  res = reduce_all_array<op, uchar,   To>(in); break;
+            case f16: res = reduce_all_array<op, half,    To>(in); break;
+            // clang-format on
+            default: TYPE_ERROR(1, type);
+        }
+        std::swap(*out, res);
+    }
+    CATCHALL;
+
+    return AF_SUCCESS;
+}
+
 template<af_op_t op>
 static af_err reduce_all_common(double *real_val, double *imag_val,
                                 const af_array in) {
@@ -625,6 +667,40 @@ static af_err reduce_all_common(double *real_val, double *imag_val,
     return AF_SUCCESS;
 }
 
+template<af_op_t op>
+static af_err reduce_all_common_array(af_array *out, const af_array in) {
+    try {
+        const ArrayInfo &in_info = getInfo(in);
+        af_dtype type            = in_info.getType();
+
+        ARG_ASSERT(2, in_info.ndims() > 0);
+        af_array res;
+
+        switch (type) {
+            // clang-format off
+            case f32: res = reduce_all_array<op, float,  float>(in); break;
+            case f64: res = reduce_all_array<op, double, double>(in); break;
+            case u32: res = reduce_all_array<op, uint,   uint>(in); break;
+            case s32: res = reduce_all_array<op, int,    int>(in); break;
+            case u64: res = reduce_all_array<op, uintl,  uintl>(in); break;
+            case s64: res = reduce_all_array<op, intl,   intl>(in); break;
+            case u16: res = reduce_all_array<op, ushort, ushort>(in); break;
+            case s16: res = reduce_all_array<op, short,  short>(in); break;
+            case b8:  res = reduce_all_array<op, char,   char>(in); break;
+            case u8:  res = reduce_all_array<op, uchar,  uchar>(in); break;
+            case f16: res = reduce_all_array<op, half,   half>(in); break;
+            // clang-format on
+            case c32: res = reduce_all_array<op, cfloat, cfloat>(in); break;
+            case c64: res = reduce_all_array<op, cdouble, cdouble>(in); break;
+            default: TYPE_ERROR(1, type);
+        }
+        std::swap(*out, res);
+    }
+    CATCHALL;
+
+    return AF_SUCCESS;
+}
+
 template<af_op_t op>
 static af_err reduce_all_promote(double *real_val, double *imag_val,
                                  const af_array in, bool change_nan = false,
@@ -686,34 +762,133 @@ static af_err reduce_all_promote(double *real_val, double *imag_val,
     return AF_SUCCESS;
 }
 
+template<af_op_t op>
+static af_err reduce_all_promote_array(af_array *out, const af_array in,
+                                       bool change_nan = false,
+                                       double nanval   = 0.0) {
+    try {
+        const ArrayInfo &in_info = getInfo(in);
+
+        af_dtype type = in_info.getType();
+        af_array res;
+
+        switch (type) {
+            case f32:
+                res =
+                    reduce_all_array<op, float, float>(in, change_nan, nanval);
+                break;
+            case f64:
+                res = reduce_all_array<op, double, double>(in, change_nan,
+                                                           nanval);
+                break;
+            case c32:
+                res = reduce_all_array<op, cfloat, cfloat>(in, change_nan,
+                                                           nanval);
+                break;
+            case c64:
+                res = reduce_all_array<op, cdouble, cdouble>(in, change_nan,
+                                                             nanval);
+                break;
+            case u32:
+                res = reduce_all_array<op, uint, uint>(in, change_nan, nanval);
+                break;
+            case s32:
+                res = reduce_all_array<op, int, int>(in, change_nan, nanval);
+                break;
+            case u64:
+                res =
+                    reduce_all_array<op, uintl, uintl>(in, change_nan, nanval);
+                break;
+            case s64:
+                res = reduce_all_array<op, intl, intl>(in, change_nan, nanval);
+                break;
+            case u16:
+                res =
+                    reduce_all_array<op, ushort, uint>(in, change_nan, nanval);
+                break;
+            case s16:
+                res = reduce_all_array<op, short, int>(in, change_nan, nanval);
+                break;
+            case u8:
+                res = reduce_all_array<op, uchar, uint>(in, change_nan, nanval);
+                break;
+            case b8: {
+                if (op == af_mul_t) {
+                    res = reduce_all_array<af_and_t, char, char>(in, change_nan,
+                                                                 nanval);
+                } else {
+                    res = reduce_all_array<af_notzero_t, char, uint>(
+                        in, change_nan, nanval);
+                }
+            } break;
+            case f16:
+                res = reduce_all_array<op, half, float>(in, change_nan, nanval);
+                break;
+            default: TYPE_ERROR(1, type);
+        }
+        std::swap(*out, res);
+    }
+    CATCHALL;
+
+    return AF_SUCCESS;
+}
+
 af_err af_min_all(double *real, double *imag, const af_array in) {
     return reduce_all_common<af_min_t>(real, imag, in);
 }
 
+af_err af_min_all_array(af_array *out, const af_array in) {
+    return reduce_all_common_array<af_min_t>(out, in);
+}
+
 af_err af_max_all(double *real, double *imag, const af_array in) {
     return reduce_all_common<af_max_t>(real, imag, in);
 }
 
+af_err af_max_all_array(af_array *out, const af_array in) {
+    return reduce_all_common_array<af_max_t>(out, in);
+}
+
 af_err af_sum_all(double *real, double *imag, const af_array in) {
     return reduce_all_promote<af_add_t>(real, imag, in);
 }
 
+af_err af_sum_all_array(af_array *out, const af_array in) {
+    return reduce_all_promote_array<af_add_t>(out, in);
+}
+
 af_err af_product_all(double *real, double *imag, const af_array in) {
     return reduce_all_promote<af_mul_t>(real, imag, in);
 }
 
+af_err af_product_all_array(af_array *out, const af_array in) {
+    return reduce_all_promote_array<af_mul_t>(out, in);
+}
+
 af_err af_count_all(double *real, double *imag, const af_array in) {
     return reduce_all_type<af_notzero_t, uint>(real, imag, in);
 }
 
+af_err af_count_all_array(af_array *out, const af_array in) {
+    return reduce_all_type_array<af_notzero_t, uint>(out, in);
+}
+
 af_err af_all_true_all(double *real, double *imag, const af_array in) {
     return reduce_all_type<af_and_t, char>(real, imag, in);
 }
 
+af_err af_all_true_all_array(af_array *out, const af_array in) {
+    return reduce_all_type_array<af_and_t, char>(out, in);
+}
+
 af_err af_any_true_all(double *real, double *imag, const af_array in) {
     return reduce_all_type<af_or_t, char>(real, imag, in);
 }
 
+af_err af_any_true_all_array(af_array *out, const af_array in) {
+    return reduce_all_type_array<af_or_t, char>(out, in);
+}
+
 template<af_op_t op, typename T>
 static inline void ireduce(af_array *res, af_array *loc, const af_array in,
                            const int dim) {
@@ -948,7 +1123,17 @@ af_err af_sum_nan_all(double *real, double *imag, const af_array in,
     return reduce_all_promote<af_add_t>(real, imag, in, true, nanval);
 }
 
+af_err af_sum_nan_all_array(af_array *out, const af_array in,
+                            const double nanval) {
+    return reduce_all_promote_array<af_add_t>(out, in, true, nanval);
+}
+
 af_err af_product_nan_all(double *real, double *imag, const af_array in,
                           const double nanval) {
     return reduce_all_promote<af_mul_t>(real, imag, in, true, nanval);
 }
+
+af_err af_product_nan_all_array(af_array *out, const af_array in,
+                                const double nanval) {
+    return reduce_all_promote_array<af_mul_t>(out, in, true, nanval);
+}
diff --git a/src/api/c/stdev.cpp b/src/api/c/stdev.cpp
index 4f66328782..3be779e544 100644
--- a/src/api/c/stdev.cpp
+++ b/src/api/c/stdev.cpp
@@ -10,6 +10,7 @@
 #include <arith.hpp>
 #include <backend.hpp>
 #include <common/cast.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <math.hpp>
 #include <mean.hpp>
@@ -31,6 +32,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createValueArray;
 using detail::division;
+using detail::getScalar;
 using detail::intl;
 using detail::mean;
 using detail::reduce;
@@ -52,9 +54,9 @@ static outType stdev(const af_array& in, const af_var_bias bias) {
         detail::arithOp<outType, af_sub_t>(input, meanCnst, input.dims());
     Array<outType> diffSq =
         detail::arithOp<outType, af_mul_t>(diff, diff, diff.dims());
-    outType result =
-        division(reduce_all<af_add_t, outType, outType>(diffSq),
-                 (input.elements() - (bias == AF_VARIANCE_SAMPLE)));
+    outType result = division(
+        getScalar<outType>(reduce_all<af_add_t, outType, outType>(diffSq)),
+        (input.elements() - (bias == AF_VARIANCE_SAMPLE)));
     return sqrt(result);
 }
 
diff --git a/src/api/c/surface.cpp b/src/api/c/surface.cpp
index 986cedae09..2f6a3eda7b 100644
--- a/src/api/c/surface.cpp
+++ b/src/api/c/surface.cpp
@@ -15,6 +15,7 @@
 #include <common/err_common.hpp>
 #include <common/graphics_common.hpp>
 #include <common/moddims.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <join.hpp>
 #include <reduce.hpp>
@@ -28,6 +29,7 @@ using detail::Array;
 using detail::copy_surface;
 using detail::createEmptyArray;
 using detail::forgeManager;
+using detail::getScalar;
 using detail::reduce_all;
 using detail::uchar;
 using detail::uint;
@@ -101,12 +103,12 @@ fg_chart setup_surface(fg_window window, const af_array xVals,
         T dmin[3], dmax[3];
         FG_CHECK(_.fg_get_chart_axes_limits(
             &cmin[0], &cmax[0], &cmin[1], &cmax[1], &cmin[2], &cmax[2], chart));
-        dmin[0] = reduce_all<af_min_t, T, T>(xIn);
-        dmax[0] = reduce_all<af_max_t, T, T>(xIn);
-        dmin[1] = reduce_all<af_min_t, T, T>(yIn);
-        dmax[1] = reduce_all<af_max_t, T, T>(yIn);
-        dmin[2] = reduce_all<af_min_t, T, T>(zIn);
-        dmax[2] = reduce_all<af_max_t, T, T>(zIn);
+        dmin[0] = getScalar<T>(reduce_all<af_min_t, T, T>(xIn));
+        dmax[0] = getScalar<T>(reduce_all<af_max_t, T, T>(xIn));
+        dmin[1] = getScalar<T>(reduce_all<af_min_t, T, T>(yIn));
+        dmax[1] = getScalar<T>(reduce_all<af_max_t, T, T>(yIn));
+        dmin[2] = getScalar<T>(reduce_all<af_min_t, T, T>(zIn));
+        dmax[2] = getScalar<T>(reduce_all<af_max_t, T, T>(zIn));
 
         if (cmin[0] == 0 && cmax[0] == 0 && cmin[1] == 0 && cmax[1] == 0 &&
             cmin[2] == 0 && cmax[2] == 0) {
diff --git a/src/api/c/var.cpp b/src/api/c/var.cpp
index fe111de5f5..efbbfc8a70 100644
--- a/src/api/c/var.cpp
+++ b/src/api/c/var.cpp
@@ -12,6 +12,7 @@
 #include <common/cast.hpp>
 #include <common/err_common.hpp>
 #include <common/half.hpp>
+#include <copy.hpp>
 #include <handle.hpp>
 #include <math.hpp>
 #include <mean.hpp>
@@ -34,6 +35,7 @@ using detail::cfloat;
 using detail::createEmptyArray;
 using detail::createValueArray;
 using detail::division;
+using detail::getScalar;
 using detail::imag;
 using detail::intl;
 using detail::mean;
@@ -64,9 +66,9 @@ static outType varAll(const af_array& in, const af_var_bias bias) {
 
     Array<outType> diffSq = arithOp<outType, af_mul_t>(diff, diff, diff.dims());
 
-    outType result =
-        division(reduce_all<af_add_t, outType, outType>(diffSq),
-                 (input.elements() - (bias == AF_VARIANCE_SAMPLE)));
+    outType result = division(
+        getScalar<outType>(reduce_all<af_add_t, outType, outType>(diffSq)),
+        (input.elements() - (bias == AF_VARIANCE_SAMPLE)));
 
     return result;
 }
@@ -78,7 +80,8 @@ static outType varAll(const af_array& in, const af_array weights) {
     Array<outType> input = cast<outType>(getArray<inType>(in));
     Array<outType> wts   = cast<outType>(getArray<bType>(weights));
 
-    bType wtsSum = reduce_all<af_add_t, bType, bType>(getArray<bType>(weights));
+    bType wtsSum = getScalar<bType>(
+        reduce_all<af_add_t, bType, bType>(getArray<bType>(weights)));
     auto wtdMean = mean<outType, bType>(input, getArray<bType>(weights));
 
     Array<outType> meanArr = createValueArray<outType>(input.dims(), wtdMean);
@@ -89,8 +92,9 @@ static outType varAll(const af_array& in, const af_array weights) {
     Array<outType> accDiffSq =
         arithOp<outType, af_mul_t>(diffSq, wts, diffSq.dims());
 
-    outType result =
-        division(reduce_all<af_add_t, outType, outType>(accDiffSq), wtsSum);
+    outType result = division(
+        getScalar<outType>(reduce_all<af_add_t, outType, outType>(accDiffSq)),
+        wtsSum);
 
     return result;
 }
diff --git a/src/api/cpp/reduce.cpp b/src/api/cpp/reduce.cpp
index 44f981982d..cfdadf85ae 100644
--- a/src/api/cpp/reduce.cpp
+++ b/src/api/cpp/reduce.cpp
@@ -212,6 +212,14 @@ void max(array &val, array &idx, const array &in, const int dim) {
         return out;                                       \
     }
 
+#define INSTANTIATE_ARRAY(fnC, fnCPP)                   \
+    template<>                                          \
+    AFAPI af::array fnCPP(const array &in) {            \
+        af_array out = 0;                               \
+        AF_THROW(af_##fnC##_all_array(&out, in.get())); \
+        return array(out);                              \
+    }
+
 INSTANTIATE(sum, sum)
 INSTANTIATE(product, product)
 INSTANTIATE(min, min)
@@ -223,8 +231,17 @@ INSTANTIATE(count, count)
 INSTANTIATE_REAL(all_true, allTrue, bool);
 INSTANTIATE_REAL(any_true, anyTrue, bool);
 
+INSTANTIATE_ARRAY(sum, sum)
+INSTANTIATE_ARRAY(product, product)
+INSTANTIATE_ARRAY(min, min)
+INSTANTIATE_ARRAY(max, max)
+INSTANTIATE_ARRAY(all_true, allTrue)
+INSTANTIATE_ARRAY(any_true, anyTrue)
+INSTANTIATE_ARRAY(count, count)
+
 #undef INSTANTIATE_REAL
 #undef INSTANTIATE_CPLX
+#undef INSTANTIATE_ARRAY
 
 #define INSTANTIATE_REAL(fnC, fnCPP, T)                           \
     template<>                                                    \
@@ -243,12 +260,23 @@ INSTANTIATE_REAL(any_true, anyTrue, bool);
         return out;                                               \
     }
 
+#define INSTANTIATE_ARRAY(fnC, fnCPP)                             \
+    template<>                                                    \
+    AFAPI af::array fnCPP(const array &in, const double nanval) { \
+        af_array out = 0;                                         \
+        AF_THROW(af_##fnC##_all_array(&out, in.get(), nanval));   \
+        return array(out);                                        \
+    }
+INSTANTIATE_ARRAY(sum_nan, sum)
+INSTANTIATE_ARRAY(product_nan, product)
+
 INSTANTIATE(sum_nan, sum)
 INSTANTIATE(product_nan, product)
 
 #undef INSTANTIATE_REAL
 #undef INSTANTIATE_CPLX
 #undef INSTANTIATE
+#undef INSTANTIATE_ARRAY
 
 #define INSTANTIATE_COMPAT(fnCPP, fnCompat, T) \
     template<>                                 \
diff --git a/src/api/unified/algorithm.cpp b/src/api/unified/algorithm.cpp
index 87f03a053a..8f990fb535 100644
--- a/src/api/unified/algorithm.cpp
+++ b/src/api/unified/algorithm.cpp
@@ -124,6 +124,33 @@ ALGO_HAPI_DEF(af_imax_all)
 
 #undef ALGO_HAPI_DEF
 
+#define ALGO_HAPI_DEF(af_func)                         \
+    af_err af_func(af_array *out, const af_array in) { \
+        CHECK_ARRAYS(in);                              \
+        CALL(af_func, out, in);                        \
+    }
+
+ALGO_HAPI_DEF(af_sum_all_array)
+ALGO_HAPI_DEF(af_product_all_array)
+ALGO_HAPI_DEF(af_min_all_array)
+ALGO_HAPI_DEF(af_max_all_array)
+ALGO_HAPI_DEF(af_count_all_array)
+ALGO_HAPI_DEF(af_any_true_all_array)
+ALGO_HAPI_DEF(af_all_true_all_array)
+
+#undef ALGO_HAPI_DEF
+
+#define ALGO_HAPI_DEF(af_func)                                              \
+    af_err af_func(af_array *out, const af_array in, const double nanval) { \
+        CHECK_ARRAYS(in);                                                   \
+        CALL(af_func, out, in, nanval);                                     \
+    }
+
+ALGO_HAPI_DEF(af_sum_nan_all_array)
+ALGO_HAPI_DEF(af_product_nan_all_array)
+
+#undef ALGO_HAPI_DEF
+
 af_err af_where(af_array *idx, const af_array in) {
     CHECK_ARRAYS(in);
     CALL(af_where, idx, in);
diff --git a/src/api/unified/symbol_manager.hpp b/src/api/unified/symbol_manager.hpp
index aeed23a415..cbf6e76861 100644
--- a/src/api/unified/symbol_manager.hpp
+++ b/src/api/unified/symbol_manager.hpp
@@ -144,6 +144,11 @@ bool checkArrays(af_backend activeBackend, T a, Args... arg) {
     if (unified::getActiveHandle()) {                                            \
         thread_local af_func func = (af_func)common::getFunctionPointer(         \
             unified::getActiveHandle(), __func__);                               \
+        if (!func) {                                                             \
+            AF_RETURN_ERROR(                                                     \
+                "requested symbol name could not be found in loaded library.",   \
+                AF_ERR_LOAD_LIB);                                                \
+        }                                                                        \
         if (index_ != unified::getActiveBackend()) {                             \
             index_ = unified::getActiveBackend();                                \
             func   = (af_func)common::getFunctionPointer(                        \
diff --git a/src/backend/cpu/kernel/reduce.hpp b/src/backend/cpu/kernel/reduce.hpp
index cd8678edda..374816102e 100644
--- a/src/backend/cpu/kernel/reduce.hpp
+++ b/src/backend/cpu/kernel/reduce.hpp
@@ -156,5 +156,46 @@ struct reduce_dim_by_key<op, Ti, Tk, To, 0> {
         }
     }
 };
+
+template<af_op_t op, typename Ti, typename To>
+struct reduce_all {
+    common::Transform<data_t<Ti>, compute_t<To>, op> transform;
+    common::Binary<compute_t<To>, op> reduce;
+    void operator()(Param<To> out, CParam<Ti> in, bool change_nan,
+                    double nanval) {
+        // Decrement dimension of select dimension
+        af::dim4 dims            = in.dims();
+        af::dim4 strides         = in.strides();
+        const data_t<Ti> *inPtr  = in.get();
+        data_t<To> *const outPtr = out.get();
+
+        compute_t<To> out_val = common::Binary<compute_t<To>, op>::init();
+
+        for (dim_t l = 0; l < dims[3]; l++) {
+            dim_t off3 = l * strides[3];
+
+            for (dim_t k = 0; k < dims[2]; k++) {
+                dim_t off2 = k * strides[2];
+
+                for (dim_t j = 0; j < dims[1]; j++) {
+                    dim_t off1 = j * strides[1];
+
+                    for (dim_t i = 0; i < dims[0]; i++) {
+                        dim_t idx = i + off1 + off2 + off3;
+
+                        compute_t<To> in_val = transform(inPtr[idx]);
+                        if (change_nan) {
+                            in_val = IS_NAN(in_val) ? nanval : in_val;
+                        }
+                        out_val = reduce(in_val, out_val);
+                    }
+                }
+            }
+        }
+
+        *outPtr = data_t<To>(out_val);
+    }
+};
+
 }  // namespace kernel
 }  // namespace cpu
diff --git a/src/backend/cpu/reduce.cpp b/src/backend/cpu/reduce.cpp
index 795390a04e..e1baf5daea 100644
--- a/src/backend/cpu/reduce.cpp
+++ b/src/backend/cpu/reduce.cpp
@@ -107,51 +107,27 @@ void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
     vals_out = ovals;
 }
 
-template<af_op_t op, typename Ti, typename Taccumulate>
-Taccumulate reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
-    in.eval();
-    getQueue().sync();
-
-    Transform<Ti, compute_t<Taccumulate>, op> transform;
-    Binary<compute_t<Taccumulate>, op> reduce;
-
-    compute_t<Taccumulate> out = Binary<compute_t<Taccumulate>, op>::init();
-
-    // Decrement dimension of select dimension
-    af::dim4 dims           = in.dims();
-    af::dim4 strides        = in.strides();
-    const data_t<Ti> *inPtr = in.get();
-
-    for (dim_t l = 0; l < dims[3]; l++) {
-        dim_t off3 = l * strides[3];
-
-        for (dim_t k = 0; k < dims[2]; k++) {
-            dim_t off2 = k * strides[2];
-
-            for (dim_t j = 0; j < dims[1]; j++) {
-                dim_t off1 = j * strides[1];
-
-                for (dim_t i = 0; i < dims[0]; i++) {
-                    dim_t idx = i + off1 + off2 + off3;
+template<af_op_t op, typename Ti, typename To>
+using reduce_all_func =
+    std::function<void(Param<To>, CParam<Ti>, bool, double)>;
 
-                    compute_t<Taccumulate> in_val = transform(inPtr[idx]);
-                    if (change_nan) {
-                        in_val = IS_NAN(in_val) ? nanval : in_val;
-                    }
-                    out = reduce(in_val, out);
-                }
-            }
-        }
-    }
+template<af_op_t op, typename Ti, typename To>
+Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
+    in.eval();
 
-    return data_t<Taccumulate>(out);
+    Array<To> out = createEmptyArray<To>(1);
+    static const reduce_all_func<op, Ti, To> reduce_all_kernel =
+        kernel::reduce_all<op, Ti, To>();
+    getQueue().enqueue(reduce_all_kernel, out, in, change_nan, nanval);
+    getQueue().sync();
+    return out;
 }
 
 #define INSTANTIATE(ROp, Ti, To)                                               \
     template Array<To> reduce<ROp, Ti, To>(const Array<Ti> &in, const int dim, \
                                            bool change_nan, double nanval);    \
-    template To reduce_all<ROp, Ti, To>(const Array<Ti> &in, bool change_nan,  \
-                                        double nanval);                        \
+    template Array<To> reduce_all<ROp, Ti, To>(                                \
+        const Array<Ti> &in, bool change_nan, double nanval);                  \
     template void reduce_by_key<ROp, Ti, int, To>(                             \
         Array<int> & keys_out, Array<To> & vals_out, const Array<int> &keys,   \
         const Array<Ti> &vals, const int dim, bool change_nan, double nanval); \
diff --git a/src/backend/cpu/reduce.hpp b/src/backend/cpu/reduce.hpp
index 9923d2aef3..3db9b0cc8a 100644
--- a/src/backend/cpu/reduce.hpp
+++ b/src/backend/cpu/reduce.hpp
@@ -21,5 +21,6 @@ void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
                    bool change_nan = false, double nanval = 0);
 
 template<af_op_t op, typename Ti, typename To>
-To reduce_all(const Array<Ti> &in, bool change_nan = false, double nanval = 0);
+Array<To> reduce_all(const Array<Ti> &in, bool change_nan = false,
+                     double nanval = 0);
 }  // namespace cpu
diff --git a/src/backend/cpu/sparse.cpp b/src/backend/cpu/sparse.cpp
index bf2565883e..30c7475292 100644
--- a/src/backend/cpu/sparse.cpp
+++ b/src/backend/cpu/sparse.cpp
@@ -40,7 +40,7 @@ using common::SparseArray;
 template<typename T, af_storage stype>
 SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
     if (stype == AF_STORAGE_CSR) {
-        uint nNZ = reduce_all<af_notzero_t, T, uint>(in);
+        uint nNZ = getScalar<uint>(reduce_all<af_notzero_t, T, uint>(in));
 
         auto sparse = createEmptySparseArray<T>(in.dims(), nNZ, stype);
         sparse.eval();
diff --git a/src/backend/cuda/kernel/reduce.hpp b/src/backend/cuda/kernel/reduce.hpp
index 02eedb4237..fb51a72851 100644
--- a/src/backend/cuda/kernel/reduce.hpp
+++ b/src/backend/cuda/kernel/reduce.hpp
@@ -21,6 +21,7 @@
 
 #include <cub/warp/warp_reduce.cuh>
 
+#include <climits>
 #include <vector>
 
 using std::unique_ptr;
@@ -258,6 +259,176 @@ __global__ static void reduce_first_kernel(Param<To> out, CParam<Ti> in,
     if (tidx == 0) optr[blockIdx_x] = data_t<To>(out_val);
 }
 
+template<typename Ti, typename To, af_op_t op, uint DIMX>
+__global__ static void reduce_all_kernel(Param<To> out,
+                                         Param<unsigned> retirementCount,
+                                         Param<To> tmp, CParam<Ti> in,
+                                         uint blocks_x, uint blocks_y,
+                                         uint repeat, bool change_nan,
+                                         To nanval) {
+    const uint tidx = threadIdx.x;
+    const uint tidy = threadIdx.y;
+    const uint tid  = tidy * DIMX + tidx;
+
+    const uint zid        = blockIdx.x / blocks_x;
+    const uint blockIdx_x = blockIdx.x - (blocks_x)*zid;
+    const uint xid        = blockIdx_x * blockDim.x * repeat + tidx;
+
+    const uint wid = (blockIdx.y + blockIdx.z * gridDim.y) / blocks_y;
+    const uint blockIdx_y =
+        (blockIdx.y + blockIdx.z * gridDim.y) - (blocks_y)*wid;
+    const uint yid = blockIdx_y * blockDim.y + tidy;
+
+    common::Binary<compute_t<To>, op> reduce;
+    common::Transform<Ti, compute_t<To>, op> transform;
+
+    const int nwarps = THREADS_PER_BLOCK / 32;
+    __shared__ compute_t<To> s_val[nwarps];
+
+    const data_t<Ti> *const iptr =
+        in.ptr +
+        (wid * in.strides[3] + zid * in.strides[2] + yid * in.strides[1]);
+
+    bool cond = yid < in.dims[1] && zid < in.dims[2] && wid < in.dims[3];
+
+    int lim = min((int)(xid + repeat * DIMX), in.dims[0]);
+
+    compute_t<To> out_val = common::Binary<compute_t<To>, op>::init();
+    for (int id = xid; cond && id < lim; id += DIMX) {
+        compute_t<To> in_val = transform(iptr[id]);
+        if (change_nan)
+            in_val =
+                !IS_NAN(in_val) ? in_val : static_cast<compute_t<To>>(nanval);
+        out_val = reduce(in_val, out_val);
+    }
+
+    const int warpid = tid / 32;
+    const int lid    = tid % 32;
+
+    typedef cub::WarpReduce<compute_t<To>> WarpReduce;
+    __shared__ typename WarpReduce::TempStorage temp_storage[nwarps];
+
+    out_val = WarpReduce(temp_storage[warpid]).Reduce(out_val, reduce);
+
+    if (cond && lid == 0) {
+        s_val[warpid] = out_val;
+    } else if (!cond) {
+        s_val[warpid] = common::Binary<compute_t<To>, op>::init();
+    }
+    __syncthreads();
+
+    if (tid < 32) {
+        out_val = tid < nwarps ? s_val[tid]
+                               : common::Binary<compute_t<To>, op>::init();
+        out_val = WarpReduce(temp_storage[0]).Reduce(out_val, reduce);
+    }
+
+    const unsigned total_blocks = (gridDim.x * gridDim.y * gridDim.z);
+    const int uubidx            = (gridDim.x * gridDim.y) * blockIdx.z +
+                       (gridDim.x * blockIdx.y) + blockIdx.x;
+    if (cond && tid == 0) {
+        if (total_blocks != 1) {
+            tmp.ptr[uubidx] = data_t<To>(out_val);
+        } else {
+            out.ptr[0] = data_t<To>(out_val);
+        }
+    }
+
+    // Last block to perform final reduction
+    if (total_blocks > 1) {
+        __shared__ bool amLast;
+
+        // wait until all outstanding memory instructions in this thread are
+        // finished
+        __threadfence();
+
+        // Thread 0 takes a ticket
+        if (tid == 0) {
+            unsigned int ticket = atomicInc(retirementCount.ptr, total_blocks);
+            // If the ticket ID == number of blocks, we are the last block
+            amLast = (ticket == (total_blocks - 1));
+        }
+        __syncthreads();  // for amlast
+
+        if (amLast) {
+            int i   = tid;
+            out_val = common::Binary<compute_t<To>, op>::init();
+
+            while (i < total_blocks) {
+                compute_t<To> in_val = compute_t<To>(tmp.ptr[i]);
+                out_val              = reduce(in_val, out_val);
+                i += THREADS_PER_BLOCK;
+            }
+
+            out_val = WarpReduce(temp_storage[warpid]).Reduce(out_val, reduce);
+            if (lid == 0) { s_val[warpid] = out_val; }
+            __syncthreads();
+
+            if (tid < 32) {
+                out_val = tid < nwarps
+                              ? s_val[tid]
+                              : common::Binary<compute_t<To>, op>::init();
+                out_val = WarpReduce(temp_storage[0]).Reduce(out_val, reduce);
+            }
+
+            if (tid == 0) {
+                out.ptr[0] = out_val;
+
+                // reset retirement count so that next run succeeds
+                retirementCount.ptr[0] = 0;
+            }
+        }
+    }
+}
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_all_launcher(Param<To> out, CParam<Ti> in, const uint blocks_x,
+                         const uint blocks_y, const uint threads_x,
+                         bool change_nan, double nanval) {
+    dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
+    dim3 blocks(blocks_x * in.dims[2], blocks_y * in.dims[3]);
+
+    uint repeat = divup(in.dims[0], (blocks_x * threads_x));
+
+    const int maxBlocksY =
+        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
+    blocks.z = divup(blocks.y, maxBlocksY);
+    blocks.y = divup(blocks.y, blocks.z);
+
+    long tmp_elements = blocks.x * blocks.y * blocks.z;
+    if (tmp_elements > UINT_MAX) {
+        AF_ERROR("Too many blocks requested (retirementCount == unsigned)",
+                 AF_ERR_RUNTIME);
+    }
+    Array<To> tmp                   = createEmptyArray<To>(tmp_elements);
+    Array<unsigned> retirementCount = createValueArray<unsigned>(1, 0);
+
+    switch (threads_x) {
+        case 32:
+            CUDA_LAUNCH((reduce_all_kernel<Ti, To, op, 32>), blocks, threads,
+                        out, retirementCount, tmp, in, blocks_x, blocks_y,
+                        repeat, change_nan, scalar<To>(nanval));
+            break;
+        case 64:
+            CUDA_LAUNCH((reduce_all_kernel<Ti, To, op, 64>), blocks, threads,
+                        out, retirementCount, tmp, in, blocks_x, blocks_y,
+                        repeat, change_nan, scalar<To>(nanval));
+            break;
+        case 128:
+            CUDA_LAUNCH((reduce_all_kernel<Ti, To, op, 128>), blocks, threads,
+                        out, retirementCount, tmp, in, blocks_x, blocks_y,
+                        repeat, change_nan, scalar<To>(nanval));
+            break;
+        case 256:
+            CUDA_LAUNCH((reduce_all_kernel<Ti, To, op, 256>), blocks, threads,
+                        out, retirementCount, tmp, in, blocks_x, blocks_y,
+                        repeat, change_nan, scalar<To>(nanval));
+            break;
+    }
+
+    POST_LAUNCH_CHECK();
+}
+
 template<typename Ti, typename To, af_op_t op>
 void reduce_first_launcher(Param<To> out, CParam<Ti> in, const uint blocks_x,
                            const uint blocks_y, const uint threads_x,
@@ -344,81 +515,33 @@ void reduce(Param<To> out, CParam<Ti> in, int dim, bool change_nan,
         case 3: return reduce_dim<Ti, To, op, 3>(out, in, change_nan, nanval);
     }
 }
-
 template<typename Ti, typename To, af_op_t op>
-To reduce_all(CParam<Ti> in, bool change_nan, double nanval) {
+void reduce_all(Param<To> out, CParam<Ti> in, bool change_nan, double nanval) {
     int in_elements = in.dims[0] * in.dims[1] * in.dims[2] * in.dims[3];
     bool is_linear  = (in.strides[0] == 1);
     for (int k = 1; k < 4; k++) {
         is_linear &= (in.strides[k] == (in.strides[k - 1] * in.dims[k - 1]));
     }
 
-    // FIXME: Use better heuristics to get to the optimum number
-    if (in_elements > 4096 || !is_linear) {
-        if (is_linear) {
-            in.dims[0] = in_elements;
-            for (int k = 1; k < 4; k++) {
-                in.dims[k]    = 1;
-                in.strides[k] = in_elements;
-            }
-        }
-        uint threads_x = nextpow2(std::max(32u, (uint)in.dims[0]));
-        threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
-        uint threads_y = THREADS_PER_BLOCK / threads_x;
-
-        Param<To> tmp;
-
-        uint blocks_x = divup(in.dims[0], threads_x * REPEAT);
-        uint blocks_y = divup(in.dims[1], threads_y);
-
-        tmp.dims[0]    = blocks_x;
-        tmp.strides[0] = 1;
-
+    if (is_linear) {
+        in.dims[0] = in_elements;
         for (int k = 1; k < 4; k++) {
-            tmp.dims[k]    = in.dims[k];
-            tmp.strides[k] = tmp.dims[k - 1] * tmp.strides[k - 1];
+            in.dims[k]    = 1;
+            in.strides[k] = in_elements;
         }
+    }
 
-        int tmp_elements = tmp.strides[3] * tmp.dims[3];
-
-        auto tmp_alloc = memAlloc<To>(tmp_elements);
-        tmp.ptr        = tmp_alloc.get();
-        reduce_first_launcher<Ti, To, op>(tmp, in, blocks_x, blocks_y,
-                                          threads_x, change_nan, nanval);
-
-        std::vector<To> h_data(tmp_elements);
-        CUDA_CHECK(
-            cudaMemcpyAsync(h_data.data(), tmp.ptr, tmp_elements * sizeof(To),
-                            cudaMemcpyDeviceToHost, cuda::getActiveStream()));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
-
-        common::Binary<compute_t<To>, op> reduce;
-        compute_t<To> out = common::Binary<compute_t<To>, op>::init();
-        for (int i = 0; i < tmp_elements; i++) {
-            out = reduce(out, compute_t<To>(h_data[i]));
-        }
+    uint threads_x = nextpow2(std::max(32u, (uint)in.dims[0]));
+    threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
+    uint threads_y = THREADS_PER_BLOCK / threads_x;
 
-        return data_t<To>(out);
-    } else {
-        std::vector<Ti> h_data(in_elements);
-        CUDA_CHECK(
-            cudaMemcpyAsync(h_data.data(), in.ptr, in_elements * sizeof(Ti),
-                            cudaMemcpyDeviceToHost, cuda::getActiveStream()));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
-
-        common::Transform<Ti, compute_t<To>, op> transform;
-        common::Binary<compute_t<To>, op> reduce;
-        compute_t<To> out       = common::Binary<compute_t<To>, op>::init();
-        compute_t<To> nanval_to = scalar<compute_t<To>>(nanval);
-
-        for (int i = 0; i < in_elements; i++) {
-            compute_t<To> in_val = transform(h_data[i]);
-            if (change_nan) in_val = !IS_NAN(in_val) ? in_val : nanval_to;
-            out = reduce(out, in_val);
-        }
+    // TODO: perf REPEAT, consider removing or runtime eval
+    // max problem size < SM resident threads, don't use REPEAT
+    uint blocks_x = divup(in.dims[0], threads_x * REPEAT);
+    uint blocks_y = divup(in.dims[1], threads_y);
 
-        return data_t<To>(out);
-    }
+    reduce_all_launcher<Ti, To, op>(out, in, blocks_x, blocks_y, threads_x,
+                                    change_nan, nanval);
 }
 
 }  // namespace kernel
diff --git a/src/backend/cuda/reduce.hpp b/src/backend/cuda/reduce.hpp
index 8f3ad82898..d606153650 100644
--- a/src/backend/cuda/reduce.hpp
+++ b/src/backend/cuda/reduce.hpp
@@ -21,5 +21,6 @@ void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
                    bool change_nan = false, double nanval = 0);
 
 template<af_op_t op, typename Ti, typename To>
-To reduce_all(const Array<Ti> &in, bool change_nan = false, double nanval = 0);
+Array<To> reduce_all(const Array<Ti> &in, bool change_nan = false,
+                     double nanval = 0);
 }  // namespace cuda
diff --git a/src/backend/cuda/reduce_impl.hpp b/src/backend/cuda/reduce_impl.hpp
index 73b0d47761..0c4e2e3e87 100644
--- a/src/backend/cuda/reduce_impl.hpp
+++ b/src/backend/cuda/reduce_impl.hpp
@@ -353,9 +353,12 @@ void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
 }
 
 template<af_op_t op, typename Ti, typename To>
-To reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
-    return kernel::reduce_all<Ti, To, op>(in, change_nan, nanval);
+Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
+    Array<To> out = createEmptyArray<To>(1);
+    kernel::reduce_all<Ti, To, op>(out, in, change_nan, nanval);
+    return out;
 }
+
 }  // namespace cuda
 
 #define INSTANTIATE(Op, Ti, To)                                                \
@@ -367,5 +370,5 @@ To reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
     template void reduce_by_key<Op, Ti, uint, To>(                             \
         Array<uint> & keys_out, Array<To> & vals_out, const Array<uint> &keys, \
         const Array<Ti> &vals, const int dim, bool change_nan, double nanval); \
-    template To reduce_all<Op, Ti, To>(const Array<Ti> &in, bool change_nan,   \
-                                       double nanval);
+    template Array<To> reduce_all<Op, Ti, To>(const Array<Ti> &in,             \
+                                              bool change_nan, double nanval);
diff --git a/src/backend/opencl/kernel/reduce.hpp b/src/backend/opencl/kernel/reduce.hpp
index 0b803ba794..f3c8022b71 100644
--- a/src/backend/opencl/kernel/reduce.hpp
+++ b/src/backend/opencl/kernel/reduce.hpp
@@ -20,6 +20,7 @@
 #include <kernel/config.hpp>
 #include <kernel/names.hpp>
 #include <kernel_headers/ops.hpp>
+#include <kernel_headers/reduce_all.hpp>
 #include <kernel_headers/reduce_dim.hpp>
 #include <kernel_headers/reduce_first.hpp>
 #include <math.hpp>
@@ -109,6 +110,54 @@ void reduceDim(Param out, Param in, int change_nan, double nanval, int dim) {
     }
 }
 
+template<typename Ti, typename To, af_op_t op>
+void reduceAllLauncher(Param out, Param in, const uint groups_x,
+                       const uint groups_y, const uint threads_x,
+                       int change_nan, double nanval) {
+    ToNumStr<To> toNumStr;
+    std::vector<TemplateArg> targs = {
+        TemplateTypename<Ti>(),
+        TemplateTypename<To>(),
+        TemplateArg(op),
+        TemplateArg(threads_x),
+    };
+    std::vector<std::string> options = {
+        DefineKeyValue(Ti, dtype_traits<Ti>::getName()),
+        DefineKeyValue(To, dtype_traits<To>::getName()),
+        DefineKeyValue(T, "To"),
+        DefineKeyValue(DIMX, threads_x),
+        DefineValue(THREADS_PER_GROUP),
+        DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
+        DefineKeyFromStr(binOpName<op>()),
+        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+    };
+    options.emplace_back(getTypeBuildDefinition<Ti, To>());
+
+    auto reduceAll = common::getKernel(
+        "reduce_all_kernel", {ops_cl_src, reduce_all_cl_src}, targs, options);
+
+    cl::NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
+    cl::NDRange global(groups_x * in.info.dims[2] * local[0],
+                       groups_y * in.info.dims[3] * local[1]);
+
+    uint repeat = divup(in.info.dims[0], (local[0] * groups_x));
+
+    long tmp_elements = groups_x * in.info.dims[2] * groups_y * in.info.dims[3];
+    if (tmp_elements > UINT_MAX) {
+        AF_ERROR("Too many blocks requested (retirementCount == unsigned)",
+                 AF_ERR_RUNTIME);
+    }
+    Array<To> tmp                   = createEmptyArray<To>(tmp_elements);
+    Array<unsigned> retirementCount = createValueArray<unsigned>(1, 0);
+    Param p_tmp(tmp);
+    Param p_Count(retirementCount);
+
+    reduceAll(cl::EnqueueArgs(getQueue(), global, local), *out.data, out.info,
+              *p_Count.data, *p_tmp.data, p_tmp.info, *in.data, in.info,
+              groups_x, groups_y, repeat, change_nan, scalar<To>(nanval));
+    CL_DEBUG_FINISH(getQueue());
+}
+
 template<typename Ti, typename To, af_op_t op>
 void reduceFirstLauncher(Param out, Param in, const uint groups_x,
                          const uint groups_y, const uint threads_x,
@@ -192,7 +241,7 @@ void reduce(Param out, Param in, int dim, int change_nan, double nanval) {
 }
 
 template<typename Ti, typename To, af_op_t op>
-To reduceAll(Param in, int change_nan, double nanval) {
+void reduceAll(Param out, Param in, int change_nan, double nanval) {
     int in_elements =
         in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
 
@@ -202,59 +251,22 @@ To reduceAll(Param in, int change_nan, double nanval) {
                       (in.info.strides[k - 1] * in.info.dims[k - 1]));
     }
 
-    // FIXME: Use better heuristics to get to the optimum number
-    if (in_elements > 4096 || !is_linear) {
-        if (is_linear) {
-            in.info.dims[0] = in_elements;
-            for (int k = 1; k < 4; k++) {
-                in.info.dims[k]    = 1;
-                in.info.strides[k] = in_elements;
-            }
+    if (is_linear) {
+        in.info.dims[0] = in_elements;
+        for (int k = 1; k < 4; k++) {
+            in.info.dims[k]    = 1;
+            in.info.strides[k] = in_elements;
         }
+    }
 
-        uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
-        threads_x      = std::min(threads_x, THREADS_PER_GROUP);
-        uint threads_y = THREADS_PER_GROUP / threads_x;
-
-        uint groups_x = divup(in.info.dims[0], threads_x * REPEAT);
-        uint groups_y = divup(in.info.dims[1], threads_y);
-        Array<To> tmp = createEmptyArray<To>(
-            {groups_x, in.info.dims[1], in.info.dims[2], in.info.dims[3]});
-
-        int tmp_elements = tmp.elements();
-
-        reduceFirstLauncher<Ti, To, op>(tmp, in, groups_x, groups_y, threads_x,
-                                        change_nan, nanval);
-
-        std::vector<To> h_ptr(tmp_elements);
-        getQueue().enqueueReadBuffer(*tmp.get(), CL_TRUE, 0,
-                                     sizeof(To) * tmp_elements, h_ptr.data());
-
-        common::Binary<compute_t<To>, op> reduce;
-        compute_t<To> out = common::Binary<compute_t<To>, op>::init();
-        for (int i = 0; i < (int)tmp_elements; i++) {
-            out = reduce(out, compute_t<To>(h_ptr[i]));
-        }
-        return data_t<To>(out);
-    } else {
-        std::vector<Ti> h_ptr(in_elements);
-        getQueue().enqueueReadBuffer(*in.data, CL_TRUE,
-                                     sizeof(Ti) * in.info.offset,
-                                     sizeof(Ti) * in_elements, h_ptr.data());
-
-        common::Transform<Ti, compute_t<To>, op> transform;
-        common::Binary<compute_t<To>, op> reduce;
-        compute_t<To> out       = common::Binary<compute_t<To>, op>::init();
-        compute_t<To> nanval_to = scalar<compute_t<To>>(nanval);
-
-        for (int i = 0; i < (int)in_elements; i++) {
-            compute_t<To> in_val = transform(h_ptr[i]);
-            if (change_nan) in_val = IS_NAN(in_val) ? nanval_to : in_val;
-            out = reduce(out, compute_t<To>(in_val));
-        }
+    uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+    threads_x      = std::min(threads_x, THREADS_PER_GROUP);
+    uint threads_y = THREADS_PER_GROUP / threads_x;
 
-        return data_t<To>(out);
-    }
+    uint groups_x = divup(in.info.dims[0], threads_x * REPEAT);
+    uint groups_y = divup(in.info.dims[1], threads_y);
+    reduceAllLauncher<Ti, To, op>(out, in, groups_x, groups_y, threads_x,
+                                  change_nan, nanval);
 }
 
 }  // namespace kernel
diff --git a/src/backend/opencl/kernel/reduce_all.cl b/src/backend/opencl/kernel/reduce_all.cl
new file mode 100644
index 0000000000..dccb0f1c69
--- /dev/null
+++ b/src/backend/opencl/kernel/reduce_all.cl
@@ -0,0 +1,160 @@
+/*******************************************************
+ * Copyright (c) 2021, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+// careful w/__threadfence substitution!
+// http://www.whatmannerofburgeristhis.com/blog/opencl-vs-cuda-gpu-memory-fences/
+
+kernel void reduce_all_kernel(global To *oData, KParam oInfo,
+                              global int* retirementCount, global To *tmp, KParam tmpInfo,
+                              const global Ti *iData, KParam iInfo,
+                              uint groups_x, uint groups_y, uint repeat,
+                              int change_nan, To nanval) {
+
+    const uint tidx = get_local_id(0);
+    const uint tidy = get_local_id(1);
+    const uint tid  = tidy * DIMX + tidx;
+
+    const uint zid       = get_group_id(0) / groups_x;
+    const uint groupId_x = get_group_id(0) - (groups_x)*zid;
+    const uint xid       = groupId_x * get_local_size(0) * repeat + tidx;
+
+    const uint wid       = get_group_id(1) / groups_y;
+    const uint groupId_y = get_group_id(1) - (groups_y)*wid;
+    const uint yid       = groupId_y * get_local_size(1) + tidy;
+
+    local To s_val[THREADS_PER_GROUP];
+    local bool amLast;
+
+    iData += wid * iInfo.strides[3] + zid * iInfo.strides[2] +
+             yid * iInfo.strides[1] + iInfo.offset;
+
+    bool cond =
+        (yid < iInfo.dims[1]) && (zid < iInfo.dims[2]) && (wid < iInfo.dims[3]);
+
+
+    int last   = (xid + repeat * DIMX);
+    int lim    = last > iInfo.dims[0] ? iInfo.dims[0] : last;
+
+    To out_val = init;
+    for (int id = xid; cond && id < lim; id += DIMX) {
+        To in_val = transform(iData[id]);
+        if (change_nan) in_val = !IS_NAN(in_val) ? in_val : nanval;
+        out_val = binOp(in_val, out_val);
+    }
+
+    s_val[tid] = out_val;
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (THREADS_PER_GROUP == 256) {
+        if (tid < 128) s_val[tid] = binOp(s_val[tid], s_val[tid + 128]);
+        barrier(CLK_LOCAL_MEM_FENCE);
+    }
+
+    if (THREADS_PER_GROUP >= 128) {
+        if (tid < 64) s_val[tid] = binOp(s_val[tid], s_val[tid + 64]);
+        barrier(CLK_LOCAL_MEM_FENCE);
+    }
+
+    if (THREADS_PER_GROUP >= 64) {
+        if (tid < 32) s_val[tid] = binOp(s_val[tid], s_val[tid + 32]);
+        barrier(CLK_LOCAL_MEM_FENCE);
+    }
+
+    if (tid < 16) s_val[tid] = binOp(s_val[tid], s_val[tid + 16]);
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (tid < 8) s_val[tid] = binOp(s_val[tid], s_val[tid + 8]);
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (tid < 4) s_val[tid] = binOp(s_val[tid], s_val[tid + 4]);
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (tid < 2) s_val[tid] = binOp(s_val[tid], s_val[tid + 2]);
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+    if (tid < 1) s_val[tid] = binOp(s_val[tid], s_val[tid + 1]);
+    barrier(CLK_LOCAL_MEM_FENCE);
+
+
+    const unsigned total_blocks = (get_num_groups(0) * get_num_groups(1) * get_num_groups(2));
+    const int uubidx = (get_num_groups(0) * get_num_groups(1)) * get_group_id(2)
+                       + (get_num_groups(0) * get_group_id(1)) + get_group_id(0);
+    if (cond && tid == 0) {
+        if(total_blocks != 1) {
+            tmp[uubidx] = s_val[0];
+        } else {
+            oData[0] = s_val[0];
+        }
+    }
+
+    // Last block to perform final reduction
+    if (total_blocks > 1) {
+
+        mem_fence(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
+
+        // Thread 0 takes a ticket
+        if (tid == 0) {
+            unsigned int ticket = atomic_inc(retirementCount);
+            // If the ticket ID == number of blocks, we are the last block
+            amLast = (ticket == (total_blocks - 1));
+        }
+        barrier(CLK_LOCAL_MEM_FENCE);
+
+        if (amLast) {
+            int i = tid;
+            To fout_val = init;
+
+            while (i < total_blocks) {
+                To in_val = tmp[i];
+                fout_val = binOp(in_val, fout_val);
+                i += THREADS_PER_GROUP;
+            }
+
+            s_val[tid] = fout_val;
+            barrier(CLK_LOCAL_MEM_FENCE);
+
+            // reduce final block
+            if (THREADS_PER_GROUP == 256) {
+                if (tid < 128) s_val[tid] = binOp(s_val[tid], s_val[tid + 128]);
+                barrier(CLK_LOCAL_MEM_FENCE);
+            }
+
+            if (THREADS_PER_GROUP >= 128) {
+                if (tid < 64) s_val[tid] = binOp(s_val[tid], s_val[tid + 64]);
+                barrier(CLK_LOCAL_MEM_FENCE);
+            }
+
+            if (THREADS_PER_GROUP >= 64) {
+                if (tid < 32) s_val[tid] = binOp(s_val[tid], s_val[tid + 32]);
+                barrier(CLK_LOCAL_MEM_FENCE);
+            }
+
+            if (tid < 16) s_val[tid] = binOp(s_val[tid], s_val[tid + 16]);
+            barrier(CLK_LOCAL_MEM_FENCE);
+
+            if (tid < 8) s_val[tid] = binOp(s_val[tid], s_val[tid + 8]);
+            barrier(CLK_LOCAL_MEM_FENCE);
+
+            if (tid < 4) s_val[tid] = binOp(s_val[tid], s_val[tid + 4]);
+            barrier(CLK_LOCAL_MEM_FENCE);
+
+            if (tid < 2) s_val[tid] = binOp(s_val[tid], s_val[tid + 2]);
+            barrier(CLK_LOCAL_MEM_FENCE);
+
+            if (tid < 1) s_val[tid] = binOp(s_val[tid], s_val[tid + 1]);
+            barrier(CLK_LOCAL_MEM_FENCE);
+
+            if (tid == 0) {
+                oData[0] = s_val[0];
+
+                // reset retirement count so that next run succeeds
+                retirementCount[0] = 0;
+            }
+        }
+    }
+}
diff --git a/src/backend/opencl/reduce.hpp b/src/backend/opencl/reduce.hpp
index 4da84d10df..4c9581c761 100644
--- a/src/backend/opencl/reduce.hpp
+++ b/src/backend/opencl/reduce.hpp
@@ -22,5 +22,6 @@ void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
                    bool change_nan = false, double nanval = 0);
 
 template<af_op_t op, typename Ti, typename To>
-To reduce_all(const Array<Ti> &in, bool change_nan = false, double nanval = 0);
+Array<To> reduce_all(const Array<Ti> &in, bool change_nan = false,
+                     double nanval = 0);
 }  // namespace opencl
diff --git a/src/backend/opencl/reduce_impl.hpp b/src/backend/opencl/reduce_impl.hpp
index f7c8c675b6..4211dc9050 100644
--- a/src/backend/opencl/reduce_impl.hpp
+++ b/src/backend/opencl/reduce_impl.hpp
@@ -37,9 +37,12 @@ void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
 }
 
 template<af_op_t op, typename Ti, typename To>
-To reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
-    return kernel::reduceAll<Ti, To, op>(in, change_nan, nanval);
+Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
+    Array<To> out = createEmptyArray<To>(1);
+    kernel::reduceAll<Ti, To, op>(out, in, change_nan, nanval);
+    return out;
 }
+
 }  // namespace opencl
 
 #define INSTANTIATE(Op, Ti, To)                                                \
@@ -51,5 +54,5 @@ To reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
     template void reduce_by_key<Op, Ti, uint, To>(                             \
         Array<uint> & keys_out, Array<To> & vals_out, const Array<uint> &keys, \
         const Array<Ti> &vals, const int dim, bool change_nan, double nanval); \
-    template To reduce_all<Op, Ti, To>(const Array<Ti> &in, bool change_nan,   \
-                                       double nanval);
+    template Array<To> reduce_all<Op, Ti, To>(const Array<Ti> &in,             \
+                                              bool change_nan, double nanval);
diff --git a/src/backend/opencl/sparse.cpp b/src/backend/opencl/sparse.cpp
index d579761a72..580822d5d1 100644
--- a/src/backend/opencl/sparse.cpp
+++ b/src/backend/opencl/sparse.cpp
@@ -61,7 +61,7 @@ template<typename T, af_storage stype>
 SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in_) {
     in_.eval();
 
-    uint nNZ = reduce_all<af_notzero_t, T, uint>(in_);
+    uint nNZ = getScalar<uint>(reduce_all<af_notzero_t, T, uint>(in_));
 
     SparseArray<T> sparse_ = createEmptySparseArray<T>(in_.dims(), nNZ, stype);
     sparse_.eval();
diff --git a/src/backend/opencl/svd.cpp b/src/backend/opencl/svd.cpp
index 2d76c46961..5aa6c0e1ed 100644
--- a/src/backend/opencl/svd.cpp
+++ b/src/backend/opencl/svd.cpp
@@ -87,7 +87,7 @@ void svd(Array<T> &arrU, Array<Tr> &arrS, Array<T> &arrVT, Array<T> &arrA,
     static const double smlnum = std::sqrt(cpu_lapack_lamch('S')) / eps;
     static const double bignum = 1. / smlnum;
 
-    Tr anrm = abs(reduce_all<af_max_t, T, T>(arrA));
+    Tr anrm = abs(getScalar<T>(reduce_all<af_max_t, T, T>(arrA)));
 
     T scale                = scalar<T>(1);
     static const int ione  = 1;
diff --git a/test/mean.cpp b/test/mean.cpp
index 22b622c868..9c4c8f7fb4 100644
--- a/test/mean.cpp
+++ b/test/mean.cpp
@@ -362,8 +362,9 @@ TEST(MeanAll, SubArray) {
     array in  = randu(inDims);
     array sub = in(0, span, span, span);
 
-    size_t nElems = sub.elements();
-    ASSERT_FLOAT_EQ(mean<float>(sub), sum<float>(sub) / nElems);
+    size_t nElems   = sub.elements();
+    float max_error = std::numeric_limits<float>::epsilon() * nElems;
+    ASSERT_NEAR(mean<float>(sub), sum<float>(sub) / nElems, max_error);
 }
 
 TEST(MeanHalf, dim0) {
diff --git a/test/reduce.cpp b/test/reduce.cpp
index 3cb1c33a55..0633bd0536 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -13,7 +13,9 @@
 #include <af/dim4.hpp>
 #include <af/traits.hpp>
 
+#include <math.h>
 #include <algorithm>
+#include <cmath>
 #include <functional>
 #include <iostream>
 #include <string>
@@ -96,7 +98,6 @@ void reduceTest(string pTestFile, int off = 0, bool isSubRef = false,
                 EXPECT_EQ(currGoldBar[elIter], outData[elIter])
                     << "at: " << elIter << " for dim " << d + off << endl;
             }
-            af_print_array(outArray);
             for (int i = 0; i < (int)nElems; i++) {
                 cout << currGoldBar[i] << ", ";
             }
@@ -1263,10 +1264,14 @@ TEST(Reduce, KernelName) {
 }
 
 TEST(Reduce, AllSmallIndexed) {
-    const int len = 1000;
-    array a       = af::range(dim4(len, 2));
-    array b       = a(seq(len / 2), span);
-    ASSERT_EQ(max<float>(b), len / 2 - 1);
+    const int len = 512;
+    for (int i = 0; i < 1000; ++i) {
+        // const int len = 10000;
+        array a = af::range(dim4(len, 2));
+        array b = a(seq(len / 2), span);
+        // af::sync();
+        ASSERT_EQ(max<float>(b), len / 2 - 1);
+    }
 }
 
 TEST(ProductAll, BoolIn_ISSUE2543_All_Ones) {
@@ -2091,3 +2096,192 @@ TEST(ReduceByKey, ISSUE_3062) {
     af::countByKey(okeys, ovalues, zeros, ones, 1);
     ASSERT_EQ(ovalues.scalar<unsigned>(), 129);
 }
+
+TEST(Reduce, Test_Sum_Global_Array) {
+    const int num = 513;
+    array a       = af::randn(num, 2, 33, 4);
+
+    float res          = af::sum<float>(a);
+    array full_reduce  = af::sum<af::array>(a);
+
+    float *h_a = a.host<float>();
+    float gold = 0.f;
+
+    for (int i = 0; i < a.elements(); i++) { gold += h_a[i]; }
+
+    float max_error = std::numeric_limits<float>::epsilon() * (float)a.elements();
+    ASSERT_NEAR(gold, res, max_error);
+    ASSERT_NEAR(res, full_reduce.scalar<float>(), max_error);
+    freeHost(h_a);
+}
+
+TEST(Reduce, Test_Product_Global_Array) {
+    const int num = 512;
+    array a       = 1 + (0.005 * af::randn(num, 2, 3, 4));
+
+    float res          = af::product<float>(a);
+    array full_reduce  = af::product<af::array>(a);
+
+    float *h_a = a.host<float>();
+    float gold = 1.f;
+
+    for (int i = 0; i < a.elements(); i++) { gold *= h_a[i]; }
+
+    float max_error = std::numeric_limits<float>::epsilon() * (float)a.elements();
+    ASSERT_NEAR(gold, res, max_error);
+    ASSERT_NEAR(res, full_reduce.scalar<float>(), max_error);
+    freeHost(h_a);
+}
+
+TEST(Reduce, Test_Count_Global_Array) {
+    const int num = 10000;
+    array a       = round(2 * randu(num, 2, 3, 4));
+    array b       = a.as(b8);
+
+    int res       = count<int>(b);
+    array res_arr = count<af::array>(b);
+    char *h_b     = b.host<char>();
+    unsigned gold      = 0;
+
+    for (int i = 0; i < a.elements(); i++) { gold += h_b[i]; }
+
+    ASSERT_EQ(gold, res);
+    ASSERT_EQ(gold, res_arr.scalar<unsigned>());
+    freeHost(h_b);
+}
+
+TEST(Reduce, Test_min_Global_Array) {
+    SUPPORTED_TYPE_CHECK(double);
+
+    const int num = 10000;
+    array a       = af::randn(num, 2, 3, 4, f64);
+    double res    = min<double>(a);
+    array res_arr = min<af::array>(a);
+    double *h_a   = a.host<double>();
+    double gold   = std::numeric_limits<double>::max();
+
+    SUPPORTED_TYPE_CHECK(double);
+
+    for (int i = 0; i < a.elements(); i++) { gold = std::min(gold, h_a[i]); }
+
+    ASSERT_EQ(gold, res);
+    ASSERT_EQ(gold, res_arr.scalar<double>());
+    freeHost(h_a);
+}
+
+TEST(Reduce, Test_max_Global_Array) {
+    const int num = 10000;
+    array a       = af::randn(num, 2, 3, 4);
+    float res     = max<float>(a);
+    array res_arr = max<af::array>(a);
+    float *h_a    = a.host<float>();
+    float gold    = -std::numeric_limits<float>::max();
+
+    for (int i = 0; i < a.elements(); i++) { gold = std::max(gold, h_a[i]); }
+
+    ASSERT_EQ(gold, res);
+    ASSERT_EQ(gold, res_arr.scalar<float>());
+    freeHost(h_a);
+}
+
+TYPED_TEST(Reduce, Test_All_Global_Array) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+
+    // Input size test
+    for (int i = 1; i < 1000; i += 100) {
+        int num = 10 * i;
+        vector<TypeParam> h_vals(num, (TypeParam) true);
+        array a(2, num / 2, &h_vals.front());
+
+        TypeParam res = allTrue<TypeParam>(a);
+        array res_arr = allTrue<array>(a);
+        typed_assert_eq((TypeParam) true, res, false);
+        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(), false);
+
+        h_vals[3] = false;
+        a         = array(2, num / 2, &h_vals.front());
+
+        res = allTrue<TypeParam>(a);
+        res_arr = allTrue<array>(a);
+        typed_assert_eq((TypeParam) false, res, false);
+        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(), false);
+    }
+
+    // false value location test
+    const int num = 10000;
+    vector<TypeParam> h_vals(num, (TypeParam) true);
+    for (int i = 1; i < 10000; i += 100) {
+        h_vals[i] = false;
+        array a(2, num / 2, &h_vals.front());
+
+        TypeParam res = allTrue<TypeParam>(a);
+        array res_arr = allTrue<array>(a);
+        typed_assert_eq((TypeParam) false, res, false);
+        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(), false);
+
+        h_vals[i] = true;
+    }
+}
+
+TYPED_TEST(Reduce, Test_Any_Global_Array) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+
+    // Input size test
+    for (int i = 1; i < 1000; i += 100) {
+        int num = 10 * i;
+        vector<TypeParam> h_vals(num, (TypeParam) false);
+        array a(2, num / 2, &h_vals.front());
+
+        TypeParam res = anyTrue<TypeParam>(a);
+        array res_arr = anyTrue<array>(a);
+        typed_assert_eq((TypeParam) false, res, false);
+        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(), false);
+
+        h_vals[3] = true;
+        a         = array(2, num / 2, &h_vals.front());
+
+        res = anyTrue<TypeParam>(a);
+        res_arr = anyTrue<array>(a);
+        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(), false);
+    }
+
+    // true value location test
+    const int num = 10000;
+    vector<TypeParam> h_vals(num, (TypeParam) false);
+    for (int i = 1; i < 10000; i += 100) {
+        h_vals[i] = true;
+        array a(2, num / 2, &h_vals.front());
+
+        TypeParam res = anyTrue<TypeParam>(a);
+        array res_arr = anyTrue<array>(a);
+        typed_assert_eq((TypeParam) true, res, false);
+        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(), false);
+
+        h_vals[i] = false;
+    }
+}
+
+
+TEST(Reduce, Test_Sum_Global_Array_nanval) {
+    const int num = 100000;
+    array a = af::randn(num, 2, 34, 4);
+    a(1, 0, 0, 0) = NAN;
+    a(0, 1, 0, 0) = NAN;
+    a(0, 0, 1, 0) = NAN;
+    a(0, 0, 0, 1) = NAN;
+
+    double nanval = 0.2;
+    float res          = af::sum<float>(a, nanval);
+    array full_reduce  = af::sum<af::array>(a, nanval);
+
+    float *h_a = a.host<float>();
+    float gold = 0.f;
+
+    for (int i = 0; i < a.elements(); i++) {
+        gold += (isnan(h_a[i])) ? nanval : h_a[i];
+    }
+    float max_error = std::numeric_limits<float>::epsilon() * (float)a.elements();
+    ASSERT_NEAR(gold, res, max_error);
+    ASSERT_NEAR(res, full_reduce.scalar<float>(), max_error);
+    freeHost(h_a);
+}

From 4868a37947672a09e138744646bbfb6b3afd87b3 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 9 Apr 2022 14:09:18 -0400
Subject: [PATCH 036/473] Fix ccache configuration issue because it was
 configured before CUDA

Ccache was configured before CUDA was setup. This caused the launch-nvcc
script to include an empty CUDA_NVCC_EXECUTABLE variable.
---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 784ed20144..cb88845889 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,7 +14,6 @@ project(ArrayFire VERSION 3.9.0 LANGUAGES C CXX)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
 
 include(AFconfigure_deps_vars)
-include(config_ccache)
 include(AFBuildConfigurations)
 include(AFInstallDirs)
 include(CMakeDependentOption)
@@ -58,6 +57,7 @@ find_package(MKL)
 find_package(spdlog 1.8.5 QUIET)
 
 include(boost_package)
+include(config_ccache)
 
 option(AF_BUILD_CPU      "Build ArrayFire with a CPU backend"        ON)
 option(AF_BUILD_CUDA     "Build ArrayFire with a CUDA backend"       ${CUDA_FOUND})

From 096e0cae2c14d879a6ef600b1a35df4eef2d86f0 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 9 Apr 2022 14:13:28 -0400
Subject: [PATCH 037/473] Fix issue with CMAKE_MODULE_PATH when it has multiple
 values

The path to some configuration files were relative to CMAKE_MODULE_PATH.
This variable can be a list of strings which causes errors when
CMAKE_MODULE_PATH was modified to include additional values.
---
 CMakeLists.txt                   | 6 +++---
 CMakeModules/InternalUtils.cmake | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index cb88845889..adfe1d59bf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -430,7 +430,7 @@ write_basic_package_version_file(
 set(INCLUDE_DIRS include)
 set(CMAKE_DIR ${AF_INSTALL_CMAKE_DIR})
 configure_package_config_file(
-  ${CMAKE_MODULE_PATH}/ArrayFireConfig.cmake.in
+  ${ArrayFire_SOURCE_DIR}/CMakeModules/ArrayFireConfig.cmake.in
   cmake/install/ArrayFireConfig.cmake
   INSTALL_DESTINATION "${AF_INSTALL_CMAKE_DIR}"
   PATH_VARS INCLUDE_DIRS CMAKE_DIR
@@ -488,7 +488,7 @@ endif()
 set(INCLUDE_DIRS "${ArrayFire_SOURCE_DIR}/include" "${ArrayFire_BINARY_DIR}/include")
 set(CMAKE_DIR "${ArrayFire_BINARY_DIR}/cmake")
 configure_package_config_file(
-  ${CMAKE_MODULE_PATH}/ArrayFireConfig.cmake.in
+  ${ArrayFire_SOURCE_DIR}/CMakeModules/ArrayFireConfig.cmake.in
   ArrayFireConfig.cmake
   INSTALL_DESTINATION "${ArrayFire_BINARY_DIR}"
   PATH_VARS INCLUDE_DIRS CMAKE_DIR
@@ -506,7 +506,7 @@ configure_package_config_file(
 unset(CMAKE_CXX_VISIBILITY_PRESET)
 
 configure_file(
-  ${CMAKE_MODULE_PATH}/CTestCustom.cmake
+  ${ArrayFire_SOURCE_DIR}/CMakeModules/CTestCustom.cmake
   ${ArrayFire_BINARY_DIR}/CTestCustom.cmake)
 
 include(CTest)
diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index 8fd21e7447..3b19485d6f 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -205,7 +205,7 @@ macro(arrayfire_set_cmake_default_variables)
   #          EPILOG ${compiler_header_epilogue}
   #          )
   configure_file(
-    ${CMAKE_MODULE_PATH}/compilers.h
+    ${ArrayFire_SOURCE_DIR}/CMakeModules/compilers.h
     ${ArrayFire_BINARY_DIR}/include/af/compilers.h)
 endmacro()
 

From 590267d2f719a0ea62901ec0fdc0d7b7a426d531 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 11 Apr 2022 13:20:07 -0400
Subject: [PATCH 038/473] Do not add cuda_unified test.

---
 test/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index af9afe4991..1c7bc8792e 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -372,7 +372,9 @@ if(CUDA_FOUND)
         FOLDER "Tests"
         OUTPUT_NAME "cuda_${backend}")
 
-      add_test(NAME ${target} COMMAND ${target})
+      if(NOT ${backend} STREQUAL "unified")
+        add_test(NAME ${target} COMMAND ${target})
+      endif()
     endif()
   endforeach()
 endif()

From a7f422dc7056528321a96d910e2395ca5266d2e5 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 19 Apr 2022 17:44:24 -0400
Subject: [PATCH 039/473] Fix static MKL. Avoid calling interface/threading
 layer functions

We only need to call the mkl_set_threading_layer and mkl_set_interface_layer
functions for shared library builds of MKL. Static builds do not need those
functions.
---
 src/api/c/device.cpp              | 2 +-
 src/backend/cpu/CMakeLists.txt    | 1 +
 src/backend/opencl/CMakeLists.txt | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp
index d77969aeb1..3ed23a0c3e 100644
--- a/src/api/c/device.cpp
+++ b/src/api/c/device.cpp
@@ -108,7 +108,7 @@ af_err af_init() {
         thread_local std::once_flag flag;
         std::call_once(flag, []() {
             getDeviceInfo();
-#if defined(USE_MKL)
+#if defined(USE_MKL) && !defined(USE_STATIC_MKL)
             int errCode = -1;
             // Have used the AF_MKL_INTERFACE_SIZE as regular if's so that
             // we will know if these are not defined when using MKL when a
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index 9707ef5f23..e3c862d169 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -320,6 +320,7 @@ if(BUILD_WITH_MKL)
 
   if(AF_WITH_STATIC_MKL)
       target_link_libraries(afcpu PRIVATE MKL::Static)
+      target_compile_definitions(afcpu PRIVATE USE_STATIC_MKL)
   else()
       target_link_libraries(afcpu PRIVATE MKL::RT)
   endif()
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 5385f4fa1f..dd557ede47 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -472,6 +472,7 @@ if(LAPACK_FOUND OR BUILD_WITH_MKL)
 
     if(AF_WITH_STATIC_MKL)
         target_link_libraries(afopencl PRIVATE MKL::Static)
+        target_compile_definitions(afopencl PRIVATE USE_STATIC_MKL)
     else()
         target_link_libraries(afopencl PRIVATE MKL::RT)
     endif()

From 18d8131537f337d9dedfbc8b2065e3bc436e40bc Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 19 Apr 2022 17:48:27 -0400
Subject: [PATCH 040/473] Remove link to OpenCL library with unified backend

The unified backend was linking to the OpenCL library. This was done
to include the header but the library was also linking. Fixed this issue
by using the INTERFACE_INCLUDE_DIRECTORIES generator expression to include
the OpenCL header
---
 src/api/unified/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/api/unified/CMakeLists.txt b/src/api/unified/CMakeLists.txt
index cc08659976..5c0cec9d6f 100644
--- a/src/api/unified/CMakeLists.txt
+++ b/src/api/unified/CMakeLists.txt
@@ -43,9 +43,9 @@ if(OpenCL_FOUND)
     ${CMAKE_CURRENT_SOURCE_DIR}/opencl.cpp
   )
 
-  target_link_libraries(af
+  target_include_directories(af
     PRIVATE
-      OpenCL::OpenCL)
+      $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>)
 
 endif()
 

From bc4919fad6ea7ae640d3acc8d3510bf0da03de62 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 19 Apr 2022 18:36:19 -0400
Subject: [PATCH 041/473] Remove complex not supported note on some trig
 function

---
 docs/details/arith.dox | 38 --------------------------------------
 include/af/arith.h     |  4 ++--
 2 files changed, 2 insertions(+), 40 deletions(-)

diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index 79e8cce0d0..f53de09a87 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -295,8 +295,6 @@ Hypotenuse of the two inputs
 
 sin of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_cos cos
 
@@ -304,8 +302,6 @@ sin of input
 
 cos of input
 
-\copydoc arith_real_only
-
 
 
 \defgroup arith_func_tan tan/tan2
@@ -314,8 +310,6 @@ cos of input
 
 tan of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_asin asin
 
@@ -323,8 +317,6 @@ tan of input
 
 arc sin of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_acos acos
 \brief Inverse cosine.
@@ -333,8 +325,6 @@ arc sin of input
 
 arc cos of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_atan atan/atan2
 
@@ -342,8 +332,6 @@ arc cos of input
 
 arc tan of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_sinh sinh
 
@@ -351,8 +339,6 @@ arc tan of input
 
 sinh of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_cosh cosh
 
@@ -360,8 +346,6 @@ sinh of input
 
 cosh of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_tanh tanh
 
@@ -369,8 +353,6 @@ cosh of input
 
 tanh of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_asinh asinh
 
@@ -378,8 +360,6 @@ tanh of input
 
 asinh of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_acosh acosh
 \brief Inverse hyperbolic cosine
@@ -388,8 +368,6 @@ asinh of input
 
 acosh of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_atanh atanh
 
@@ -397,8 +375,6 @@ acosh of input
 
 atanh of input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_cplx complex
 
@@ -439,8 +415,6 @@ Get complex conjugate
 
 Find root of an input
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_pow pow
 
@@ -464,8 +438,6 @@ point types used to compute power is given below.
 
 The output array will be of the same type as input.
 
-\copydoc arith_real_only
-
 
 
 \defgroup arith_func_exp exp
@@ -509,8 +481,6 @@ Complementary Error function value
 
 Natural logarithm
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_log1p log1p
 
@@ -536,8 +506,6 @@ logarithm base 10
 
 Square root of input arrays
 
-\copydoc arith_real_only
-
 \defgroup arith_func_rsqrt rsqrt
 
 \ingroup explog_mat
@@ -590,8 +558,6 @@ Logarithm of absolute values of Gamma function
 
 Check if values are zero
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_isinf isinf
 
@@ -599,8 +565,6 @@ Check if values are zero
 
 Check if values are infinite
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_isnan isNan
 
@@ -608,8 +572,6 @@ Check if values are infinite
 
 Check if values are Nan
 
-\copydoc arith_real_only
-
 
 \defgroup arith_func_cast cast
 
diff --git a/include/af/arith.h b/include/af/arith.h
index 83240ffc6d..319bda674b 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -473,7 +473,7 @@ namespace af
     /// \param[in] in is input
     /// \return the natural logarithm of (1 + input)
     ///
-    /// \note This function is useful when \p is small
+    /// \note This function is useful when \p in is small
     /// \ingroup arith_func_log1p
     AFAPI array log1p  (const array &in);
 
@@ -488,7 +488,7 @@ namespace af
     /// C++ Interface for logarithm base 2
     ///
     /// \param[in] in is input
-    /// \return the logarithm of input in base 2
+    /// \return the logarithm of input \p in base 2
     ///
     /// \ingroup explog_func_log2
     AFAPI array log2   (const array &in);

From df57c5679829d1e4e5122c3acc0630b651a4a642 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 21 Apr 2022 19:15:11 -0400
Subject: [PATCH 042/473] Release notes for v3.8.2

---
 docs/pages/release_notes.md | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 259b927772..fe893c564c 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -1,6 +1,37 @@
 Release Notes {#releasenotes}
 ==============
 
+v3.8.2
+======
+
+## Improvements
+
+- Optimize JIT by removing some consecutive cast operations \PR{3031}
+- Add driver checks checks for CUDA 11.5 and 11.6 \PR{3203}
+- Improve the timing algorithm used for timeit \PR{3185}
+- Dynamically link against CUDA numeric libraries by default \PR{3205}
+- Add support for pruning CUDA binaries to reduce static binary sizes \PR{3234} \PR{3237}
+- Remove unused cuDNN libraries from installations \PR{3235}
+- Add support to staticly link NVRTC libraries after CUDA 11.5 \PR{3236}
+- Add support for compiling with ccache when building the CUDA backend \PR{3241}
+- Make cuSparse an optional runtime dependency \PR{3240}
+
+## Fixes
+
+- Fix issue with consecutive moddims operations in the CPU backend \PR{3232}
+- Better floating point comparisons for tests \PR{3212}
+- Fix several warnings and inconsistencies with doxygen and documentation \PR{3226}
+- Fix issue when passing empty arrays into join \PR{3211}
+- Fix default value for the `AF_COMPUTE_LIBRARY` when not set \PR{3228}
+- Fix missing symbol issue when MKL is staticly linked \PR{3244}
+- Remove linking of OpenCL's library to the unified backend \PR{3244}
+
+## Contributions
+
+Special thanks to our contributors:
+[Jacob Kahn](https://github.com/jacobkahn)
+[Willy Born](https://github.com/willyborn)
+
 v3.8.1
 ======
 

From 2e36e8ce848fcb6c1e3e9fa569d1c0574461d917 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 May 2022 00:45:03 -0400
Subject: [PATCH 043/473] Improve vcpkg support

* Improves vcpkg support with new packages
* Fix spdlog dependency version number
* Add features for MKL and forge
* Remove unused packages
---
 CMakeLists.txt                                |  6 ++-
 CMakeModules/AF_vcpkg_options.cmake           | 12 +++++
 CMakeModules/vcpkg-triplets/x64-windows.cmake |  9 ++++
 src/backend/common/CMakeLists.txt             |  5 +-
 src/backend/common/debug.hpp                  |  1 -
 vcpkg.json                                    | 53 +++++++++++--------
 6 files changed, 61 insertions(+), 25 deletions(-)
 create mode 100644 CMakeModules/vcpkg-triplets/x64-windows.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index adfe1d59bf..8dfee21544 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -201,10 +201,14 @@ endif()
 #otherwise, forge is not built at all
 include(AFconfigure_forge_dep)
 add_library(af_spdlog INTERFACE)
+set_target_properties(af_spdlog
+  PROPERTIES
+    INTERFACE_COMPILE_DEFINITIONS FMT_HEADER_ONLY)
+
 if(TARGET spdlog::spdlog_header_only)
   target_include_directories(af_spdlog
     SYSTEM INTERFACE
-    $<TARGET_PROPERTY:spdlog::spdlog_header_only,INTERFACE_INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:spdlog::spdlog_header_only,INTERFACE_INCLUDE_DIRECTORIES>
     )
 else()
   af_dep_check_and_populate(${spdlog_prefix}
diff --git a/CMakeModules/AF_vcpkg_options.cmake b/CMakeModules/AF_vcpkg_options.cmake
index 0639c377a4..75297a02b6 100644
--- a/CMakeModules/AF_vcpkg_options.cmake
+++ b/CMakeModules/AF_vcpkg_options.cmake
@@ -7,14 +7,26 @@
 
 set(ENV{VCPKG_FEATURE_FLAGS} "versions")
 set(ENV{VCPKG_KEEP_ENV_VARS} "MKLROOT")
+set(VCPKG_MANIFEST_NO_DEFAULT_FEATURES ON)
+
+set(VCPKG_OVERLAY_TRIPLETS ${ArrayFire_SOURCE_DIR}/CMakeModules/vcpkg-triplets)
 
 if(AF_BUILD_CUDA)
   list(APPEND VCPKG_MANIFEST_FEATURES "cuda")
 endif()
+
 if(AF_BUILD_OPENCL)
   list(APPEND VCPKG_MANIFEST_FEATURES "opencl")
 endif()
 
+if(AF_BUILD_FORGE)
+  list(APPEND VCPKG_MANIFEST_FEATURES "forge")
+endif()
+
+if(AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL")
+  list(APPEND VCPKG_MANIFEST_FEATURES "mkl")
+endif()
+
 if(DEFINED VCPKG_ROOT AND NOT DEFINED CMAKE_TOOLCHAIN_FILE)
   set(CMAKE_TOOLCHAIN_FILE "${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "")
 elseif(DEFINED ENV{VCPKG_ROOT} AND NOT DEFINED CMAKE_TOOLCHAIN_FILE)
diff --git a/CMakeModules/vcpkg-triplets/x64-windows.cmake b/CMakeModules/vcpkg-triplets/x64-windows.cmake
new file mode 100644
index 0000000000..67dfc468eb
--- /dev/null
+++ b/CMakeModules/vcpkg-triplets/x64-windows.cmake
@@ -0,0 +1,9 @@
+set(VCPKG_TARGET_ARCHITECTURE x64)
+
+if(PORT MATCHES "freetype")
+  set(VCPKG_CRT_LINKAGE static)
+  set(VCPKG_LIBRARY_LINKAGE static)
+else()
+  set(VCPKG_CRT_LINKAGE dynamic)
+  set(VCPKG_LIBRARY_LINKAGE dynamic)
+endif()
diff --git a/src/backend/common/CMakeLists.txt b/src/backend/common/CMakeLists.txt
index 125c620754..d12823c6a3 100644
--- a/src/backend/common/CMakeLists.txt
+++ b/src/backend/common/CMakeLists.txt
@@ -91,6 +91,7 @@ target_link_libraries(afcommon_interface
     Boost::boost
     ${CMAKE_DL_LIBS}
 )
+
 if(TARGET glad::glad)
   target_link_libraries(afcommon_interface INTERFACE glad::glad)
 else()
@@ -105,7 +106,9 @@ target_include_directories(afcommon_interface
   INTERFACE
     ${ArrayFire_SOURCE_DIR}/src/backend
     ${span-lite_SOURCE_DIR}/include
-    ${ArrayFire_BINARY_DIR}
+    ${ArrayFire_BINARY_DIR})
+
+target_include_directories(afcommon_interface
   SYSTEM INTERFACE
     $<$<PLATFORM_ID:Darwin>:${OPENGL_INCLUDE_DIR}>
   )
diff --git a/src/backend/common/debug.hpp b/src/backend/common/debug.hpp
index 6c2c6cbfb8..e91c903d53 100644
--- a/src/backend/common/debug.hpp
+++ b/src/backend/common/debug.hpp
@@ -9,7 +9,6 @@
 
 #pragma once
 
-#define FMT_HEADER_ONLY
 #include <boost/stacktrace.hpp>
 #include <common/ArrayFireTypesIO.hpp>
 #include <common/jit/NodeIO.hpp>
diff --git a/vcpkg.json b/vcpkg.json
index a3fafdecf2..654d9ad8b6 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -5,34 +5,37 @@
     "description": "ArrayFire is a HPC general-purpose library targeting parallel and massively-parallel architectures such as CPUs, GPUs, etc.",
     "supports": "x64",
     "dependencies": [
-        "boost-compute",
-        "boost-functional",
+        "boost-math",
         "boost-stacktrace",
-        {
-            "name": "forge",
-            "version>=": "1.0.8",
-            "platform": "windows"
-        },
-        "freeimage",
-        {
-            "name": "fontconfig",
-            "platform": "!windows"
-        },
-        "glad",
-        "intel-mkl",
-        "spdlog"
+        "spdlog",
+        "freeimage"
     ],
     "overrides": [
-       {
-           "name": "fmt",
-           "version": "6.2.1"
-       },
+        {
+            "name": "fmt",
+            "version": "7.1.3"
+        },
         {
             "name": "spdlog",
-            "version": "1.6.1"
+            "version": "1.8.5"
         }
     ],
     "features": {
+        "forge": {
+            "description": "Build Forge",
+            "dependencies": [
+                {
+                    "name": "freetype",
+                    "default-features": false
+                },
+                {
+                    "name": "fontconfig",
+                    "platform": "!windows"
+                },
+                "glfw3",
+                "glad"
+            ]
+        },
         "cuda": {
             "description": "Build CUDA backend",
             "dependencies": [
@@ -43,10 +46,16 @@
         "opencl": {
             "description": "Build OpenCL backend",
             "dependencies": [
-                "boost-program-options",
+                "boost-compute",
                 "opencl"
             ]
+        },
+        "mkl": {
+            "description": "Build with MKL",
+            "dependencies": [
+                "intel-mkl"
+            ]
         }
     },
-    "builtin-baseline": "5568f110b509a9fd90711978a7cb76bae75bb092"
+    "builtin-baseline": "14e7bb4ae24616ec54ff6b2f6ef4e8659434ea44"
 }

From a8c5dea2058c587f09e5ee0a28e2f7c36622a8a7 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 May 2022 11:40:31 -0400
Subject: [PATCH 044/473] Fix trivial warnings in gcc 12

---
 src/api/c/data.cpp                           |  6 +++---
 src/backend/common/util.cpp                  |  8 ++++++++
 src/backend/common/util.hpp                  |  2 ++
 src/backend/cpu/platform.cpp                 | 10 ----------
 src/backend/opencl/jit/kernel_generators.hpp |  2 +-
 src/backend/opencl/platform.cpp              |  9 ---------
 6 files changed, 14 insertions(+), 23 deletions(-)

diff --git a/src/api/c/data.cpp b/src/api/c/data.cpp
index 6a82d419c5..f231c7b300 100644
--- a/src/api/c/data.cpp
+++ b/src/api/c/data.cpp
@@ -325,7 +325,7 @@ af_err af_diag_extract(af_array *out, const af_array in, const int num) {
 
         DIM_ASSERT(1, in_info.ndims() >= 2);
 
-        af_array result;
+        af_array result = nullptr;
         switch (type) {
             case f32: result = diagExtract<float>(in, num); break;
             case c32: result = diagExtract<cfloat>(in, num); break;
@@ -367,7 +367,7 @@ af_err af_lower(af_array *out, const af_array in, bool is_unit_diag) {
 
         if (info.ndims() == 0) { return af_retain_array(out, in); }
 
-        af_array res;
+        af_array res = nullptr;
         switch (type) {
             case f32: res = triangle<float>(in, false, is_unit_diag); break;
             case f64: res = triangle<double>(in, false, is_unit_diag); break;
@@ -396,7 +396,7 @@ af_err af_upper(af_array *out, const af_array in, bool is_unit_diag) {
 
         if (info.ndims() == 0) { return af_retain_array(out, in); }
 
-        af_array res;
+        af_array res = nullptr;
         switch (type) {
             case f32: res = triangle<float>(in, true, is_unit_diag); break;
             case f64: res = triangle<double>(in, true, is_unit_diag); break;
diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp
index c0d1d30cc9..ee579d67ac 100644
--- a/src/backend/common/util.cpp
+++ b/src/backend/common/util.cpp
@@ -35,6 +35,14 @@ using std::accumulate;
 using std::string;
 using std::vector;
 
+// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring/217605#217605
+// trim from start
+string& ltrim(string& s) {
+    s.erase(s.begin(),
+            find_if(s.begin(), s.end(), [](char c) { return !isspace(c); }));
+    return s;
+}
+
 string getEnvVar(const std::string& key) {
 #if defined(OS_WIN)
     DWORD bufSize =
diff --git a/src/backend/common/util.hpp b/src/backend/common/util.hpp
index bb197e2af3..c0f712ec0e 100644
--- a/src/backend/common/util.hpp
+++ b/src/backend/common/util.hpp
@@ -31,6 +31,8 @@ constexpr const char* JIT_KERNEL_CACHE_DIRECTORY_ENV_NAME =
 
 std::string getEnvVar(const std::string& key);
 
+std::string& ltrim(std::string& s);
+
 // Dump the kernel sources only if the environment variable is defined
 void saveKernel(const std::string& funcName, const std::string& jit_ker,
                 const std::string& ext);
diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index 179ff7a659..523737b07a 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -23,9 +23,7 @@
 
 using common::memory::MemoryManagerBase;
 using std::endl;
-using std::not1;
 using std::ostringstream;
-using std::ptr_fun;
 using std::stoi;
 using std::string;
 using std::unique_ptr;
@@ -45,14 +43,6 @@ static string get_system() {
 #endif
 }
 
-// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring/217605#217605
-// trim from start
-static inline string& ltrim(string& s) {
-    s.erase(s.begin(),
-            find_if(s.begin(), s.end(), not1(ptr_fun<int, int>(isspace))));
-    return s;
-}
-
 int getBackend() { return AF_BACKEND_CPU; }
 
 string getDeviceInfo() noexcept {
diff --git a/src/backend/opencl/jit/kernel_generators.hpp b/src/backend/opencl/jit/kernel_generators.hpp
index 54ebc69720..c2eb711c1b 100644
--- a/src/backend/opencl/jit/kernel_generators.hpp
+++ b/src/backend/opencl/jit/kernel_generators.hpp
@@ -28,7 +28,7 @@ void generateParamDeclaration(std::stringstream& kerStream, int id,
 }
 
 /// Calls the setArg function to set the arguments for a kernel call
-int setKernelArguments(
+inline int setKernelArguments(
     int start_id, bool is_linear,
     std::function<void(int id, const void* ptr, size_t arg_size)>& setArg,
     const std::shared_ptr<cl::Buffer>& ptr, const KParam& info) {
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 94706135ea..e2c4571995 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -60,7 +60,6 @@ using std::move;
 using std::once_flag;
 using std::ostringstream;
 using std::pair;
-using std::ptr_fun;
 using std::string;
 using std::to_string;
 using std::unique_ptr;
@@ -87,14 +86,6 @@ static string get_system() {
 
 int getBackend() { return AF_BACKEND_OPENCL; }
 
-// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring/217605#217605
-// trim from start
-static inline string& ltrim(string& s) {
-    s.erase(s.begin(),
-            find_if(s.begin(), s.end(), not1(ptr_fun<int, int>(isspace))));
-    return s;
-}
-
 bool verify_present(const string& pname, const string ref) {
     auto iter =
         search(begin(pname), end(pname), begin(ref), end(ref),

From 077a52a7e04fce8d9946f43c365ef6cc82a1e248 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 May 2022 11:44:10 -0400
Subject: [PATCH 045/473] Add reset function to unique_handle

---
 src/backend/common/unique_handle.hpp | 12 +++++++++---
 src/backend/cuda/platform.cpp        |  8 ++++----
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/backend/common/unique_handle.hpp b/src/backend/common/unique_handle.hpp
index d8da5c7d67..52d0acfeda 100644
--- a/src/backend/common/unique_handle.hpp
+++ b/src/backend/common/unique_handle.hpp
@@ -50,9 +50,15 @@ class unique_handle {
     explicit constexpr unique_handle(T handle) noexcept : handle_(handle){};
 
     /// \brief Deletes the handle if created.
-    ~unique_handle() noexcept {
-        if (handle_) { ResourceHandler<T>::destroyHandle(handle_); }
-    };
+    ~unique_handle() noexcept { reset(); }
+
+    /// \brief Deletes the handle if created.
+    void reset() noexcept {
+        if (handle_) {
+            ResourceHandler<T>::destroyHandle(handle_);
+            handle_ = 0;
+        }
+    }
 
     unique_handle(const unique_handle &other) noexcept = delete;
     unique_handle &operator=(unique_handle &other) noexcept = delete;
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index ab94cf298f..0e639ec62d 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -177,12 +177,12 @@ DeviceManager::~DeviceManager() {
         // handles of all devices
         for (int i = 0; i < nDevices; ++i) {
             setDevice(i);
-            delete cusolverManager(i);
-            delete cusparseManager(i);
+            cusolverManager(i)->reset();
+            cusparseManager(i)->reset();
             cufftManager(i).reset();
-            delete cublasManager(i);
+            cublasManager(i)->reset();
 #ifdef WITH_CUDNN
-            delete nnManager(i);
+            nnManager(i)->reset();
 #endif
         }
     } catch (const AfError &err) {

From 92badad9e35a9bbc460caac4643607cb3a9fbd28 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 May 2022 11:46:58 -0400
Subject: [PATCH 046/473] Update license date

---
 LICENSE | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/LICENSE b/LICENSE
index f7b9cfdcf7..8f4c645ca1 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2014-2018, ArrayFire
+Copyright (c) 2014-2022, ArrayFire
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

From a84fa2ea7aa466ec4f2f9ddf8e9195b5ed27c362 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 May 2022 11:48:36 -0400
Subject: [PATCH 047/473] Fix NSIS template, MaybeSelectionChanged should be in
 quotes

---
 CMakeModules/nsis/NSIS.template.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeModules/nsis/NSIS.template.in b/CMakeModules/nsis/NSIS.template.in
index f45b01127a..bc3a44f233 100644
--- a/CMakeModules/nsis/NSIS.template.in
+++ b/CMakeModules/nsis/NSIS.template.in
@@ -815,7 +815,7 @@ SectionEnd
 ;--------------------------------
 ; Component dependencies
 Function .onSelChange
-  !insertmacro SectionList MaybeSelectionChanged
+  !insertmacro SectionList "MaybeSelectionChanged"
 FunctionEnd
 
 ;--------------------------------

From 252767fd2b5316bce3c34d466394f91e27a1a59d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 May 2022 12:49:25 -0400
Subject: [PATCH 048/473] Several CPack changes to improve NSIS and DEB
 installers

---
 CMakeLists.txt                        |   2 +-
 CMakeModules/CPackConfig.cmake        | 310 ++------------
 CMakeModules/CPackProjectConfig.cmake | 560 ++++++++++++++++++++++++++
 CMakeModules/debian/postinst          |   9 +
 4 files changed, 607 insertions(+), 274 deletions(-)
 create mode 100644 CMakeModules/CPackProjectConfig.cmake
 create mode 100644 CMakeModules/debian/postinst

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8dfee21544..537ae9a736 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -415,7 +415,7 @@ foreach(backend CPU CUDA OpenCL Unified)
     install(EXPORT ArrayFire${backend}Targets
             NAMESPACE ArrayFire::
             DESTINATION ${AF_INSTALL_CMAKE_DIR}
-            COMPONENT ${lower_backend})
+            COMPONENT ${lower_backend}_dev)
 
     export( EXPORT ArrayFire${backend}Targets
             NAMESPACE ArrayFire::
diff --git a/CMakeModules/CPackConfig.cmake b/CMakeModules/CPackConfig.cmake
index 07d1d46962..d073527089 100644
--- a/CMakeModules/CPackConfig.cmake
+++ b/CMakeModules/CPackConfig.cmake
@@ -10,10 +10,10 @@ cmake_minimum_required(VERSION 3.5)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/CMakeModules/nsis")
 
 include(Version)
-include(CPackIFW)
+
+set(CPACK_THREADS 8)
 
 set(CPACK_GENERATOR "STGZ;TGZ" CACHE STRING "STGZ;TGZ;DEB;RPM;productbuild")
-set_property(CACHE CPACK_GENERATOR PROPERTY STRINGS STGZ DEB RPM productbuild)
 mark_as_advanced(CPACK_GENERATOR)
 
 set(VENDOR_NAME "ArrayFire")
@@ -42,7 +42,7 @@ set(CPACK_PREFIX_DIR ${CMAKE_INSTALL_PREFIX})
 set(CPACK_PACKAGE_NAME "${LIBRARY_NAME}")
 set(CPACK_PACKAGE_VENDOR "${VENDOR_NAME}")
 set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY ${LIBRARY_NAME})
-set(CPACK_PACKAGE_CONTACT "ArrayFire Development Group <technical@arrayfire.com>")
+set(CPACK_PACKAGE_CONTACT "ArrayFire <technical@arrayfire.com>")
 set(MY_CPACK_PACKAGE_ICON "${CMAKE_SOURCE_DIR}/assets/${APP_LOW_NAME}.ico")
 
 file(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/assets/" NATIVE_ASSETS_PATH)
@@ -55,14 +55,38 @@ set(CPACK_PACKAGE_VERSION_PATCH "${ArrayFire_VERSION_PATCH}")
 
 set(CPACK_PACKAGE_INSTALL_DIRECTORY "${LIBRARY_NAME}")
 
-set(inst_pkg_name ${APP_LOW_NAME})
-set(inst_pkg_hash "")
-if (WIN32)
-  set(inst_pkg_name ${CPACK_PACKAGE_NAME})
-  set(inst_pkg_hash "-${GIT_COMMIT_HASH}")
-endif ()
-
-set(CPACK_PACKAGE_FILE_NAME "${inst_pkg_name}${inst_pkg_hash}")
+set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT)
+set(CPACK_DEB_COMPONENT_INSTALL ON)
+set(CPACK_DEBIAN_DEBUGINFO_PACKAGE OFF)
+set(CPACK_DEBIAN_PACKAGE_DEBUG ON)
+set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS ON)
+set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS_POLICY ">=")
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE http://www.arrayfire.com)
+set(CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION TRUE)
+set(CPACK_DEBIAN_COMPRESSION_TYPE xz)
+set(CPACK_DEBIAN_DEBUGINFO_PACKAGE ON)
+
+# Creates a variable from a ArrayFire variable so that it can be passed
+# into cpack project file. This is done by prepending CPACK_ before the
+# variable name
+macro(to_cpack_variable variable)
+  set(CPACK_${variable} ${${variable}})
+endmacro()
+
+to_cpack_variable(AF_COMPUTE_LIBRARY)
+to_cpack_variable(ArrayFire_SOURCE_DIR)
+to_cpack_variable(ArrayFire_BINARY_DIR)
+to_cpack_variable(CUDA_VERSION_MAJOR)
+to_cpack_variable(CUDA_VERSION_MINOR)
+
+# Create a arrayfire component so that Debian package has a top level
+# package that installs all the backends. This package needs to have
+# some files associated with it so that it doesn't get deleted by
+# APT after its installed.
+file(WRITE ${ArrayFire_BINARY_DIR}/arrayfire_version.txt ${ArrayFire_VERSION})
+install(FILES ${ArrayFire_BINARY_DIR}/arrayfire_version.txt
+	DESTINATION ${CMAKE_INSTALL_SYSCONFDIR}
+  COMPONENT arrayfire)
 
 # Platform specific settings for CPACK generators
 # - OSX specific
@@ -107,6 +131,7 @@ elseif(WIN32)
   set(CPACK_NSIS_HELP_LINK "${SITE_URL}")
   set(CPACK_NSIS_URL_INFO_ABOUT "${SITE_URL}")
   set(CPACK_NSIS_INSTALLED_ICON_NAME "${MY_CPACK_PACKAGE_ICON}")
+  set(CPACK_NSIS_COMPRESSOR "lzma")
   if (CMAKE_CL_64)
     set(CPACK_NSIS_INSTALL_ROOT "$PROGRAMFILES64")
   else (CMAKE_CL_64)
@@ -117,267 +142,6 @@ else()
   set(CPACK_RESOURCE_FILE_README "${ArrayFire_SOURCE_DIR}/README.md")
 endif()
 
-# Set the default components installed in the package
-get_cmake_property(CPACK_COMPONENTS_ALL COMPONENTS)
-
-include(CPackComponent)
-
-cpack_add_install_type(All DISPLAY_NAME "All Components")
-cpack_add_install_type(Development DISPLAY_NAME "Development")
-cpack_add_install_type(Extra DISPLAY_NAME "Extra")
-cpack_add_install_type(Runtime DISPLAY_NAME "Runtime")
-
-cpack_add_component_group(backends
-  DISPLAY_NAME "ArrayFire"
-  DESCRIPTION "ArrayFire backend libraries"
-  EXPANDED)
-cpack_add_component_group(cpu-backend
-  DISPLAY_NAME "CPU backend"
-  DESCRIPTION "Libraries and dependencies of the CPU backend."
-  PARENT_GROUP backends)
-cpack_add_component_group(cuda-backend
-  DISPLAY_NAME "CUDA backend"
-  DESCRIPTION "Libraries and dependencies of the CUDA backend."
-  PARENT_GROUP backends)
-cpack_add_component_group(opencl-backend
-  DISPLAY_NAME "OpenCL backend"
-  DESCRIPTION "Libraries and dependencies of the OpenCL backend."
-  PARENT_GROUP backends)
-
-set(PACKAGE_MKL_DEPS OFF)
-
-if ((USE_CPU_MKL OR USE_OPENCL_MKL) AND TARGET MKL::Shared)
-  set(PACKAGE_MKL_DEPS ON)
-  cpack_add_component(mkl_dependencies
-    DISPLAY_NAME "Intel MKL"
-	DESCRIPTION "Intel Math Kernel Libraries for FFTW, BLAS, and LAPACK routines."
-	GROUP backends
-    INSTALL_TYPES All Development Runtime)
-endif ()
-
-cpack_add_component(common_backend_dependencies
-  DISPLAY_NAME "Dependencies"
-  DESCRIPTION "Libraries commonly required by all ArrayFire backends."
-  GROUP backends
-  INSTALL_TYPES All Development Runtime)
-
-cpack_add_component(opencl_dependencies
-  DISPLAY_NAME "OpenCL Dependencies"
-  DESCRIPTION "Libraries required by the OpenCL backend."
-  GROUP opencl-backend
-  INSTALL_TYPES All Development Runtime)
-if (NOT APPLE) #TODO(pradeep) Remove check after OSX support addition
-  cpack_add_component(afopencl_debug_symbols
-    DISPLAY_NAME "OpenCL Backend Debug Symbols"
-    DESCRIPTION "File containing debug symbols for afopencl dll/so/dylib file"
-    GROUP opencl-backend
-    DISABLED
-    INSTALL_TYPES Development)
-endif ()
-
-cpack_add_component(cuda_dependencies
-  DISPLAY_NAME "CUDA Dependencies"
-  DESCRIPTION "CUDA runtime and libraries required by the CUDA backend."
-  GROUP cuda-backend
-  INSTALL_TYPES All Development Runtime)
-if (NOT APPLE) #TODO(pradeep) Remove check after OSX support addition
-  cpack_add_component(afcuda_debug_symbols
-    DISPLAY_NAME "CUDA Backend Debug Symbols"
-    DESCRIPTION "File containing debug symbols for afcuda dll/so/dylib file"
-    GROUP cuda-backend
-    DISABLED
-    INSTALL_TYPES Development)
-endif ()
-
-if (NOT APPLE) #TODO(pradeep) Remove check after OSX support addition
-  cpack_add_component(afcpu_debug_symbols
-    DISPLAY_NAME "CPU Backend Debug Symbols"
-    DESCRIPTION "File containing debug symbols for afcpu dll/so/dylib file"
-    GROUP cpu-backend
-    DISABLED
-    INSTALL_TYPES Development)
-endif ()
-
-cpack_add_component(cuda
-  DISPLAY_NAME "CUDA Backend"
-  DESCRIPTION "The CUDA backend allows you to run ArrayFire code on CUDA-enabled GPUs. Verify that you have the CUDA toolkit installed or install the CUDA dependencies component."
-  GROUP cuda-backend
-  DEPENDS common_backend_dependencies cuda_dependencies
-  INSTALL_TYPES All Development Runtime)
-
-list(APPEND cpu_deps_comps common_backend_dependencies)
-list(APPEND ocl_deps_comps common_backend_dependencies)
-
-if (NOT APPLE)
-  list(APPEND ocl_deps_comps opencl_dependencies)
-endif ()
-
-if (PACKAGE_MKL_DEPS)
-  list(APPEND cpu_deps_comps mkl_dependencies)
-  list(APPEND ocl_deps_comps mkl_dependencies)
-endif ()
-
-cpack_add_component(cpu
-  DISPLAY_NAME "CPU Backend"
-  DESCRIPTION "The CPU backend allows you to run ArrayFire code on your CPU."
-  GROUP cpu-backend
-  DEPENDS ${cpu_deps_comps}
-  INSTALL_TYPES All Development Runtime)
-
-cpack_add_component(opencl
-  DISPLAY_NAME "OpenCL Backend"
-  DESCRIPTION "The OpenCL backend allows you to run ArrayFire code on OpenCL-capable GPUs. Note: ArrayFire does not currently support OpenCL for Intel CPUs on OSX."
-  GROUP opencl-backend
-  DEPENDS ${ocl_deps_comps}
-  INSTALL_TYPES All Development Runtime)
-
-if (NOT APPLE) #TODO(pradeep) Remove check after OSX support addition
-  cpack_add_component(af_debug_symbols
-    DISPLAY_NAME "Unified Backend Debug Symbols"
-    DESCRIPTION "File containing debug symbols for af dll/so/dylib file"
-    GROUP backends
-    DISABLED
-    INSTALL_TYPES Development)
-endif ()
-cpack_add_component(unified
-  DISPLAY_NAME "Unified Backend"
-  DESCRIPTION "The Unified backend allows you to choose between any of the installed backends (CUDA, OpenCL, or CPU) at runtime."
-  GROUP backends
-  INSTALL_TYPES All Development Runtime)
-
-cpack_add_component(headers
-  DISPLAY_NAME "C/C++ Headers"
-  DESCRIPTION "Headers for the ArrayFire libraries."
-  GROUP backends
-  INSTALL_TYPES All Development)
-cpack_add_component(cmake
-  DISPLAY_NAME "CMake Support"
-  DESCRIPTION "Configuration files to use ArrayFire using CMake."
-  INSTALL_TYPES All Development)
-cpack_add_component(documentation
-  DISPLAY_NAME "Documentation"
-  DESCRIPTION "ArrayFire html documentation"
-  INSTALL_TYPES All Extra)
-cpack_add_component(examples
-  DISPLAY_NAME "ArrayFire Examples"
-  DESCRIPTION "Various examples using ArrayFire."
-  INSTALL_TYPES All Extra)
-cpack_add_component(licenses
-  DISPLAY_NAME "Licenses"
-  DESCRIPTION "License files for ArrayFire and its upstream libraries."
-  REQUIRED)
-
-if (AF_INSTALL_FORGE_DEV)
-  cpack_add_component(forge
-    DISPLAY_NAME "Forge"
-    DESCRIPTION "High Performance Visualization Library"
-    INSTALL_TYPES Extra)
-endif ()
-
-##
-# IFW CPACK generator
-# Uses Qt installer framework, cross platform installer generator.
-# Uniform installer GUI on all major desktop platforms: Windows, OSX & Linux.
-##
-set(CPACK_IFW_PACKAGE_TITLE "${CPACK_PACKAGE_NAME}")
-set(CPACK_IFW_PACKAGE_PUBLISHER "${CPACK_PACKAGE_VENDOR}")
-set(CPACK_IFW_PRODUCT_URL "${SITE_URL}")
-set(CPACK_IFW_PACKAGE_ICON "${MY_CPACK_PACKAGE_ICON}")
-set(CPACK_IFW_PACKAGE_WINDOW_ICON "${CMAKE_SOURCE_DIR}/assets/${APP_LOW_NAME}_icon.png")
-set(CPACK_IFW_PACKAGE_WIZARD_DEFAULT_WIDTH 640)
-set(CPACK_IFW_PACKAGE_WIZARD_DEFAULT_HEIGHT 480)
-if (WIN32)
-    set(CPACK_IFW_ADMIN_TARGET_DIRECTORY "@ApplicationsDirX64@/${CPACK_PACKAGE_INSTALL_DIRECTORY}")
-else ()
-    set(CPACK_IFW_ADMIN_TARGET_DIRECTORY "/opt/${CPACK_PACKAGE_INSTALL_DIRECTORY}")
-endif ()
-
-get_native_path(zlib_lic_path "${CMAKE_SOURCE_DIR}/LICENSES/zlib-libpng License.txt")
-get_native_path(boost_lic_path "${CMAKE_SOURCE_DIR}/LICENSES/Boost Software License.txt")
-get_native_path(fimg_lic_path "${CMAKE_SOURCE_DIR}/LICENSES/FreeImage Public License.txt")
-get_native_path(apache_lic_path "${CMAKE_SOURCE_DIR}/LICENSES/Apache-2.0.txt")
-get_native_path(sift_lic_path "${CMAKE_SOURCE_DIR}/LICENSES/OpenSIFT License.txt")
-get_native_path(bsd3_lic_path "${CMAKE_SOURCE_DIR}/LICENSES/BSD 3-Clause.txt")
-get_native_path(issl_lic_path "${CMAKE_SOURCE_DIR}/LICENSES/ISSL License.txt")
-
-cpack_ifw_configure_component_group(backends)
-cpack_ifw_configure_component_group(cpu-backend)
-cpack_ifw_configure_component_group(cuda-backend)
-cpack_ifw_configure_component_group(opencl-backend)
-if (PACKAGE_MKL_DEPS)
-  cpack_ifw_configure_component(mkl_dependencies)
-endif ()
-if (NOT APPLE)
-  cpack_ifw_configure_component(opencl_dependencies)
-endif ()
-cpack_ifw_configure_component(common_backend_dependencies)
-cpack_ifw_configure_component(cuda_dependencies)
-cpack_ifw_configure_component(cpu)
-cpack_ifw_configure_component(cuda)
-cpack_ifw_configure_component(opencl)
-cpack_ifw_configure_component(unified)
-cpack_ifw_configure_component(headers)
-cpack_ifw_configure_component(cmake)
-cpack_ifw_configure_component(documentation)
-cpack_ifw_configure_component(examples)
-cpack_ifw_configure_component(licenses FORCED_INSTALLATION
-  LICENSES "GLFW" ${zlib_lic_path} "FreeImage" ${fimg_lic_path}
-  "Boost" ${boost_lic_path} "CLBlast, clFFT" ${apache_lic_path} "SIFT" ${sift_lic_path}
-  "BSD3" ${bsd3_lic_path} "Intel MKL" ${issl_lic_path}
-)
-if (AF_INSTALL_FORGE_DEV)
-  cpack_ifw_configure_component(forge)
-endif ()
-
-##
-# Debian package
-##
-set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT)
-set(CPACK_DEB_COMPONENT_INSTALL ON)
-#set(CMAKE_INSTALL_RPATH /usr/lib;${ArrayFire_BUILD_DIR}/third_party/forge/lib)
-#set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
-set(CPACK_DEBIAN_PACKAGE_HOMEPAGE http://www.arrayfire.com)
-
-##
-# RPM package
-##
-set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
-set(CPACK_RPM_PACKAGE_AUTOREQPROV " no")
-set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries")
-set(CPACK_RPM_PACKAGE_LICENSE "BSD")
-set(CPACK_RPM_PACKAGE_URL "${SITE_URL}")
-if(AF_BUILD_FORGE)
-    set(CPACK_RPM_PACKAGE_SUGGESTS "fontconfig-devel, libX11, libXrandr, libXinerama, libXxf86vm, libXcursor, mesa-libGL-devel")
-endif()
-
-##
-# Source package
-##
-set(CPACK_SOURCE_GENERATOR "TGZ")
-set(CPACK_SOURCE_PACKAGE_FILE_NAME
-    ${CPACK_PACKAGE_NAME}_src_${GIT_COMMIT_HASH}_${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR})
-set(CPACK_SOURCE_IGNORE_FILES
-    "/build"
-    "CMakeFiles"
-    "/\\\\.dir"
-    "/\\\\.git"
-    "/\\\\.gitignore$"
-    ".*~$"
-    "\\\\.bak$"
-    "\\\\.swp$"
-    "\\\\.orig$"
-    "/\\\\.DS_Store$"
-    "/Thumbs\\\\.db"
-    "/CMakeLists.txt.user$"
-    ${CPACK_SOURCE_IGNORE_FILES})
-# Ignore build directories that may be in the source tree
-file(GLOB_RECURSE CACHES "${CMAKE_SOURCE_DIR}/CMakeCache.txt")
-
-if (WIN32)
-  # Configure file with custom definitions for NSIS.
-  configure_file(
-    ${PROJECT_SOURCE_DIR}/CMakeModules/nsis/NSIS.definitions.nsh.in
-    ${CMAKE_CURRENT_BINARY_DIR}/NSIS.definitions.nsh)
-endif ()
+set(CPACK_PROJECT_CONFIG_FILE "${CMAKE_SOURCE_DIR}/CMakeModules/CPackProjectConfig.cmake")
 
 include(CPack)
diff --git a/CMakeModules/CPackProjectConfig.cmake b/CMakeModules/CPackProjectConfig.cmake
new file mode 100644
index 0000000000..6cd6e20088
--- /dev/null
+++ b/CMakeModules/CPackProjectConfig.cmake
@@ -0,0 +1,560 @@
+
+include(CPackIFW)
+include(CPackComponent)
+
+# Only install the components created using the af_component macro
+set(CPACK_COMPONENTS_ALL "")
+
+# This is necessary if you don't have a cuda driver installed on your system
+# but you are still building the cuda package. You need the libcuda.so library
+# which is installed by the driver. This tell the dpkg-shlibs to ignore
+# this library because it is a private library
+set (CPACK_DEBIAN_PACKAGE_SHLIBDEPS_PRIVATE_DIRS
+  "/usr/local/cuda-${CPACK_CUDA_VERSION_MAJOR}.${CPACK_CUDA_VERSION_MINOR}/lib64/stubs")
+
+
+# Create an ArrayFire component with a set of properties for each package manager
+# This function sets all the variables for each component in ArrayFire.
+#
+# ``COMPONENT``
+# The name of the ArrayFire component used in the install(XXX) commands
+#
+# ``DISPLAY_NAME``
+# The name that will appear in the GUI installers for this component
+#
+# ``SUMMARY``
+# A short one line summary of the package
+#
+# ``DESCRIPTION``
+# A longer description of the package
+#
+# ``GROUP``
+# Used to combine packages in GUI installers. Ignored in DEB and RPM installers
+#
+# ``DEB_PACKAGE_NAME``
+# Name of the package for the DEB installers. This is the first component of the
+# file name.
+#
+# ``DEB_PROVIDES``
+# The virtual packages provided by the deb package. This is a higher level name
+# of the file that can be used across version numbers. also includes the version
+# information about the package
+#
+# ``DEB_REPLACES``
+# The packages and virtual packages this will replace. Used if there is a package
+# that is installed as part of the base debian installation
+#
+# ``REQUIRES``
+# The components required for the GUI installers
+#
+# ``OPTIONAL``
+# Optional packages that this component can use.
+#
+# ``INSTALL_TYPE``
+# A group of components that will be selected in GUI installers from a drop down
+#
+# ``DEB_REQUIRES``
+# Set of packages required by the debian package. This is slighly different from
+# REQUIRES because it also takes into account external dependencies that can be
+# installed by apt
+#
+# ``DEB_OPTIONAL``
+# Same as OPTIONAL but for debian packages
+#
+# ``DEB_RECOMMENDS``
+# Packages that should be installed but are not required. These packages will
+# be installed by default but if removed will not also delete this package
+#
+# ``HIDDEN``
+# If set, the package will not appear in the GUI installers like NSIS. Usually
+# components that install dependencies
+macro(af_component)
+  cmake_parse_arguments(RC
+    "HIDDEN;DISABLED;DEB_USE_SHLIBDEPS;DEB_ADD_POSTINST"
+    "COMPONENT;DISPLAY_NAME;SUMMARY;DESCRIPTION;GROUP;DEB_PACKAGE_NAME;DEB_PROVIDES;DEB_REPLACES"
+    "REQUIRES;OPTIONAL;INSTALL_TYPES;DEB_REQUIRES;DEB_OPTIONAL;DEB_RECOMMENDS" ${ARGN})
+
+  list(APPEND CPACK_COMPONENTS_ALL ${RC_COMPONENT})
+
+  string(TOUPPER ${RC_COMPONENT} COMPONENT_UPPER)
+  string(REPLACE ";" ", " DEB_REQ "${RC_DEB_REQUIRES}")
+  string(REPLACE ";" ", " DEB_REC "${RC_DEB_RECOMMENDS}")
+  string(REPLACE ";" ", " DEB_OPT "${RC_DEB_OPTIONAL}")
+  string(REPLACE ";" ", " DEB_PROVIDES "${RC_DEB_PROVIDES}")
+
+  if(CPACK_GENERATOR MATCHES "DEB")
+    cpack_add_component(${RC_COMPONENT}
+      DISPLAY_NAME "${RC_DISPLAY_NAME}"
+      INSTALL_TYPES ${RC_INSTALL_TYPES}
+      DESCRIPTION ${RC_DESCRIPTION})
+
+    if(RC_DEB_RECOMMENDS)
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_RECOMMENDS ${DEB_REC})
+    endif()
+
+    if(RC_DEB_PACKAGE_NAME)
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_NAME "${RC_DEB_PACKAGE_NAME}")
+    endif()
+
+    set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_SUGGESTS ${DEB_OPT})
+
+    if(RC_DEB_REQUIRES)
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_DEPENDS "${DEB_REQ}")
+    endif()
+
+    if(RC_DEB_USE_SHLIBDEPS)
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_SHLIBDEPS ON)
+    else()
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_SHLIBDEPS OFF)
+    endif()
+
+    if(RC_DEB_PROVIDES)
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_PROVIDES ${DEB_PROVIDES})
+    endif()
+
+    if(RC_DEB_REPLACES)
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_REPLACES ${RC_DEB_REPLACES})
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_CONFLICTS ${RC_DEB_REPLACES})
+    endif()
+
+    if(RC_DEB_ADD_POSTINST)
+      configure_file(
+        "${CPACK_ArrayFire_SOURCE_DIR}/CMakeModules/debian/postinst"
+        "${CPACK_ArrayFire_BINARY_DIR}/cpack/${COMPONENT_UPPER}/postinst")
+
+      set(CPACK_DEBIAN_${COMPONENT_UPPER}_PACKAGE_CONTROL_EXTRA
+        "${CPACK_ArrayFire_BINARY_DIR}/cpack/${COMPONENT_UPPER}/postinst")
+    endif()
+  else()
+    cpack_add_component(${RC_COMPONENT}
+      DISPLAY_NAME "${RC_DISPLAY_NAME}"
+      DEPENDS ${RC_REQUIRES}
+      GROUP ${RC_GROUP}
+      INSTALL_TYPES ${RC_INSTALL_TYPES}
+      DESCRIPTION ${RC_DESCRIPTION})
+  endif()
+
+  set(CPACK_COMPONENT_${RC_COMPONENT}_DESCRIPTION_SUMMARY ${RC_SUMMARY})
+  set(CPACK_COMPONENT_${COMPONENT_UPPER}_DESCRIPTION ${RC_DESCRIPTION})
+
+  set(CPACK_COMPONENT_${COMPONENT_UPPER}_HIDDEN ${RC_HIDDEN})
+  set(CPACK_COMPONENT_${COMPONENT_UPPER}_DISABLED ${RC_DISABLED})
+
+  # Does not work with RPM for some reason using
+  # CPACK_RPM_${COMPONENT_UPPER}_PACKAGE_REQUIRES  instead
+
+endmacro()
+
+cpack_add_install_type(All DISPLAY_NAME "All Components")
+cpack_add_install_type(Development DISPLAY_NAME "Development")
+cpack_add_install_type(Runtime DISPLAY_NAME "Runtime")
+
+# Groups on debian packages will combine all the packages into one
+# debian component
+if(NOT CPACK_GENERATOR MATCHES "DEB")
+  cpack_add_component_group(afruntime
+    DISPLAY_NAME "ArrayFire Runtime"
+    DESCRIPTION "ArrayFire runtime libraries")
+
+  cpack_add_component_group(afdevelopment
+    DISPLAY_NAME "ArrayFire Development"
+    DESCRIPTION "ArrayFire development files including headers and configuration files"
+    EXPANDED)
+
+  cpack_add_component_group(debug
+    DISPLAY_NAME "ArrayFire Debug Symbols"
+    DESCRIPTION "ArrayFire Debug symbols")
+endif()
+
+set(arrayfire_cuda_runtime_name "CUDA Runtime(${CPACK_CUDA_VERSION_MAJOR}.${CPACK_CUDA_VERSION_MINOR})")
+set(arrayfire_cuda_dev_name "CUDA Dev")
+
+if(CPACK_GENERATOR MATCHES "DEB")
+  af_component(
+    COMPONENT arrayfire
+    REQUIRES cpu_dev cuda_dev opencl_dev examples documentation
+    SUMMARY  "ArrayFire high performance library"
+    DESCRIPTION  "ArrayFire
+ArrayFire is a general-purpose library that simplifies software
+development that targets parallel and massively-parallel architectures
+including CPUs, GPUs, and other hardware acceleration devices."
+
+    DEB_PACKAGE_NAME arrayfire
+    DEB_REQUIRES arrayfire-cpu3-dev
+                 arrayfire-headers
+
+    DEB_RECOMMENDS arrayfire-cuda3-dev
+                   arrayfire-opencl3-dev
+                   arrayfire-unified3-dev
+                   arrayfire-examples
+                   arrayfire-cmake
+                   arrayfire-doc
+  )
+endif()
+
+
+list(APPEND cpu_deps_comps common_backend_dependencies)
+list(APPEND ocl_deps_comps common_backend_dependencies)
+
+if (NOT APPLE)
+  list(APPEND ocl_deps_comps opencl_dependencies)
+endif ()
+
+set(PACKAGE_MKL_DEPS OFF)
+
+if(CPACK_CUDA_VERSION_MAJOR STREQUAL "10" AND CPACK_GENERATOR MATCHES "DEB")
+  set(deb_cuda_runtime_requirements "libcublas${CPACK_CUDA_VERSION_MAJOR}")
+elseif(CPACK_CUDA_VERSION_MAJOR STREQUAL "11" AND CPACK_GENERATOR MATCHES "DEB")
+  set(deb_cuda_runtime_requirements "libcublas-${CPACK_CUDA_VERSION_MAJOR}-${CPACK_CUDA_VERSION_MINOR}")
+elseif(CPACK_GENERATOR MATCHES "DEB")
+  message(FATAL_ERROR "THIS CUDA VERSION NOT ADDRESSED FOR DEBIN PACKAGES")
+endif()
+
+if (CPACK_AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL")
+  set(PACKAGE_MKL_DEPS ON)
+  if(NOT CPACK_GENERATOR STREQUAL "DEB")
+    af_component(
+      COMPONENT mkl_dependencies
+      DISPLAY_NAME "Intel MKL Libraries"
+            DESCRIPTION "Intel Math Kernel Libraries for FFTW, BLAS, and LAPACK routines."
+      HIDDEN
+      INSTALL_TYPES All Runtime)
+    list(APPEND cpu_deps_comps mkl_dependencies)
+    list(APPEND ocl_deps_comps mkl_dependencies)
+  endif()
+  set(deb_opencl_runtime_package_name arrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR}-mkl)
+  set(deb_opencl_runtime_requirements "intel-mkl-core-rt-2020.0-166, intel-mkl-gnu-rt-2020.0-166")
+  set(deb_cpu_runtime_package_name arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR}-mkl)
+  set(deb_cpu_runtime_requirements "intel-mkl-core-rt-2020.0-166, intel-mkl-gnu-rt-2020.0-166")
+else()
+  # OpenCL and CPU runtime dependencies are detected using
+  # SHLIBDEPS
+  set(deb_opencl_runtime_package_name arrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR}-openblas)
+  set(deb_opencl_runtime_requirements "")
+  set(deb_cpu_runtime_package_name arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR}-openblas)
+  set(deb_cpu_runtime_requirements "")
+endif ()
+
+af_component(
+  COMPONENT cpu
+  DISPLAY_NAME "CPU Runtime"
+  SUMMARY "ArrayFire CPU backend shared libraries"
+  DESCRIPTION "ArrayFire CPU backend shared libraries"
+  OPTIONAL forge
+  GROUP afruntime
+  REQUIRES ${cpu_deps_comps} licenses
+  INSTALL_TYPES All Runtime
+
+  DEB_PACKAGE_NAME ${deb_cpu_runtime_package_name}
+  DEB_REQUIRES ${deb_cpu_runtime_requirements}
+  DEB_PROVIDES "arrayfire-cpu (= ${CPACK_PACKAGE_VERSION}), arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION}), libarrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-cpu, arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_USE_SHLIBDEPS
+  DEB_ADD_POSTINST
+  DEB_OPTIONAL forge libfreeimage3
+)
+
+af_component(
+  COMPONENT cpu_dev
+  DISPLAY_NAME "CPU Dev"
+  SUMMARY  "ArrayFire CPU backend development files"
+  DESCRIPTION  "ArrayFire CPU backend development files"
+  REQUIRES cpu headers cmake
+  GROUP afdevelopment
+  INSTALL_TYPES All Development
+
+  DEB_PACKAGE_NAME arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR}-dev
+  DEB_PROVIDES "arrayfire-cpu-dev (= ${CPACK_PACKAGE_VERSION}), arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR}-dev (= ${CPACK_PACKAGE_VERSION}), libarrayfire-cpu-dev (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-cpu-dev (<< ${CPACK_PACKAGE_VERSION}), arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR}-dev (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-cpu3-dev (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_REQUIRES "arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR}-openblas (>= ${CPACK_PACKAGE_VERSION}) | arrayfire-cpu${CPACK_PACKAGE_VERSION_MAJOR}-mkl (>= ${CPACK_PACKAGE_VERSION}), arrayfire-headers (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_RECOMMENDS "arrayfire-cmake (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_OPTIONAL "cmake (>= 3.0)"
+)
+
+af_component(
+  COMPONENT cuda
+  DISPLAY_NAME "${arrayfire_cuda_runtime_name}"
+  SUMMARY "ArrayFire CUDA backend shared libraries"
+  DESCRIPTION "ArrayFire CUDA backend shared libraries"
+  OPTIONAL forge
+  REQUIRES common_backend_dependencies cuda_dependencies licenses
+  GROUP afruntime
+  INSTALL_TYPES All Runtime
+
+  DEB_PACKAGE_NAME arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR}-cuda-${CPACK_CUDA_VERSION_MAJOR}-${CPACK_CUDA_VERSION_MINOR}
+  DEB_REQUIRES ${deb_cuda_runtime_requirements}
+  DEB_ADD_POSTINST
+  DEB_USE_SHLIBDEPS
+  DEB_PROVIDES "arrayfire-cuda (= ${CPACK_PACKAGE_VERSION}), arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION}), libarrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-cuda (<< ${CPACK_PACKAGE_VERSION}), arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_OPTIONAL libcudnn8 forge libfreeimage3
+)
+
+af_component(
+  COMPONENT cuda_dev
+  DISPLAY_NAME "${arrayfire_cuda_dev_name}"
+  SUMMARY  "ArrayFire CUDA backend development files"
+  DESCRIPTION  "ArrayFire CUDA backend development files"
+  REQUIRES cuda headers cmake
+  GROUP afdevelopment
+  INSTALL_TYPES All Development
+
+  DEB_PACKAGE_NAME arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR}-dev
+  DEB_PROVIDES "arrayfire-cuda-dev (= ${CPACK_PACKAGE_VERSION}), arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR}-dev (= ${CPACK_PACKAGE_VERSION}), libarrayfire-cuda-dev (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-cuda-dev (<< ${CPACK_PACKAGE_VERSION}), arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR}-dev (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_REQUIRES "arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (>= ${CPACK_PACKAGE_VERSION}), arrayfire-headers (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_RECOMMENDS "arrayfire-cmake (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_OPTIONAL "cmake (>= 3.0)"
+)
+
+af_component(
+  COMPONENT opencl
+  DISPLAY_NAME "OpenCL Runtime"
+  SUMMARY "ArrayFire OpenCL backend shared libraries"
+  DESCRIPTION "ArrayFire OpenCL backend shared libraries"
+  REQUIRES ${opencl_deps_comps} licenses
+  OPTIONAL forge
+  GROUP afruntime
+  INSTALL_TYPES All Runtime
+
+  DEB_PACKAGE_NAME ${deb_opencl_runtime_package_name}
+  DEB_PROVIDES "arrayfire-opencl (= ${CPACK_PACKAGE_VERSION}), arrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION}), libarrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-opencl (<< ${CPACK_PACKAGE_VERSION}), arrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_REQUIRES ${deb_opencl_runtime_requirements}
+  DEB_USE_SHLIBDEPS
+  DEB_ADD_POSTINST
+  DEB_OPTIONAL forge libfreeimage3
+)
+
+af_component(
+  COMPONENT opencl_dev
+  DISPLAY_NAME "OpenCL Dev"
+  SUMMARY  "ArrayFire OpenCL backend development files"
+  DESCRIPTION  "ArrayFire OpenCL backend development files"
+  REQUIRES opencl headers cmake
+  GROUP afdevelopment
+  INSTALL_TYPES All Development
+
+  DEB_PACKAGE_NAME arrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR}-dev
+  DEB_PROVIDES "arrayfire-opencl-dev (= ${CPACK_PACKAGE_VERSION}), arrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR}-dev (= ${CPACK_PACKAGE_VERSION}), libarrayfire-opencl-dev (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-opencl-dev (<< ${CPACK_PACKAGE_VERSION}), arrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR}-dev (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-opencl-dev (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_REQUIRES "arrayfire-opencl${CPACK_PACKAGE_VERSION_MAJOR} (>= ${CPACK_PACKAGE_VERSION}), arrayfire-headers (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_RECOMMENDS "arrayfire-cmake (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_OPTIONAL "cmake (>= 3.0)"
+)
+
+af_component(
+  COMPONENT unified
+  DISPLAY_NAME "Unified Runtime"
+  SUMMARY "ArrayFire Unified backend shared libraries."
+  DESCRIPTION "ArrayFire Unified backend shared libraries. Requires other backends to function."
+  OPTIONAL forge
+  REQUIRES licenses
+  GROUP afruntime
+  INSTALL_TYPES All Runtime
+
+  DEB_PACKAGE_NAME arrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR}
+  DEB_PROVIDES "arrayfire-unified (= ${CPACK_PACKAGE_VERSION}), arrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION}), libarrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-unified (<< ${CPACK_PACKAGE_VERSION}), arrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_REQUIRES "arrayfire-cpu (>= ${CPACK_PACKAGE_VERSION}) | arrayfire-cuda (>= ${CPACK_PACKAGE_VERSION}) | arrayfire-opencl (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_USE_SHLIBDEPS
+)
+
+af_component(
+  COMPONENT unified_dev
+  DISPLAY_NAME "Unified Dev"
+  SUMMARY  "ArrayFire Unified backend development files"
+  DESCRIPTION  "ArrayFire Unified backend development files"
+  REQUIRES unified headers cmake
+  OPTIONAL forge
+  GROUP afdevelopment
+  INSTALL_TYPES All Development
+
+  DEB_PACKAGE_NAME arrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR}-dev
+  DEB_PROVIDES "arrayfire-unified-dev (= ${CPACK_PACKAGE_VERSION}), arrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR}-dev (= ${CPACK_PACKAGE_VERSION}), libarrayfire-unified-dev (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-unified-dev (<< ${CPACK_PACKAGE_VERSION}), arrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR}-dev (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-unified-dev (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_REQUIRES "arrayfire-unified${CPACK_PACKAGE_VERSION_MAJOR} (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_RECOMMENDS "arrayfire-cmake (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_OPTIONAL "cmake (>= 3.0)"
+)
+
+af_component(
+  COMPONENT documentation
+  DISPLAY_NAME "Documentation"
+  SUMMARY  "ArrayFire Documentation"
+  INSTALL_TYPES All
+  DESCRIPTION  "ArrayFire Doxygen Documentation"
+
+  DEB_PACKAGE_NAME arrayfire-doc
+  DEB_REPLACES "arrayfire-doc (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-doc (<< ${CPACK_PACKAGE_VERSION})"
+)
+
+af_component(
+  COMPONENT headers
+  DISPLAY_NAME "C/C++ Headers"
+  HIDDEN
+  INSTALL_TYPES All Development
+  DESCRIPTION "Headers for the ArrayFire libraries.")
+
+af_component(
+  COMPONENT examples
+  DISPLAY_NAME "ArrayFire Examples"
+  INSTALL_TYPES All
+  DESCRIPTION "Various examples using ArrayFire.")
+
+af_component(
+  COMPONENT cmake
+  DISPLAY_NAME "CMake Files"
+  HIDDEN
+  INSTALL_TYPES All Development
+  DESCRIPTION "Configuration files to use ArrayFire using CMake.")
+
+af_component(
+  COMPONENT licenses
+  DISPLAY_NAME "Licenses"
+  DESCRIPTION "License files for ArrayFire and its upstream libraries."
+  HIDDEN
+  REQUIRED)
+
+if(NOT CPACK_GENERATOR MATCHES "DEB")
+  af_component(
+    COMPONENT common_backend_dependencies
+    DISPLAY_NAME "Common Dependencies"
+    DESCRIPTION "Libraries commonly required by all ArrayFire backends."
+    HIDDEN
+    INSTALL_TYPES All Development Runtime)
+
+  af_component(
+    COMPONENT cuda_dependencies
+    DISPLAY_NAME "CUDA Dependencies"
+    DESCRIPTION "Shared libraries required for the CUDA backend."
+    HIDDEN
+    INSTALL_TYPES All Development Runtime)
+
+endif()
+
+#TODO(pradeep) Remove check after OSX support addition
+# Debug symbols in debian installers are created using the DEBINFO property
+if(NOT APPLE AND
+   NOT CPACK_GENERATOR MATCHES "DEB")
+  af_component(
+    COMPONENT afopencl_debug_symbols
+    DISPLAY_NAME "OpenCL Debug Symbols"
+    DESCRIPTION "Debug symbols for the OpenCL backend."
+    GROUP debug
+    DISABLED
+    INSTALL_TYPES Development)
+
+  af_component(
+    COMPONENT afcuda_debug_symbols
+    DISPLAY_NAME "CUDA Debug Symbols"
+    DESCRIPTION "Debug symbols for CUDA backend backend."
+    GROUP debug
+    DISABLED
+    INSTALL_TYPES Development)
+
+  af_component(
+    COMPONENT afcpu_debug_symbols
+    DISPLAY_NAME "CPU Debug Symbols"
+    DESCRIPTION "Debug symbols for CPU backend backend."
+    GROUP debug
+    DISABLED
+    INSTALL_TYPES Development)
+
+  af_component(
+    COMPONENT af_debug_symbols
+    DISPLAY_NAME "Unified Debug Symbols"
+    DESCRIPTION "Debug symbols for the Unified backend."
+    GROUP debug
+    DISABLED
+    INSTALL_TYPES Development)
+endif()
+
+# if (AF_INSTALL_FORGE_DEV)
+#   list(APPEND CPACK_COMPONENTS_ALL forge)
+#   af_component(
+#     COMPONENT forge
+#     DISPLAY_NAME "Forge Vizualiation"
+#     DESCRIPTION "Visualization Library"
+#     INSTALL_TYPES Extra)
+# endif ()
+#
+#set(LIBRARY_NAME ${PROJECT_NAME})
+#string(TOLOWER "${LIBRARY_NAME}" APP_LOW_NAME)
+#set(SITE_URL "https://arrayfire.com")
+#
+# set(inst_pkg_name ${APP_LOW_NAME})
+# set(inst_pkg_hash "")
+# if (WIN32)
+#   set(inst_pkg_name ${CPACK_PACKAGE_NAME})
+#   set(inst_pkg_hash "-${GIT_COMMIT_HASH}")
+# endif ()
+#
+#set(CPACK_PACKAGE_FILE_NAME "${inst_pkg_name}${inst_pkg_hash}")
+
+# ##
+# # IFW CPACK generator
+# # Uses Qt installer framework, cross platform installer generator.
+# # Uniform installer GUI on all major desktop platforms: Windows, OSX & Linux.
+# ##
+# set(CPACK_IFW_PACKAGE_TITLE "${CPACK_PACKAGE_NAME}")
+# set(CPACK_IFW_PACKAGE_PUBLISHER "${CPACK_PACKAGE_VENDOR}")
+# set(CPACK_IFW_PRODUCT_URL "${SITE_URL}")
+# set(CPACK_IFW_PACKAGE_ICON "${MY_CPACK_PACKAGE_ICON}")
+# set(CPACK_IFW_PACKAGE_WINDOW_ICON "${CMAKE_SOURCE_DIR}/assets/${APP_LOW_NAME}_icon.png")
+# set(CPACK_IFW_PACKAGE_WIZARD_DEFAULT_WIDTH 640)
+# set(CPACK_IFW_PACKAGE_WIZARD_DEFAULT_HEIGHT 480)
+# if (WIN32)
+#     set(CPACK_IFW_ADMIN_TARGET_DIRECTORY "@ApplicationsDirX64@/${CPACK_PACKAGE_INSTALL_DIRECTORY}")
+# else ()
+#     set(CPACK_IFW_ADMIN_TARGET_DIRECTORY "/opt/${CPACK_PACKAGE_INSTALL_DIRECTORY}")
+# endif ()
+#
+# function(get_native_path out_path path)
+#   file(TO_NATIVE_PATH ${path} native_path)
+#   if (WIN32)
+#     string(REPLACE "\\" "\\\\" native_path  ${native_path})
+#     set(${out_path} ${native_path} PARENT_SCOPE)
+#   else ()
+#     set(${out_path} ${path} PARENT_SCOPE)
+#   endif ()
+# endfunction()
+#
+# get_native_path(zlib_lic_path "${CPACK_ArrayFire_SOURCE_DIR}/LICENSES/zlib-libpng License.txt")
+# get_native_path(boost_lic_path "${CPACK_ArrayFire_SOURCE_DIR}/LICENSES/Boost Software License.txt")
+# get_native_path(fimg_lic_path "${CPACK_ArrayFire_SOURCE_DIR}/LICENSES/FreeImage Public License.txt")
+# get_native_path(apache_lic_path "${CPACK_ArrayFire_SOURCE_DIR}/LICENSES/Apache-2.0.txt")
+# get_native_path(sift_lic_path "${CPACK_ArrayFire_SOURCE_DIR}/LICENSES/OpenSIFT License.txt")
+# get_native_path(bsd3_lic_path "${CPACK_ArrayFire_SOURCE_DIR}/LICENSES/BSD 3-Clause.txt")
+# get_native_path(issl_lic_path "${CPACK_ArrayFire_SOURCE_DIR}/LICENSES/ISSL License.txt")
+
+#cpack_ifw_configure_component_group(backends)
+#cpack_ifw_configure_component_group(cpu-backend)
+#cpack_ifw_configure_component_group(cuda-backend)
+#cpack_ifw_configure_component_group(opencl-backend)
+#if (PACKAGE_MKL_DEPS)
+#  cpack_ifw_configure_component(mkl_dependencies)
+#endif ()
+#if (NOT APPLE)
+#  cpack_ifw_configure_component(opencl_dependencies)
+#endif ()
+#cpack_ifw_configure_component(common_backend_dependencies)
+#cpack_ifw_configure_component(cuda_dependencies)
+#cpack_ifw_configure_component(cpu)
+#cpack_ifw_configure_component(cuda)
+#cpack_ifw_configure_component(opencl)
+#cpack_ifw_configure_component(unified)
+#cpack_ifw_configure_component(headers)
+#cpack_ifw_configure_component(cmake)
+#cpack_ifw_configure_component(documentation)
+#cpack_ifw_configure_component(examples)
+#cpack_ifw_configure_component(licenses FORCED_INSTALLATION
+#  LICENSES "GLFW" ${zlib_lic_path} "FreeImage" ${fimg_lic_path}
+#  "Boost" ${boost_lic_path} "CLBlast, clFFT" ${apache_lic_path} "SIFT" ${sift_lic_path}
+#  "BSD3" ${bsd3_lic_path} "Intel MKL" ${issl_lic_path}
+#)
+#if (AF_INSTALL_FORGE_DEV)
+#  cpack_ifw_configure_component(forge)
+#endif ()
+
+
diff --git a/CMakeModules/debian/postinst b/CMakeModules/debian/postinst
new file mode 100644
index 0000000000..093371bd32
--- /dev/null
+++ b/CMakeModules/debian/postinst
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+if [ "$1" = "configure" ]; then
+    echo "/opt/intel/compilers_and_libraries/linux/mkl/lib/intel64_lin" >> /etc/ld.so.conf.d/99_arrayfire_${RC_COMPONENT}.conf
+    echo "/usr/local/cuda-${CPACK_CUDA_VERSION_MAJOR}.${CPACK_CUDA_VERSION_MINOR}/lib64" >> /etc/ld.so.conf.d/99_arrayfire_${RC_COMPONENT}.conf
+    ldconfig
+fi

From b0a322a9d3c0af4cbdfe4dc7ae6ba0067955988a Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 May 2022 17:27:04 -0400
Subject: [PATCH 049/473] Update GitHub workflow with updated hash and freetype
 features

---
 .github/workflows/win_cpu_build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/win_cpu_build.yml b/.github/workflows/win_cpu_build.yml
index 72c6955238..067f951fff 100644
--- a/.github/workflows/win_cpu_build.yml
+++ b/.github/workflows/win_cpu_build.yml
@@ -14,7 +14,7 @@ jobs:
         runs-on: windows-latest
         env:
 
-          VCPKG_HASH: 4428702c1c56fdb7cb779584efdcba254d7b57ca #[neon2sse] create a new port; Has forge v1.0.8 and other cmake/vcpkg fixes
+          VCPKG_HASH: 14e7bb4ae24616ec54ff6b2f6ef4e8659434ea44
 
           VCPKG_DEFAULT_TRIPLET: x64-windows
         steps:
@@ -36,7 +36,7 @@ jobs:
                 cd vcpkg
                 git checkout $env:VCPKG_HASH
                 .\bootstrap-vcpkg.bat
-                .\vcpkg.exe install boost-compute boost-functional boost-stacktrace fftw3 forge freeimage freetype glfw3 openblas
+                .\vcpkg.exe install boost-compute boost-math boost-stacktrace fftw3 freeimage freetype[core] forge glfw3 openblas
                 Remove-Item .\downloads,.\buildtrees,.\packages -Recurse -Force
 
             - name: CMake Configure

From 20aaff0f490143953243ce789d3ffc44d6c4e63d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar456@gmail.com>
Date: Sat, 28 May 2022 23:12:50 -0400
Subject: [PATCH 050/473] Add driver information for CUDA 11.7

---
 src/backend/cuda/device_manager.cpp | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index ca46388484..354a216741 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -86,6 +86,7 @@ struct ToolkitDriverVersions {
 
 // clang-format off
 static const int jetsonComputeCapabilities[] = {
+    8070,
     7020,
     6020,
     5030,
@@ -95,6 +96,7 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {11070, 8, 7, 0},
     {11060, 8, 6, 0},
     {11050, 8, 6, 0},
     {11040, 8, 6, 0},
@@ -129,13 +131,14 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
-        {11060, 510.39f, 511.23f},
-        {11050, 495.29f, 496.13f},
-        {11040, 470.42f, 471.11f},
-        {11030, 465.19f, 465.89f},
-        {11020, 460.27f, 460.82f},
-        {11010, 455.23f, 456.38f},
-        {11000, 450.51f, 451.48f},
+        {11070, 450.80f, 452.39f},
+        {11060, 450.80f, 452.39f},
+        {11050, 450.80f, 452.39f},
+        {11040, 450.80f, 452.39f},
+        {11030, 450.80f, 452.39f},
+        {11020, 450.80f, 452.39f},
+        {11010, 450.80f, 452.39f},
+        {11000, 450.36f, 451.22f},
         {10020, 440.33f, 441.22f},
         {10010, 418.39f, 418.96f},
         {10000, 410.48f, 411.31f},
@@ -156,7 +159,7 @@ static ComputeCapabilityToStreamingProcessors gpus[] = {
     {0x21, 48},  {0x30, 192}, {0x32, 192}, {0x35, 192}, {0x37, 192},
     {0x50, 128}, {0x52, 128}, {0x53, 128}, {0x60, 64},  {0x61, 128},
     {0x62, 128}, {0x70, 64},  {0x75, 64},  {0x80, 64},  {0x86, 128},
-    {-1, -1},
+    {0x87, 128}, {-1, -1},
 };
 
 // pulled from CUTIL from CUDA SDK

From c2f24a8bfc6ae1268553221cda43c80066d98dde Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar456@gmail.com>
Date: Tue, 31 May 2022 11:16:46 -0400
Subject: [PATCH 051/473] Fix search for cuSparse libraries on Windows

cuSparse libraries on windows encode the cuda version in the
DLL names. This commit adds the suffixes to the cuSparse module
class
---
 src/backend/cuda/cusparseModule.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/cuda/cusparseModule.cpp b/src/backend/cuda/cusparseModule.cpp
index f229372b43..e7b8105221 100644
--- a/src/backend/cuda/cusparseModule.cpp
+++ b/src/backend/cuda/cusparseModule.cpp
@@ -22,7 +22,7 @@ cusparseModule::cusparseModule()
 #ifdef AF_cusparse_STATIC_LINKING
     module(nullptr, nullptr)
 #else
-    module("cusparse", nullptr)
+    module({"cusparse"}, {"64_11", "64_10", "64_9", "64_8"}, {""})
 #endif
 {
 #ifdef AF_cusparse_STATIC_LINKING

From f2f68edebdceb561be27082adae3ad40e4f71950 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar456@gmail.com>
Date: Tue, 31 May 2022 11:19:32 -0400
Subject: [PATCH 052/473] Add support for ccache on Windows

Ccache has support for windows. This seems to work with the Windows
binaries of the ccache program with the Ninja generator. I don't
think this is working in Visual Studio.
---
 CMakeModules/config_ccache.cmake | 69 ++++++++++++++++----------------
 1 file changed, 34 insertions(+), 35 deletions(-)

diff --git a/CMakeModules/config_ccache.cmake b/CMakeModules/config_ccache.cmake
index 1bf3adaef6..80783b06c1 100644
--- a/CMakeModules/config_ccache.cmake
+++ b/CMakeModules/config_ccache.cmake
@@ -1,42 +1,41 @@
 # picked up original content from https://crascit.com/2016/04/09/using-ccache-with-cmake/
 
-if (UNIX)
-  find_program(CCACHE_PROGRAM ccache)
+find_program(CCACHE_PROGRAM ccache)
 
-  set(CCACHE_FOUND OFF)
-  if(CCACHE_PROGRAM)
-    set(CCACHE_FOUND ON)
-  endif()
+set(CCACHE_FOUND OFF)
+if(CCACHE_PROGRAM)
+  set(CCACHE_FOUND ON)
+endif()
 
-  option(AF_USE_CCACHE "Use ccache when compiling" ${CCACHE_FOUND})
+option(AF_USE_CCACHE "Use ccache when compiling" ${CCACHE_FOUND})
 
-  if(${AF_USE_CCACHE})
-    # Set up wrapper scripts
-    set(C_LAUNCHER   "${CCACHE_PROGRAM}")
-    set(CXX_LAUNCHER "${CCACHE_PROGRAM}")
-    set(NVCC_LAUNCHER "${CCACHE_PROGRAM}")
-    configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-c.in   launch-c)
-    configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-cxx.in launch-cxx)
-    configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-nvcc.in launch-nvcc)
-    execute_process(COMMAND chmod a+rx
-        "${ArrayFire_BINARY_DIR}/launch-c"
-        "${ArrayFire_BINARY_DIR}/launch-cxx"
-        "${ArrayFire_BINARY_DIR}/launch-nvcc"
-      )
-    if(CMAKE_GENERATOR STREQUAL "Xcode")
-      # Set Xcode project attributes to route compilation and linking
-      # through our scripts
-      set(CMAKE_XCODE_ATTRIBUTE_CC         "${ArrayFire_BINARY_DIR}/launch-c")
-      set(CMAKE_XCODE_ATTRIBUTE_CXX        "${ArrayFire_BINARY_DIR}/launch-cxx")
-      set(CMAKE_XCODE_ATTRIBUTE_LD         "${ArrayFire_BINARY_DIR}/launch-c")
-      set(CMAKE_XCODE_ATTRIBUTE_LDPLUSPLUS "${ArrayFire_BINARY_DIR}/launch-cxx")
-    else()
-      # Support Unix Makefiles and Ninja
-      set(CMAKE_C_COMPILER_LAUNCHER   "${ArrayFire_BINARY_DIR}/launch-c")
-      set(CMAKE_CXX_COMPILER_LAUNCHER "${ArrayFire_BINARY_DIR}/launch-cxx")
-      set(CUDA_NVCC_EXECUTABLE "${ArrayFire_BINARY_DIR}/launch-nvcc")
-    endif()
+if(${AF_USE_CCACHE})
+  message(STATUS "ccache FOUND: ${CCACHE_PROGRAM}")
+  # Set up wrapper scripts
+  set(C_LAUNCHER   "${CCACHE_PROGRAM}")
+  set(CXX_LAUNCHER "${CCACHE_PROGRAM}")
+  set(NVCC_LAUNCHER "${CCACHE_PROGRAM}")
+  configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-c.in   launch-c)
+  configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-cxx.in launch-cxx)
+  configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/launch-nvcc.in launch-nvcc)
+  execute_process(COMMAND chmod a+rx
+      "${ArrayFire_BINARY_DIR}/launch-c"
+      "${ArrayFire_BINARY_DIR}/launch-cxx"
+      "${ArrayFire_BINARY_DIR}/launch-nvcc"
+    )
+  if(CMAKE_GENERATOR STREQUAL "Xcode")
+    # Set Xcode project attributes to route compilation and linking
+    # through our scripts
+    set(CMAKE_XCODE_ATTRIBUTE_CC         "${ArrayFire_BINARY_DIR}/launch-c")
+    set(CMAKE_XCODE_ATTRIBUTE_CXX        "${ArrayFire_BINARY_DIR}/launch-cxx")
+    set(CMAKE_XCODE_ATTRIBUTE_LD         "${ArrayFire_BINARY_DIR}/launch-c")
+    set(CMAKE_XCODE_ATTRIBUTE_LDPLUSPLUS "${ArrayFire_BINARY_DIR}/launch-cxx")
+  else()
+    # Support Unix Makefiles and Ninja
+    set(CMAKE_C_COMPILER_LAUNCHER   "${CCACHE_PROGRAM}")
+    set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
+    set(CUDA_NVCC_EXECUTABLE ${CCACHE_PROGRAM} "${CUDA_NVCC_EXECUTABLE}")
   endif()
-  mark_as_advanced(CCACHE_PROGRAM)
-  mark_as_advanced(AF_USE_CCACHE)
 endif()
+mark_as_advanced(CCACHE_PROGRAM)
+mark_as_advanced(AF_USE_CCACHE)

From 338a1adb13b9ce7291f6ec7b8c6dba6c7ad09275 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar456@gmail.com>
Date: Tue, 31 May 2022 12:00:33 -0400
Subject: [PATCH 053/473] Catch errors when creating OCL contexts from device

Catch OpenCL errors when creating Contexts from OpenCL devices.
This change is necessary because some platforms(Intel FPGA) were
crashing if certain environment variables were not set when
crating contexts even though the platform returned the device
from the platform. We catch errors for particular devices and
then we remove them from the device list.
---
 src/backend/opencl/device_manager.cpp | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index 9404614f42..6452ee590e 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -244,20 +244,29 @@ DeviceManager::DeviceManager()
     // Sort OpenCL devices based on default criteria
     stable_sort(mDevices.begin(), mDevices.end(), compare_default);
 
+    auto devices = move(mDevices);
+    mDevices.clear();
+
     // Create contexts and queues once the sort is done
     for (int i = 0; i < nDevices; i++) {
         cl_platform_id device_platform =
-            mDevices[i]->getInfo<CL_DEVICE_PLATFORM>();
+            devices[i]->getInfo<CL_DEVICE_PLATFORM>();
         cl_context_properties cps[3] = {
             CL_CONTEXT_PLATFORM, (cl_context_properties)(device_platform), 0};
-
-        mContexts.push_back(make_unique<Context>(*mDevices[i], cps));
-        mQueues.push_back(make_unique<CommandQueue>(
-            *mContexts.back(), *mDevices[i], cl::QueueProperties::None));
-        mIsGLSharingOn.push_back(false);
-        mDeviceTypes.push_back(getDeviceTypeEnum(*mDevices[i]));
-        mPlatforms.push_back(getPlatformEnum(*mDevices[i]));
+        try {
+            mContexts.push_back(make_unique<Context>(*devices[i], cps));
+            mQueues.push_back(make_unique<CommandQueue>(
+                *mContexts.back(), *devices[i], cl::QueueProperties::None));
+            mIsGLSharingOn.push_back(false);
+            mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
+            mPlatforms.push_back(getPlatformEnum(*devices[i]));
+            mDevices.emplace_back(std::move(devices[i]));
+        } catch (const cl::Error& err) {
+            AF_TRACE("Error creating context for device {} with error {}\n",
+                     devices[i]->getInfo<CL_DEVICE_NAME>(), err.what());
+        }
     }
+    nDevices = mDevices.size();
 
     bool default_device_set = false;
     deviceENV               = getEnvVar("AF_OPENCL_DEFAULT_DEVICE");

From 6228a4d43439d14cec3d59e51f2c342d94704621 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar456@gmail.com>
Date: Tue, 31 May 2022 12:14:18 -0400
Subject: [PATCH 054/473] Make cuDNN an optional feature in vcpkg

---
 vcpkg.json | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/vcpkg.json b/vcpkg.json
index 654d9ad8b6..8986d52dbe 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -39,8 +39,7 @@
         "cuda": {
             "description": "Build CUDA backend",
             "dependencies": [
-                "cuda",
-                "cudnn"
+                "cuda"
             ]
         },
         "opencl": {
@@ -55,6 +54,12 @@
             "dependencies": [
                 "intel-mkl"
             ]
+        },
+        "cudnn": {
+            "description": "Build CUDA with support for cuDNN",
+            "dependencies": [
+                "cudnn"
+            ]
         }
     },
     "builtin-baseline": "14e7bb4ae24616ec54ff6b2f6ef4e8659434ea44"

From 20982dfd448cb75c8787754eaca111c84d5d718b Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar456@gmail.com>
Date: Tue, 31 May 2022 12:22:09 -0400
Subject: [PATCH 055/473] Fix linear jit workgroup calculations for CPU devices

---
 src/backend/opencl/jit.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index b8b486cae0..06d2b41b08 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -288,7 +288,7 @@ void evalNodes(vector<Param> &outputs, const vector<Node *> &output_nodes) {
         uint out_elements = outDims[3] * out_info.strides[3];
         uint groups       = divup(out_elements, local_0);
 
-        global_1 = divup(groups, 1000) * local_1;
+        global_1 = divup(groups, work_group_size) * local_1;
         global_0 = divup(groups, global_1) * local_0;
 
     } else {

From bd0b86448ccb590a650a2b16ce9f205c988deff8 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 15 Jun 2022 13:44:37 -0400
Subject: [PATCH 056/473] fixes nanval substitution on new keys

---
 src/backend/cpu/kernel/reduce.hpp |  1 +
 test/reduce.cpp                   | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/src/backend/cpu/kernel/reduce.hpp b/src/backend/cpu/kernel/reduce.hpp
index 374816102e..db39dbc8b8 100644
--- a/src/backend/cpu/kernel/reduce.hpp
+++ b/src/backend/cpu/kernel/reduce.hpp
@@ -147,6 +147,7 @@ struct reduce_dim_by_key<op, Ti, Tk, To, 0> {
 
                 current_key = keyval;
                 out_val     = transform(inValsPtr[vOffset + (i * istride)]);
+                if (change_nan) out_val = IS_NAN(out_val) ? nanval : out_val;
                 ++keyidx;
             }
 
diff --git a/test/reduce.cpp b/test/reduce.cpp
index 0633bd0536..c9e09f53fd 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -2285,3 +2285,28 @@ TEST(Reduce, Test_Sum_Global_Array_nanval) {
     ASSERT_NEAR(res, full_reduce.scalar<float>(), max_error);
     freeHost(h_a);
 }
+
+TEST(Reduce, nanval_issue_3255) {
+    char *info_str;
+    af_array  ikeys, ivals, okeys, ovals;
+    dim_t dims[1] = {8};
+
+    int ikeys_src[8] = {0, 0,  1, 1, 1,  2, 2,  0};
+    af_create_array(&ikeys, ikeys_src, 1, dims, u32);
+
+    int i;
+    for (i=0; i<8; i++) {
+        double ivals_src[8] = {1, 2,  3, 4, 5,  6, 7,  8};
+        ivals_src[i] = NAN;
+        af_create_array(&ivals, ivals_src, 1, dims, f64);
+
+        af_product_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0);
+        af::array ovals_cpp(ovals);
+        ASSERT_FALSE(af::anyTrue<bool>(af::isNaN(ovals_cpp)));
+
+        af_sum_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0);
+        ovals_cpp = af::array(ovals);
+
+        ASSERT_FALSE(af::anyTrue<bool>(af::isNaN(ovals_cpp)));
+    }
+}

From 2688275d2de79ad114a4b115e3594fb6d28c8033 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 21 Jun 2022 16:26:23 -0400
Subject: [PATCH 057/473] Restrict initializer list to fundamental types

This commit limits the types that can be used in the initializer
list to fundamental types. This change is necessary because when
we use the uniform initialization syntax and pass in an array, the
compiler incorrectly uses the initialization list constructor instead
of the other array constructor.
---
 include/af/array.h |  7 +++++--
 test/array.cpp     | 21 +++++++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/include/af/array.h b/include/af/array.h
index bdd9ac4e9c..b1405c903c 100644
--- a/include/af/array.h
+++ b/include/af/array.h
@@ -522,7 +522,9 @@ namespace af
 #if AF_API_VERSION >= 38
 #if AF_COMPILER_CXX_GENERALIZED_INITIALIZERS
         /// \brief Initializer list constructor
-        template <typename T> array(std::initializer_list<T> list)
+        template <typename T, typename = typename std::enable_if<
+                                  std::is_fundamental<T>::value, void>::type>
+        array(std::initializer_list<T> list)
         : arr(nullptr) {
           dim_t size = list.size();
           if (af_err __aferr = af_create_array(&arr, list.begin(), 1, &size,
@@ -537,7 +539,8 @@ namespace af
         }
 
         /// \brief Initializer list constructor
-        template <typename T>
+        template <typename T, typename = typename std::enable_if<
+                                  std::is_fundamental<T>::value, void>::type>
         array(const af::dim4 &dims, std::initializer_list<T> list)
             : arr(nullptr) {
           const dim_t *size = dims.get();
diff --git a/test/array.cpp b/test/array.cpp
index 9770549d2d..7d45cf1ea7 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -640,3 +640,24 @@ TEST(Array, ReferenceCount2) {
         ASSERT_REF(d, 0) << "After d = c;";
     }
 }
+
+// This tests situations where the compiler incorrectly assumes the initializer
+// list constructor instead of the regular constructor when using the uniform
+// initilization syntax
+TEST(Array, InitializerListFixAFArray) {
+    array a = randu(1);
+    array b{a};
+
+    ASSERT_ARRAYS_EQ(a, b);
+}
+
+// This tests situations where the compiler incorrectly assumes the initializer
+// list constructor instead of the regular constructor when using the uniform
+// initilization syntax
+TEST(Array, InitializerListFixDim4) {
+    array a            = randu(1);
+    vector<float> data = {3.14f, 3.14f, 3.14f, 3.14f, 3.14f,
+                          3.14f, 3.14f, 3.14f, 3.14f};
+    array b{dim4(3, 3), data.data()};
+    ASSERT_ARRAYS_EQ(constant(3.14, 3, 3), b);
+}

From ef69c518a7bef74859b88e8c929955450978ff24 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 4 Jul 2022 19:58:44 -0400
Subject: [PATCH 058/473] Move tile function to common namespace. Avoid calling
 from detail

This commit moves the implementation of the tile funciton to the common
namespace. This is done because the tile funciton in detail does not perform JIT
optimization. It instead calls the tile kernel directly. This is undesirable
because there are some instances where tile funciton can be performed by
indexing. This commit also updates several calls to tile in the codebase
to use this new version.

It is still fairly easy to call the detail::tile function and we need to address
this at some point. Perhaps it should be deprecated and only called by the
common::tile function. This commit does not address this issue.
---
 src/api/c/assign.cpp        |  4 ++--
 src/api/c/canny.cpp         |  5 ++--
 src/api/c/convolve.cpp      |  6 ++---
 src/api/c/rgb_gray.cpp      |  4 ++--
 src/api/c/surface.cpp       |  6 ++---
 src/api/c/tile.cpp          | 27 ++-------------------
 src/backend/common/tile.hpp | 48 +++++++++++++++++++++++++++++++++++++
 7 files changed, 63 insertions(+), 37 deletions(-)
 create mode 100644 src/backend/common/tile.hpp

diff --git a/src/api/c/assign.cpp b/src/api/c/assign.cpp
index ef7bacd821..20aa69e629 100644
--- a/src/api/c/assign.cpp
+++ b/src/api/c/assign.cpp
@@ -15,11 +15,11 @@
 #include <common/err_common.hpp>
 #include <common/half.hpp>
 #include <common/moddims.hpp>
+#include <common/tile.hpp>
 #include <copy.hpp>
 #include <handle.hpp>
 #include <indexing_common.hpp>
 #include <math.hpp>
-#include <tile.hpp>
 #include <af/data.h>
 #include <af/defines.h>
 #include <af/dim4.hpp>
@@ -78,7 +78,7 @@ static void assign(Array<Tout>& out, const vector<af_seq> seqs,
         // If both out and in are vectors of equal elements,
         // reshape in to out dims
         Array<Tin> in_ =
-            in.elements() == 1 ? tile(in, oDims) : modDims(in, oDims);
+            in.elements() == 1 ? common::tile(in, oDims) : modDims(in, oDims);
         auto dst = createSubArray<Tout>(out, seqs, false);
 
         copyArray<Tin, Tout>(dst, in_);
diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp
index d9d74da7d9..625ce748fa 100644
--- a/src/api/c/canny.cpp
+++ b/src/api/c/canny.cpp
@@ -14,6 +14,7 @@
 #include <backend.hpp>
 #include <common/cast.hpp>
 #include <common/err_common.hpp>
+#include <common/tile.hpp>
 #include <complex.hpp>
 #include <convolve.hpp>
 #include <copy.hpp>
@@ -25,7 +26,6 @@
 #include <reduce.hpp>
 #include <scan.hpp>
 #include <sobel.hpp>
-#include <tile.hpp>
 #include <transpose.hpp>
 #include <unary.hpp>
 #include <af/defines.h>
@@ -37,6 +37,7 @@
 
 using af::dim4;
 using common::cast;
+using common::tile;
 using detail::arithOp;
 using detail::Array;
 using detail::convolve2;
@@ -137,7 +138,7 @@ Array<float> otsuThreshold(const Array<float>& in, const unsigned NUM_BINS,
 
     ireduce<af_max_t, float>(thresh, locs, sigmas, 0);
 
-    return cast<float, uint>(tile(locs, dim4(inDims[0], inDims[1])));
+    return cast<float, uint>(common::tile(locs, dim4(inDims[0], inDims[1])));
 }
 
 Array<float> normalize(const Array<float>& supEdges, const float minVal,
diff --git a/src/api/c/convolve.cpp b/src/api/c/convolve.cpp
index b7581dd484..ddcd916ae6 100644
--- a/src/api/c/convolve.cpp
+++ b/src/api/c/convolve.cpp
@@ -13,9 +13,9 @@
 #include <common/cast.hpp>
 #include <common/err_common.hpp>
 #include <common/half.hpp>
+#include <common/tile.hpp>
 #include <fftconvolve.hpp>
 #include <handle.hpp>
-#include <tile.hpp>
 #include <af/data.h>
 #include <af/defines.h>
 #include <af/dim4.hpp>
@@ -54,8 +54,8 @@ inline af_array convolve2(const af_array &s, const af_array &c_f,
     const Array<accT> signal    = castArray<accT>(s);
 
     if (colFilter.isScalar() && rowFilter.isScalar()) {
-        Array<accT> colArray = detail::tile(colFilter, signal.dims());
-        Array<accT> rowArray = detail::tile(rowFilter, signal.dims());
+        Array<accT> colArray = common::tile(colFilter, signal.dims());
+        Array<accT> rowArray = common::tile(rowFilter, signal.dims());
 
         Array<accT> filter =
             arithOp<accT, af_mul_t>(colArray, rowArray, signal.dims());
diff --git a/src/api/c/rgb_gray.cpp b/src/api/c/rgb_gray.cpp
index 635474e846..3c189af5df 100644
--- a/src/api/c/rgb_gray.cpp
+++ b/src/api/c/rgb_gray.cpp
@@ -17,10 +17,10 @@
 #include <backend.hpp>
 #include <common/ArrayInfo.hpp>
 #include <common/cast.hpp>
+#include <common/tile.hpp>
 #include <handle.hpp>
 #include <join.hpp>
 #include <math.hpp>
-#include <tile.hpp>
 
 using af::dim4;
 using common::cast;
@@ -75,7 +75,7 @@ static af_array gray2rgb(const af_array& in, const float r, const float g,
                          const float b) {
     if (r == 1.0 && g == 1.0 && b == 1.0) {
         dim4 tileDims(1, 1, 3, 1);
-        return getHandle(tile(getArray<T>(in), tileDims));
+        return getHandle(common::tile(getArray<T>(in), tileDims));
     }
 
     af_array mod_input = 0;
diff --git a/src/api/c/surface.cpp b/src/api/c/surface.cpp
index 2f6a3eda7b..58cc9476aa 100644
--- a/src/api/c/surface.cpp
+++ b/src/api/c/surface.cpp
@@ -15,13 +15,13 @@
 #include <common/err_common.hpp>
 #include <common/graphics_common.hpp>
 #include <common/moddims.hpp>
+#include <common/tile.hpp>
 #include <copy.hpp>
 #include <handle.hpp>
 #include <join.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
 #include <surface.hpp>
-#include <tile.hpp>
 
 using af::dim4;
 using common::modDims;
@@ -58,13 +58,13 @@ fg_chart setup_surface(fg_window window, const af_array xVals,
         xIn = modDims(xIn, xIn.elements());
         // Now tile along second dimension
         dim4 x_tdims(1, Y_dims[0], 1, 1);
-        xIn = tile(xIn, x_tdims);
+        xIn = common::tile(xIn, x_tdims);
 
         // Convert yIn to a row vector
         yIn = modDims(yIn, dim4(1, yIn.elements()));
         // Now tile along first dimension
         dim4 y_tdims(X_dims[0], 1, 1, 1);
-        yIn = tile(yIn, y_tdims);
+        yIn = common::tile(yIn, y_tdims);
     }
 
     // Flatten xIn, yIn and zIn into row vectors
diff --git a/src/api/c/tile.cpp b/src/api/c/tile.cpp
index db3d456691..443419b540 100644
--- a/src/api/c/tile.cpp
+++ b/src/api/c/tile.cpp
@@ -7,7 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <tile.hpp>
+#include <common/tile.hpp>
 
 #include <arith.hpp>
 #include <backend.hpp>
@@ -33,30 +33,7 @@ using detail::ushort;
 
 template<typename T>
 static inline af_array tile(const af_array in, const af::dim4 &tileDims) {
-    const Array<T> inArray = getArray<T>(in);
-    const dim4 &inDims     = inArray.dims();
-
-    // FIXME: Always use JIT instead of checking for the condition.
-    // The current limitation exists for performance reasons. it should change
-    // in the future.
-
-    bool take_jit_path = true;
-    dim4 outDims(1, 1, 1, 1);
-
-    // Check if JIT path can be taken. JIT path can only be taken if tiling a
-    // singleton dimension.
-    for (int i = 0; i < 4; i++) {
-        take_jit_path &= (inDims[i] == 1 || tileDims[i] == 1);
-        outDims[i] = inDims[i] * tileDims[i];
-    }
-
-    af_array out = nullptr;
-    if (take_jit_path) {
-        out = getHandle(unaryOp<T, af_noop_t>(inArray, outDims));
-    } else {
-        out = getHandle(tile<T>(inArray, tileDims));
-    }
-    return out;
+    return getHandle(common::tile<T>(getArray<T>(in), tileDims));
 }
 
 af_err af_tile(af_array *out, const af_array in, const af::dim4 &tileDims) {
diff --git a/src/backend/common/tile.hpp b/src/backend/common/tile.hpp
new file mode 100644
index 0000000000..512d14b62b
--- /dev/null
+++ b/src/backend/common/tile.hpp
@@ -0,0 +1,48 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <tile.hpp>
+
+#include <Array.hpp>
+#include <arith.hpp>
+#include <backend.hpp>
+#include <optypes.hpp>
+#include <unary.hpp>
+
+#include <af/dim4.hpp>
+
+namespace common {
+
+/// duplicates the elements of an Array<T> array.
+template<typename T>
+detail::Array<T> tile(const detail::Array<T> &in, const af::dim4 tileDims) {
+    const af::dim4 &inDims = in.dims();
+
+    // FIXME: Always use JIT instead of checking for the condition.
+    // The current limitation exists for performance reasons. it should change
+    // in the future.
+
+    bool take_jit_path = true;
+    af::dim4 outDims(1, 1, 1, 1);
+
+    // Check if JIT path can be taken. JIT path can only be taken if tiling a
+    // singleton dimension.
+    for (int i = 0; i < 4; i++) {
+        take_jit_path &= (inDims[i] == 1 || tileDims[i] == 1);
+        outDims[i] = inDims[i] * tileDims[i];
+    }
+
+    if (take_jit_path) {
+        return detail::unaryOp<T, af_noop_t>(in, outDims);
+    } else {
+        return detail::tile<T>(in, tileDims);
+    }
+}
+
+}  // namespace common

From c115cbcb2d1532939d2ede27cf678a8cfca1644c Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 5 Jul 2022 18:55:05 -0400
Subject: [PATCH 059/473] broadcasting in af_arith for binary operations
 (#2871)

This commit adds broadcasting capabilities to the arithmetic functions without
the need of tile. Automatic broadcasting is performed for binary operations when
one of the operands has one element across one dimension and another is greater
than one. In this case ArrayFire will automatically perform a tiling operation.
Multiple broadcasts can be performed at one time.

Here are a couple of examples:

```
array c(10)         = randu(10) + randu(1);
array c(10, 10)     = randu(10, 10) + randu(1);
array c(10, 10, 10) = randu(10, 10, 10) + randu(1);
array c(10, 10, 10) = randu(10, 10, 10) + randu(10);
array c(10, 10, 10) = randu(1 , 10, 10) + randu(10);

array c(10, 1, 10) = randu(1, 1, 10) + randu(10);
```

Co-authored-by: pradeep <pradeep@arrayfire.com>
Co-authored-by: Umar Arshad <umar@arrayfire.com>
---
 src/api/c/binary.cpp |  89 +++++++++---
 test/array.cpp       |  20 +--
 test/binary.cpp      | 325 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 407 insertions(+), 27 deletions(-)

diff --git a/src/api/c/binary.cpp b/src/api/c/binary.cpp
index ffe21e2591..fc24fd64eb 100644
--- a/src/api/c/binary.cpp
+++ b/src/api/c/binary.cpp
@@ -10,6 +10,8 @@
 #include <backend.hpp>
 #include <common/ArrayInfo.hpp>
 #include <common/err_common.hpp>
+#include <common/moddims.hpp>
+#include <common/tile.hpp>
 #include <handle.hpp>
 #include <implicit.hpp>
 #include <optypes.hpp>
@@ -29,8 +31,11 @@
 using af::dim4;
 using af::dtype;
 using common::half;
+using common::modDims;
+using common::tile;
 using detail::arithOp;
 using detail::arithOpD;
+using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
@@ -55,6 +60,36 @@ static inline af_array arithOp(const af_array lhs, const af_array rhs,
     return getHandle(arithOp<T, op>(l, r, odims));
 }
 
+template<typename T, af_op_t op>
+static inline af_array arithOpBroadcast(const af_array lhs,
+                                        const af_array rhs) {
+    const ArrayInfo &linfo = getInfo(lhs);
+    const ArrayInfo &rinfo = getInfo(rhs);
+
+    dim4 odims(1), ltile(1), rtile(1);
+    dim4 lshape = linfo.dims();
+    dim4 rshape = rinfo.dims();
+
+    for (int d = 0; d < AF_MAX_DIMS; ++d) {
+        DIM_ASSERT(
+            1, ((lshape[d] == rshape[d]) || (lshape[d] == 1 && rshape[d] > 1) ||
+                (lshape[d] > 1 && rshape[d] == 1)));
+        odims[d] = std::max(lshape[d], rshape[d]);
+        if (lshape[d] == rshape[d]) {
+            ltile[d] = rtile[d] = 1;
+        } else if (lshape[d] == 1 && rshape[d] > 1) {
+            ltile[d] = odims[d];
+        } else if (lshape[d] > 1 && rshape[d] == 1) {
+            rtile[d] = odims[d];
+        }
+    }
+
+    Array<T> lhst = common::tile<T>(modDims(getArray<T>(lhs), lshape), ltile);
+    Array<T> rhst = common::tile<T>(modDims(getArray<T>(rhs), rshape), rtile);
+
+    return getHandle(arithOp<T, op>(lhst, rhst, odims));
+}
+
 template<typename T, af_op_t op>
 static inline af_array sparseArithOp(const af_array lhs, const af_array rhs) {
     auto res = arithOp<T, op>(getSparseArray<T>(lhs), getSparseArray<T>(rhs));
@@ -82,25 +117,45 @@ static af_err af_arith(af_array *out, const af_array lhs, const af_array rhs,
         const ArrayInfo &linfo = getInfo(lhs);
         const ArrayInfo &rinfo = getInfo(rhs);
 
-        dim4 odims = getOutDims(linfo.dims(), rinfo.dims(), batchMode);
-
         const af_dtype otype = implicit(linfo.getType(), rinfo.getType());
         af_array res;
-        switch (otype) {
-            case f32: res = arithOp<float, op>(lhs, rhs, odims); break;
-            case f64: res = arithOp<double, op>(lhs, rhs, odims); break;
-            case c32: res = arithOp<cfloat, op>(lhs, rhs, odims); break;
-            case c64: res = arithOp<cdouble, op>(lhs, rhs, odims); break;
-            case s32: res = arithOp<int, op>(lhs, rhs, odims); break;
-            case u32: res = arithOp<uint, op>(lhs, rhs, odims); break;
-            case u8: res = arithOp<uchar, op>(lhs, rhs, odims); break;
-            case b8: res = arithOp<char, op>(lhs, rhs, odims); break;
-            case s64: res = arithOp<intl, op>(lhs, rhs, odims); break;
-            case u64: res = arithOp<uintl, op>(lhs, rhs, odims); break;
-            case s16: res = arithOp<short, op>(lhs, rhs, odims); break;
-            case u16: res = arithOp<ushort, op>(lhs, rhs, odims); break;
-            case f16: res = arithOp<half, op>(lhs, rhs, odims); break;
-            default: TYPE_ERROR(0, otype);
+
+        if (batchMode || linfo.dims() == rinfo.dims()) {
+            dim4 odims = getOutDims(linfo.dims(), rinfo.dims(), batchMode);
+
+            switch (otype) {
+                case f32: res = arithOp<float, op>(lhs, rhs, odims); break;
+                case f64: res = arithOp<double, op>(lhs, rhs, odims); break;
+                case c32: res = arithOp<cfloat, op>(lhs, rhs, odims); break;
+                case c64: res = arithOp<cdouble, op>(lhs, rhs, odims); break;
+                case s32: res = arithOp<int, op>(lhs, rhs, odims); break;
+                case u32: res = arithOp<uint, op>(lhs, rhs, odims); break;
+                case u8: res = arithOp<uchar, op>(lhs, rhs, odims); break;
+                case b8: res = arithOp<char, op>(lhs, rhs, odims); break;
+                case s64: res = arithOp<intl, op>(lhs, rhs, odims); break;
+                case u64: res = arithOp<uintl, op>(lhs, rhs, odims); break;
+                case s16: res = arithOp<short, op>(lhs, rhs, odims); break;
+                case u16: res = arithOp<ushort, op>(lhs, rhs, odims); break;
+                case f16: res = arithOp<half, op>(lhs, rhs, odims); break;
+                default: TYPE_ERROR(0, otype);
+            }
+        } else {
+            switch (otype) {
+                case f32: res = arithOpBroadcast<float, op>(lhs, rhs); break;
+                case f64: res = arithOpBroadcast<double, op>(lhs, rhs); break;
+                case c32: res = arithOpBroadcast<cfloat, op>(lhs, rhs); break;
+                case c64: res = arithOpBroadcast<cdouble, op>(lhs, rhs); break;
+                case s32: res = arithOpBroadcast<int, op>(lhs, rhs); break;
+                case u32: res = arithOpBroadcast<uint, op>(lhs, rhs); break;
+                case u8: res = arithOpBroadcast<uchar, op>(lhs, rhs); break;
+                case b8: res = arithOpBroadcast<char, op>(lhs, rhs); break;
+                case s64: res = arithOpBroadcast<intl, op>(lhs, rhs); break;
+                case u64: res = arithOpBroadcast<uintl, op>(lhs, rhs); break;
+                case s16: res = arithOpBroadcast<short, op>(lhs, rhs); break;
+                case u16: res = arithOpBroadcast<ushort, op>(lhs, rhs); break;
+                case f16: res = arithOpBroadcast<half, op>(lhs, rhs); break;
+                default: TYPE_ERROR(0, otype);
+            }
         }
 
         std::swap(*out, res);
diff --git a/test/array.cpp b/test/array.cpp
index 7d45cf1ea7..deb85e2e22 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -641,23 +641,23 @@ TEST(Array, ReferenceCount2) {
     }
 }
 
-// This tests situations where the compiler incorrectly assumes the initializer
-// list constructor instead of the regular constructor when using the uniform
-// initilization syntax
+// This tests situations where the compiler incorrectly assumes the
+// initializer list constructor instead of the regular constructor when
+// using the uniform initilization syntax
 TEST(Array, InitializerListFixAFArray) {
-    array a = randu(1);
-    array b{a};
+    af::array a = randu(1);
+    af::array b{a};
 
     ASSERT_ARRAYS_EQ(a, b);
 }
 
-// This tests situations where the compiler incorrectly assumes the initializer
-// list constructor instead of the regular constructor when using the uniform
-// initilization syntax
+// This tests situations where the compiler incorrectly assumes the
+// initializer list constructor instead of the regular constructor when
+// using the uniform initilization syntax
 TEST(Array, InitializerListFixDim4) {
-    array a            = randu(1);
+    af::array a        = randu(1);
     vector<float> data = {3.14f, 3.14f, 3.14f, 3.14f, 3.14f,
                           3.14f, 3.14f, 3.14f, 3.14f};
-    array b{dim4(3, 3), data.data()};
+    af::array b{dim4(3, 3), data.data()};
     ASSERT_ARRAYS_EQ(constant(3.14, 3, 3), b);
 }
diff --git a/test/binary.cpp b/test/binary.cpp
index 2bc2a1a62a..06e720ed8e 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -577,3 +577,328 @@ TYPED_TEST(ResultTypeScalar, FloatMultiplication) {
 TYPED_TEST(ResultTypeScalar, FloatDivision) {
     ASSERT_EQ(f32, (af::array(10, f32) / this->scalar).type());
 }
+
+class Broadcast : public ::testing::TestWithParam<std::tuple<dim4, dim4> > {
+    void SetUp() override {}
+};
+/// clang-format off
+
+INSTANTIATE_TEST_CASE_P(
+    CorrectCases, Broadcast,
+    ::testing::Combine(
+        ::testing::Values(dim4(1), dim4(10), dim4(1, 10), dim4(1, 1, 10),
+                          dim4(1, 1, 1, 10), dim4(10, 10), dim4(1, 10, 10),
+                          dim4(1, 1, 10, 10), dim4(10, 1, 10),
+                          dim4(1, 10, 1, 10), dim4(10, 1, 1, 10),
+                          dim4(10, 10, 10), dim4(1, 10, 10, 10),
+                          dim4(10, 1, 10, 10), dim4(10, 10, 1, 10),
+                          dim4(10, 10, 10, 10)),
+        ::testing::Values(dim4(1), dim4(10), dim4(1, 10), dim4(1, 1, 10),
+                          dim4(1, 1, 1, 10), dim4(10, 10), dim4(1, 10, 10),
+                          dim4(1, 1, 10, 10), dim4(10, 1, 10),
+                          dim4(1, 10, 1, 10), dim4(10, 1, 1, 10),
+                          dim4(10, 10, 10), dim4(1, 10, 10, 10),
+                          dim4(10, 1, 10, 10), dim4(10, 10, 1, 10),
+                          dim4(10, 10, 10, 10))),
+    [](const ::testing::TestParamInfo<Broadcast::ParamType> info) {
+        stringstream ss;
+        ss << "lhs_" << get<0>(info.param) << "_rhs_" << get<1>(info.param);
+        string s = ss.str();
+        std::replace(begin(s), std::end(s), ' ', '_');
+        return s;
+    });
+/// clang-format on
+
+af::dim4 broadcastOut(dim4 lhs, dim4 rhs) {
+    dim4 out(1);
+    for (int i = 0; i < AF_MAX_DIMS; i++) {
+        if (lhs[i] == rhs[i])
+            out[i] = lhs[i];
+        else if (lhs[i] == 1 && rhs[i] > 1)
+            out[i] = rhs[i];
+        else if (lhs[i] > 1 && rhs[i] == 1)
+            out[i] = lhs[i];
+        else {
+            std::cout << "incorrect dimension" << lhs << " op " << rhs;
+            return dim4(0);
+        }
+    }
+    return out;
+}
+
+af::dim4 tileRepeations(dim4 in, dim4 other) {
+    af::dim4 out;
+    for (int i = 0; i < AF_MAX_DIMS; i++) {
+        out[i] = std::max(dim_t(1), other[i] / in[i]);
+    }
+    return out;
+}
+
+TEST_P(Broadcast, Addition) {
+    auto params   = GetParam();
+    af::array lhs = iota(get<0>(params));
+    af::array rhs = constant(1, get<1>(params));
+
+    af::array out = lhs + rhs;
+
+    af::dim4 outdims       = broadcastOut(lhs.dims(), rhs.dims());
+    af::dim4 tilerepetions = tileRepeations(lhs.dims(), rhs.dims());
+    af::array tiledlhs     = tile(lhs, tilerepetions);
+
+    vector<float> outvec(outdims.elements());
+    tiledlhs.host(outvec.data());
+    for (auto &out : outvec) { out += 1; }
+
+    ASSERT_VEC_ARRAY_EQ(outvec, outdims, out);
+}
+
+TEST_P(Broadcast, Subtraction) {
+    auto params   = GetParam();
+    af::array lhs = range(get<0>(params));
+    af::array rhs = constant(1, get<1>(params));
+
+    af::array out          = lhs - rhs;
+    af::dim4 outdims       = broadcastOut(lhs.dims(), rhs.dims());
+    af::dim4 tilerepetions = tileRepeations(lhs.dims(), rhs.dims());
+    af::array tiledlhs     = tile(lhs, tilerepetions);
+
+    vector<float> outvec(outdims.elements());
+    tiledlhs.host(outvec.data());
+    for (auto &out : outvec) { out -= 1; }
+
+    ASSERT_VEC_ARRAY_EQ(outvec, outdims, out);
+}
+
+TEST_P(Broadcast, Multiplication) {
+    auto params   = GetParam();
+    af::array lhs = range(get<0>(params));
+    af::array rhs = constant(2, get<1>(params));
+
+    af::array out          = lhs * rhs;
+    af::dim4 outdims       = broadcastOut(lhs.dims(), rhs.dims());
+    af::dim4 tilerepetions = tileRepeations(lhs.dims(), rhs.dims());
+    af::array tiledlhs     = tile(lhs, tilerepetions);
+
+    vector<float> outvec(outdims.elements());
+    tiledlhs.host(outvec.data());
+    for (auto &out : outvec) { out *= 2; }
+
+    ASSERT_VEC_ARRAY_EQ(outvec, outdims, out);
+}
+
+TEST_P(Broadcast, Division) {
+    auto params   = GetParam();
+    af::array lhs = range(get<0>(params));
+    af::array rhs = constant(2, get<1>(params));
+
+    af::array out          = lhs / rhs;
+    af::dim4 outdims       = broadcastOut(lhs.dims(), rhs.dims());
+    af::dim4 tilerepetions = tileRepeations(lhs.dims(), rhs.dims());
+    af::array tiledlhs     = tile(lhs, tilerepetions);
+
+    vector<float> outvec(outdims.elements());
+    tiledlhs.host(outvec.data());
+    for (auto &out : outvec) { out /= 2; }
+
+    ASSERT_VEC_ARRAY_EQ(outvec, outdims, out);
+}
+
+TEST_P(Broadcast, AdditionLHSIndexed) {
+    auto params   = GetParam();
+    af::array lhs = iota(get<0>(params) * 2);
+    af::array rhs = constant(1, get<1>(params));
+
+    dim4 lhs_dims = get<0>(params);
+    af::array out = lhs(seq(lhs_dims[0]), seq(lhs_dims[1]), seq(lhs_dims[2]),
+                        seq(lhs_dims[3])) +
+                    rhs;
+
+    af::dim4 outdims       = broadcastOut(get<0>(params), rhs.dims());
+    af::array indexedlhs   = lhs(seq(lhs_dims[0]), seq(lhs_dims[1]),
+                               seq(lhs_dims[2]), seq(lhs_dims[3]));
+    af::dim4 tilerepetions = tileRepeations(get<0>(params), rhs.dims());
+    af::array tiledlhs     = tile(indexedlhs, tilerepetions);
+
+    vector<float> outvec(outdims.elements());
+    tiledlhs.host(outvec.data());
+    for (auto &out : outvec) { out += 1; }
+
+    ASSERT_VEC_ARRAY_EQ(outvec, outdims, out);
+}
+
+TEST_P(Broadcast, AdditionRHSIndexed) {
+    auto params   = GetParam();
+    af::array lhs = iota(get<0>(params));
+    af::array rhs = constant(1, get<1>(params) * 2);
+
+    dim4 rhs_dims = get<1>(params);
+    af::array out = lhs + rhs(seq(rhs_dims[0]), seq(rhs_dims[1]),
+                              seq(rhs_dims[2]), seq(rhs_dims[3]));
+
+    af::dim4 outdims       = broadcastOut(get<0>(params), get<1>(params));
+    af::dim4 tilerepetions = tileRepeations(get<0>(params), get<1>(params));
+    af::array tiledlhs     = tile(lhs, tilerepetions);
+
+    vector<float> outvec(outdims.elements());
+    tiledlhs.host(outvec.data());
+    for (auto &out : outvec) { out += 1; }
+
+    ASSERT_VEC_ARRAY_EQ(outvec, outdims, out);
+}
+
+TEST_P(Broadcast, AdditionBothIndexed) {
+    auto params   = GetParam();
+    af::array lhs = iota(get<0>(params) * 2);
+    af::array rhs = constant(1, get<1>(params) * 2);
+
+    dim4 lhs_dims = get<0>(params);
+    dim4 rhs_dims = get<1>(params);
+    af::array out = lhs(seq(lhs_dims[0]), seq(lhs_dims[1]), seq(lhs_dims[2]),
+                        seq(lhs_dims[3])) +
+                    rhs(seq(rhs_dims[0]), seq(rhs_dims[1]), seq(rhs_dims[2]),
+                        seq(rhs_dims[3]));
+
+    af::dim4 outdims       = broadcastOut(lhs_dims, rhs_dims);
+    af::array indexedlhs   = lhs(seq(lhs_dims[0]), seq(lhs_dims[1]),
+                               seq(lhs_dims[2]), seq(lhs_dims[3]));
+    af::dim4 tilerepetions = tileRepeations(get<0>(params), get<1>(params));
+    af::array tiledlhs     = tile(indexedlhs, tilerepetions);
+
+    vector<float> outvec(outdims.elements());
+    tiledlhs.host(outvec.data());
+    for (auto &out : outvec) { out += 1; }
+
+    ASSERT_VEC_ARRAY_EQ(outvec, outdims, out);
+}
+
+TEST(Broadcast, VectorMatrix2d) {
+    dim_t s     = 10;
+    af::array A = range(dim4(s, 3), 1);
+    af::array B = -range(dim4(3));
+
+    try {
+        A + B;
+        FAIL();
+    } catch (af::exception &e) { ASSERT_EQ(e.err(), AF_ERR_SIZE); }
+    try {
+        B + A;
+        FAIL();
+    } catch (af::exception &e) { ASSERT_EQ(e.err(), AF_ERR_SIZE); }
+}
+
+TEST(Broadcast, VectorMatrix3d) {
+    dim_t s     = 10;
+    af::array A = range(dim4(s, s, 3), 2);
+    af::array B = -range(dim4(3));
+
+    try {
+        A + B;
+        FAIL();
+    } catch (af::exception &e) { ASSERT_EQ(e.err(), AF_ERR_SIZE); }
+    try {
+        B + A;
+        FAIL();
+    } catch (af::exception &e) { ASSERT_EQ(e.err(), AF_ERR_SIZE); }
+}
+
+TEST(Broadcast, VectorMatrix4d) {
+    dim_t s     = 10;
+    af::array A = range(dim4(s, s, s, 3), 3);
+    af::array B = -range(dim4(3));
+
+    try {
+        A + B;
+        FAIL();
+    } catch (af::exception &e) { ASSERT_EQ(e.err(), AF_ERR_SIZE); }
+    try {
+        B + A;
+        FAIL();
+    } catch (af::exception &e) { ASSERT_EQ(e.err(), AF_ERR_SIZE); }
+}
+
+void testAllBroadcast(dim4 dims) {
+    af::array A = constant(1, dims);
+    for (int k = 0; k < dims.ndims(); ++k) {
+        dim4 rdims  = dims;
+        rdims[k]    = 1;
+        af::array B = constant(-1, rdims);
+        af::array C = A + B;
+        ASSERT_ARRAYS_EQ(C, constant(0, dims));
+
+        C = B + A;
+        ASSERT_ARRAYS_EQ(C, constant(0, dims));
+    }
+}
+
+TEST(Broadcast, MatrixMatrix2d) { testAllBroadcast(dim4(10, 15)); }
+
+TEST(Broadcast, MatrixMatrix3d) { testAllBroadcast(dim4(10, 15, 20)); }
+
+TEST(Broadcast, MatrixMatrix4d) { testAllBroadcast(dim4(10, 15, 20, 25)); }
+
+TEST(Broadcast, MismatchingDim0) {
+    af::array A = range(dim4(2, 3, 5), 1);
+    af::array B = -range(dim4(3, 5), 0);
+
+    try {
+        A + B;
+    } catch (af::exception &e) { ASSERT_EQ(e.err(), AF_ERR_SIZE); }
+}
+
+TEST(Broadcast, TestFirstMatchingDim) {
+    af::array A = range(dim4(3, 2, 2, 4), 1);
+    af::array B = -range(dim4(2));
+
+    try {
+        A + B;
+    } catch (af::exception &e) { ASSERT_EQ(e.err(), AF_ERR_SIZE); }
+}
+
+TEST(Broadcast, ManySlicesVsOneSlice) {
+    af::array A = constant(1, dim4(3, 3, 2));
+    af::array B = constant(2, dim4(3, 3));
+    af::array C = A + B;
+
+    ASSERT_ARRAYS_EQ(C, constant(3, dim4(3, 3, 2)));
+
+    C = B + A;
+    ASSERT_ARRAYS_EQ(C, constant(3, dim4(3, 3, 2)));
+}
+
+TEST(Broadcast, SubArray) {
+    dim_t subdim = 5;
+    af::array A  = constant(1, dim4(10, 10, 2));
+    af::array B  = constant(2, dim4(5, 5));
+    af::array C  = A(seq(subdim), seq(subdim), span) + B;
+
+    ASSERT_ARRAYS_EQ(C, constant(3, dim4(subdim, subdim, 2)));
+
+    C = B + A(seq(subdim), seq(subdim), span);
+    ASSERT_ARRAYS_EQ(C, constant(3, dim4(subdim, subdim, 2)));
+}
+
+TEST(Broadcast, SubArrays) {
+    dim_t subdim = 5;
+    af::array A  = constant(1, dim4(10, 10, 2));
+    af::array B  = constant(2, dim4(15, 15));
+
+    af::array C =
+        A(seq(subdim), seq(subdim), span) + B(seq(subdim), seq(subdim));
+    ASSERT_ARRAYS_EQ(C, constant(3, dim4(subdim, subdim, 2)));
+
+    C = B(seq(subdim), seq(subdim)) + A(seq(subdim), seq(subdim), span);
+    ASSERT_ARRAYS_EQ(C, constant(3, dim4(subdim, subdim, 2)));
+}
+
+TEST(Broadcast, IndexedArray) {
+    af::array A = constant(1, dim4(2, 2, 2, 2));
+    af::array B = constant(-1, dim4(1, 5));
+
+    af::array idx = range(dim4(2, 2, 2, 2), 0);
+
+    af::array C = A(idx % 2 == 0) + B;
+    ASSERT_ARRAYS_EQ(C, constant(0, dim4(8, 5)));
+
+    C = B + A(idx % 2 == 0);
+    ASSERT_ARRAYS_EQ(C, constant(0, dim4(8, 5)));
+}

From f199a5d1d9074e0c77d140a9ef3aee66bf871422 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 1 Jul 2022 20:58:58 -0400
Subject: [PATCH 060/473] Call setDevice on each thread at entry point.

CUDA requires that cudaSetDevice be called in each thread before
any other calls are made to the CUDA API. This is done by default
on the main thread but it is not done on new threads created. This
commit changes the behavior or the af_init function so that it call
the cudaSetDevice when creating a new object in ArrayFire.

This commit also refactors the af_init function so that it calls a lower
overhead init function which initializes the device manager.
---
 src/api/c/device.cpp            | 3 ++-
 src/backend/cpu/platform.cpp    | 5 +++++
 src/backend/cpu/platform.hpp    | 2 ++
 src/backend/cuda/platform.cpp   | 6 ++++++
 src/backend/cuda/platform.hpp   | 2 ++
 src/backend/opencl/platform.cpp | 5 +++++
 src/backend/opencl/platform.hpp | 2 ++
 7 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp
index 3ed23a0c3e..cf65bfd81c 100644
--- a/src/api/c/device.cpp
+++ b/src/api/c/device.cpp
@@ -39,6 +39,7 @@ using detail::getActiveDeviceId;
 using detail::getBackend;
 using detail::getDeviceCount;
 using detail::getDeviceInfo;
+using detail::init;
 using detail::intl;
 using detail::isDoubleSupported;
 using detail::isHalfSupported;
@@ -107,7 +108,7 @@ af_err af_init() {
     try {
         thread_local std::once_flag flag;
         std::call_once(flag, []() {
-            getDeviceInfo();
+            init();
 #if defined(USE_MKL) && !defined(USE_STATIC_MKL)
             int errCode = -1;
             // Have used the AF_MKL_INTERFACE_SIZE as regular if's so that
diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index 523737b07a..3f83956b91 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -112,6 +112,11 @@ int& getMaxJitSize() {
 
 int getDeviceCount() { return DeviceManager::NUM_DEVICES; }
 
+void init() {
+    thread_local const auto& instance = DeviceManager::getInstance();
+    UNUSED(instance);
+}
+
 // Get the currently active device id
 unsigned getActiveDeviceId() { return DeviceManager::ACTIVE_DEVICE_ID; }
 
diff --git a/src/backend/cpu/platform.hpp b/src/backend/cpu/platform.hpp
index a37f12351f..f50e16461b 100644
--- a/src/backend/cpu/platform.hpp
+++ b/src/backend/cpu/platform.hpp
@@ -40,6 +40,8 @@ int& getMaxJitSize();
 
 int getDeviceCount();
 
+void init();
+
 unsigned getActiveDeviceId();
 
 size_t getDeviceMemorySize(int device);
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 0e639ec62d..647566eb2a 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -348,6 +348,12 @@ int getDeviceCount() {
     }
 }
 
+void init() {
+    thread_local auto err =
+        cudaSetDevice(getDeviceNativeId(getActiveDeviceId()));
+    UNUSED(err);
+}
+
 unsigned getActiveDeviceId() { return tlocalActiveDeviceId(); }
 
 int getDeviceNativeId(int device) {
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index b4e9dd2360..6d1778b3ab 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -80,6 +80,8 @@ int& getMaxJitSize();
 
 int getDeviceCount();
 
+void init();
+
 unsigned getActiveDeviceId();
 
 int getDeviceNativeId(int device);
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index e2c4571995..b159758b37 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -218,6 +218,11 @@ int getDeviceCount() noexcept try {
     return 0;
 }
 
+void init() {
+    thread_local const DeviceManager& devMngr = DeviceManager::getInstance();
+    UNUSED(devMngr);
+}
+
 unsigned getActiveDeviceId() {
     // Second element is the queue id, which is
     // what we mean by active device id in opencl backend
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 6292c1331d..8ea6ca2540 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -55,6 +55,8 @@ std::string getDeviceInfo() noexcept;
 
 int getDeviceCount() noexcept;
 
+void init();
+
 unsigned getActiveDeviceId();
 
 int& getMaxJitSize();

From ae348f5e931919cf77f8dafdfce0de3a6e26a8b0 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 8 Jul 2022 15:47:08 -0400
Subject: [PATCH 061/473] Fix missing release_array calls in the reduce tests

---
 test/reduce.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/reduce.cpp b/test/reduce.cpp
index c9e09f53fd..69e6573d3c 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -2303,10 +2303,14 @@ TEST(Reduce, nanval_issue_3255) {
         af_product_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0);
         af::array ovals_cpp(ovals);
         ASSERT_FALSE(af::anyTrue<bool>(af::isNaN(ovals_cpp)));
+        ASSERT_SUCCESS(af_release_array(okeys));
 
         af_sum_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0);
         ovals_cpp = af::array(ovals);
 
         ASSERT_FALSE(af::anyTrue<bool>(af::isNaN(ovals_cpp)));
+        ASSERT_SUCCESS(af_release_array(ivals));
+        ASSERT_SUCCESS(af_release_array(okeys));
     }
+    ASSERT_SUCCESS(af_release_array(ikeys));
 }

From 4d04cd3ed87cb6ad0857b50c50b78f2d7c305f90 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 8 Jul 2022 15:47:42 -0400
Subject: [PATCH 062/473] Remove unnecessary death test in test/array.cpp

---
 test/array.cpp | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/test/array.cpp b/test/array.cpp
index deb85e2e22..08b5a568d7 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -584,14 +584,10 @@ TEST(Array, CopyListInitializerListDim4Assignment) {
 }
 
 TEST(Array, EmptyArrayHostCopy) {
-    EXPECT_EXIT(
-        {
-            af::array empty;
-            std::vector<float> hdata(100);
-            empty.host(hdata.data());
-            exit(0);
-        },
-        ::testing::ExitedWithCode(0), ".*");
+    af::array empty;
+    std::vector<float> hdata(100);
+    empty.host(hdata.data());
+    SUCCEED();
 }
 
 TEST(Array, ReferenceCount1) {

From aefe79addd7c54ade1788417ce9b37f509512049 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 8 Jul 2022 15:48:28 -0400
Subject: [PATCH 063/473] Refactor SIFT for memory usage and fix memory leak in
 GLOH and SIFT tests

---
 src/backend/opencl/kernel/sift.hpp | 129 ++++++++++-------------------
 src/backend/opencl/memory.hpp      |   5 +-
 2 files changed, 46 insertions(+), 88 deletions(-)

diff --git a/src/backend/opencl/kernel/sift.hpp b/src/backend/opencl/kernel/sift.hpp
index bd10faa1ce..4b1609514e 100644
--- a/src/backend/opencl/kernel/sift.hpp
+++ b/src/backend/opencl/kernel/sift.hpp
@@ -400,13 +400,20 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
     vector<Param> dog_pyr =
         buildDoGPyr<T>(gauss_pyr, n_octaves, n_layers, kernels[0]);
 
-    vector<Buffer*> d_x_pyr(n_octaves, NULL);
-    vector<Buffer*> d_y_pyr(n_octaves, NULL);
-    vector<Buffer*> d_response_pyr(n_octaves, NULL);
-    vector<Buffer*> d_size_pyr(n_octaves, NULL);
-    vector<Buffer*> d_ori_pyr(n_octaves, NULL);
-    vector<Buffer*> d_desc_pyr(n_octaves, NULL);
+    vector<bufptr> d_x_pyr;
+    vector<bufptr> d_y_pyr;
+    vector<bufptr> d_response_pyr;
+    vector<bufptr> d_size_pyr;
+    vector<bufptr> d_ori_pyr;
+    vector<bufptr> d_desc_pyr;
     vector<unsigned> feat_pyr(n_octaves, 0);
+
+    d_x_pyr.reserve(n_octaves);
+    d_y_pyr.reserve(n_octaves);
+    d_response_pyr.reserve(n_octaves);
+    d_size_pyr.reserve(n_octaves);
+    d_ori_pyr.reserve(n_octaves);
+    d_desc_pyr.reserve(n_octaves);
     unsigned total_feat = 0;
 
     const unsigned d  = DescrWidth;
@@ -417,7 +424,7 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
     const unsigned desc_len =
         (compute_GLOH) ? (1 + (rb - 1) * ab) * hb : d * d * n;
 
-    Buffer* d_count = bufferAlloc(sizeof(unsigned));
+    auto d_count = memAlloc<unsigned>(1);
 
     for (unsigned o = 0; o < n_octaves; o++) {
         if (dog_pyr[o].info.dims[0] - 2 * ImgBorder < 1 ||
@@ -427,9 +434,9 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
         const unsigned imel = dog_pyr[o].info.dims[0] * dog_pyr[o].info.dims[1];
         const unsigned max_feat = ceil(imel * feature_ratio);
 
-        Buffer* d_extrema_x     = bufferAlloc(max_feat * sizeof(float));
-        Buffer* d_extrema_y     = bufferAlloc(max_feat * sizeof(float));
-        Buffer* d_extrema_layer = bufferAlloc(max_feat * sizeof(unsigned));
+        auto d_extrema_x     = memAlloc<float>(max_feat);
+        auto d_extrema_y     = memAlloc<float>(max_feat);
+        auto d_extrema_layer = memAlloc<unsigned>(max_feat);
 
         unsigned extrema_feat = 0;
         getQueue().enqueueWriteBuffer(*d_count, CL_FALSE, 0, sizeof(unsigned),
@@ -458,23 +465,17 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
                                      &extrema_feat);
         extrema_feat = std::min(extrema_feat, max_feat);
 
-        if (extrema_feat == 0) {
-            bufferFree(d_extrema_x);
-            bufferFree(d_extrema_y);
-            bufferFree(d_extrema_layer);
-
-            continue;
-        }
+        if (extrema_feat == 0) { continue; }
 
         unsigned interp_feat = 0;
         getQueue().enqueueWriteBuffer(*d_count, CL_FALSE, 0, sizeof(unsigned),
                                       &interp_feat);
 
-        Buffer* d_interp_x     = bufferAlloc(extrema_feat * sizeof(float));
-        Buffer* d_interp_y     = bufferAlloc(extrema_feat * sizeof(float));
-        Buffer* d_interp_layer = bufferAlloc(extrema_feat * sizeof(unsigned));
-        Buffer* d_interp_response = bufferAlloc(extrema_feat * sizeof(float));
-        Buffer* d_interp_size     = bufferAlloc(extrema_feat * sizeof(float));
+        auto d_interp_x        = memAlloc<float>(extrema_feat);
+        auto d_interp_y        = memAlloc<float>(extrema_feat);
+        auto d_interp_layer    = memAlloc<unsigned>(extrema_feat);
+        auto d_interp_response = memAlloc<float>(extrema_feat);
+        auto d_interp_size     = memAlloc<float>(extrema_feat);
 
         const int blk_x_interp = divup(extrema_feat, SIFT_THREADS);
         const NDRange local_interp(SIFT_THREADS, 1);
@@ -489,23 +490,11 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
              n_layers, contrast_thr, edge_thr, init_sigma, img_scale);
         CL_DEBUG_FINISH(getQueue());
 
-        bufferFree(d_extrema_x);
-        bufferFree(d_extrema_y);
-        bufferFree(d_extrema_layer);
-
         getQueue().enqueueReadBuffer(*d_count, CL_TRUE, 0, sizeof(unsigned),
                                      &interp_feat);
         interp_feat = std::min(interp_feat, extrema_feat);
 
-        if (interp_feat == 0) {
-            bufferFree(d_interp_x);
-            bufferFree(d_interp_y);
-            bufferFree(d_interp_layer);
-            bufferFree(d_interp_response);
-            bufferFree(d_interp_size);
-
-            continue;
-        }
+        if (interp_feat == 0) { continue; }
 
         compute::command_queue queue(getQueue()());
         compute::context context(getContext()());
@@ -546,11 +535,11 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
         getQueue().enqueueWriteBuffer(*d_count, CL_FALSE, 0, sizeof(unsigned),
                                       &nodup_feat);
 
-        Buffer* d_nodup_x        = bufferAlloc(interp_feat * sizeof(float));
-        Buffer* d_nodup_y        = bufferAlloc(interp_feat * sizeof(float));
-        Buffer* d_nodup_layer    = bufferAlloc(interp_feat * sizeof(unsigned));
-        Buffer* d_nodup_response = bufferAlloc(interp_feat * sizeof(float));
-        Buffer* d_nodup_size     = bufferAlloc(interp_feat * sizeof(float));
+        auto d_nodup_x        = memAlloc<float>(interp_feat);
+        auto d_nodup_y        = memAlloc<float>(interp_feat);
+        auto d_nodup_layer    = memAlloc<unsigned>(interp_feat);
+        auto d_nodup_response = memAlloc<float>(interp_feat);
+        auto d_nodup_size     = memAlloc<float>(interp_feat);
 
         const int blk_x_nodup = divup(extrema_feat, SIFT_THREADS);
         const NDRange local_nodup(SIFT_THREADS, 1);
@@ -568,26 +557,17 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
                                      &nodup_feat);
         nodup_feat = std::min(nodup_feat, interp_feat);
 
-        bufferFree(d_interp_x);
-        bufferFree(d_interp_y);
-        bufferFree(d_interp_layer);
-        bufferFree(d_interp_response);
-        bufferFree(d_interp_size);
-
         unsigned oriented_feat = 0;
         getQueue().enqueueWriteBuffer(*d_count, CL_FALSE, 0, sizeof(unsigned),
                                       &oriented_feat);
         const unsigned max_oriented_feat = nodup_feat * 3;
 
-        Buffer* d_oriented_x = bufferAlloc(max_oriented_feat * sizeof(float));
-        Buffer* d_oriented_y = bufferAlloc(max_oriented_feat * sizeof(float));
-        Buffer* d_oriented_layer =
-            bufferAlloc(max_oriented_feat * sizeof(unsigned));
-        Buffer* d_oriented_response =
-            bufferAlloc(max_oriented_feat * sizeof(float));
-        Buffer* d_oriented_size =
-            bufferAlloc(max_oriented_feat * sizeof(float));
-        Buffer* d_oriented_ori = bufferAlloc(max_oriented_feat * sizeof(float));
+        auto d_oriented_x        = memAlloc<float>(max_oriented_feat);
+        auto d_oriented_y        = memAlloc<float>(max_oriented_feat);
+        auto d_oriented_layer    = memAlloc<unsigned>(max_oriented_feat);
+        auto d_oriented_response = memAlloc<float>(max_oriented_feat);
+        auto d_oriented_size     = memAlloc<float>(max_oriented_feat);
+        auto d_oriented_ori      = memAlloc<float>(max_oriented_feat);
 
         const int blk_x_ori = divup(nodup_feat, SIFT_THREADS_Y);
         const NDRange local_ori(SIFT_THREADS_X, SIFT_THREADS_Y);
@@ -604,27 +584,13 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
              Local(OriHistBins * SIFT_THREADS_Y * 2 * sizeof(float)));
         CL_DEBUG_FINISH(getQueue());
 
-        bufferFree(d_nodup_x);
-        bufferFree(d_nodup_y);
-        bufferFree(d_nodup_layer);
-        bufferFree(d_nodup_response);
-        bufferFree(d_nodup_size);
-
         getQueue().enqueueReadBuffer(*d_count, CL_TRUE, 0, sizeof(unsigned),
                                      &oriented_feat);
         oriented_feat = std::min(oriented_feat, max_oriented_feat);
 
-        if (oriented_feat == 0) {
-            bufferFree(d_oriented_x);
-            bufferFree(d_oriented_y);
-            bufferFree(d_oriented_layer);
-            bufferFree(d_oriented_response);
-            bufferFree(d_oriented_size);
+        if (oriented_feat == 0) { continue; }
 
-            continue;
-        }
-
-        Buffer* d_desc = bufferAlloc(oriented_feat * desc_len * sizeof(float));
+        auto d_desc = memAlloc<float>(oriented_feat * desc_len);
 
         float scale = 1.f / (1 << o);
         if (double_input) scale *= 2.f;
@@ -660,17 +626,15 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
         feat_pyr[o] = oriented_feat;
 
         if (oriented_feat > 0) {
-            d_x_pyr[o]        = d_oriented_x;
-            d_y_pyr[o]        = d_oriented_y;
-            d_response_pyr[o] = d_oriented_response;
-            d_ori_pyr[o]      = d_oriented_ori;
-            d_size_pyr[o]     = d_oriented_size;
-            d_desc_pyr[o]     = d_desc;
+            d_x_pyr.emplace_back(std::move(d_oriented_x));
+            d_y_pyr.emplace_back(std::move(d_oriented_y));
+            d_response_pyr.emplace_back(std::move(d_oriented_response));
+            d_ori_pyr.emplace_back(std::move(d_oriented_ori));
+            d_size_pyr.emplace_back(std::move(d_oriented_size));
+            d_desc_pyr.emplace_back(std::move(d_desc));
         }
     }
 
-    bufferFree(d_count);
-
     for (size_t i = 0; i < gauss_pyr.size(); i++) bufferFree(gauss_pyr[i].data);
     for (size_t i = 0; i < dog_pyr.size(); i++) bufferFree(dog_pyr[i].data);
 
@@ -755,13 +719,6 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
                                      offset * desc_len * sizeof(unsigned),
                                      feat_pyr[i] * desc_len * sizeof(unsigned));
 
-        bufferFree(d_x_pyr[i]);
-        bufferFree(d_y_pyr[i]);
-        bufferFree(d_response_pyr[i]);
-        bufferFree(d_ori_pyr[i]);
-        bufferFree(d_size_pyr[i]);
-        bufferFree(d_desc_pyr[i]);
-
         offset += feat_pyr[i];
     }
 
diff --git a/src/backend/opencl/memory.hpp b/src/backend/opencl/memory.hpp
index 778c611ad9..ba7e340d32 100644
--- a/src/backend/opencl/memory.hpp
+++ b/src/backend/opencl/memory.hpp
@@ -24,9 +24,10 @@ namespace opencl {
 cl::Buffer *bufferAlloc(const size_t &bytes);
 void bufferFree(cl::Buffer *buf);
 
+using bufptr = std::unique_ptr<cl::Buffer, std::function<void(cl::Buffer *)>>;
+
 template<typename T>
-std::unique_ptr<cl::Buffer, std::function<void(cl::Buffer *)>> memAlloc(
-    const size_t &elements);
+bufptr memAlloc(const size_t &elements);
 void *memAllocUser(const size_t &bytes);
 
 // Need these as 2 separate function and not a default argument

From 9ca49de3db6271866315adff917e57255aea019e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 8 Jul 2022 15:49:57 -0400
Subject: [PATCH 064/473] Rename the name for the basic_c.cpp tests

---
 test/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 1c7bc8792e..09a794c63b 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -432,7 +432,7 @@ make_test(SRC write.cpp)
 make_test(SRC ycbcr_rgb.cpp)
 
 foreach(backend ${enabled_backends})
-  set(target "test_basic_c_${backend}")
+  set(target "basic_c_${backend}")
   add_executable(${target} basic_c.c)
   if(${backend} STREQUAL "unified")
     target_link_libraries(${target}
@@ -443,7 +443,7 @@ foreach(backend ${enabled_backends})
       PRIVATE
       ArrayFire::af${backend})
   endif()
-  add_test(NAME ${target} COMMAND ${target})
+  add_test(NAME test_${target} COMMAND ${target})
 endforeach()
 
 if(AF_TEST_WITH_MTX_FILES)

From b05da694a3f789579af25887108a214f1a978326 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 8 Jul 2022 17:41:46 -0400
Subject: [PATCH 065/473] Fix leaks in clFFT and update reference. Update
 LSANSuppressions

---
 CMakeModules/LSANSuppression.txt | 2 +-
 CMakeModules/build_clFFT.cmake   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeModules/LSANSuppression.txt b/CMakeModules/LSANSuppression.txt
index 43ac584d10..b305e805f3 100644
--- a/CMakeModules/LSANSuppression.txt
+++ b/CMakeModules/LSANSuppression.txt
@@ -2,11 +2,11 @@
 leak:libnvidia-ptxjitcompile
 leak:tbb::internal::task_stream
 leak:libnvidia-opencl.so
-leak:FFTRepo::FFTRepoKey::privatizeData
 
 # Allocated by Intel's OpenMP implementation during inverse_dense_cpu
 # This is not something we can control in ArrayFire
 leak:kmp_alloc_cpp*::bget
+leak:kmp_b_alloc
 
 # ArrayFire leaks the default random engine on each thread. This is to avoid
 # errors on exit on Windows.
diff --git a/CMakeModules/build_clFFT.cmake b/CMakeModules/build_clFFT.cmake
index 380357e02e..dc29e22ced 100644
--- a/CMakeModules/build_clFFT.cmake
+++ b/CMakeModules/build_clFFT.cmake
@@ -7,7 +7,7 @@
 
 af_dep_check_and_populate(${clfft_prefix}
   URI https://github.com/arrayfire/clFFT.git
-  REF cmake_fixes
+  REF arrayfire-release
 )
 
 set(current_build_type ${BUILD_SHARED_LIBS})

From 5a512056921929d2dbce1a1449a32115bd123588 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 23 Jul 2022 16:56:06 -0400
Subject: [PATCH 066/473] Fix issue where ndims was incorrectly used to
 calculate shape of input

---
 src/api/c/convolve.cpp | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/api/c/convolve.cpp b/src/api/c/convolve.cpp
index ddcd916ae6..9a496633b0 100644
--- a/src/api/c/convolve.cpp
+++ b/src/api/c/convolve.cpp
@@ -344,14 +344,17 @@ af_err af_convolve2_nn(af_array *out, const af_array signal,
 
         const af_dtype signalType = sInfo.getType();
 
-        ARG_ASSERT(3, stride_dims > 0 && stride_dims <= 2);
-        ARG_ASSERT(5, padding_dims > 0 && padding_dims <= 2);
-        ARG_ASSERT(7, dilation_dims > 0 && dilation_dims <= 2);
-
         dim4 stride(stride_dims, strides);
         dim4 padding(padding_dims, paddings);
         dim4 dilation(dilation_dims, dilations);
 
+        size_t stride_ndims   = stride.ndims();
+        size_t padding_ndims  = padding.ndims();
+        size_t dilation_ndims = dilation.ndims();
+        ARG_ASSERT(3, stride_ndims > 0 && stride_ndims <= 2);
+        ARG_ASSERT(5, padding_ndims >= 0 && padding_ndims <= 2);
+        ARG_ASSERT(7, dilation_ndims > 0 && dilation_ndims <= 2);
+
         // assert number of features matches between signal and filter
         DIM_ASSERT(1, sDims[2] == fDims[2]);
 
@@ -424,14 +427,17 @@ af_err af_convolve2_gradient_nn(
 
         af_array output;
 
-        ARG_ASSERT(3, stride_dims > 0 && stride_dims <= 2);
-        ARG_ASSERT(5, padding_dims > 0 && padding_dims <= 2);
-        ARG_ASSERT(7, dilation_dims > 0 && dilation_dims <= 2);
-
         af::dim4 stride(stride_dims, strides);
         af::dim4 padding(padding_dims, paddings);
         af::dim4 dilation(dilation_dims, dilations);
 
+        size_t stride_ndims   = stride.ndims();
+        size_t padding_ndims  = padding.ndims();
+        size_t dilation_ndims = dilation.ndims();
+        ARG_ASSERT(3, stride_ndims > 0 && stride_ndims <= 2);
+        ARG_ASSERT(5, padding_ndims > 0 && padding_ndims <= 2);
+        ARG_ASSERT(7, dilation_ndims > 0 && dilation_ndims <= 2);
+
         af_dtype type = oinfo.getType();
         switch (type) {
             case f32:

From be7f2d93de3796050e56037cc0c340a2ef34e813 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 10 Jun 2022 18:43:52 -0400
Subject: [PATCH 067/473] Make constructors that accept simple types explicit

Some of the ArrayFire constructors that accept the dim_t type are not
marked explicit. this allows the initialization of the ArrayFire's array
using integer types. For example

```
af::array a = 5
```

will create an af::array with 5 elements. This is not intended behavior.

I have looked into the ABI for this change and it doesn't seem to
be affected on GCC. I have to still test this on MSVC. This CAN
break some existing code because it does change the API but ArrayFire
was never designed with this code in mind.
---
 .../machine_learning/geneticalgorithm.cpp     | 21 ++++++++++---------
 include/af/array.h                            | 16 ++++++++++----
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/examples/machine_learning/geneticalgorithm.cpp b/examples/machine_learning/geneticalgorithm.cpp
index d930a9cd44..184bc9914e 100644
--- a/examples/machine_learning/geneticalgorithm.cpp
+++ b/examples/machine_learning/geneticalgorithm.cpp
@@ -123,8 +123,8 @@ void reproducePrint(float& currentMax, array& searchSpace, array& sampleX,
 }
 
 void geneticSearch(bool console, const int nSamples, const int n) {
-    array searchSpaceXDisplay = 0;
-    array searchSpaceYDisplay = 0;
+    array searchSpaceXDisplay;
+    array searchSpaceYDisplay;
     array searchSpace;
     array sampleX;
     array sampleY;
@@ -170,17 +170,18 @@ int main(int argc, char** argv) {
 
     try {
         af::info();
-        printf("** ArrayFire Genetic Algorithm Search Demo **\n\n");
         printf(
-            "Search for trueMax in a search space where the objective function "
-            "is defined as :\n\n");
-        printf("SS(x ,y) = min(x, n - (x + 1)) + min(y, n - (y + 1))\n\n");
-        printf("(x, y) belongs to RxR; R = [0, n); n = %d\n\n", n);
+            "** ArrayFire Genetic Algorithm Search Demo **\n\n"
+            "Search for trueMax in a search space where the objective "
+            "function is defined as :\n\n"
+            "SS(x ,y) = min(x, n - (x + 1)) + min(y, n - (y + 1))\n\n"
+            "(x, y) belongs to RxR; R = [0, n); n = %d\n\n",
+            n);
         if (!console) {
-            printf("The left figure shows the objective function.\n");
             printf(
-                "The figure on the right shows current generation's parameters "
-                "and function values.\n\n");
+                "The left figure shows the objective function.\n"
+                "The right figure shows current generation's "
+                "parameters and function values.\n\n");
         }
         geneticSearch(console, nSamples, n);
     } catch (af::exception& e) { fprintf(stderr, "%s\n", e.what()); }
diff --git a/include/af/array.h b/include/af/array.h
index b1405c903c..0edb9558e1 100644
--- a/include/af/array.h
+++ b/include/af/array.h
@@ -246,6 +246,7 @@ namespace af
                        (default is f32)
 
         */
+        explicit
         array(dim_t dim0, dtype ty = f32);
 
         /**
@@ -271,6 +272,7 @@ namespace af
                        (default is f32)
 
         */
+        explicit
         array(dim_t dim0, dim_t dim1, dtype ty = f32);
 
         /**
@@ -297,6 +299,7 @@ namespace af
                        (default is f32)
 
         */
+        explicit
         array(dim_t dim0, dim_t dim1, dim_t dim2, dtype ty = f32);
 
         /**
@@ -324,6 +327,7 @@ namespace af
                        (default is f32)
 
         */
+        explicit
         array(dim_t dim0, dim_t dim1, dim_t dim2, dim_t dim3, dtype ty = f32);
 
         /**
@@ -368,10 +372,10 @@ namespace af
 
             array A(4, h_buffer);   // copy host data to device
                                     //
-                                    // A = 23
-                                    //   = 34
-                                    //   = 18
-                                    //   = 99
+                                    // A = [23]
+                                    //     [34]
+                                    //     [18]
+                                    //     [99]
 
             \endcode
 
@@ -382,6 +386,7 @@ namespace af
 
         */
         template<typename T>
+        explicit
         array(dim_t dim0,
               const T *pointer, af::source src=afHost);
 
@@ -409,6 +414,7 @@ namespace af
                   format when performing linear algebra operations.
         */
         template<typename T>
+        explicit
         array(dim_t dim0, dim_t dim1,
               const T *pointer, af::source src=afHost);
 
@@ -440,6 +446,7 @@ namespace af
             \image html 3dArray.png
         */
         template<typename T>
+        explicit
         array(dim_t dim0, dim_t dim1, dim_t dim2,
               const T *pointer, af::source src=afHost);
 
@@ -473,6 +480,7 @@ namespace af
 
         */
         template<typename T>
+        explicit
         array(dim_t dim0, dim_t dim1, dim_t dim2, dim_t dim3,
               const T *pointer, af::source src=afHost);
 

From 04bcd18aa4851e0fd933b5164249634f3243cebd Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 2 Sep 2022 13:22:41 -0400
Subject: [PATCH 068/473] Update cmake minimum version to 3.10.2

---
 .github/workflows/unix_cpu_build.yml | 4 ++--
 CMakeLists.txt                       | 2 +-
 CMakeModules/CPackConfig.cmake       | 2 +-
 test/mmio/CMakeLists.txt             | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 47dff97a42..ad616ddd3d 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -14,7 +14,7 @@ jobs:
         runs-on: ${{ matrix.os }}
         env:
           NINJA_VER: 1.10.2
-          CMAKE_VER: 3.5.1
+          CMAKE_VER: 3.10.2
         strategy:
             fail-fast: false
             matrix:
@@ -39,7 +39,7 @@ jobs:
                   chmod +x ninja
                   ${GITHUB_WORKSPACE}/ninja --version
 
-            - name: Download CMake 3.5.1 for Linux
+            - name: Download CMake 3.10.2 for Linux
               if: matrix.os != 'macos-latest'
               env:
                   OS_NAME: ${{ matrix.os }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 537ae9a736..721b9136fc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10.2)
 
 include(CMakeModules/AF_vcpkg_options.cmake)
 
diff --git a/CMakeModules/CPackConfig.cmake b/CMakeModules/CPackConfig.cmake
index d073527089..6cd13a1d71 100644
--- a/CMakeModules/CPackConfig.cmake
+++ b/CMakeModules/CPackConfig.cmake
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # https://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10.2)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/CMakeModules/nsis")
 
diff --git a/test/mmio/CMakeLists.txt b/test/mmio/CMakeLists.txt
index 5ef52292ad..5f4bd419f0 100644
--- a/test/mmio/CMakeLists.txt
+++ b/test/mmio/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.10.2)
 
 project(MatrixMarketIO LANGUAGES C)
 

From d3c02906a6f93a98bf71c799a190037e3f4180db Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 2 Sep 2022 11:44:04 -0400
Subject: [PATCH 069/473] Fix LAPACKE warnings and Update OpenCL library
 directory

---
 CMakeLists.txt                    | 11 ++++++-
 CMakeModules/FindLAPACKE.cmake    | 53 ++++---------------------------
 CMakeModules/FindOpenCL.cmake     |  3 +-
 src/backend/cpu/CMakeLists.txt    |  5 ++-
 src/backend/opencl/CMakeLists.txt |  6 ++--
 5 files changed, 23 insertions(+), 55 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 721b9136fc..973508e280 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -156,6 +156,7 @@ mark_as_advanced(
   AF_TEST_WITH_MTX_FILES
   ArrayFire_DIR
   Boost_INCLUDE_DIR
+  CLEAR CUDA_VERSION
   CUDA_HOST_COMPILER
   CUDA_SDK_ROOT_DIR
   CUDA_USE_STATIC_CUDA_RUNTIME
@@ -171,7 +172,15 @@ mark_as_advanced(
   spdlog_DIR
   FG_BUILD_OFFLINE
   )
-mark_as_advanced(CLEAR CUDA_VERSION)
+
+if(MKL_FOUND)
+  set(BLA_VENDOR "Intel10_64lp")
+  if(MKL_THREAD_LAYER STREQUAL "Sequential")
+    set(BLA_VENDOR "${BLA_VENDOR}_seq")
+  endif()
+endif()
+find_package(BLAS)
+find_package(LAPACK)
 
 # IF: the old USE_CPU_MKL/USE_OPENCL_MKL flags are present,
 # THEN Irrespective of AF_COMPUTE_LIBRARY value, continue with MKL to preserve old
diff --git a/CMakeModules/FindLAPACKE.cmake b/CMakeModules/FindLAPACKE.cmake
index 84e20fe7e9..65c513abb2 100644
--- a/CMakeModules/FindLAPACKE.cmake
+++ b/CMakeModules/FindLAPACKE.cmake
@@ -3,12 +3,8 @@
 # Usage:
 #   FIND_PACKAGE(LAPACKE [REQUIRED] [QUIET] )
 #
-# It sets the following variables:
-#   LAPACK_FOUND               ... true if LAPACKE is found on the system
-#   LAPACK_LIBRARIES           ... full path to LAPACKE library
-#   LAPACK_INCLUDES            ... LAPACKE include directory
-#
 
+INCLUDE(FindPackageHandleStandardArgs)
 SET(LAPACKE_ROOT_DIR CACHE STRING
   "Root directory for custom LAPACK implementation")
 
@@ -77,14 +73,6 @@ ELSE(PC_LAPACKE_FOUND)
             DOC "LAPACKE Library"
             NO_DEFAULT_PATH
             )
-        FIND_LIBRARY(
-            LAPACK_LIB
-            NAMES "lapack" "LAPACK" "liblapack" "mkl_rt"
-            PATHS ${LAPACKE_ROOT_DIR}
-            PATH_SUFFIXES "lib" "lib64" "lib/${MKL_LIB_DIR_SUFFIX}"
-            DOC "LAPACK Library"
-            NO_DEFAULT_PATH
-            )
         FIND_PATH(
             LAPACKE_INCLUDES
             NAMES "lapacke.h" "mkl_lapacke.h"
@@ -109,21 +97,6 @@ ELSE(PC_LAPACKE_FOUND)
             /opt/local/lib
             DOC "LAPACKE Library"
             )
-        FIND_LIBRARY(
-           LAPACK_LIB
-            NAMES "lapack" "liblapack" "openblas" "mkl_rt"
-            PATHS
-            ${PC_LAPACKE_LIBRARY_DIRS}
-            ${LIB_INSTALL_DIR}
-            /opt/intel/mkl/lib/${MKL_LIB_DIR_SUFFIX}
-            /usr/lib64
-            /usr/lib
-            /usr/local/lib64
-            /usr/local/lib
-            /sw/lib
-            /opt/local/lib
-            DOC "LAPACK Library"
-            )
         FIND_PATH(
             LAPACKE_INCLUDES
             NAMES "lapacke.h" "mkl_lapacke.h"
@@ -140,34 +113,20 @@ ELSE(PC_LAPACKE_FOUND)
             lapacke
             )
     ENDIF(LAPACKE_ROOT_DIR)
+    find_package_handle_standard_args(LAPACKE DEFAULT_MSG LAPACKE_LIB LAPACKE_INCLUDES)
 ENDIF(PC_LAPACKE_FOUND)
 
-IF(PC_LAPACKE_FOUND OR (LAPACKE_LIB AND LAPACK_LIB))
-    SET(LAPACK_LIBRARIES ${LAPACKE_LIB} ${LAPACK_LIB})
-ENDIF()
-IF(LAPACKE_INCLUDES)
-    SET(LAPACK_INCLUDE_DIR ${LAPACKE_INCLUDES})
-ENDIF()
-
-INCLUDE(FindPackageHandleStandardArgs)
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(LAPACK DEFAULT_MSG
-  LAPACK_INCLUDE_DIR LAPACK_LIBRARIES)
-
 MARK_AS_ADVANCED(
   LAPACKE_ROOT_DIR
-  LAPACK_INCLUDES
-  LAPACK_LIBRARIES
-  LAPACK_LIB
   LAPACKE_INCLUDES
   LAPACKE_LIB
-  lapack_LIBRARY
   lapacke_LIBRARY)
 
-if(LAPACK_FOUND)
+if(PC_LAPACKE_FOUND OR (LAPACKE_LIB AND LAPACKE_INCLUDES))
   add_library(LAPACKE::LAPACKE UNKNOWN IMPORTED)
   set_target_properties(LAPACKE::LAPACKE PROPERTIES
       IMPORTED_LINK_INTERFACE_LANGUAGE "C"
-      IMPORTED_LOCATION "${LAPACK_LIBRARIES}"
-      INTERFACE_INCLUDE_DIRECTORIES "${LAPACK_INCLUDE_DIR}"
+      IMPORTED_LOCATION "${LAPACKE_LIB}"
+      INTERFACE_INCLUDE_DIRECTORIES "${LAPACKE_INCLUDES}"
     )
-endif(LAPACK_FOUND)
+endif()
diff --git a/CMakeModules/FindOpenCL.cmake b/CMakeModules/FindOpenCL.cmake
index 54c26e5c84..cdaeba20cc 100644
--- a/CMakeModules/FindOpenCL.cmake
+++ b/CMakeModules/FindOpenCL.cmake
@@ -117,7 +117,8 @@ if(WIN32)
   endif()
 else()
   find_library(OpenCL_LIBRARY
-    NAMES OpenCL)
+    NAMES OpenCL
+    PATH_SUFFIXES lib64/)
 endif()
 
 set(OpenCL_LIBRARIES ${OpenCL_LIBRARY})
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index e3c862d169..7aa10bc529 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -331,9 +331,8 @@ else()
       FFTW::FFTW
       FFTW::FFTWF
     )
-  if(LAPACK_FOUND)
-    target_link_libraries(afcpu PRIVATE ${LAPACK_LIBRARIES})
-    target_include_directories(afcpu PRIVATE ${LAPACK_INCLUDE_DIR})
+  if(LAPACK_FOUND AND LAPACKE_FOUND)
+    target_link_libraries(afcpu PRIVATE LAPACKE::LAPACKE ${LAPACK_LIBRARIES})
   endif()
 endif()
 
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index dd557ede47..4660b99754 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -483,12 +483,12 @@ if(LAPACK_FOUND OR BUILD_WITH_MKL)
 
     target_include_directories(afopencl
       PRIVATE
-        ${CBLAS_INCLUDE_DIR}
-        ${LAPACK_INCLUDE_DIR})
+        ${CBLAS_INCLUDE_DIR})
     target_link_libraries(afopencl
       PRIVATE
         ${CBLAS_LIBRARIES}
-        ${LAPACK_LIBRARIES})
+        ${LAPACK_LIBRARIES}
+        LAPACKE::LAPACKE)
   endif()
 
   target_compile_definitions(afopencl PRIVATE WITH_LINEAR_ALGEBRA)

From 293ce5c220acde98f6bafb9259d6f09e37a30d33 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 1 Sep 2022 14:28:53 -0400
Subject: [PATCH 070/473] Add option to use external dependencies instead of
 fetchcontent

This commit adds the ability to search for already installed software on the
system instead of downloading the required libraries using fetchcontent. This
allows package managers to select dependencies that are more compatible with the
system than the one targeted by the ArrayFire build system. One disadvantage of
this approach is the increase build failures and version incompatibilities
---
 CMakeLists.txt                                |  82 +++++++----
 CMakeModules/bin2cpp.cpp                      |   5 +-
 CMakeModules/boost_package.cmake              |   5 +-
 CMakeModules/build_CLBlast.cmake              | 137 ++++++++++--------
 CMakeModules/build_cl2hpp.cmake               |  15 +-
 examples/CMakeLists.txt                       |   2 +-
 src/api/unified/CMakeLists.txt                |   7 +
 src/backend/common/CMakeLists.txt             |   9 +-
 src/backend/common/util.cpp                   |  80 +++++-----
 src/backend/cuda/CMakeLists.txt               |   9 +-
 src/backend/opencl/CMakeLists.txt             |   4 +-
 .../opencl/kernel/scan_by_key/CMakeLists.txt  |   1 +
 .../opencl/kernel/sort_by_key/CMakeLists.txt  |   1 +
 test/CMakeLists.txt                           |  12 +-
 14 files changed, 221 insertions(+), 148 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 973508e280..c79cc691e5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -39,6 +39,11 @@ set_policies(
            CMP0079)
 arrayfire_set_cmake_default_variables()
 
+option(AF_WITH_EXTERNAL_PACKAGES_ONLY "Build ArrayFire with External packages only" OFF)
+if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
+  set(AF_REQUIRED REQUIRED)
+endif()
+
 #Set Intel OpenMP as default MKL thread layer
 set(MKL_THREAD_LAYER "Intel OpenMP" CACHE STRING "The thread layer to choose for MKL")
 
@@ -54,7 +59,15 @@ find_package(CBLAS)
 find_package(LAPACKE)
 find_package(Doxygen)
 find_package(MKL)
-find_package(spdlog 1.8.5 QUIET)
+find_package(spdlog QUIET ${AF_REQUIRED})
+find_package(fmt QUIET ${AF_REQUIRED})
+find_package(span-lite QUIET)
+find_package(GTest)
+find_package(CLBlast QUIET)
+find_package(Boost 1.70 ${AF_REQUIRED})
+
+# CLFFT used in ArrayFire requires a specific fork
+#find_package(clFFT QUIET)
 
 include(boost_package)
 include(config_ccache)
@@ -75,6 +88,8 @@ option(AF_WITH_STACKTRACE  "Add stacktraces to the error messages." ON)
 option(AF_CACHE_KERNELS_TO_DISK "Enable caching kernels to disk" ON)
 option(AF_WITH_STATIC_MKL "Link against static Intel MKL libraries" OFF)
 option(AF_WITH_STATIC_CUDA_NUMERIC_LIBS "Link libafcuda with static numeric libraries(cublas, cufft, etc.)" OFF)
+option(AF_WITH_SPDLOG_HEADER_ONLY "Build ArrayFire with header only version of spdlog" OFF)
+option(AF_WITH_FMT_HEADER_ONLY "Build ArrayFire with header only version of fmt" OFF)
 
 if(AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   option(AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS "Prune CUDA static libraries to reduce binary size.(WARNING: May break some libs on older CUDA toolkits for some compute arch)" OFF)
@@ -173,7 +188,7 @@ mark_as_advanced(
   FG_BUILD_OFFLINE
   )
 
-if(MKL_FOUND)
+if(AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL")
   set(BLA_VENDOR "Intel10_64lp")
   if(MKL_THREAD_LAYER STREQUAL "Sequential")
     set(BLA_VENDOR "${BLA_VENDOR}_seq")
@@ -209,22 +224,38 @@ endif()
 #forge is included in ALL target if AF_BUILD_FORGE is ON
 #otherwise, forge is not built at all
 include(AFconfigure_forge_dep)
-add_library(af_spdlog INTERFACE)
-set_target_properties(af_spdlog
-  PROPERTIES
-    INTERFACE_COMPILE_DEFINITIONS FMT_HEADER_ONLY)
-
-if(TARGET spdlog::spdlog_header_only)
-  target_include_directories(af_spdlog
-    SYSTEM INTERFACE
-      $<TARGET_PROPERTY:spdlog::spdlog_header_only,INTERFACE_INCLUDE_DIRECTORIES>
-    )
+
+if(TARGET fmt::fmt AND AF_WITH_FMT_HEADER_ONLY)
+  set_target_properties(fmt::fmt
+    PROPERTIES
+      INTERFACE_COMPILE_DEFINITIONS "FMT_HEADER_ONLY=1")
+endif()
+
+if(TARGET spdlog::spdlog OR AF_WITH_EXTERNAL_PACKAGES_ONLY)
+  if(AF_WITH_SPDLOG_HEADER_ONLY)
+    add_library(af_spdlog ALIAS spdlog::spdlog_header_only)
+  else()
+    add_library(af_spdlog ALIAS spdlog::spdlog)
+  endif()
 else()
+  add_library(af_spdlog INTERFACE)
   af_dep_check_and_populate(${spdlog_prefix}
     URI https://github.com/gabime/spdlog.git
-    REF v1.8.5
+    REF v1.9.2
   )
+  add_subdirectory(${${spdlog_prefix}_SOURCE_DIR} ${${spdlog_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
+
   target_include_directories(af_spdlog INTERFACE "${${spdlog_prefix}_SOURCE_DIR}/include")
+  if(TARGET fmt::fmt)
+    set_target_properties(af_spdlog
+      PROPERTIES
+        INTERFACE_COMPILE_DEFINITIONS "SPDLOG_FMT_EXTERNAL")
+  endif()
+  if(AF_WITH_SPDLOG_HEADER_ONLY)
+    set_target_properties(af_spdlog
+      PROPERTIES
+        INTERFACE_COMPILE_DEFINITIONS "$<TARGET_PROPERTY:af_spdlog,INTERFACE_COMPILE_DEFINITIONS>;SPDLOG_HEADER_ONLY")
+  endif()
 endif()
 
 if(NOT TARGET glad::glad)
@@ -237,15 +268,17 @@ if(NOT TARGET glad::glad)
   add_library(af_glad STATIC $<TARGET_OBJECTS:af_glad_obj_lib>)
   target_link_libraries(af_glad PUBLIC ${CMAKE_DL_LIBS})
   target_include_directories(af_glad
-    PUBLIC
-    $<BUILD_INTERFACE:$<TARGET_PROPERTY:af_glad_obj_lib,INTERFACE_INCLUDE_DIRECTORIES>>
-    )
+    SYSTEM PUBLIC
+      $<BUILD_INTERFACE:$<TARGET_PROPERTY:af_glad_obj_lib,INTERFACE_INCLUDE_DIRECTORIES>>)
 endif()
 
-af_dep_check_and_populate(span-lite
-  URI https://github.com/martinmoene/span-lite
-  REF "ccf2351"
-  )
+if(NOT TARGET nonstd::span-lite)
+  af_dep_check_and_populate(span-lite
+    URI https://github.com/martinmoene/span-lite
+    REF "ccf2351"
+    )
+  add_subdirectory(${span-lite_SOURCE_DIR} EXCLUDE_FROM_ALL)
+endif()
 
 af_dep_check_and_populate(${assets_prefix}
   URI https://github.com/arrayfire/assets.git
@@ -271,6 +304,9 @@ if(CMAKE_CROSSCOMPILING)
 else()
   add_executable(bin2cpp ${ArrayFire_SOURCE_DIR}/CMakeModules/bin2cpp.cpp
                          ${ArrayFire_SOURCE_DIR}/src/backend/common/util.cpp)
+
+  # NOSPDLOG is used to remove the spdlog dependency from bin2cpp
+  target_compile_definitions(bin2cpp PRIVATE NOSPDLOG)
   if(WIN32)
     target_compile_definitions(bin2cpp PRIVATE OS_WIN)
   elseif(APPLE)
@@ -282,11 +318,6 @@ else()
                              ${ArrayFire_SOURCE_DIR}/include
                              ${ArrayFire_BINARY_DIR}/include
                              ${ArrayFire_SOURCE_DIR}/src/backend)
-  if(TARGET spdlog::spdlog_header_only)
-    target_link_libraries(bin2cpp PRIVATE spdlog::spdlog_header_only)
-  else()
-    target_link_libraries(bin2cpp PRIVATE af_spdlog)
-  endif()
   export(TARGETS bin2cpp FILE ${CMAKE_BINARY_DIR}/ImportExecutables.cmake)
 endif()
 
@@ -298,7 +329,6 @@ if(NOT LAPACK_FOUND)
         unset(LAPACK_LIB CACHE)
         unset(LAPACKE_INCLUDES CACHE)
         unset(LAPACKE_ROOT_DIR CACHE)
-        find_package(LAPACK)
     endif()
 endif()
 
diff --git a/CMakeModules/bin2cpp.cpp b/CMakeModules/bin2cpp.cpp
index b72a02e636..217b3efe14 100644
--- a/CMakeModules/bin2cpp.cpp
+++ b/CMakeModules/bin2cpp.cpp
@@ -14,9 +14,8 @@
 #define STRTOK_CALL(...) strtok_r(__VA_ARGS__)
 #endif
 
-#include <assert.h>
-#include <common/util.hpp>
 #include <algorithm>
+#include <cassert>
 #include <cstdlib>
 #include <cstring>
 #include <fstream>
@@ -29,6 +28,8 @@
 #include <utility>
 #include <vector>
 
+#include <common/util.hpp>
+
 using namespace std;
 using std::cout;
 typedef map<string, string> opt_t;
diff --git a/CMakeModules/boost_package.cmake b/CMakeModules/boost_package.cmake
index a0b1c84329..f6fa995c7f 100644
--- a/CMakeModules/boost_package.cmake
+++ b/CMakeModules/boost_package.cmake
@@ -5,8 +5,6 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-find_package(Boost 1.66 REQUIRED)
-
 set(Boost_MIN_VER 107000)
 set(Boost_MIN_VER_STR "1.70")
 
@@ -16,7 +14,8 @@ if(NOT
    (Boost_VERSION_STRING VERSION_GREATER Boost_MIN_VER_STR OR
     Boost_VERSION_STRING VERSION_EQUAL Boost_MIN_VER_STR) OR
    (Boost_VERSION_MACRO VERSION_GREATER Boost_MIN_VER OR
-    Boost_VERSION_MACRO VERSION_EQUAL Boost_MIN_VER)))
+    Boost_VERSION_MACRO VERSION_EQUAL Boost_MIN_VER))
+  AND NOT AF_WITH_EXTERNAL_PACKAGES_ONLY)
   set(VER 1.70.0)
   message(WARNING
       "WARN: Found Boost v${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}."
diff --git a/CMakeModules/build_CLBlast.cmake b/CMakeModules/build_CLBlast.cmake
index eaa0908ca8..780cddbaaf 100644
--- a/CMakeModules/build_CLBlast.cmake
+++ b/CMakeModules/build_CLBlast.cmake
@@ -5,76 +5,89 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-af_dep_check_and_populate(${clblast_prefix}
-  URI https://github.com/cnugteren/CLBlast.git
-  REF 4500a03440e2cc54998c0edab366babf5e504d67
-)
+if(TARGET clblast OR AF_WITH_EXTERNAL_PACKAGES_ONLY)
+  if(TARGET clblast)
+    # CLBlast has a broken imported link interface where it lists
+    # the full path to the OpenCL library. OpenCL is imported by
+    # another package so we dont need this property to link against
+    # CLBlast.
+    set_target_properties(clblast PROPERTIES
+      IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE "")
+  else()
+    message(ERROR "CLBlast now found")
+  endif()
+else()
+  af_dep_check_and_populate(${clblast_prefix}
+    URI https://github.com/cnugteren/CLBlast.git
+    REF 4500a03440e2cc54998c0edab366babf5e504d67
+  )
 
-include(ExternalProject)
-find_program(GIT git)
+  include(ExternalProject)
+  find_program(GIT git)
 
-set(prefix ${PROJECT_BINARY_DIR}/third_party/CLBlast)
-set(CLBlast_libname ${CMAKE_STATIC_LIBRARY_PREFIX}clblast${CMAKE_STATIC_LIBRARY_SUFFIX})
-set(CLBlast_location ${${clblast_prefix}_BINARY_DIR}/pkg/lib/${CLBlast_libname})
+  set(prefix ${PROJECT_BINARY_DIR}/third_party/CLBlast)
+  set(CLBlast_libname ${CMAKE_STATIC_LIBRARY_PREFIX}clblast${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(CLBlast_location ${${clblast_prefix}_BINARY_DIR}/pkg/lib/${CLBlast_libname})
 
-set(extproj_gen_opts "-G${CMAKE_GENERATOR}")
-if(WIN32 AND CMAKE_GENERATOR_PLATFORM AND NOT CMAKE_GENERATOR MATCHES "Ninja")
-  list(APPEND extproj_gen_opts "-A${CMAKE_GENERATOR_PLATFORM}")
-  if(CMAKE_GENERATOR_TOOLSET)
-    list(APPEND extproj_gen_opts "-T${CMAKE_GENERATOR_TOOLSET}")
+  set(extproj_gen_opts "-G${CMAKE_GENERATOR}")
+  if(WIN32 AND CMAKE_GENERATOR_PLATFORM AND NOT CMAKE_GENERATOR MATCHES "Ninja")
+    list(APPEND extproj_gen_opts "-A${CMAKE_GENERATOR_PLATFORM}")
+    if(CMAKE_GENERATOR_TOOLSET)
+      list(APPEND extproj_gen_opts "-T${CMAKE_GENERATOR_TOOLSET}")
+    endif()
+  endif()
+  if(VCPKG_TARGET_TRIPLET)
+    list(APPEND extproj_gen_opts "-DOPENCL_ROOT:PATH=${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}")
   endif()
-endif()
-if(VCPKG_TARGET_TRIPLET)
-  list(APPEND extproj_gen_opts "-DOPENCL_ROOT:PATH=${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}")
-endif()
 
-set(extproj_build_type_option "")
-if(NOT isMultiConfig)
-  if("${CMAKE_BUILD_TYPE}" MATCHES "Release|RelWithDebInfo")
-    set(extproj_build_type "Release")
-  else()
-    set(extproj_build_type ${CMAKE_BUILD_TYPE})
+  set(extproj_build_type_option "")
+  if(NOT isMultiConfig)
+    if("${CMAKE_BUILD_TYPE}" MATCHES "Release|RelWithDebInfo")
+      set(extproj_build_type "Release")
+    else()
+      set(extproj_build_type ${CMAKE_BUILD_TYPE})
+    endif()
+    set(extproj_build_type_option "-DCMAKE_BUILD_TYPE:STRING=${extproj_build_type}")
   endif()
-  set(extproj_build_type_option "-DCMAKE_BUILD_TYPE:STRING=${extproj_build_type}")
-endif()
 
-ExternalProject_Add(
-    CLBlast-ext
-    DOWNLOAD_COMMAND ""
-    UPDATE_COMMAND ""
-    PATCH_COMMAND ""
-    SOURCE_DIR "${${clblast_prefix}_SOURCE_DIR}"
-    BINARY_DIR "${${clblast_prefix}_BINARY_DIR}"
-    PREFIX "${prefix}"
-    INSTALL_DIR "${${clblast_prefix}_BINARY_DIR}/pkg"
-    BUILD_BYPRODUCTS ${CLBlast_location}
-    CONFIGURE_COMMAND ${CMAKE_COMMAND} ${extproj_gen_opts}
-      -Wno-dev <SOURCE_DIR>
-      -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}
-      "-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} -w -fPIC"
-      -DOVERRIDE_MSVC_FLAGS_TO_MT:BOOL=OFF
-      -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}
-      "-DCMAKE_C_FLAGS:STRING=${CMAKE_C_FLAGS} -w -fPIC"
-      ${extproj_build_type_option}
-      -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
-      -DCMAKE_INSTALL_LIBDIR:PATH=lib
-      -DBUILD_SHARED_LIBS:BOOL=OFF
-      -DSAMPLES:BOOL=OFF
-      -DTUNERS:BOOL=OFF
-      -DCLIENTS:BOOL=OFF
-      -DTESTS:BOOL=OFF
-      -DNETLIB:BOOL=OFF
-    )
+  ExternalProject_Add(
+      CLBlast-ext
+      DOWNLOAD_COMMAND ""
+      UPDATE_COMMAND ""
+      PATCH_COMMAND ""
+      SOURCE_DIR "${${clblast_prefix}_SOURCE_DIR}"
+      BINARY_DIR "${${clblast_prefix}_BINARY_DIR}"
+      PREFIX "${prefix}"
+      INSTALL_DIR "${${clblast_prefix}_BINARY_DIR}/pkg"
+      BUILD_BYPRODUCTS ${CLBlast_location}
+      CONFIGURE_COMMAND ${CMAKE_COMMAND} ${extproj_gen_opts}
+        -Wno-dev <SOURCE_DIR>
+        -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}
+        "-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} -w -fPIC"
+        -DOVERRIDE_MSVC_FLAGS_TO_MT:BOOL=OFF
+        -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}
+        "-DCMAKE_C_FLAGS:STRING=${CMAKE_C_FLAGS} -w -fPIC"
+        ${extproj_build_type_option}
+        -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+        -DCMAKE_INSTALL_LIBDIR:PATH=lib
+        -DBUILD_SHARED_LIBS:BOOL=OFF
+        -DSAMPLES:BOOL=OFF
+        -DTUNERS:BOOL=OFF
+        -DCLIENTS:BOOL=OFF
+        -DTESTS:BOOL=OFF
+        -DNETLIB:BOOL=OFF
+      )
 
-set(CLBLAST_INCLUDE_DIRS "${${clblast_prefix}_BINARY_DIR}/pkg/include")
-set(CLBLAST_LIBRARIES CLBlast)
-set(CLBLAST_FOUND ON)
+  set(CLBLAST_INCLUDE_DIRS "${${clblast_prefix}_BINARY_DIR}/pkg/include")
+  set(CLBLAST_LIBRARIES CLBlast)
+  set(CLBLAST_FOUND ON)
 
-make_directory("${CLBLAST_INCLUDE_DIRS}")
+  make_directory("${CLBLAST_INCLUDE_DIRS}")
 
-add_library(CLBlast UNKNOWN IMPORTED)
-set_target_properties(CLBlast PROPERTIES
-  IMPORTED_LOCATION "${CLBlast_location}"
-  INTERFACE_INCLUDE_DIRECTORIES "${CLBLAST_INCLUDE_DIRS}")
+  add_library(clblast UNKNOWN IMPORTED)
+  set_target_properties(clblast PROPERTIES
+    IMPORTED_LOCATION "${CLBlast_location}"
+    INTERFACE_INCLUDE_DIRECTORIES "${CLBLAST_INCLUDE_DIRS}")
 
-add_dependencies(CLBlast CLBlast-ext)
+  add_dependencies(clblast CLBlast-ext)
+endif()
diff --git a/CMakeModules/build_cl2hpp.cmake b/CMakeModules/build_cl2hpp.cmake
index fd8709fb02..e090dd0800 100644
--- a/CMakeModules/build_cl2hpp.cmake
+++ b/CMakeModules/build_cl2hpp.cmake
@@ -13,15 +13,18 @@
 
 find_package(OpenCL)
 
-af_dep_check_and_populate(${cl2hpp_prefix}
-  URI https://github.com/KhronosGroup/OpenCL-CLHPP.git
-  REF v2.0.12
-)
-
 if (NOT TARGET OpenCL::cl2hpp OR NOT TARGET cl2hpp)
+  af_dep_check_and_populate(${cl2hpp_prefix}
+    URI https://github.com/KhronosGroup/OpenCL-CLHPP.git
+    REF v2.0.12)
+
+  find_path(cl2hpp_var
+    NAMES CL/cl2.hpp
+    PATHS ${ArrayFire_BINARY_DIR}/extern/${cl2hpp_prefix}-src/include)
+
   add_library(cl2hpp IMPORTED INTERFACE GLOBAL)
   add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
 
   set_target_properties(cl2hpp OpenCL::cl2hpp PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES ${${cl2hpp_prefix}_SOURCE_DIR}/include)
+    INTERFACE_INCLUDE_DIRECTORIES ${cl2hpp_var})
 endif()
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index e6bf747554..f69eff6e1f 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -11,7 +11,7 @@ project(ArrayFire-Examples
   VERSION 3.7.0
   LANGUAGES CXX)
 
-set(CMAKE_CXX_STANDARD 98)
+set(CMAKE_CXX_STANDARD 14)
 if(NOT EXISTS "${ArrayFire_SOURCE_DIR}/CMakeLists.txt")
   set(ASSETS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/..")
 endif()
diff --git a/src/api/unified/CMakeLists.txt b/src/api/unified/CMakeLists.txt
index 5c0cec9d6f..522a19ba2a 100644
--- a/src/api/unified/CMakeLists.txt
+++ b/src/api/unified/CMakeLists.txt
@@ -107,6 +107,13 @@ target_link_libraries(af
     ${CMAKE_DL_LIBS}
   )
 
+if(TARGET fmt::fmt)
+  target_link_libraries(af
+    PRIVATE
+      fmt::fmt
+  )
+endif()
+
 install(TARGETS af
   EXPORT ArrayFireUnifiedTargets
   COMPONENT unified
diff --git a/src/backend/common/CMakeLists.txt b/src/backend/common/CMakeLists.txt
index d12823c6a3..8f553814e7 100644
--- a/src/backend/common/CMakeLists.txt
+++ b/src/backend/common/CMakeLists.txt
@@ -89,9 +89,17 @@ target_link_libraries(afcommon_interface
   INTERFACE
     af_spdlog
     Boost::boost
+    nonstd::span-lite
     ${CMAKE_DL_LIBS}
 )
 
+if(TARGET fmt::fmt)
+  target_link_libraries(afcommon_interface
+    INTERFACE
+      fmt::fmt
+  )
+endif()
+
 if(TARGET glad::glad)
   target_link_libraries(afcommon_interface INTERFACE glad::glad)
 else()
@@ -105,7 +113,6 @@ endif()
 target_include_directories(afcommon_interface
   INTERFACE
     ${ArrayFire_SOURCE_DIR}/src/backend
-    ${span-lite_SOURCE_DIR}/include
     ${ArrayFire_BINARY_DIR})
 
 target_include_directories(afcommon_interface
diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp
index ee579d67ac..a5af7f80e6 100644
--- a/src/backend/common/util.cpp
+++ b/src/backend/common/util.cpp
@@ -15,7 +15,10 @@
 #include <unistd.h>
 #endif
 
+#ifndef NOSPDLOG
 #include <common/Logger.hpp>
+#endif
+
 #include <common/defines.hpp>
 #include <common/util.hpp>
 #include <af/defines.h>
@@ -32,7 +35,15 @@
 #include <vector>
 
 using std::accumulate;
+using std::hash;
+using std::ofstream;
+using std::once_flag;
+using std::rename;
+using std::size_t;
 using std::string;
+using std::thread;
+using std::to_string;
+using std::uint8_t;
 using std::vector;
 
 // http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring/217605#217605
@@ -43,7 +54,7 @@ string& ltrim(string& s) {
     return s;
 }
 
-string getEnvVar(const std::string& key) {
+string getEnvVar(const string& key) {
 #if defined(OS_WIN)
     DWORD bufSize =
         32767;  // limit according to GetEnvironment Variable documentation
@@ -80,23 +91,23 @@ const char* getName(af_dtype type) {
     }
 }
 
-void saveKernel(const std::string& funcName, const std::string& jit_ker,
-                const std::string& ext) {
+void saveKernel(const string& funcName, const string& jit_ker,
+                const string& ext) {
     static constexpr const char* saveJitKernelsEnvVarName =
         "AF_JIT_KERNEL_TRACE";
     static const char* jitKernelsOutput = getenv(saveJitKernelsEnvVarName);
     if (!jitKernelsOutput) { return; }
-    if (std::strcmp(jitKernelsOutput, "stdout") == 0) {
+    if (strcmp(jitKernelsOutput, "stdout") == 0) {
         fputs(jit_ker.c_str(), stdout);
         return;
     }
-    if (std::strcmp(jitKernelsOutput, "stderr") == 0) {
+    if (strcmp(jitKernelsOutput, "stderr") == 0) {
         fputs(jit_ker.c_str(), stderr);
         return;
     }
     // Path to a folder
-    const std::string ffp =
-        std::string(jitKernelsOutput) + AF_PATH_SEPARATOR + funcName + ext;
+    const string ffp =
+        string(jitKernelsOutput) + AF_PATH_SEPARATOR + funcName + ext;
     FILE* f = fopen(ffp.c_str(), "we");
     if (!f) {
         fprintf(stderr, "Cannot open file %s\n", ffp.c_str());
@@ -108,9 +119,9 @@ void saveKernel(const std::string& funcName, const std::string& jit_ker,
     fclose(f);
 }
 
-std::string int_version_to_string(int version) {
-    return std::to_string(version / 1000) + "." +
-           std::to_string(static_cast<int>((version % 1000) / 10.));
+string int_version_to_string(int version) {
+    return to_string(version / 1000) + "." +
+           to_string(static_cast<int>((version % 1000) / 10.));
 }
 
 #if defined(OS_WIN)
@@ -162,25 +173,26 @@ bool removeFile(const string& path) {
 }
 
 bool renameFile(const string& sourcePath, const string& destPath) {
-    return std::rename(sourcePath.c_str(), destPath.c_str()) == 0;
+    return rename(sourcePath.c_str(), destPath.c_str()) == 0;
 }
 
 bool isDirectoryWritable(const string& path) {
     if (!directoryExists(path) && !createDirectory(path)) { return false; }
 
     const string testPath = path + AF_PATH_SEPARATOR + "test";
-    if (!std::ofstream(testPath).is_open()) { return false; }
+    if (!ofstream(testPath).is_open()) { return false; }
     removeFile(testPath);
 
     return true;
 }
 
+#ifndef NOSPDLOG
 string& getCacheDirectory() {
-    static std::once_flag flag;
+    static once_flag flag;
     static string cacheDirectory;
 
-    std::call_once(flag, []() {
-        std::string pathList[] = {
+    call_once(flag, []() {
+        string pathList[] = {
 #if defined(OS_WIN)
             getTemporaryDirectory() + "\\ArrayFire"
 #else
@@ -200,8 +212,8 @@ string& getCacheDirectory() {
         }
 
         if (env_path.empty()) {
-            auto iterDir = std::find_if(begin(pathList), end(pathList),
-                                        isDirectoryWritable);
+            auto iterDir =
+                find_if(begin(pathList), end(pathList), isDirectoryWritable);
 
             cacheDirectory = iterDir != end(pathList) ? *iterDir : "";
         } else {
@@ -211,44 +223,40 @@ string& getCacheDirectory() {
 
     return cacheDirectory;
 }
+#endif
 
 string makeTempFilename() {
-    thread_local std::size_t fileCount = 0u;
+    thread_local size_t fileCount = 0u;
 
     ++fileCount;
-    const std::size_t threadID =
-        std::hash<std::thread::id>{}(std::this_thread::get_id());
+    const size_t threadID = hash<thread::id>{}(std::this_thread::get_id());
 
-    return std::to_string(std::hash<string>{}(std::to_string(threadID) + "_" +
-                                              std::to_string(fileCount)));
+    return to_string(
+        hash<string>{}(to_string(threadID) + "_" + to_string(fileCount)));
 }
 
-std::size_t deterministicHash(const void* data, std::size_t byteSize,
-                              std::size_t prevHash) {
+size_t deterministicHash(const void* data, size_t byteSize, size_t prevHash) {
     // Fowler-Noll-Vo "1a" 32 bit hash
     // https://en.wikipedia.org/wiki/Fowler-Noll-Vo_hash_function
-    const auto* byteData = static_cast<const std::uint8_t*>(data);
-    return std::accumulate(byteData, byteData + byteSize, prevHash,
-                           [&](std::size_t hash, std::uint8_t data) {
-                               return (hash ^ data) * FNV1A_PRIME;
-                           });
+    const auto* byteData = static_cast<const uint8_t*>(data);
+    return accumulate(
+        byteData, byteData + byteSize, prevHash,
+        [&](size_t hash, uint8_t data) { return (hash ^ data) * FNV1A_PRIME; });
 }
 
-std::size_t deterministicHash(const std::string& data,
-                              const std::size_t prevHash) {
+size_t deterministicHash(const string& data, const size_t prevHash) {
     return deterministicHash(data.data(), data.size(), prevHash);
 }
 
-std::size_t deterministicHash(const vector<std::string>& list,
-                              const std::size_t prevHash) {
-    std::size_t hash = prevHash;
+size_t deterministicHash(const vector<string>& list, const size_t prevHash) {
+    size_t hash = prevHash;
     for (auto s : list) { hash = deterministicHash(s.data(), s.size(), hash); }
     return hash;
 }
 
-std::size_t deterministicHash(const std::vector<common::Source>& list) {
+size_t deterministicHash(const vector<common::Source>& list) {
     // Combine the different source codes, via their hashes
-    std::size_t hash = FNV1A_BASE_OFFSET;
+    size_t hash = FNV1A_BASE_OFFSET;
     for (auto s : list) {
         size_t h = s.hash ? s.hash : deterministicHash(s.ptr, s.length);
         hash     = deterministicHash(&h, sizeof(size_t), hash);
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 8f25f1bea1..3fcf1d2259 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -334,6 +334,12 @@ if(CUDA_VERSION_MAJOR VERSION_GREATER 10 OR
   target_compile_definitions(af_cuda_static_cuda_library PRIVATE AF_USE_NEW_CUSPARSE_API)
 endif()
 
+target_link_libraries(af_cuda_static_cuda_library
+  PRIVATE
+    Boost::boost
+    af_spdlog
+    nonstd::span-lite)
+
 if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   check_cxx_compiler_flag("-Wl,--start-group -Werror" group_flags)
   if(group_flags)
@@ -343,8 +349,6 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
 
   target_link_libraries(af_cuda_static_cuda_library
     PRIVATE
-      af_spdlog
-      Boost::boost
       ${CMAKE_DL_LIBS}
       ${cusolver_lib}
       ${START_GROUP}
@@ -373,7 +377,6 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
 else()
   target_link_libraries(af_cuda_static_cuda_library
     PUBLIC
-      Boost::boost
       ${CUDA_CUBLAS_LIBRARIES}
       ${CUDA_CUFFT_LIBRARIES}
       ${CUDA_cusolver_LIBRARY}
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 4660b99754..506b9b3f55 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -402,8 +402,6 @@ target_include_directories(afopencl
 arrayfire_set_default_cxx_flags(afopencl)
 
 add_dependencies(afopencl ${cl_kernel_targets} CLBlast-ext)
-add_dependencies(opencl_scan_by_key ${cl_kernel_targets} cl2hpp Boost::boost)
-add_dependencies(opencl_sort_by_key ${cl_kernel_targets} cl2hpp Boost::boost)
 
 set_target_properties(afopencl PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
@@ -421,7 +419,7 @@ target_link_libraries(afopencl
     OpenCL::cl2hpp
     afcommon_interface
     clFFT
-    CLBlast
+    clblast
     opencl_scan_by_key
     opencl_sort_by_key
     Threads::Threads
diff --git a/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt b/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
index 6add18a881..91f1cc9ffc 100644
--- a/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
+++ b/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
@@ -76,6 +76,7 @@ foreach(SBK_BINARY_OP ${SBK_BINARY_OPS})
       PRIVATE
         ${opencl_compile_definitions}
         $<TARGET_PROPERTY:Boost::boost,INTERFACE_COMPILE_DEFINITIONS>
+        $<TARGET_PROPERTY:af_spdlog,INTERFACE_COMPILE_DEFINITIONS>
         TYPE=${SBK_BINARY_OP} AFDLL)
     target_sources(opencl_scan_by_key
       INTERFACE $<TARGET_OBJECTS:opencl_scan_by_key_${SBK_BINARY_OP}>)
diff --git a/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt b/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
index e7a7ca27f3..0d55ffce4e 100644
--- a/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
+++ b/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
@@ -38,6 +38,7 @@ foreach(SBK_TYPE ${SBK_TYPES})
         $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>
         $<TARGET_PROPERTY:OpenCL::cl2hpp,INTERFACE_INCLUDE_DIRECTORIES>
         $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
+        $<TARGET_PROPERTY:af_spdlog,INTERFACE_INCLUDE_DIRECTORIES>
         ${ArrayFire_BINARY_DIR}/include
       )
     if(TARGET Forge::forge)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 09a794c63b..c7add80ca3 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -15,7 +15,9 @@ if(AF_TEST_WITH_MTX_FILES)
   include(download_sparse_datasets)
 endif()
 
-if(NOT TARGET gtest)
+if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
+    dependency_check(GTest_FOUND)
+else()
   af_dep_check_and_populate(${gtest_prefix}
     URI https://github.com/google/googletest.git
     REF release-1.8.1
@@ -34,6 +36,7 @@ if(NOT TARGET gtest)
   set_target_properties(gtest gtest_main
     PROPERTIES
       FOLDER "ExternalProjectTargets/gtest")
+  add_library(GTest::gtest ALIAS gtest)
   if(UNIX)
     if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND
       CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "10.3.0")
@@ -109,7 +112,7 @@ target_include_directories(arrayfire_test
     ${ArrayFire_SOURCE_DIR}/extern/half/include
     mmio
     $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
-    ${${gtest_prefix}_SOURCE_DIR}/googletest/include)
+    $<TARGET_PROPERTY:GTest::gtest,INTERFACE_INCLUDE_DIRECTORIES>)
 
 if(WIN32)
   target_compile_options(arrayfire_test
@@ -169,7 +172,7 @@ function(make_test)
     target_link_libraries(${target}
       PRIVATE
         ${mt_args_LIBRARIES}
-        gtest
+	      GTest::gtest
       )
 
     if(${backend} STREQUAL "unified")
@@ -340,7 +343,6 @@ if(CUDA_FOUND)
           ${ArrayFire_BINARY_DIR}/include
           ${ArrayFire_SOURCE_DIR}/extern/half/include
           ${CMAKE_CURRENT_SOURCE_DIR}
-          ${${gtest_prefix}_SOURCE_DIR}/googletest/include
         )
       endif()
       cuda_add_executable(${target} cuda.cu  $<TARGET_OBJECTS:arrayfire_test>)
@@ -357,7 +359,7 @@ if(CUDA_FOUND)
       endif()
       target_link_libraries(${target}
         mmio
-        gtest)
+        GTest::gtest)
 
       # Couldn't get Threads::Threads to work with this cuda binary. The import
       # target would not add the -pthread flag which is required for this

From 2dff454176565900621787ebec8f52c8df426266 Mon Sep 17 00:00:00 2001
From: Carlo Cabrera <30379873+carlocab@users.noreply.github.com>
Date: Tue, 13 Sep 2022 01:37:41 +0800
Subject: [PATCH 071/473] Avoid overriding `CMAKE_INSTALL_RPATH` on macOS.
 (#3283)

* Avoid overriding `CMAKE_INSTALL_RPATH` on macOS.

Currently, `InternalUtils.cmake` sets `CMAKE_INSTALL_RPATH` on macOS to
`/opt/arrayfire/lib`. This is not always the install location (e.g. if a
user sets `CMAKE_INSTALL_PREFIX`), nor does it always make sense to only
have a single `LC_RPATH` command inside the libraries on macOS.

In particular, if a user passes `CMAKE_INSTALL_RPATH` from the
command-line on macOS, it would be good to avoid overriding that, since
the user is more likely to supply the correct paths for their system
than keeping a fixed value of `/opt/arrayfire/lib`.

This PR emits a warning if `CMAKE_INSTALL_RPATH` is not set on macOS to
warn the user to set it through the command line.
---
 .github/workflows/unix_cpu_build.yml | 2 ++
 CMakeModules/InternalUtils.cmake     | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index ad616ddd3d..1962db4891 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -102,6 +102,7 @@ jobs:
                   dashboard=$(if [ -z "$prnum" ]; then echo "Continuous"; else echo "Experimental"; fi)
                   backend=$(if [ "$USE_MKL" == 1 ]; then echo "Intel-MKL"; else echo "FFTW/LAPACK/BLAS"; fi)
                   buildname="$buildname-cpu-$BLAS_BACKEND"
+                  cmake_rpath=$(if [ $OS_NAME == 'macos-latest' ]; then echo "-DCMAKE_INSTALL_RPATH=/opt/arrayfire/lib"; fi)
                   mkdir build && cd build
                   ${CMAKE_PROGRAM} -G Ninja \
                       -DCMAKE_MAKE_PROGRAM:FILEPATH=${GITHUB_WORKSPACE}/ninja \
@@ -109,6 +110,7 @@ jobs:
                       -DAF_BUILD_UNIFIED:BOOL=OFF -DAF_BUILD_EXAMPLES:BOOL=ON \
                       -DAF_BUILD_FORGE:BOOL=ON \
                       -DAF_COMPUTE_LIBRARY:STRING=${backend} \
+                      "$cmake_rpath" \
                       -DBUILDNAME:STRING=${buildname} ..
                   echo "CTEST_DASHBOARD=${dashboard}" >> $GITHUB_ENV
 
diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index 3b19485d6f..f212c50750 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -177,8 +177,8 @@ macro(arrayfire_set_cmake_default_variables)
     set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${ArrayFire_BINARY_DIR}/bin)
   endif()
 
-  if(APPLE)
-    set(CMAKE_INSTALL_RPATH "/opt/arrayfire/lib")
+  if(APPLE AND (NOT DEFINED CMAKE_INSTALL_RPATH))
+      message(WARNING "CMAKE_INSTALL_RPATH is required when installing ArrayFire to the local system. Set it to /opt/arrayfire/lib if making the installer or your own custom install path.")
   endif()
 
   # This code is used to generate the compilers.h file in CMakeModules. Not all

From 5216b7a40acf53fa7d9113803cf31d83d1565cf2 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 4 Aug 2022 01:09:21 +0200
Subject: [PATCH 072/473] Threads management General Threads/Blocks
 (Local/Global) calculations when all available dimensions are used.,
 including optimized number of active parallel GPU threads.

---
 src/backend/common/dispatch.hpp     | 151 ++++++++++++-
 src/backend/cuda/device_manager.hpp |   2 +-
 src/backend/cuda/platform.cpp       |  48 ++--
 src/backend/cuda/platform.hpp       |  19 +-
 src/backend/cuda/threadsMgt.hpp     | 327 +++++++++++++++++++++++++++
 src/backend/opencl/platform.cpp     |  17 +-
 src/backend/opencl/platform.hpp     |  51 ++++-
 src/backend/opencl/threadsMgt.hpp   | 328 ++++++++++++++++++++++++++++
 8 files changed, 908 insertions(+), 35 deletions(-)
 create mode 100644 src/backend/cuda/threadsMgt.hpp
 create mode 100644 src/backend/opencl/threadsMgt.hpp

diff --git a/src/backend/common/dispatch.hpp b/src/backend/common/dispatch.hpp
index 099b0aa6a5..e248a22a97 100644
--- a/src/backend/common/dispatch.hpp
+++ b/src/backend/common/dispatch.hpp
@@ -9,6 +9,10 @@
 
 #pragma once
 
+#include <assert.h>
+#include <platform.hpp>
+#include <af/defines.h>
+#include <algorithm>
 #include <cmath>
 
 #define divup(a, b) (((a) + (b)-1) / (b))
@@ -21,8 +25,8 @@ template<typename T>
 inline bool isPrime(T n) {
     if (n <= 1) return false;
 
-    const T last = (T)std::sqrt((double)n);
-    for (T x = 2; x <= last; ++x) {
+    const T last{(T)std::sqrt((double)n)};
+    for (T x{2}; x <= last; ++x) {
         if (n % x == 0) return false;
     }
 
@@ -31,7 +35,7 @@ inline bool isPrime(T n) {
 
 template<typename T>
 inline T greatestPrimeFactor(T n) {
-    T v = 2;
+    T v{2};
 
     while (v <= n) {
         if (n % v == 0 && isPrime(v))
@@ -42,3 +46,144 @@ inline T greatestPrimeFactor(T n) {
 
     return v;
 }
+// Empty columns (dim==1) in refDims are removed from dims & strides.
+// INPUT: refDims, refNdims
+// UPDATE: dims, strides
+// RETURN: ndims
+template<typename T>
+T removeEmptyColumns(const T refDims[AF_MAX_DIMS], const T refNdims,
+                     T dims[AF_MAX_DIMS], T strides[AF_MAX_DIMS]) {
+    T ndims{0};
+    const T* refPtr{refDims};
+    const T* refPtr_end{refDims + refNdims};
+    // Search for first dimension == 1
+    while (refPtr != refPtr_end && *refPtr != 1) {
+        ++refPtr;
+        ++ndims;
+    }
+    if (ndims != refNdims) {
+        T* dPtr_out{dims + ndims};
+        const T* dPtr_in{dPtr_out};
+        T* sPtr_out{strides + ndims};
+        const T* sPtr_in{sPtr_out};
+        // Compress all remaining dimensions
+        while (refPtr != refPtr_end) {
+            if (*refPtr != 1) {
+                *(dPtr_out++) = *dPtr_in;
+                *(sPtr_out++) = *sPtr_in;
+                ++ndims;
+            }
+            ++refPtr;
+            ++dPtr_in;
+            ++sPtr_in;
+        }
+        // Fill remaining dimensions with 1 and calculate corresponding strides
+        // lastStride = last written dim * last written stride
+        const T lastStride{*(dPtr_out - 1) * *(sPtr_out - 1)};
+        const T lastDim{1};
+        for (const T* dPtr_end{dims + AF_MAX_DIMS}; dPtr_out != dPtr_end;
+             ++dPtr_out, ++sPtr_out) {
+            *dPtr_out = lastDim;
+            *sPtr_out = lastStride;
+        }
+    }
+    return ndims;
+}
+
+// Empty columns (dim==1) in refDims are removed from strides
+// ASSUMPTION: dims are equal to refDims, so are not provided
+// INPUT: refDims, refNdims
+// UPDATE: strides
+// RETURN: ndims
+template<typename T>
+T removeEmptyColumns(const T refDims[AF_MAX_DIMS], const T refNdims,
+                     T strides[AF_MAX_DIMS]) {
+    T ndims{0};
+    const T* refPtr{refDims};
+    const T* refPtr_end{refDims + refNdims};
+    // Search for first dimension == 1
+    while (refPtr != refPtr_end && *refPtr != 1) {
+        ++refPtr;
+        ++ndims;
+    }
+    if (ndims != refNdims) {
+        T* sPtr_out{strides + ndims};
+        const T* sPtr_in{sPtr_out};
+        // Compress all remaining dimensions
+        while (refPtr != refPtr_end) {
+            if (*refPtr != 1) {
+                *(sPtr_out++) = *sPtr_in;
+                ++ndims;
+            };
+            ++refPtr;
+            ++sPtr_in;
+        }
+        // Calculate remaining strides
+        // lastStride = last written dim * last written stride
+        const T lastStride{*(refPtr - 1) * *(sPtr_out - 1)};
+        for (const T* sPtr_end{strides + AF_MAX_DIMS}; sPtr_out != sPtr_end;
+             ++sPtr_out) {
+            *sPtr_out = lastStride;
+        }
+    }
+    return ndims;
+}
+
+// Columns with the same stride in both arrays are combined.  Both arrays will
+// remain in sync and will return the same ndims.
+// ASSUMPTION: both arrays have the same ndims
+// UPDATE: dims1, strides1, UPDATE: dims2, strides2, ndims
+// RETURN: ndims
+template<typename T>
+T combineColumns(T dims1[AF_MAX_DIMS], T strides1[AF_MAX_DIMS], T& ndims,
+                 T dims2[AF_MAX_DIMS], T strides2[AF_MAX_DIMS]) {
+    for (T c{0}; c < ndims - 1; ++c) {
+        if (dims1[c] == dims2[c] && dims1[c] * strides1[c] == strides1[c + 1] &&
+            dims1[c] * strides2[c] == strides2[c + 1]) {
+            // Combine columns, since they are linear
+            // This will increase the dimension of the resulting column,
+            // given more opportunities for kernel optimization
+            dims1[c] *= dims1[c + 1];
+            dims2[c] *= dims2[c + 1];
+            --ndims;
+            for (T i{c + 1}; i < ndims; ++i) {
+                dims1[i]    = dims1[i + 1];
+                dims2[i]    = dims2[i + 1];
+                strides1[i] = strides1[i + 1];
+                strides2[i] = strides2[i + 1];
+            }
+            dims1[ndims] = 1;
+            dims2[ndims] = 1;
+            --c;  // Redo this colum, since it is removed now
+        }
+    }
+    return ndims;
+}
+// Columns with the same stride in both arrays are combined.  Both arrays will
+// remain in sync and will return the same ndims.
+// ASSUMPTION: both arrays have the same dims
+// UPDATE: dims1, strides1,
+// UPDATE: strides2, ndims
+// RETURN: ndims
+template<typename T>
+T combineColumns(T dims1[AF_MAX_DIMS], T strides1[AF_MAX_DIMS], T& ndims,
+                 T strides2[AF_MAX_DIMS]) {
+    for (T c{0}; c < ndims - 1; ++c) {
+        if (dims1[c] * strides1[c] == strides1[c + 1] &&
+            dims1[c] * strides2[c] == strides2[c + 1]) {
+            // Combine columns, since they are linear
+            // This will increase the dimension of the resulting column,
+            // given more opportunities for kernel optimization
+            dims1[c] *= dims1[c + 1];
+            --ndims;
+            for (T i{c + 1}; i < ndims; ++i) {
+                dims1[i]    = dims1[i + 1];
+                strides1[i] = strides1[i + 1];
+                strides2[i] = strides2[i + 1];
+            }
+            dims1[ndims] = 1;
+            --c;  // Redo this colum, since it is removed now
+        }
+    }
+    return ndims;
+}
\ No newline at end of file
diff --git a/src/backend/cuda/device_manager.hpp b/src/backend/cuda/device_manager.hpp
index c6009337d2..5ea6d3a2f6 100644
--- a/src/backend/cuda/device_manager.hpp
+++ b/src/backend/cuda/device_manager.hpp
@@ -90,7 +90,7 @@ class DeviceManager {
 
     friend int setDevice(int device);
 
-    friend cudaDeviceProp getDeviceProp(int device);
+    friend const cudaDeviceProp& getDeviceProp(int device);
 
     friend std::pair<int, int> getComputeCapability(const int device);
 
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 647566eb2a..520d4f90f5 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -58,6 +58,7 @@ using std::runtime_error;
 using std::string;
 using std::to_string;
 using std::unique_ptr;
+using std::vector;
 
 using common::unique_handle;
 using common::memory::MemoryManagerBase;
@@ -202,7 +203,7 @@ DeviceManager::~DeviceManager() {
 int getBackend() { return AF_BACKEND_CUDA; }
 
 string getDeviceInfo(int device) noexcept {
-    cudaDeviceProp dev = getDeviceProp(device);
+    const cudaDeviceProp &dev = getDeviceProp(device);
 
     size_t mem_gpu_total = dev.totalGlobalMem;
     // double cc = double(dev.major) + double(dev.minor) / 10;
@@ -244,19 +245,19 @@ string getPlatformInfo() noexcept {
     return platform;
 }
 
-bool isDoubleSupported(int device) {
+bool isDoubleSupported(int device) noexcept {
     UNUSED(device);
     return true;
 }
 
 bool isHalfSupported(int device) {
-    std::array<bool, DeviceManager::MAX_DEVICES> half_supported = []() {
+    static std::array<bool, DeviceManager::MAX_DEVICES> half_supported = []() {
         std::array<bool, DeviceManager::MAX_DEVICES> out{};
         int count = getDeviceCount();
         for (int i = 0; i < count; i++) {
-            auto prop   = getDeviceProp(i);
-            int compute = prop.major * 1000 + prop.minor * 10;
-            out[i]      = compute >= 5030;
+            const auto &prop = getDeviceProp(i);
+            int compute      = prop.major * 1000 + prop.minor * 10;
+            out[i]           = compute >= 5030;
         }
         return out;
     }();
@@ -266,7 +267,7 @@ bool isHalfSupported(int device) {
 void devprop(char *d_name, char *d_platform, char *d_toolkit, char *d_compute) {
     if (getDeviceCount() <= 0) { return; }
 
-    cudaDeviceProp dev = getDeviceProp(getActiveDeviceId());
+    const cudaDeviceProp &dev = getDeviceProp(getActiveDeviceId());
 
     // Name
     snprintf(d_name, 256, "%s", dev.name);
@@ -354,7 +355,7 @@ void init() {
     UNUSED(err);
 }
 
-unsigned getActiveDeviceId() { return tlocalActiveDeviceId(); }
+int getActiveDeviceId() { return tlocalActiveDeviceId(); }
 
 int getDeviceNativeId(int device) {
     if (device <
@@ -397,12 +398,31 @@ int setDevice(int device) {
     return DeviceManager::getInstance().setActiveDevice(device);
 }
 
-cudaDeviceProp getDeviceProp(int device) {
-    if (device <
-        static_cast<int>(DeviceManager::getInstance().cuDevices.size())) {
-        return DeviceManager::getInstance().cuDevices[device].prop;
-    }
-    return DeviceManager::getInstance().cuDevices[0].prop;
+size_t getL2CacheSize(const int device) {
+    return getDeviceProp(device).l2CacheSize;
+}
+
+const int *getMaxGridSize(const int device) {
+    return getDeviceProp(device).maxGridSize;
+}
+
+unsigned getMemoryBusWidth(const int device) {
+    return getDeviceProp(device).memoryBusWidth;
+}
+
+unsigned getMultiProcessorCount(const int device) {
+    return getDeviceProp(device).multiProcessorCount;
+}
+
+unsigned getMaxParallelThreads(const int device) {
+    const cudaDeviceProp &prop{getDeviceProp(device)};
+    return prop.multiProcessorCount * prop.maxThreadsPerMultiProcessor;
+}
+
+const cudaDeviceProp &getDeviceProp(const int device) {
+    const vector<cudaDevice_t> &devs = DeviceManager::getInstance().cuDevices;
+    if (device < static_cast<int>(devs.size())) { return devs[device].prop; }
+    return devs[0].prop;
 }
 
 MemoryManagerBase &memoryManager() {
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 6d1778b3ab..bbdf5a8d6d 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -69,7 +69,7 @@ std::string getDriverVersion() noexcept;
 std::string getCUDARuntimeVersion() noexcept;
 
 // Returns true if double is supported by the device
-bool isDoubleSupported(int device);
+bool isDoubleSupported(int device) noexcept;
 
 // Returns true if half is supported by the device
 bool isHalfSupported(int device);
@@ -82,7 +82,7 @@ int getDeviceCount();
 
 void init();
 
-unsigned getActiveDeviceId();
+int getActiveDeviceId();
 
 int getDeviceNativeId(int device);
 
@@ -94,6 +94,19 @@ size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();
 
+size_t getL2CacheSize(const int device);
+
+// Returns int[3] of maxGridSize
+const int* getMaxGridSize(const int device);
+
+unsigned getMemoryBusWidth(const int device);
+
+// maximum nr of threads the device really can run in parallel, without
+// scheduling
+unsigned getMaxParallelThreads(const int device);
+
+unsigned getMultiProcessorCount(const int device);
+
 int setDevice(int device);
 
 void sync(int device);
@@ -101,7 +114,7 @@ void sync(int device);
 // Returns true if the AF_SYNCHRONIZE_CALLS environment variable is set to 1
 bool synchronize_calls();
 
-cudaDeviceProp getDeviceProp(int device);
+const cudaDeviceProp& getDeviceProp(const int device);
 
 std::pair<int, int> getComputeCapability(const int device);
 
diff --git a/src/backend/cuda/threadsMgt.hpp b/src/backend/cuda/threadsMgt.hpp
new file mode 100644
index 0000000000..06fccdb0a3
--- /dev/null
+++ b/src/backend/cuda/threadsMgt.hpp
@@ -0,0 +1,327 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <common/dispatch.hpp>
+#include <platform.hpp>
+
+namespace cuda {
+// OVERALL USAGE (With looping):
+// ...                                                      // OWN CODE
+// threadsMgt<T> th(...);                                   // backend.hpp
+// const dim3 threads{th.genThreads()};                     // backend.hpp
+// const dim3 blocks{th.genBlocks(threads,..)};             // backend.hpp
+// cuda::Kernel KER{GETKERNEL(..., th.loop0, th.loop1, th.loop2,
+//                               th.loop3)};                // OWN CODE
+// KER(threads,blocks,...);                                 // OWN CODE
+// ...                                                      // OWN CODE
+//
+// OVERALL USAGE (without looping):
+// ...                                                      // OWN CODE
+// threadsMgt<T> th(...);                                   // backend.hpp
+// const dim3 threads{th.genThreads()};                     // backend.hpp
+// const dim3 blocks{th.genBlocksFull(threads,...)};        // backend.hpp
+// cuda::Kernel KER{GETKERNEL(...)};                        // OWN CODE
+// KER(threads,blocks,...);                                 // OWN CODE
+// ...                                                      // OWN CODE
+template<typename T>
+class threadsMgt {
+   public:
+    bool loop0, loop1, loop2, loop3;
+
+   private:
+    const unsigned d0, d1, d2, d3;
+    const T ndims;
+    const unsigned maxParallelThreads;
+
+   public:
+    // INPUT: dims of the output array
+    // INPUT: ndims of previous dims
+    threadsMgt(const T dims[4], const T ndims);
+
+    // Generate optimal thread values
+    inline const dim3 genThreads() const;
+
+    // INPUT threads, generated by genThreads()
+    // OUTPUT blocks, supposing that each element results in 1 thread
+    inline dim3 genBlocksFull(const dim3& threads) const;
+
+    // Generate the optimal block values
+    // INPUT threads, generated by genThreads()
+    // INPUT nrInputs = number of input buffers read by kernel in parallel
+    // INPUT nrOutputs = number of output buffers written by kernel in parallel
+    // INPUT totalSize = size of all input arrays and all output arrays together
+    // INPUT sizeofT = size of 1 element TO BE WRITTEN
+    // OUTPUT blocks, assuming that the previously calculated loopings will be
+    // executed in the kernel
+    inline dim3 genBlocks(const dim3& threads, const unsigned nrInputs,
+                          const unsigned nrOutputs, const size_t totalSize,
+                          const size_t sizeofT);
+};
+
+// INPUT: dims of the output array
+// INPUT: ndims of previous dims
+template<typename T>
+threadsMgt<T>::threadsMgt(const T dims[4], const T ndims)
+    : loop0(false)
+    , loop1(false)
+    , loop2(false)
+    , loop3(false)
+    , d0(static_cast<unsigned>(dims[0]))
+    , d1(static_cast<unsigned>(dims[1]))
+    , d2(static_cast<unsigned>(dims[2]))
+    , d3(static_cast<unsigned>(dims[3]))
+    , ndims(ndims)
+    , maxParallelThreads(getMaxParallelThreads(getActiveDeviceId())){};
+
+// Generate optimal thread values
+template<typename T>
+const dim3 threadsMgt<T>::genThreads() const {
+    // Performance is mainly dependend on:
+    //    - reducing memory latency, by preferring a sequential read of
+    //    cachelines (principally dim0)
+    //    - more parallel threads --> higher occupation of available
+    //    threads
+    //    - more I/O operations per thread --> dims[3] indicates the #
+    //    of I/Os handled by the kernel inside each thread, and outside
+    //    the scope of the block scheduler
+    // High performance is achievable with occupation rates as low as
+    // 30%. Here we aim at 50%, to also cover older hardware with slower
+    // cores.
+    // https://stackoverflow.com/questions/7737772/improving-kernel-performance-by-increasing-occupancy
+    // http://www.nvidia.com/content/gtc-2010/pdfs/2238_gtc2010.pdf
+    // https://www.cvg.ethz.ch/teaching/2011spring/gpgpu/GPU-Optimization.pdf
+    // https://en.wikipedia.org/wiki/Graphics_Core_Next#SIMD_Vector_Unit
+
+    // The performance for vectors is independent from array sizes.
+    if ((d1 == 1) & (d2 == 1)) return dim3(128U);
+
+    // TOTAL OCCUPATION = occup(dim0) * occup(dim1) * occup(dim2).
+    // For linearized arrays, each linear block is allocated to a dim,
+    // resulting in large numbers for dim0 & dim1.
+    // - For dim2, we only return exact dividers of the array dim[3], so
+    // occup(dim2)=100%
+    // - For dim0 & dim1, we aim somewhere between 30% and 50%
+    //      * Having 2 blocks filled + 1 thread in block 3 --> occup >
+    //      2/3=66%
+    //      * Having 3 blocks filled + 1 thread in block 4 --> occup >
+    //      3/4=75%
+    //      * Having 4 blocks filled + 1 thread in block 5 --> occup >
+    //      4/5=80%
+    constexpr unsigned OCCUPANCY_FACTOR{2U};  // at least 2 blocks filled
+
+    // NVIDIA:
+    //  warp             = 32
+    //  possible blocks  = [32, 64, 96, 128, 160, 192, 224, 256, ..
+    //  1024] best performance = [32, 64, 96, 128] optimal perf     =
+    //  128; any combination
+    //   NIVIDA always processes full wavefronts.  Allocating partial
+    //   warps
+    //   (<32) reduces throughput.  Performance reaches a plateau from
+    //   128 with a slightly slowing for very large sizes.
+    // For algorithm below:
+    //  parallelThreads  = [32, 64, 96, 128]
+    constexpr unsigned minThreads{32};
+    const unsigned relevantElements{d0 * d1 * d2};
+    constexpr unsigned warp{32};
+
+    // For small array's, we reduce the maximum threads in 1 block to
+    // improve parallelisme.  In worst case the scheduler can have 1
+    // block per CU, even when only partly loaded. Range for block is:
+    // [minThreads ... 4 * warp multiple]
+    //   * NVIDIA: [4*32=128 threads]
+    // At 4 * warp multiple, full wavefronts (queue of 4 partial
+    // wavefronts) are all occupied.
+
+    // We need at least maxParallelThreads to occupy all the CU's.
+    const unsigned parallelThreads{
+        relevantElements <= maxParallelThreads
+            ? minThreads
+            : std::min(4U, relevantElements / maxParallelThreads) * warp};
+
+    // Priority 1: keep cachelines filled.  Aparrantly sharing
+    // cachelines between CU's has a heavy cost. Testing confirmed that
+    // the occupation is mostly > 50%
+    const unsigned threads0{d0 == 1 ? 1
+                            : d0 <= minThreads
+                                ? minThreads  // better distribution
+                                : std::min(128U, (divup(d0, warp) * warp))};
+
+    // Priority 2: Fill the block, while respecting the occupation limit
+    // (>66%) (through parallelThreads limit)
+    const unsigned threads1{
+        (threads0 * 64U <= parallelThreads) &&
+                (!(d1 & (64U - 1U)) || (d1 > OCCUPANCY_FACTOR * 64U))
+            ? 64U
+        : (threads0 * 32U <= parallelThreads) &&
+                (!(d1 & (32U - 1U)) || (d1 > OCCUPANCY_FACTOR * 32U))
+            ? 32U
+        : (threads0 * 16U <= parallelThreads) &&
+                (!(d1 & (16U - 1U)) || (d1 > OCCUPANCY_FACTOR * 16U))
+            ? 16U
+        : (threads0 * 8U <= parallelThreads) &&
+                (!(d1 & (8U - 1U)) || (d1 > OCCUPANCY_FACTOR * 8U))
+            ? 8U
+        : (threads0 * 4U <= parallelThreads) &&
+                (!(d1 & (4U - 1U)) || (d1 > OCCUPANCY_FACTOR * 4U))
+            ? 4U
+        : (threads0 * 2U <= parallelThreads) &&
+                (!(d1 & (2U - 1U)) || (d1 > OCCUPANCY_FACTOR * 2U))
+            ? 2U
+            : 1U};
+
+    const unsigned threads01{threads0 * threads1};
+    if ((d2 == 1) | (threads01 * 2 > parallelThreads))
+        return dim3(threads0, threads1);
+
+    // Priority 3: Only exact dividers are used, so that
+    //  - overflow checking is not needed in the kernel.
+    //  - occupation rate never is reduced
+    // Chances are low that threads2 will be different from 1.
+    const unsigned threads2{
+        (threads01 * 8 <= parallelThreads) && !(d2 & (8U - 1U))   ? 8U
+        : (threads01 * 4 <= parallelThreads) && !(d2 & (4U - 1U)) ? 4U
+        : (threads01 * 2 <= parallelThreads) && !(d2 & (2U - 1U)) ? 2U
+                                                                  : 1U};
+    return dim3(threads0, threads1, threads2);
+};
+
+// INPUT threads, generated by genThreads()
+// OUTPUT blocks, supposing that each element results in 1 thread
+template<typename T>
+inline dim3 threadsMgt<T>::genBlocksFull(const dim3& threads) const {
+    const dim3 blocks{divup(d0, threads.x), divup(d1, threads.y),
+                      divup(d2, threads.z)};
+    return dim3(divup(d0, threads.x), divup(d1, threads.y),
+                divup(d2, threads.z));
+};
+
+// Generate the optimal block values
+// INPUT threads, generated by genThreads()
+// INPUT nrInputs = number of input buffers read by kernel in parallel
+// INPUT nrOutputs = number of output buffers written by kernel in parallel
+// INPUT totalSize = size of all input arrays and all output arrays together
+// INPUT sizeofT = size of 1 element TO BE WRITTEN
+// OUTPUT blocks, assuming that the previously calculated loopings will be
+// executed in the kernel
+template<typename T>
+inline dim3 threadsMgt<T>::genBlocks(const dim3& threads,
+                                     const unsigned nrInputs,
+                                     const unsigned nrOutputs,
+                                     const size_t totalSize,
+                                     const size_t sizeofT) {
+    // The bottleneck of anykernel is dependent on the type of memory
+    // used.
+    // a) For very small arrays (elements < maxParallelThreads), each
+    //  element receives it individual thread.
+    // b) For arrays (in+out) smaller than 3/2 L2cache, memory access no
+    //  longer is the bottleneck, because enough L2cache is available at any
+    //  time. Threads are limited to reduce scheduling overhead.
+    // c) For very large arrays and type sizes (<long double), 1 thread will
+    //  not generate enough data to keep the memory sync mechanism
+    //  saturated, so we start loooping inside each thread.
+    dim3 blocks{1};
+    const int activeDeviceId{getActiveDeviceId()};
+    const unsigned* maxGridSize{
+        reinterpret_cast<const unsigned*>(getMaxGridSize(activeDeviceId))};
+    const size_t L2CacheSize{getL2CacheSize(activeDeviceId)};
+    const unsigned cacheLine{getMemoryBusWidth(activeDeviceId)};
+    const unsigned multiProcessorCount{getMultiProcessorCount(activeDeviceId)};
+    const unsigned maxThreads{maxParallelThreads *
+                              (sizeofT * nrInputs * nrInputs > 8 ? 1 : 2)};
+
+    if (ndims == 1) {
+        if (d0 > maxThreads) {
+            if (totalSize * 2 > L2CacheSize * 3) {
+                // General formula to calculate best #loops
+                // Dedicated GPUs:
+                //  32/sizeof(T)**2/#outBuffers*(3/4)**(#inBuffers-1)
+                // Integrated GPUs:
+                //  4/sizeof(T)/#outBuffers*(3/4)**(#inBuffers-1)
+                unsigned largeVolDivider{cacheLine == 64
+                                             ? sizeofT == 1   ? 4
+                                               : sizeofT == 2 ? 2
+                                                              : 1
+                                             : (sizeofT == 1   ? 32
+                                                : sizeofT == 2 ? 8
+                                                               : 1) /
+                                                   nrOutputs};
+                for (unsigned i{1}; i < nrInputs; ++i)
+                    largeVolDivider = largeVolDivider * 3 / 4;
+                if (largeVolDivider > 1) {
+                    blocks.x = d0 / (largeVolDivider * threads.x);
+                    if (blocks.x == 0) blocks.x = 1;
+                    loop0 = true;
+                }
+            } else {
+                // A reduction to (1|2*)maxParallelThreads will be
+                // performed
+                blocks.x = maxThreads / threads.x;
+                if (blocks.x == 0) blocks.x = 1;
+                loop0 = true;
+            }
+        }
+        if (!loop0) { blocks.x = divup(d0, threads.x); }
+    } else {
+        loop3    = d3 != 1;
+        blocks.x = divup(d0, threads.x);
+        blocks.z = divup(d2, threads.z);
+        // contains the mandatory loops introduced by dim3 and dim2
+        // gridSize overflow
+        unsigned dim2and3Multiplier{d3};
+        if (blocks.z > maxGridSize[2]) {
+            dim2and3Multiplier = dim2and3Multiplier * blocks.z / maxGridSize[2];
+            blocks.z           = maxGridSize[2];
+            loop2              = true;
+        }
+        if ((d1 > threads.y) &
+            (threads.x * blocks.x * d1 * threads.z * blocks.z > maxThreads)) {
+            if ((d0 * sizeofT * 8 > cacheLine * multiProcessorCount) &
+                (totalSize * 2 > L2CacheSize * 3)) {
+                // General formula to calculate best #loops
+                // Dedicated GPUs:
+                //  32/sizeof(T)**2/#outBuffers*(3/4)**(#inBuffers-1)
+                // Integrated GPUs:
+                //  4/sizeof(T)/#outBuffers*(3/4)**(#inBuffers-1)
+                unsigned largeVolDivider{
+                    cacheLine == 64 ? sizeofT == 1   ? 4
+                                      : sizeofT == 2 ? 2
+                                                     : 1
+                                    : (sizeofT == 1   ? 32
+                                       : sizeofT == 2 ? 8
+                                       : sizeofT == 4 ? 2
+                                                      : 1) /
+                                          (dim2and3Multiplier * nrOutputs)};
+                for (unsigned i{1}; i < nrInputs; ++i)
+                    largeVolDivider = largeVolDivider * 3 / 4;
+                if (largeVolDivider > 1) {
+                    blocks.y = d1 / (largeVolDivider * threads.y);
+                    if (blocks.y == 0) blocks.y = 1;
+                    loop1 = true;
+                }
+            } else {
+                // A reduction to (1|2*)maxParallelThreads will be
+                // performed
+                blocks.y = maxThreads / (threads.x * blocks.x * threads.z *
+                                         blocks.z * threads.y);
+                if (blocks.y == 0) blocks.y = 1;
+                loop1 = true;
+            }
+        }
+        if (!loop1) { blocks.y = divup(d1, threads.y); }
+        // Check on new overflows
+        if (blocks.y > maxGridSize[1]) {
+            blocks.y = maxGridSize[1];
+            loop1    = true;
+        }
+    }
+
+    return blocks;
+};
+}  // namespace cuda
\ No newline at end of file
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index b159758b37..0f0f19764b 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -21,9 +21,9 @@
 #include <device_manager.hpp>
 #include <err_opencl.hpp>
 #include <errorcodes.hpp>
+#include <platform.hpp>
 #include <version.hpp>
 #include <af/version.h>
-#include <memory>
 
 #ifdef OS_MAC
 #include <OpenGL/CGLCurrent.h>
@@ -36,6 +36,7 @@
 #include <cstdlib>
 #include <functional>
 #include <map>
+#include <memory>
 #include <mutex>
 #include <sstream>
 #include <string>
@@ -223,7 +224,7 @@ void init() {
     UNUSED(devMngr);
 }
 
-unsigned getActiveDeviceId() {
+int getActiveDeviceId() {
     // Second element is the queue id, which is
     // what we mean by active device id in opencl backend
     return get<1>(tlocalActiveDeviceId());
@@ -314,10 +315,6 @@ cl_device_type getDeviceType() {
     return type;
 }
 
-bool isHostUnifiedMemory(const cl::Device& device) {
-    return device.getInfo<CL_DEVICE_HOST_UNIFIED_MEMORY>();
-}
-
 bool OpenCLCPUOffload(bool forceOffloadOSX) {
     static const bool offloadEnv = getEnvVar("AF_OPENCL_CPU_OFFLOAD") != "0";
     bool offload                 = false;
@@ -360,9 +357,7 @@ bool isDoubleSupported(unsigned device) {
         common::lock_guard_t lock(devMngr.deviceMutex);
         dev = *devMngr.mDevices[device];
     }
-    // 64bit fp is an optional extension
-    return (dev.getInfo<CL_DEVICE_EXTENSIONS>().find("cl_khr_fp64") !=
-            string::npos);
+    return isDoubleSupported(dev);
 }
 
 bool isHalfSupported(unsigned device) {
@@ -373,9 +368,7 @@ bool isHalfSupported(unsigned device) {
         common::lock_guard_t lock(devMngr.deviceMutex);
         dev = *devMngr.mDevices[device];
     }
-    // 16bit fp is an option extension
-    return (dev.getInfo<CL_DEVICE_EXTENSIONS>().find("cl_khr_fp16") !=
-            string::npos);
+    return isHalfSupported(dev);
 }
 
 void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute) {
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 8ea6ca2540..fa937b0e0f 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -57,7 +57,7 @@ int getDeviceCount() noexcept;
 
 void init();
 
-unsigned getActiveDeviceId();
+int getActiveDeviceId();
 
 int& getMaxJitSize();
 
@@ -71,18 +71,65 @@ size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();
 
+inline unsigned getMemoryBusWidth(const cl::Device& device) {
+    return device.getInfo<CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE>();
+}
+
+// OCL only reports on L1 cache, so we have to estimate the L2 Cache
+// size. From studying many GPU cards, it is noticed that their is a
+// direct correlation between Cache line and L2 Cache size:
+//      - 16KB L2 Cache for each bit in Cache line.
+//        Example: RTX3070 (4096KB of L2 Cache, 256Bit of Cache
+//        line)
+//                   --> 256*16KB = 4096KB
+//      - This is also valid for all AMD GPU's
+//      - Exceptions
+//          * GTX10XX series have 8KB per bit of cache line
+//          * iGPU (64bit cacheline) have 5KB per bit of cache line
+inline size_t getL2CacheSize(const cl::Device& device) {
+    const unsigned cacheLine{getMemoryBusWidth(device)};
+    return cacheLine * 1024ULL *
+           (cacheLine == 64 ? 5
+            : device.getInfo<CL_DEVICE_NAME>().find("GTX 10") ==
+                    std::string::npos
+                ? 16
+                : 8);
+}
+
+inline unsigned getComputeUnits(const cl::Device& device) {
+    return device.getInfo<CL_DEVICE_MAX_COMPUTE_UNITS>();
+}
+
+// maximum nr of threads the device really can run in parallel, without
+// scheduling
+inline unsigned getMaxParallelThreads(const cl::Device& device) {
+    return getComputeUnits(device) * 2048;
+}
+
 cl_device_type getDeviceType();
 
-bool isHostUnifiedMemory(const cl::Device& device);
+inline bool isHostUnifiedMemory(const cl::Device& device) {
+    return device.getInfo<CL_DEVICE_HOST_UNIFIED_MEMORY>();
+}
 
 bool OpenCLCPUOffload(bool forceOffloadOSX = true);
 
 bool isGLSharingSupported();
 
 bool isDoubleSupported(unsigned device);
+inline bool isDoubleSupported(const cl::Device& device) {
+    // 64bit fp is an optional extension
+    return (device.getInfo<CL_DEVICE_EXTENSIONS>().find("cl_khr_fp64") !=
+            std::string::npos);
+}
 
 // Returns true if 16-bit precision floats are supported by the device
 bool isHalfSupported(unsigned device);
+inline bool isHalfSupported(const cl::Device& device) {
+    // 16bit fp is an option extension
+    return (device.getInfo<CL_DEVICE_EXTENSIONS>().find("cl_khr_fp16") !=
+            std::string::npos);
+}
 
 void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute);
 
diff --git a/src/backend/opencl/threadsMgt.hpp b/src/backend/opencl/threadsMgt.hpp
new file mode 100644
index 0000000000..4fb3838e5b
--- /dev/null
+++ b/src/backend/opencl/threadsMgt.hpp
@@ -0,0 +1,328 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/dispatch.hpp>
+#include <platform.hpp>
+#include <af/defines.h>
+
+namespace opencl {
+// OVERALL USAGE (With looping):
+// ...                                                      // OWN CODE
+// threadsMgt<T> th(...);                                   // backend.hpp
+// cl::Kernel KER{GETKERNEL(..., th.loop0, th.loop1,
+//                               th.loop3)};                // OWN CODE
+// const cl::NDRange local{th.genLocal(KER)};               // backend.hpp
+// const cl::NDRange global{th.genGlobal(local)};           // backend.hpp
+// KER(local,global,...);                                   // OWN CODE
+// ...                                                      // OWN CODE
+//
+// OVERALL USAGE (without looping):
+// ...                                                      // OWN CODE
+// threadsMgt<T> th(...);                                   // backend.hpp
+// cl::Kernel KER{GETKERNEL(...)};                          // OWN CODE
+// const cl::NDRange local{th.genLocal(KER)};               // backend.hpp
+// const cl::NDRange global{th.genGlobalFull(local)};       // backend.hpp
+// KER(local,global,...);                                   // OWN CODE
+// ...                                                      // OWN CODE
+template<typename T>
+class threadsMgt {
+   public:
+    bool loop0, loop1, loop3;
+
+   private:
+    const unsigned d0, d1, d2, d3;
+    const T ndims;
+    const size_t totalSize;
+    const cl::Device dev;
+    const unsigned maxParallelThreads;
+    const unsigned maxThreads;
+    unsigned largeVolDivider;
+
+   public:
+    // INPUT dims = dims of output array
+    // INPUT ndims = ndims of output array
+    // INPUT nrInputs = number of buffers read by kernel in parallel
+    // INPUT nrOutputs = number of buffer written by kernel in parallel
+    // INPUT totalSize = size of all input & output arrays
+    // INPUT sizeofT = size of 1 element to be written
+    // OUTPUT this.loop0, this.loop1, this.loop3 are ready to create the kernel
+    threadsMgt(const T dims[4], const T ndims, const unsigned nrInputs,
+               const unsigned nrOutputs, const size_t totalSize,
+               const size_t sizeofT);
+
+    // The generated local is only best for independent element operations,
+    //  as are: copying, scaling, math on independent elements,
+    // ... Since vector dimensions can be returned, it is NOT USABLE FOR
+    // BLOCK OPERATIONS, as are: matmul, etc.
+    inline cl::NDRange genLocal(const cl::Kernel& ker) const;
+
+    // INPUT local generated by genLocal()
+    // OUTPUT global, supposing that each element results in 1 thread
+    inline cl::NDRange genGlobalFull(const cl::NDRange& local) const;
+
+    // INPUT local generated by genLocal()
+    // OUTPUT global, assuming the the previous calculated looping will be
+    // executed in the kernel
+    inline cl::NDRange genGlobal(const cl::NDRange& local) const;
+};
+
+// INPUT dims = dims of output array
+// INPUT ndims = ndims of output array
+// INPUT nrInputs = number of buffers read by kernel in parallel
+// INPUT nrOutputs = number of buffer written by kernel in parallel
+// INPUT totalSize = size of all input & output arrays
+// INPUT sizeofT = size of 1 element to be written
+// OUTPUT this.loop0, this.loop1, this.loop3 are ready to create the kernel
+template<typename T>
+threadsMgt<T>::threadsMgt(const T dims[4], const T ndims,
+                          const unsigned nrInputs, const unsigned nrOutputs,
+                          const size_t totalSize, const size_t sizeofT)
+    : loop0(false)
+    , loop1(false)
+    , loop3(false)
+    , d0(static_cast<unsigned>(dims[0]))
+    , d1(static_cast<unsigned>(dims[1]))
+    , d2(static_cast<unsigned>(dims[2]))
+    , d3(static_cast<unsigned>(dims[3]))
+    , ndims(ndims)
+    , totalSize(totalSize)
+    , dev(opencl::getDevice())
+    , maxParallelThreads(getMaxParallelThreads(dev))
+    , maxThreads(maxParallelThreads *
+                 (sizeofT * nrInputs * nrInputs > 8 ? 1 : 2))
+    , largeVolDivider(1) {
+    const unsigned cacheLine{getMemoryBusWidth(dev)};
+    const size_t L2CacheSize{getL2CacheSize(dev)};
+    // The bottleneck of anykernel is dependent on the type of memory
+    // used.
+    // a) For very small arrays (elements < maxParallelThreads), each
+    //  element receives it individual thread
+    // b) For arrays (in+out) smaller
+    //  than 3/2 L2cache, memory access no longer is the bottleneck,
+    //  because enough L2cache is available at any time. Threads are
+    //  limited to reduce scheduling overhead.
+    // c) For very large arrays and type sizes
+    //  (<long double), 1 thread will not generate enough data to keep
+    //  the memory sync mechanism saturated, so we start loooping inside
+    //  each thread.
+    //
+    if (ndims == 1) {
+        if (d0 > maxThreads) {
+            loop0 = true;
+            if (totalSize * 2 > L2CacheSize * 3) {
+                // General formula to calculate best #loops
+                // Dedicated GPUs:
+                //  32/sizeof(T)**2/#outBuffers*(3/4)**(#inBuffers-1)
+                // Integrated GPUs:
+                //  4/sizeof(T)/#outBuffers*(3/4)**(#inBuffers-1)
+                largeVolDivider = cacheLine == 64 ? sizeofT == 1   ? 4
+                                                    : sizeofT == 2 ? 2
+                                                                   : 1
+                                                  : (sizeofT == 1   ? 32
+                                                     : sizeofT == 2 ? 8
+                                                                    : 1) /
+                                                        nrOutputs;
+                for (unsigned i = 1; i < nrInputs; ++i)
+                    largeVolDivider = largeVolDivider * 3 / 4;
+                loop0 = largeVolDivider > 1;
+            }
+        }
+    } else {
+        loop3 = d3 != 1;
+        if ((d1 > 1) & (d0 * d1 * d2 > maxThreads)) {
+            loop1 = true;
+            if ((d0 * sizeofT * 8 > cacheLine * getComputeUnits(dev)) &
+                (totalSize * 2 > L2CacheSize * 3)) {
+                // General formula to calculate best #loops
+                // Dedicated GPUs:
+                //  32/sizeof(T)**2/#outBuffers*(3/4)**(#inBuffers-1)
+                // Integrated GPUs:
+                //  4/sizeof(T)/#outBuffers*(3/4)**(#inBuffers-1)
+                //
+                // dims[3] already loops, so the remaining #loops needs
+                // to be divided
+                largeVolDivider = cacheLine == 64 ? sizeofT == 1   ? 4
+                                                    : sizeofT == 2 ? 2
+                                                                   : 1
+                                                  : (sizeofT == 1   ? 32
+                                                     : sizeofT == 2 ? 8
+                                                     : sizeofT == 4 ? 2
+                                                                    : 1) /
+                                                        (d3 * nrOutputs);
+                for (unsigned i{1}; i < nrInputs; ++i)
+                    largeVolDivider = largeVolDivider * 3 / 4;
+                loop1 = largeVolDivider > 1;
+            }
+        }
+    }
+};
+
+// The generated local is only best for independent element operations,
+//  as are: copying, scaling, math on independent elements,
+// ... Since vector dimensions can be returned, it is NOT USABLE FOR
+// BLOCK OPERATIONS, as are: matmul, etc.
+template<typename T>
+inline cl::NDRange threadsMgt<T>::genLocal(const cl::Kernel& ker) const {
+    // Performance is mainly dependend on:
+    //    - reducing memory latency, by preferring a sequential read of
+    //    cachelines (principally dim0)
+    //    - more parallel threads --> higher occupation of available
+    //    threads
+    //    - more I/O operations per thread --> dims[3] indicates the #
+    //    of I/Os handled by the kernel inside each thread, and outside
+    //    the scope of the block scheduler
+    // High performance is achievable with occupation rates as low as
+    // 30%. Here we aim at 50%, to also cover older hardware with slower
+    // cores.
+    // https://stackoverflow.com/questions/7737772/improving-kernel-performance-by-increasing-occupancy
+    // http://www.nvidia.com/content/gtc-2010/pdfs/2238_gtc2010.pdf
+    // https://www.cvg.ethz.ch/teaching/2011spring/gpgpu/GPU-Optimization.pdf
+    // https://en.wikipedia.org/wiki/Graphics_Core_Next#SIMD_Vector_Unit
+
+    // The performance for vectors is independent from array sizes.
+    if ((d1 == 1) & (d2 == 1)) return cl::NDRange{128ULL};
+
+    // TOTAL OCCUPATION = occup(dim0) * occup(dim1) * occup(dim2).
+    // For linearized arrays, each linear block is allocated to a dim,
+    // resulting in large numbers for dim0 & dim1.
+    // - For dim2, we only return exact dividers of the array dim[3], so
+    // occup(dim2)=100%
+    // - For dim0 & dim1, we aim somewhere between 30% and 50%
+    //      * Having 2 blocks filled + 1 thread in block 3 --> occup >
+    //      2/3=66%
+    //      * Having 3 blocks filled + 1 thread in block 4 --> occup >
+    //      3/4=75%
+    //      * Having 4 blocks filled + 1 thread in block 5 --> occup >
+    //      4/5=80%
+    constexpr unsigned OCCUPANCY_FACTOR{2U};  // at least 2 blocks filled
+
+    // NVIDIA:
+    //  WG multiple      = 32
+    //  possible blocks  = [32, 64, 96, 128, 160, 192, 224, 256, .. 1024]
+    //  best performance = [32, 64, 96, 128]
+    //  optimal perf     = 128; any combination
+    //   NIVIDA always processes full wavefronts.  Allocating partial WG
+    //   (<32) reduces throughput.  Performance reaches a plateau from
+    //   128 with a slightly slowing for very large sizes.
+    // AMD:
+    //  WG multiple      = 64
+    //  possible block   = [16, 32, 48, 64, 128, 192, 256]
+    //  best performance = [(32, low #threads) 64, 128, 256]
+    //  optimal perf     = (128,2,1); max 128 for 1 dimension
+    //   AMD can process partial wavefronts (multiple of 16), although
+    //   all threads of a full WG are allocated, only the active ones
+    //   are executed, so the same number of WGs will fit a CU. When we
+    //   have insufficent threads to occupy all the CU's, partial
+    //   wavefronts (<64) are usefull to distribute all threads over the
+    //   available CU's iso all concentrating on the 1st CU.
+    // For algorithm below:
+    //  parallelThreads  = [32, 64, (96 for NIVIDA), 128, (256 for AMD)]
+    constexpr unsigned minThreads{32};
+    const unsigned relevantElements{d0 * d1 * d2};
+    const unsigned WG{static_cast<unsigned>(
+        ker.getWorkGroupInfo<CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE>(
+            dev))};
+
+    // For small array's, we reduce the maximum threads in 1 block to
+    // improve parallelisme.  In worst case the scheduler can have 1
+    // block per CU, even when only partly loaded. Range for block is:
+    //   [minThreads ... 4 * WG multiple]
+    //   * NVIDIA: [4*32=128 threads]
+    //   * AMD:    [4*64=256 threads]
+    // At 4 * WG multiple, full wavefronts (queue of 4 partial
+    // wavefronts) are all occupied.
+
+    // We need at least maxParallelThreads to occupy all the CU's.
+    const unsigned parallelThreads{
+        relevantElements <= maxParallelThreads
+            ? minThreads
+            : std::min(4U, relevantElements / maxParallelThreads) * WG};
+
+    // Priority 1: keep cachelines filled.  Aparrantly sharing
+    // cachelines between CU's has a cost. Testing confirmed that the
+    // occupation is mostly > 50%
+    const unsigned threads0{d0 == 1 ? 1
+                            : d0 <= minThreads
+                                ? minThreads  // better distribution
+                                : std::min(128U, (divup(d0, WG) * WG))};
+
+    // Priority 2: Fill the block, while respecting the occupation limit
+    // (>66%) (through parallelThreads limit)
+    const unsigned threads1{
+        (threads0 * 64U <= parallelThreads) &&
+                (!(d1 & (64U - 1U)) || (d1 > OCCUPANCY_FACTOR * 64U))
+            ? 64U
+        : (threads0 * 32U <= parallelThreads) &&
+                (!(d1 & (32U - 1U)) || (d1 > OCCUPANCY_FACTOR * 32U))
+            ? 32U
+        : (threads0 * 16U <= parallelThreads) &&
+                (!(d1 & (16U - 1U)) || (d1 > OCCUPANCY_FACTOR * 16U))
+            ? 16U
+        : (threads0 * 8U <= parallelThreads) &&
+                (!(d1 & (8U - 1U)) || (d1 > OCCUPANCY_FACTOR * 8U))
+            ? 8U
+        : (threads0 * 4U <= parallelThreads) &&
+                (!(d1 & (4U - 1U)) || (d1 > OCCUPANCY_FACTOR * 4U))
+            ? 4U
+        : (threads0 * 2U <= parallelThreads) &&
+                (!(d1 & (2U - 1U)) || (d1 > OCCUPANCY_FACTOR * 2U))
+            ? 2U
+            : 1U};
+
+    const unsigned threads01{threads0 * threads1};
+    if ((d2 == 1) | (threads01 * 2 > parallelThreads))
+        return cl::NDRange(threads0, threads1);
+
+    // Priority 3: Only exact dividers are used, so that
+    //  - overflow checking is not needed in the kernel.
+    //  - occupation rate never is reduced
+    // Chances are low that threads2 will be different from 1.
+    const unsigned threads2{
+        (threads01 * 8 <= parallelThreads) && !(d2 & (8U - 1U))   ? 8U
+        : (threads01 * 4 <= parallelThreads) && !(d2 & (4U - 1U)) ? 4U
+        : (threads01 * 2 <= parallelThreads) && !(d2 & (2U - 1U)) ? 2U
+                                                                  : 1U};
+    return cl::NDRange(threads0, threads1, threads2);
+};
+
+// INPUT local generated by genLocal()
+// OUTPUT global, supposing that each element results in 1 thread
+template<typename T>
+inline cl::NDRange threadsMgt<T>::genGlobalFull(
+    const cl::NDRange& local) const {
+    return cl::NDRange(divup(d0, local[0]) * local[0],
+                       divup(d1, local[1]) * local[1],
+                       divup(d2, local[2]) * local[2]);
+};
+
+// INPUT local generated by genLocal()
+// OUTPUT global, assuming the the previous calculated looping will be
+// executed in the kernel
+template<typename T>
+inline cl::NDRange threadsMgt<T>::genGlobal(const cl::NDRange& local) const {
+    if (loop0) {
+        const size_t blocks0{largeVolDivider > 1
+                                 ? d0 / (largeVolDivider * local[0])
+                                 : maxThreads / local[0]};
+        return cl::NDRange(blocks0 == 0 ? local[0] : blocks0 * local[0]);
+    } else if (loop1) {
+        const size_t global0{divup(d0, local[0]) * local[0]};
+        const size_t global2{divup(d2, local[2]) * local[2]};
+        const size_t blocks1{largeVolDivider > 1
+                                 ? d1 / (largeVolDivider * local[1])
+                                 : maxThreads / (global0 * local[1] * global2)};
+        return cl::NDRange(
+            global0, blocks1 == 0 ? local[1] : blocks1 * local[1], global2);
+    } else {
+        return genGlobalFull(local);
+    }
+};
+}  // namespace opencl
\ No newline at end of file

From 5fdf4283f204fb4a14ede36d33d89b206035ab8e Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 4 Aug 2022 01:10:12 +0200
Subject: [PATCH 073/473] OPT: memcopy

---
 src/backend/cuda/copy.cpp             | 108 +++++-----
 src/backend/cuda/kernel/copy.cuh      | 222 +++++++++++++++++---
 src/backend/cuda/kernel/memcopy.cuh   | 228 ++++++++++++++++++---
 src/backend/cuda/kernel/memcopy.hpp   | 219 +++++++++++++++-----
 src/backend/cuda/reshape.cpp          |   4 +-
 src/backend/opencl/copy.cpp           | 136 +++++++------
 src/backend/opencl/kernel/copy.cl     | 225 ++++++++++++++++----
 src/backend/opencl/kernel/memcopy.cl  | 186 ++++++++++++++---
 src/backend/opencl/kernel/memcopy.hpp | 283 +++++++++++++++++++-------
 src/backend/opencl/reshape.cpp        |   5 +-
 10 files changed, 1243 insertions(+), 373 deletions(-)

diff --git a/src/backend/cuda/copy.cpp b/src/backend/cuda/copy.cpp
index 12ec5e93e0..dbcf1284fe 100644
--- a/src/backend/cuda/copy.cpp
+++ b/src/backend/cuda/copy.cpp
@@ -22,87 +22,89 @@ using common::is_complex;
 namespace cuda {
 
 template<typename T>
-void copyData(T *dst, const Array<T> &src) {
-    if (src.elements() == 0) { return; }
-
-    // FIXME: Merge this with copyArray
-    src.eval();
-
-    Array<T> out = src;
-    const T *ptr = NULL;
-
-    if (src.isLinear() ||  // No offsets, No strides
-        src.ndims() == 1   // Simple offset, no strides.
-    ) {
-        // A.get() gets data with offsets
-        ptr = src.get();
-    } else {
-        // FIXME: Think about implementing eval
-        out = copyArray(src);
-        ptr = out.get();
+void copyData(T *data, const Array<T> &src) {
+    if (src.elements() > 0) {
+        Array<T> lin = src.isReady() && src.isLinear() ? src : copyArray(src);
+        // out is now guaranteed linear
+        auto stream = cuda::getActiveStream();
+        CUDA_CHECK(cudaMemcpyAsync(data, lin.get(), lin.elements() * sizeof(T),
+                                   cudaMemcpyDeviceToHost, stream));
+        CUDA_CHECK(cudaStreamSynchronize(stream));
     }
-
-    auto stream = cuda::getActiveStream();
-    CUDA_CHECK(cudaMemcpyAsync(dst, ptr, src.elements() * sizeof(T),
-                               cudaMemcpyDeviceToHost, stream));
-    CUDA_CHECK(cudaStreamSynchronize(stream));
 }
 
 template<typename T>
 Array<T> copyArray(const Array<T> &src) {
     Array<T> out = createEmptyArray<T>(src.dims());
-    if (src.elements() == 0) { return out; }
-
-    if (src.isLinear()) {
-        CUDA_CHECK(
-            cudaMemcpyAsync(out.get(), src.get(), src.elements() * sizeof(T),
-                            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
-    } else {
-        kernel::memcopy<T>(out, src, src.ndims());
+    if (src.elements() > 0) {
+        if (src.isReady()) {
+            if (src.isLinear()) {
+                CUDA_CHECK(cudaMemcpyAsync(
+                    out.get(), src.get(), src.elements() * sizeof(T),
+                    cudaMemcpyDeviceToDevice, getActiveStream()));
+            } else {
+                kernel::memcopy<T>(out, src, src.ndims());
+            }
+        } else {
+            evalNodes<T>(out, src.getNode().get());
+        }
     }
     return out;
 }
 
 template<typename T>
-void multiply_inplace(Array<T> &in, double val) {
-    kernel::copy<T, T>(in, in, in.ndims(), scalar<T>(0), val);
+void multiply_inplace(Array<T> &src, double norm) {
+    if (src.elements() > 0) {
+        kernel::copy<T, T>(src, src, src.ndims(), scalar<T>(0), norm);
+    }
 }
 
 template<typename inType, typename outType>
 struct copyWrapper {
-    void operator()(Array<outType> &out, Array<inType> const &in) {
-        kernel::copy<inType, outType>(out, in, in.ndims(), scalar<outType>(0),
-                                      1);
+    void operator()(Array<outType> &dst, Array<inType> const &src) {
+        kernel::copy<inType, outType>(dst, src, dst.ndims(), scalar<outType>(0),
+                                      1.0);
     }
 };
 
 template<typename T>
 struct copyWrapper<T, T> {
-    void operator()(Array<T> &out, Array<T> const &in) {
-        if (out.isLinear() && in.isLinear() &&
-            out.elements() == in.elements()) {
-            CUDA_CHECK(cudaMemcpyAsync(
-                out.get(), in.get(), in.elements() * sizeof(T),
-                cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
-        } else {
-            kernel::copy<T, T>(out, in, in.ndims(), scalar<T>(0), 1);
+    void operator()(Array<T> &dst, Array<T> const &src) {
+        if (src.elements() > 0) {
+            if (dst.dims() == src.dims()) {
+                if (src.isReady()) {
+                    if (dst.isLinear() && src.isLinear()) {
+                        CUDA_CHECK(cudaMemcpyAsync(
+                            dst.get(), src.get(), src.elements() * sizeof(T),
+                            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+                    } else {
+                        kernel::memcopy<T>(dst, src, src.ndims());
+                    }
+                } else {
+                    Param<T> info(dst.get(), src.dims().dims,
+                                  dst.strides().dims);
+                    evalNodes(info, src.getNode().get());
+                }
+            } else {
+                // dst has more elements than src, so default has to be applied
+                kernel::copy<T, T>(dst, src, dst.ndims(), scalar<T>(0), 1.0);
+            }
         }
     }
 };
 
 template<typename inType, typename outType>
-void copyArray(Array<outType> &out, Array<inType> const &in) {
+void copyArray(Array<outType> &dst, Array<inType> const &src) {
     static_assert(!(is_complex<inType>::value && !is_complex<outType>::value),
                   "Cannot copy from complex value to a non complex value");
-    ARG_ASSERT(1, (in.ndims() == out.dims().ndims()));
     copyWrapper<inType, outType> copyFn;
-    copyFn(out, in);
+    copyFn(dst, src);
 }
 
-#define INSTANTIATE(T)                                       \
-    template void copyData<T>(T * dst, const Array<T> &src); \
-    template Array<T> copyArray<T>(const Array<T> &src);     \
-    template void multiply_inplace<T>(Array<T> & in, double norm);
+#define INSTANTIATE(T)                                        \
+    template void copyData<T>(T * data, const Array<T> &src); \
+    template Array<T> copyArray<T>(const Array<T> &src);      \
+    template void multiply_inplace<T>(Array<T> & src, double norm);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
@@ -168,9 +170,9 @@ INSTANTIATE_COPY_ARRAY_COMPLEX(cfloat)
 INSTANTIATE_COPY_ARRAY_COMPLEX(cdouble)
 
 template<typename T>
-T getScalar(const Array<T> &in) {
+T getScalar(const Array<T> &src) {
     T retVal{};
-    CUDA_CHECK(cudaMemcpyAsync(&retVal, in.get(), sizeof(T),
+    CUDA_CHECK(cudaMemcpyAsync(&retVal, src.get(), sizeof(T),
                                cudaMemcpyDeviceToHost,
                                cuda::getActiveStream()));
     CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
diff --git a/src/backend/cuda/kernel/copy.cuh b/src/backend/cuda/kernel/copy.cuh
index 628a898904..5c6b6e485a 100644
--- a/src/backend/cuda/kernel/copy.cuh
+++ b/src/backend/cuda/kernel/copy.cuh
@@ -94,41 +94,199 @@ OTHER_SPECIALIZATIONS(uchar)
 OTHER_SPECIALIZATIONS(char)
 OTHER_SPECIALIZATIONS(common::half)
 
-template<typename inType, typename outType, bool same_dims>
-__global__ void copy(Param<outType> dst, CParam<inType> src,
-                     outType default_value, double factor, const dims_t trgt,
-                     uint blk_x, uint blk_y) {
-    const uint lx = threadIdx.x;
-    const uint ly = threadIdx.y;
-
-    const uint gz         = blockIdx.x / blk_x;
-    const uint gw         = (blockIdx.y + (blockIdx.z * gridDim.y)) / blk_y;
-    const uint blockIdx_x = blockIdx.x - (blk_x)*gz;
-    const uint blockIdx_y =
-        (blockIdx.y + (blockIdx.z * gridDim.y)) - (blk_y)*gw;
-    const uint gx = blockIdx_x * blockDim.x + lx;
-    const uint gy = blockIdx_y * blockDim.y + ly;
-
-    const inType *in = src.ptr + (gw * src.strides[3] + gz * src.strides[2] +
-                                  gy * src.strides[1]);
-    outType *out     = dst.ptr + (gw * dst.strides[3] + gz * dst.strides[2] +
-                              gy * dst.strides[1]);
-
-    int istride0 = src.strides[0];
-    int ostride0 = dst.strides[0];
-
-    if (gy < dst.dims[1] && gz < dst.dims[2] && gw < dst.dims[3]) {
-        int loop_offset = blockDim.x * blk_x;
-        bool cond = gy < trgt.dim[1] && gz < trgt.dim[2] && gw < trgt.dim[3];
-        for (int rep = gx; rep < dst.dims[0]; rep += loop_offset) {
-            outType temp = default_value;
-            if (same_dims || (rep < trgt.dim[0] && cond)) {
-                temp = convertType<inType, outType>(
-                    scale<inType>(in[rep * istride0], factor));
+// scaledCopy without looping, so dim3 has to be 1.
+// conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] >= dims[1]
+//      global dims[2] == dims[2]
+//      only dims[3] == 1 will be processed!!
+template<typename inType, typename outType, bool SAME_DIMS, bool FACTOR>
+__global__ void scaledCopy(Param<outType> dst, CParam<inType> src,
+                           const outType default_value, const double factor) {
+    const int id0 = blockIdx.x * blockDim.x + threadIdx.x;
+    const int id1 = blockIdx.y * blockDim.y + threadIdx.y;
+    if ((id0 < (int)dst.dims[0]) & (id1 < (int)dst.dims[1])) {
+        const int id2 = blockIdx.z * blockDim.z + threadIdx.z;
+
+        const int idx_in =
+            id0 * src.strides[0] + id1 * src.strides[1] + id2 * src.strides[2];
+        const int idx_out =
+            id0 * dst.strides[0] + id1 * dst.strides[1] + id2 * dst.strides[2];
+
+        if (SAME_DIMS | ((id0 < (int)src.dims[0]) & (id1 < (int)src.dims[1]) &
+                         (id2 < (int)src.dims[2]))) {
+            dst.ptr[idx_out] = convertType<inType, outType>(
+                FACTOR ? scale<inType>(src.ptr[idx_in], factor)
+                       : src.ptr[idx_in]);
+        } else {
+            dst.ptr[idx_out] = default_value;
+        }
+    }
+}
+
+// scaledCopy with looping over dims[0] -- VECTOR ONLY
+// Conditions:
+//      global dims[0] has no restrictions
+//      only dims[1] == 1 will be processed!!
+//      only dims[2] == 1 will be processed!!
+//      only dims[3] == 1 will be processed!!
+template<typename inType, typename outType, bool SAME_DIMS, bool FACTOR>
+__global__ void scaledCopyLoop0(Param<outType> dst, CParam<inType> src,
+                                const outType default_value,
+                                const double factor) {
+    int id0              = blockIdx.x * blockDim.x + threadIdx.x;
+    const int id0End_out = dst.dims[0];
+    if (id0 < id0End_out) {
+        const int id0End_in     = src.dims[0];
+        const int istrides0     = src.strides[0];
+        const int ostrides0     = dst.strides[0];
+        const int id0Inc        = gridDim.x * blockDim.x;
+        int idx_in              = id0 * istrides0;
+        const int idxID0Inc_in  = id0Inc * istrides0;
+        int idx_out             = id0 * ostrides0;
+        const int idxID0Inc_out = id0Inc * ostrides0;
+
+        while (id0 < id0End_in) {
+            // inside input array, so convert
+            dst.ptr[idx_out] = convertType<inType, outType>(
+                FACTOR ? scale<inType>(src.ptr[idx_in], factor)
+                       : src.ptr[idx_in]);
+            id0 += id0Inc;
+            idx_in += idxID0Inc_in;
+            idx_out += idxID0Inc_out;
+        }
+        if (!SAME_DIMS) {
+            while (id0 < id0End_out) {
+                // outside the input array, so copy default value
+                dst.ptr[idx_out] = default_value;
+                id0 += id0Inc;
+                idx_out += idxID0Inc_out;
             }
-            out[rep * ostride0] = temp;
         }
     }
 }
 
+// scaledCopy with looping over dims[1]
+// Conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] has no restrictions
+//      global dims[2] == dims[2]
+//      only dims[3] == 1 will be processed!!
+template<typename inType, typename outType, bool SAME_DIMS, bool FACTOR>
+__global__ void scaledCopyLoop1(Param<outType> dst, CParam<inType> src,
+                                const outType default_value,
+                                const double factor) {
+    const int id0        = blockIdx.x * blockDim.x + threadIdx.x;
+    int id1              = blockIdx.y * blockDim.y + threadIdx.y;
+    const int id1End_out = dst.dims[1];
+    if ((id0 < (int)dst.dims[0]) & (id1 < id1End_out)) {
+        const int id2       = blockIdx.z * blockDim.z + threadIdx.z;
+        const int ostrides1 = dst.strides[1];
+        const int id1Inc    = gridDim.y * blockDim.y;
+        int idx_out         = id0 * (int)dst.strides[0] + id1 * ostrides1 +
+                      id2 * (int)dst.strides[2];
+        const int idxID1Inc_out = id1Inc * ostrides1;
+        const int id1End_in     = src.dims[1];
+        const int istrides1     = src.strides[1];
+        int idx_in              = id0 * (int)src.strides[0] + id1 * istrides1 +
+                     id2 * (int)src.strides[2];
+        const int idxID1Inc_in = id1Inc * istrides1;
+
+        if (SAME_DIMS | ((id0 < (int)src.dims[0]) & (id2 < src.dims[2]))) {
+            while (id1 < id1End_in) {
+                // inside input array, so convert
+                dst.ptr[idx_out] = convertType<inType, outType>(
+                    FACTOR ? scale<inType>(src.ptr[idx_in], factor)
+                           : src.ptr[idx_in]);
+                id1 += id1Inc;
+                idx_in += idxID1Inc_in;
+                idx_out += idxID1Inc_out;
+            }
+        }
+        if (!SAME_DIMS) {
+            while (id1 < id1End_out) {
+                // outside the input array, so copy default value
+                dst.ptr[idx_out] = default_value;
+                id1 += id1Inc;
+                idx_out += idxID1Inc_out;
+            }
+        }
+    }
+}
+
+// scaledCopy with looping over dims[1], dims[2] and dims[3]
+// Conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] has no restrictions
+//      global dims[2] <= dims[2]
+template<typename inType, typename outType, bool SAME_DIMS, bool FACTOR>
+__global__ void scaledCopyLoop123(Param<outType> out, CParam<inType> in,
+                                  outType default_value, double factor) {
+    const int id0    = blockIdx.x * blockDim.x + threadIdx.x;  // Limit 2G
+    int id1          = blockIdx.y * blockDim.y + threadIdx.y;  // Limit 64K
+    const int odims0 = out.dims[0];
+    const int odims1 = out.dims[1];
+    if ((id0 < odims0) & (id1 < odims1)) {
+        int id2 = blockIdx.z * blockDim.z + threadIdx.z;  // Limit 64K
+        int idxBaseBase_out = id0 * (int)out.strides[0] +
+                              id1 * (int)out.strides[1] +
+                              id2 * (int)out.strides[2];
+        const int idxIncID3_out     = out.strides[3];
+        const int odims2            = out.dims[2];
+        const int idxEndIncID3_out  = out.dims[3] * idxIncID3_out;
+        const int incID1            = gridDim.y * blockDim.y;
+        const int idxBaseIncID1_out = incID1 * (int)out.strides[1];
+        const int incID2            = gridDim.z * blockDim.z;
+        const int idxBaseIncID2_out = incID2 * (int)out.strides[2];
+
+        int idxBaseBase_in = id0 * (int)in.strides[0] +
+                             id1 * (int)in.strides[1] +
+                             id2 * (int)in.strides[2];
+        const int idxIncID3_in     = in.strides[3];
+        const int idims0           = in.dims[0];
+        const int idims1           = in.dims[1];
+        const int idims2           = in.dims[2];
+        const int idxEndIncID3_in  = in.dims[3] * idxIncID3_in;
+        const int idxBaseIncID1_in = incID1 * (int)in.strides[1];
+        const int idxBaseIncID2_in = incID2 * (int)in.strides[2];
+
+        do {
+            int idxBase_in  = idxBaseBase_in;
+            int idxBase_out = idxBaseBase_out;
+            do {
+                int idxEndID3_in  = idxEndIncID3_in + idxBase_in;
+                int idxEndID3_out = idxEndIncID3_out + idxBase_out;
+                int idx_in        = idxBase_in;
+                int idx_out       = idxBase_out;
+                if (SAME_DIMS |
+                    ((id0 < idims0) & (id1 < idims1) & (id2 < idims2))) {
+                    // inside input array, so convert
+                    do {
+                        out.ptr[idx_out] = convertType<inType, outType>(
+                            FACTOR ? scale<inType>(in.ptr[idx_in], factor)
+                                   : in.ptr[idx_in]);
+                        idx_in += idxIncID3_in;
+                        idx_out += idxIncID3_out;
+                    } while (idx_in != idxEndID3_in);
+                }
+                if (!SAME_DIMS) {
+                    while (idx_out != idxEndID3_out) {
+                        // outside the input array, so copy default value
+                        out.ptr[idx_out] = default_value;
+                        idx_out += idxIncID3_out;
+                    }
+                }
+                id1 += incID1;
+                if (id1 >= odims1) break;
+                idxBase_in += idxBaseIncID1_in;
+                idxBase_out += idxBaseIncID1_out;
+            } while (true);
+            id2 += incID2;
+            if (id2 >= odims2) break;
+            idxBaseBase_in += idxBaseIncID2_in;
+            idxBaseBase_out += idxBaseIncID2_out;
+        } while (true);
+    }
+}
+
 }  // namespace cuda
diff --git a/src/backend/cuda/kernel/memcopy.cuh b/src/backend/cuda/kernel/memcopy.cuh
index f22a013279..ecef444cce 100644
--- a/src/backend/cuda/kernel/memcopy.cuh
+++ b/src/backend/cuda/kernel/memcopy.cuh
@@ -13,31 +13,213 @@
 
 namespace cuda {
 
+// memCopy without looping, so dim3 has to be 1.
+// conditions:
+//      kernel dims[0] >= dims[0]
+//      kernel dims[1] >= dims[1]
+//      kernel dims[2] == dims[2]
+//      only dims[3] == 1 will be processed!!
 template<typename T>
-__global__ void memcopy(Param<T> out, CParam<T> in, uint blocks_x,
-                        uint blocks_y) {
-    const int tidx = threadIdx.x;
-    const int tidy = threadIdx.y;
-
-    const int zid        = blockIdx.x / blocks_x;
-    const int blockIdx_x = blockIdx.x - (blocks_x)*zid;
-    const int xid        = blockIdx_x * blockDim.x + tidx;
-
-    const int wid = (blockIdx.y + blockIdx.z * gridDim.y) / blocks_y;
-    const int blockIdx_y =
-        (blockIdx.y + blockIdx.z * gridDim.y) - (blocks_y)*wid;
-    const int yid = blockIdx_y * blockDim.y + tidy;
-    // FIXME: Do more work per block
-    T *const optr = out.ptr + wid * out.strides[3] + zid * out.strides[2] +
-                    yid * out.strides[1];
-    const T *iptr = in.ptr + wid * in.strides[3] + zid * in.strides[2] +
-                    yid * in.strides[1];
-
-    int istride0 = in.strides[0];
-    if (xid < in.dims[0] && yid < in.dims[1] && zid < in.dims[2] &&
-        wid < in.dims[3]) {
-        optr[xid] = iptr[xid * istride0];
+__global__ void memCopy(Param<T> out, CParam<T> in) {
+    const int id0 = blockIdx.x * blockDim.x + threadIdx.x;  // Limit 2G
+    const int id1 = blockIdx.y * blockDim.y + threadIdx.y;  // Limit 64K
+    if ((id0 < (int)in.dims[0]) & (id1 < (int)in.dims[1])) {
+        const int id2 = blockIdx.z * blockDim.z + threadIdx.z;  // Limit 64K
+
+        out.ptr[id0 * (int)out.strides[0] + id1 * (int)out.strides[1] +
+                id2 * (int)out.strides[2]] =
+            in.ptr[id0 * (int)in.strides[0] + id1 * (int)in.strides[1] +
+                   id2 * (int)in.strides[2]];
+    }
+}
+
+// memCopy with looping over dims[0] -- VECTOR ONLY
+// Conditions:
+//      kernel dims[0] has no restrictions
+//      only dims[1] == 1 will be processed!!
+//      only dims[2] == 1 will be procesed!!
+//      only dims[3] == 1 will be processed!!
+template<typename T>
+__global__ void memCopyLoop0(Param<T> out, CParam<T> in) {
+    int id0          = blockIdx.x * blockDim.x + threadIdx.x;  // Limit 2G
+    const int idims0 = in.dims[0];
+    if (id0 < idims0) {
+        const int incID0        = gridDim.x * blockDim.x;
+        const int istrides0     = in.strides[0];
+        int idx_in              = id0 * istrides0;
+        const int idxIncID0_in  = incID0 * istrides0;
+        const int ostrides0     = out.strides[0];
+        int idx_out             = id0 * ostrides0;
+        const int idxIncID0_out = incID0 * ostrides0;
+
+        do {
+            out.ptr[idx_out] = in.ptr[idx_in];
+            id0 += incID0;
+            if (id0 >= idims0) break;
+            idx_in += idxIncID0_in;
+            idx_out += idxIncID0_out;
+        } while (true);
+    }
+}
+
+// memCopy with looping over dims[1]
+// Conditions:
+//      kernel dims[0] >= dims[0]
+//      kernel dims[1] has no restrictions
+//      kernel dims[2] == dims[2]
+//      only dims[3] == 1 will be processed!!
+template<typename T>
+__global__ void memCopyLoop1(Param<T> out, CParam<T> in) {
+    const int id0    = blockIdx.x * blockDim.x + threadIdx.x;  // Limit 2G
+    int id1          = blockIdx.y * blockDim.y + threadIdx.y;  // Limit 64K
+    const int idims1 = in.dims[1];
+    if ((id0 < (int)in.dims[0]) & (id1 < idims1)) {
+        const int id2 = blockIdx.z * blockDim.z + threadIdx.z;  // Limit 64K
+        const int istrides1 = in.strides[1];
+        int idx_in          = id0 * (int)in.strides[0] + id1 * istrides1 +
+                     id2 * (int)in.strides[2];
+        const int incID1       = gridDim.y * blockDim.y;
+        const int idxIncID1_in = incID1 * istrides1;
+        const int ostrides1    = out.strides[1];
+        int idx_out            = id0 * (int)out.strides[0] + id1 * ostrides1 +
+                      id2 * (int)out.strides[2];
+        const int idxIncID1_out = incID1 * ostrides1;
+
+        do {
+            out.ptr[idx_out] = in.ptr[idx_in];
+            id1 += incID1;
+            if (id1 >= idims1) break;
+            idx_in += idxIncID1_in;
+            idx_out += idxIncID1_out;
+        } while (true);
+    }
+}
+
+// memCopy with looping over dims[3]
+// Conditions:
+//      kernel dims[0] >= dims[0]
+//      kernel dims[1] >= dims[1]
+//      kernel dims[2] == dims[2]
+template<typename T>
+__global__ void memCopyLoop3(Param<T> out, CParam<T> in) {
+    const int id0 = blockIdx.x * blockDim.x + threadIdx.x;  // Limit 2G
+    const int id1 = blockIdx.y * blockDim.y + threadIdx.y;  // Limit 64K
+    if ((id0 < (int)in.dims[0]) & (id1 < (int)in.dims[1])) {
+        const int id2 = blockIdx.z * blockDim.z + threadIdx.z;  // Limit 64K
+        int idx_in    = id0 * (int)in.strides[0] + id1 * (int)in.strides[1] +
+                     id2 * (int)in.strides[2];
+        const int idxIncID3_in = in.strides[3];
+        const int idxEnd_in    = (int)in.dims[3] * idxIncID3_in + idx_in;
+        int idx_out = id0 * (int)out.strides[0] + id1 * (int)out.strides[1] +
+                      id2 * (int)out.strides[2];
+        const int idxIncID3_out = out.strides[3];
+
+        do {
+            out.ptr[idx_out] = in.ptr[idx_in];
+            idx_in += idxIncID3_in;
+            if (idx_in == idxEnd_in) break;
+            idx_out += idxIncID3_out;
+        } while (true);
     }
 }
 
+// memCopy with looping over dims[1] and dims[3]
+// Conditions:
+//      kernel dims[0] >= dims[0]
+//      kernel dims[1] has no restrictions
+//      kernel dims[2] == dims[2]
+template<typename T>
+__global__ void memCopyLoop13(Param<T> out, CParam<T> in) {
+    const int id0    = blockIdx.x * blockDim.x + threadIdx.x;  // Limit 2G
+    int id1          = blockIdx.y * blockDim.y + threadIdx.y;  // Limit 64K
+    const int idims1 = in.dims[1];
+    if ((id0 < (int)in.dims[0]) & (g1 < idims1)) {
+        const int id2 = blockIdx.z * blockDim.z + threadIdx.z;  // Limit 64K
+        const int istrides1 = in.strides[1];
+        int idxBase_in      = id0 * (int)in.strides[0] + id1 * istrides1 +
+                         id2 * (int)in.strides[2];
+        const int incID1           = gridDim.y * blockDim.y;
+        const int idxBaseIncID1_in = incID1 * istrides1;
+        const int idxIncID3_in     = (int)in.strides[3];
+        int idxEndID3_in = (int)in.dims[3] * idxIncID3_in + idxBase_in;
+        int idxBase_out  = id0 * (int)out.strides[0] +
+                          id1 * (int)out.strides[1] + id2 * (int)out.strides[2];
+        const int idxBaseIncID1_out = incID1 * (int)out.strides[1];
+        const int idxIncID3_out     = (int)out.strides[3];
+
+        do {
+            int idx_in  = idxBase_in;
+            int idx_out = idxBase_out;
+            while (true) {
+                out.ptr[idx_out] = in.ptr[idx_in];
+                idx_in += idxIncID3_in;
+                if (idx_in == idxEndID3_in) break;
+                idx_out += idxIncID3_out;
+            }
+            id1 += incID1;
+            if (id1 >= idims1) break;
+            idxBase_in += idxBaseIncID1_in;
+            idxEndID3_in += idxBaseIncID1_in;
+            idxBase_out += idxBaseIncID1_out;
+        } while (true);
+    }
+}
+
+// memCopy with looping over dims[1],dims[2] and dims[3]
+// Conditions:
+//      kernel dims[0] >= dims[0]
+//      kernel dims[1] has no restrictions
+//      kernel dims[2] <= dims[2]
+template<typename T>
+__global__ void memCopyLoop123(Param<T> out, CParam<T> in) {
+    const int id0    = blockIdx.x * blockDim.x + threadIdx.x;  // Limit 2G
+    int id1          = blockIdx.y * blockDim.y + threadIdx.y;  // Limit 64K
+    const int idims1 = in.dims[1];
+    if ((id0 < (int)in.dims[0]) & (id1 < idims1)) {
+        int id2 = blockIdx.z * blockDim.z + threadIdx.z;  // Limit 64K
+        const int istrides1 = in.strides[1];
+        const int istrides2 = in.strides[2];
+        int idxBaseBase_in =
+            id0 * (int)in.strides[0] + id1 * istrides1 + id2 * istrides2;
+        const int incID1           = gridDim.y * blockDim.y;
+        const int idxBaseIncID1_in = incID1 * istrides1;
+        const int incID2           = gridDim.z * blockDim.z;
+        const int idxBaseIncID2_in = incID2 * istrides2;
+        const int idxIncID3_in     = in.strides[3];
+        const int idxEndIncID3_in  = (int)in.dims[3] * idxIncID3_in;
+
+        const int ostrides1 = out.strides[1];
+        const int ostrides2 = out.strides[2];
+        int idxBaseBase_out =
+            id0 * (int)out.strides[0] + id1 * ostrides1 + id2 * ostrides2;
+        const int idxBaseIncID1_out = incID1 * ostrides1;
+        const int idxBaseIncID2_out = incID2 * ostrides2;
+        const int idxIncID3_out     = out.strides[3];
+        const int idims2            = in.dims[2];
+
+        do {
+            int idxBase_in  = idxBaseBase_in;
+            int idxBase_out = idxBaseBase_out;
+            do {
+                int idxEndID3_in = idxEndIncID3_in + idxBase_in;
+                int idx_in       = idxBase_in;
+                int idx_out      = idxBase_out;
+                do {
+                    out.ptr[idx_out] = in.ptr[idx_in];
+                    idx_in += idxIncID3_in;
+                    if (idx_in == idxEndID3_in) break;
+                    idx_out += idxIncID3_out;
+                } while (true);
+                id1 += incID1;
+                if (id1 >= idims1) break;
+                idxBase_in += idxBaseIncID1_in;
+                idxBase_out += idxBaseIncID1_out;
+            } while (true);
+            id2 += incID2;
+            if (id2 >= idims2) break;
+            idxBaseBase_in += idxBaseIncID2_in;
+            idxBaseBase_out += idxBaseIncID2_out;
+        } while (true);
+    }
+}
 }  // namespace cuda
diff --git a/src/backend/cuda/kernel/memcopy.hpp b/src/backend/cuda/kernel/memcopy.hpp
index 49d18f7fa3..f37252c633 100644
--- a/src/backend/cuda/kernel/memcopy.hpp
+++ b/src/backend/cuda/kernel/memcopy.hpp
@@ -11,92 +11,199 @@
 
 #include <Param.hpp>
 #include <backend.hpp>
-#include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_cuda.hpp>
 #include <dims_param.hpp>
 #include <nvrtc_kernel_headers/copy_cuh.hpp>
 #include <nvrtc_kernel_headers/memcopy_cuh.hpp>
+#include <threadsMgt.hpp>
 
 #include <algorithm>
 
 namespace cuda {
 namespace kernel {
 
-constexpr uint DIMX = 32;
-constexpr uint DIMY = 8;
-
+// Increase vectorization by increasing the used type up to maxVectorWidth.
+// Example:
+//  input array<int> with return value = 4, means that the array became
+//  array<int4>.
+//
+// Parameters
+//  - IN     maxVectorWidth: maximum vectorisation desired
+//  - IN/OUT dims[4]: dimensions of the array
+//  - IN/OUT istrides[4]: strides of the input array
+//  - IN/OUT indims: ndims of the input array.  Updates when dim[0] becomes 1
+//  - IN/OUT ioffset: offset of the input array
+//  - IN/OUT ostrides[4]: strides of the output array
+//  - IN/OUT ooffset: offset of the output array
+//
+// Returns
+//  - maximum obtained vectorization.
+//  - All the parameters are updated accordingly
+//
 template<typename T>
-void memcopy(Param<T> out, CParam<T> in, const dim_t ndims) {
-    auto memCopy = common::getKernel("cuda::memcopy", {memcopy_cuh_src},
-                                     {TemplateTypename<T>()});
-
-    dim3 threads(DIMX, DIMY);
-
-    if (ndims == 1) {
-        threads.x *= threads.y;
-        threads.y = 1;
+dim_t vectorizeShape(const dim_t maxVectorWidth, Param<T> &out, dim_t &indims,
+                     CParam<T> &in) {
+    dim_t vectorWidth{1};
+    if ((maxVectorWidth != 1) & (in.strides[0] == 1) & (out.strides[0] == 1)) {
+        // Only adjacent items can be grouped into a base vector type
+        void *in_ptr{(void *)in.ptr};
+        void *out_ptr{(void *)out.ptr};
+        // - global is the OR of the values to be checked.  When global is
+        // divisable by 2, than all source values are also
+        dim_t global{in.dims[0]};
+        for (int i{1}; i < indims; ++i) {
+            global |= in.strides[i] | out.strides[i];
+        }
+        // - The buffers are always aligned at 128 Bytes.  The pointers in the
+        // Param<T> structure are however, direct pointers (including the
+        // offset), so the final pointer has to be chedked on alignment
+        size_t filler{64};  // give enough space for the align to move
+        unsigned count{0};
+        while (((global & 1) == 0) & (vectorWidth < maxVectorWidth) &&
+               (in.ptr ==
+                std::align(alignof(T) * vectorWidth * 2, 1, in_ptr, filler)) &&
+               (out.ptr ==
+                std::align(alignof(T) * vectorWidth * 2, 1, out_ptr, filler))) {
+            ++count;
+            vectorWidth <<= 1;
+            global >>= 1;
+        }
+        if (count != 0) {
+            // update the dimensions, to compensate for the vector base
+            // type change
+            in.dims[0] >>= count;
+            for (int i{1}; i < indims; ++i) {
+                in.strides[i] >>= count;
+                out.strides[i] >>= count;
+            }
+            if (in.dims[0] == 1) {
+                // Vectorization has absorbed the full dim0, so eliminate
+                // this dimension
+                --indims;
+                for (int i{0}; i < indims; ++i) {
+                    in.dims[i]     = in.dims[i + 1];
+                    in.strides[i]  = in.strides[i + 1];
+                    out.strides[i] = out.strides[i + 1];
+                }
+                in.dims[indims] = 1;
+            }
+        }
     }
+    return vectorWidth;
+}
 
-    // FIXME: DO more work per block
-    uint blocks_x = divup(in.dims[0], threads.x);
-    uint blocks_y = divup(in.dims[1], threads.y);
+template<typename T>
+void memcopy(Param<T> out, CParam<T> in, dim_t indims) {
+    const size_t totalSize{in.elements() * sizeof(T) * 2};
+    removeEmptyColumns(in.dims, indims, out.strides);
+    indims = removeEmptyColumns(in.dims, indims, in.dims, in.strides);
+    indims = combineColumns(in.dims, in.strides, indims, out.strides);
 
-    dim3 blocks(blocks_x * in.dims[2], blocks_y * in.dims[3]);
+    // Optimization memory access and caching.
+    // Best performance is achieved with the highest vectorization
+    // (<int> --> <int2>,<int4>, ...), since more data is processed per IO.
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    // 16 Bytes gives best performance (=cdouble)
+    const dim_t maxVectorWidth{sizeof(T) > 8 ? 1 : 16 / sizeof(T)};
+    const dim_t vectorWidth{vectorizeShape(maxVectorWidth, out, indims, in)};
+    const size_t sizeofNewT{sizeof(T) * vectorWidth};
 
-    EnqueueArgs qArgs(blocks, threads, getActiveStream());
+    threadsMgt<dim_t> th(in.dims, indims);
+    const dim3 threads{th.genThreads()};
+    const dim3 blocks{th.genBlocks(threads, 1, 1, totalSize, sizeofNewT)};
 
-    memCopy(qArgs, out, in, blocks_x, blocks_y);
+    EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
+    // select the kernel with the necessary loopings
+    const char *kernelName{th.loop0   ? "cuda::memCopyLoop0"
+                           : th.loop2 ? "cuda::memCopyLoop123"
+                           : th.loop1 ? th.loop3 ? "cuda::memCopyLoop13"
+                                                 : "cuda::memCopyLoop1"
+                           : th.loop3 ? "cuda::memCopyLoop3"
+                                      : "cuda::memCopy"};
+
+    // Conversion to cuda base vector types.
+    switch (sizeofNewT) {
+        case 1: {
+            auto memCopy{
+                common::getKernel(kernelName, {memcopy_cuh_src}, {"char"})};
+            memCopy(qArgs, Param<char>((char *)out.ptr, out.dims, out.strides),
+                    CParam<char>((const char *)in.ptr, in.dims, in.strides));
+        } break;
+        case 2: {
+            auto memCopy{
+                common::getKernel(kernelName, {memcopy_cuh_src}, {"short"})};
+            memCopy(qArgs,
+                    Param<short>((short *)out.ptr, out.dims, out.strides),
+                    CParam<short>((const short *)in.ptr, in.dims, in.strides));
+        } break;
+        case 4: {
+            auto memCopy{
+                common::getKernel(kernelName, {memcopy_cuh_src}, {"float"})};
+            memCopy(qArgs,
+                    Param<float>((float *)out.ptr, out.dims, out.strides),
+                    CParam<float>((const float *)in.ptr, in.dims, in.strides));
+        } break;
+        case 8: {
+            auto memCopy{
+                common::getKernel(kernelName, {memcopy_cuh_src}, {"float2"})};
+            memCopy(
+                qArgs, Param<float2>((float2 *)out.ptr, out.dims, out.strides),
+                CParam<float2>((const float2 *)in.ptr, in.dims, in.strides));
+        } break;
+        case 16: {
+            auto memCopy{
+                common::getKernel(kernelName, {memcopy_cuh_src}, {"float4"})};
+            memCopy(
+                qArgs, Param<float4>((float4 *)out.ptr, out.dims, out.strides),
+                CParam<float4>((const float4 *)in.ptr, in.dims, in.strides));
+        } break;
+        default: assert("type is larger than 16 bytes, which is unsupported");
+    }
     POST_LAUNCH_CHECK();
 }
 
 template<typename inType, typename outType>
-void copy(Param<outType> dst, CParam<inType> src, int ndims,
+void copy(Param<outType> dst, CParam<inType> src, dim_t ondims,
           outType default_value, double factor) {
-    dim3 threads(DIMX, DIMY);
-    size_t local_size[] = {DIMX, DIMY};
-
-    // FIXME: Why isn't threads being updated??
-    local_size[0] *= local_size[1];
-    if (ndims == 1) { local_size[1] = 1; }
-
-    uint blk_x = divup(dst.dims[0], local_size[0]);
-    uint blk_y = divup(dst.dims[1], local_size[1]);
-
-    dim3 blocks(blk_x * dst.dims[2], blk_y * dst.dims[3]);
-
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
-
-    int trgt_l       = std::min(dst.dims[3], src.dims[3]);
-    int trgt_k       = std::min(dst.dims[2], src.dims[2]);
-    int trgt_j       = std::min(dst.dims[1], src.dims[1]);
-    int trgt_i       = std::min(dst.dims[0], src.dims[0]);
-    dims_t trgt_dims = {{trgt_i, trgt_j, trgt_k, trgt_l}};
-
-    bool same_dims =
-        ((src.dims[0] == dst.dims[0]) && (src.dims[1] == dst.dims[1]) &&
-         (src.dims[2] == dst.dims[2]) && (src.dims[3] == dst.dims[3]));
+    const size_t totalSize{dst.elements() * sizeof(outType) +
+                           src.elements() * sizeof(inType)};
+    bool same_dims{true};
+    for (dim_t i{0}; i < ondims; ++i) {
+        if (src.dims[i] > dst.dims[i]) {
+            src.dims[i] = dst.dims[i];
+        } else if (src.dims[i] != dst.dims[i]) {
+            same_dims = false;
+        }
+    }
+    removeEmptyColumns(dst.dims, ondims, src.dims, src.strides);
+    ondims = removeEmptyColumns(dst.dims, ondims, dst.dims, dst.strides);
+    ondims =
+        combineColumns(dst.dims, dst.strides, ondims, src.dims, src.strides);
 
-    auto copy = common::getKernel(
-        "cuda::copy", {copy_cuh_src},
-        {TemplateTypename<inType>(), TemplateTypename<outType>(),
-         TemplateArg(same_dims)});
+    threadsMgt<dim_t> th(dst.dims, ondims);
+    const dim3 threads{th.genThreads()};
+    const dim3 blocks{th.genBlocks(threads, 1, 1, totalSize, sizeof(outType))};
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
-    copy(qArgs, dst, src, default_value, factor, trgt_dims, blk_x, blk_y);
+    auto copy{common::getKernel(th.loop0 ? "cuda::scaledCopyLoop0"
+                                : th.loop2 | th.loop3
+                                    ? "cuda::scaledCopyLoop123"
+                                : th.loop1 ? "cuda::scaledCopyLoop1"
+                                           : "cuda::scaledCopy",
+                                {copy_cuh_src},
+                                {
+                                    TemplateTypename<inType>(),
+                                    TemplateTypename<outType>(),
+                                    TemplateArg(same_dims),
+                                    TemplateArg(factor != 1.0),
+                                })};
+
+    copy(qArgs, dst, src, default_value, factor);
 
     POST_LAUNCH_CHECK();
 }
-
 }  // namespace kernel
 }  // namespace cuda
diff --git a/src/backend/cuda/reshape.cpp b/src/backend/cuda/reshape.cpp
index 6e4c541adc..8d48000457 100644
--- a/src/backend/cuda/reshape.cpp
+++ b/src/backend/cuda/reshape.cpp
@@ -21,7 +21,9 @@ template<typename inType, typename outType>
 Array<outType> reshape(const Array<inType> &in, const dim4 &outDims,
                        outType defaultValue, double scale) {
     Array<outType> out = createEmptyArray<outType>(outDims);
-    kernel::copy<inType, outType>(out, in, in.ndims(), defaultValue, scale);
+    if (out.elements() > 0) {
+        kernel::copy<inType, outType>(out, in, in.ndims(), defaultValue, scale);
+    }
     return out;
 }
 
diff --git a/src/backend/opencl/copy.cpp b/src/backend/opencl/copy.cpp
index 44eac01444..cfb5e5b61d 100644
--- a/src/backend/opencl/copy.cpp
+++ b/src/backend/opencl/copy.cpp
@@ -21,93 +21,105 @@ using common::is_complex;
 namespace opencl {
 
 template<typename T>
-void copyData(T *data, const Array<T> &A) {
-    if (A.elements() == 0) { return; }
-
-    // FIXME: Merge this with copyArray
-    A.eval();
-
-    dim_t offset = 0;
-    cl::Buffer buf;
-    Array<T> out = A;
-
-    if (A.isLinear() ||  // No offsets, No strides
-        A.ndims() == 1   // Simple offset, no strides.
-    ) {
-        buf    = *A.get();
-        offset = A.getOffset();
-    } else {
-        // FIXME: Think about implementing eval
-        out    = copyArray(A);
-        buf    = *out.get();
-        offset = 0;
+void copyData(T *data, const Array<T> &src) {
+    if (src.elements() > 0) {
+        Array<T> out = src.isReady() && src.isLinear() ? src : copyArray(src);
+        // out is now guaranteed linear
+        getQueue().enqueueReadBuffer(*out.get(), CL_TRUE,
+                                     sizeof(T) * out.getOffset(),
+                                     sizeof(T) * out.elements(), data);
     }
-
-    // FIXME: Add checks
-    getQueue().enqueueReadBuffer(buf, CL_TRUE, sizeof(T) * offset,
-                                 sizeof(T) * A.elements(), data);
 }
 
 template<typename T>
-Array<T> copyArray(const Array<T> &A) {
-    Array<T> out = createEmptyArray<T>(A.dims());
-    if (A.elements() == 0) { return out; }
-
-    dim_t offset = A.getOffset();
-    if (A.isLinear()) {
-        // FIXME: Add checks
-        getQueue().enqueueCopyBuffer(*A.get(), *out.get(), sizeof(T) * offset,
-                                     0, A.elements() * sizeof(T));
-    } else {
-        kernel::memcopy<T>(*out.get(), out.strides().get(), *A.get(),
-                           A.dims().get(), A.strides().get(), offset,
-                           (uint)A.ndims());
+Array<T> copyArray(const Array<T> &src) {
+    Array<T> out = createEmptyArray<T>(src.dims());
+    if (src.elements() > 0) {
+        if (src.isReady()) {
+            if (src.isLinear()) {
+                getQueue().enqueueCopyBuffer(
+                    *src.get(), *out.get(), src.getOffset() * sizeof(T), 0,
+                    src.elements() * sizeof(T), nullptr, nullptr);
+            } else {
+                kernel::memcopy<T>(*out.get(), out.strides(), *src.get(),
+                                   src.dims(), src.strides(), src.getOffset(),
+                                   src.ndims());
+            }
+        } else {
+            Param info = {out.get(),
+                          {{src.dims().dims[0], src.dims().dims[1],
+                            src.dims().dims[2], src.dims().dims[3]},
+                           {out.strides().dims[0], out.strides().dims[1],
+                            out.strides().dims[2], out.strides().dims[3]},
+                           0}};
+            evalNodes(info, src.getNode().get());
+        }
     }
     return out;
 }
 
 template<typename T>
-void multiply_inplace(Array<T> &in, double val) {
-    kernel::copy<T, T>(in, in, in.ndims(), scalar<T>(0), val, true);
+void multiply_inplace(Array<T> &src, double norm) {
+    if (src.elements() > 0) {
+        kernel::copy<T, T>(src, src, src.ndims(), scalar<T>(0), norm);
+    }
 }
 
 template<typename inType, typename outType>
 struct copyWrapper {
-    void operator()(Array<outType> &out, Array<inType> const &in) {
-        kernel::copy<inType, outType>(out, in, in.ndims(), scalar<outType>(0),
-                                      1, in.dims() == out.dims());
+    void operator()(Array<outType> &dst, Array<inType> const &src) {
+        kernel::copy<inType, outType>(dst, src, dst.ndims(), scalar<outType>(0),
+                                      1.0);
     }
 };
 
 template<typename T>
 struct copyWrapper<T, T> {
-    void operator()(Array<T> &out, Array<T> const &in) {
-        if (out.isLinear() && in.isLinear() &&
-            out.elements() == in.elements()) {
-            dim_t in_offset  = in.getOffset() * sizeof(T);
-            dim_t out_offset = out.getOffset() * sizeof(T);
-
-            getQueue().enqueueCopyBuffer(*in.get(), *out.get(), in_offset,
-                                         out_offset, in.elements() * sizeof(T));
-        } else {
-            kernel::copy<T, T>(out, in, in.ndims(), scalar<T>(0), 1,
-                               in.dims() == out.dims());
+    void operator()(Array<T> &dst, Array<T> const &src) {
+        if (src.elements() > 0) {
+            if (dst.dims() == src.dims()) {
+                if (src.isReady()) {
+                    if (dst.isLinear() && src.isLinear()) {
+                        getQueue().enqueueCopyBuffer(
+                            *src.get(), *dst.get(), src.getOffset() * sizeof(T),
+                            dst.getOffset() * sizeof(T),
+                            src.elements() * sizeof(T), nullptr, nullptr);
+                    } else {
+                        kernel::memcopy<T>(*dst.get(), dst.strides(),
+                                           *src.get(), src.dims(),
+                                           src.strides(), src.getOffset(),
+                                           src.ndims(), dst.getOffset());
+                    }
+                } else {
+                    Param info = {
+                        dst.get(),
+                        {{src.dims().dims[0], src.dims().dims[1],
+                          src.dims().dims[2], src.dims().dims[3]},
+                         {dst.strides().dims[0], dst.strides().dims[1],
+                          dst.strides().dims[2], dst.strides().dims[3]},
+                         dst.getOffset()}};
+                    evalNodes(info, src.getNode().get());
+                }
+            } else {
+                // dst has more elements than src, so default has to be applied
+                kernel::copy<T, T>(dst, src, dst.ndims(), scalar<T>(0), 1.0);
+            }
         }
     }
 };
 
 template<typename inType, typename outType>
-void copyArray(Array<outType> &out, Array<inType> const &in) {
+void copyArray(Array<outType> &dst, Array<inType> const &src) {
     static_assert(!(is_complex<inType>::value && !is_complex<outType>::value),
                   "Cannot copy from complex value to a non complex value");
     copyWrapper<inType, outType> copyFn;
-    copyFn(out, in);
+    copyFn(dst, src);
 }
 
-#define INSTANTIATE(T)                                         \
-    template void copyData<T>(T * data, const Array<T> &from); \
-    template Array<T> copyArray<T>(const Array<T> &A);         \
-    template void multiply_inplace<T>(Array<T> & in, double norm);
+#define INSTANTIATE(T)                                        \
+    template void copyData<T>(T * data, const Array<T> &src); \
+    template Array<T> copyArray<T>(const Array<T> &src);      \
+    template void multiply_inplace<T>(Array<T> & src, double norm);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
@@ -173,10 +185,10 @@ INSTANTIATE_COPY_ARRAY_COMPLEX(cfloat)
 INSTANTIATE_COPY_ARRAY_COMPLEX(cdouble)
 
 template<typename T>
-T getScalar(const Array<T> &in) {
+T getScalar(const Array<T> &src) {
     T retVal{};
-    getQueue().enqueueReadBuffer(*in.get(), CL_TRUE, sizeof(T) * in.getOffset(),
-                                 sizeof(T), &retVal);
+    getQueue().enqueueReadBuffer(
+        *src.get(), CL_TRUE, sizeof(T) * src.getOffset(), sizeof(T), &retVal);
     return retVal;
 }
 
diff --git a/src/backend/opencl/kernel/copy.cl b/src/backend/opencl/kernel/copy.cl
index 308f177d94..8cbe2cbf93 100644
--- a/src/backend/opencl/kernel/copy.cl
+++ b/src/backend/opencl/kernel/copy.cl
@@ -8,16 +8,14 @@
  ********************************************************/
 
 typedef struct {
-    dim_t dim[4];
-} dims_t;
+    int dims[4];
+} dims_type;
 
-inType scale(inType value, float factor) {
-#ifdef inType_float2
-    return (inType)(value.s0 * factor, value.s1 * factor);
+#ifdef FACTOR
+#define SCALE(value, factor) (value * factor)
 #else
-    return (inType)(value * factor);
+#define SCALE(value, factor) (value)
 #endif
-}
 
 #if defined(outType_double2)
 
@@ -47,42 +45,185 @@ inType scale(inType value, float factor) {
 
 #endif
 
-kernel void reshapeCopy(global outType *dst, KParam oInfo,
-                        global const inType *src, KParam iInfo,
-                        outType default_value, float factor, dims_t trgt,
-                        int blk_x, int blk_y) {
-    uint lx = get_local_id(0);
-    uint ly = get_local_id(1);
-
-    uint gz         = get_group_id(0) / blk_x;
-    uint gw         = get_group_id(1) / blk_y;
-    uint blockIdx_x = get_group_id(0) - (blk_x)*gz;
-    uint blockIdx_y = get_group_id(1) - (blk_y)*gw;
-    uint gx         = blockIdx_x * get_local_size(0) + lx;
-    uint gy         = blockIdx_y * get_local_size(1) + ly;
-
-    global const inType *in =
-        src + (gw * iInfo.strides[3] + gz * iInfo.strides[2] +
-               gy * iInfo.strides[1] + iInfo.offset);
-    global outType *out = dst + (gw * oInfo.strides[3] + gz * oInfo.strides[2] +
-                                 gy * oInfo.strides[1] + oInfo.offset);
-
-    uint istride0 = iInfo.strides[0];
-    uint ostride0 = oInfo.strides[0];
-
-    if (gy < oInfo.dims[1] && gz < oInfo.dims[2] && gw < oInfo.dims[3]) {
-        int loop_offset = get_local_size(0) * blk_x;
-        bool cond = gy < trgt.dim[1] && gz < trgt.dim[2] && gw < trgt.dim[3];
-        for (int rep = gx; rep < oInfo.dims[0]; rep += loop_offset) {
-            outType temp = default_value;
-#if SAME_DIMS
-            temp = CONVERT(scale(in[rep * istride0], factor));
-#else
-            if (rep < trgt.dim[0] && cond) {
-                temp = CONVERT(scale(in[rep * istride0], factor));
+// scaledCopy without looping, so dim3 has to be 1.
+// conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] >= dims[1]
+//      global dims[2] == dims[2]
+//      only dims[3] == 1 will be processed!!
+kernel void scaledCopy(global outType *out, const dims_type odims,
+                       const dims_type ostrides, const int ooffset,
+                       global const inType *in, const dims_type idims,
+                       const dims_type istrides, const int ioffset,
+                       const outType default_value, const factorType factor) {
+    const int g0 = get_global_id(0);
+    const int g1 = get_global_id(1);
+    if ((g0 < (int)odims.dims[0]) & (g1 < (int)odims.dims[1])) {
+        const int g2 = get_global_id(2);
+
+        int idx_in = g0 * (int)istrides.dims[0] + g1 * (int)istrides.dims[1] +
+                     g2 * (int)istrides.dims[2] + ioffset;
+        int idx_out = g0 * (int)ostrides.dims[0] + g1 * (int)ostrides.dims[1] +
+                      g2 * (int)ostrides.dims[2] + ooffset;
+
+        if (SAME_DIMS | ((g0 < (int)idims.dims[0]) & (g1 < (int)idims.dims[1]) &
+                         (g2 < (int)idims.dims[2]))) {
+            out[idx_out] = CONVERT(SCALE(in[idx_in], factor));
+        } else {
+            out[idx_out] = default_value;
+        }
+    }
+}
+
+// scaledCopy with looping over dims[0] -- VECTOR ONLY
+// Conditions:
+//      global dims[0] has no restrictions
+//      only dims[1] == 1 will be processed!!
+//      only dims[2] == 1 will be processed!!
+//      only dims[3] == 1 will be processed!!
+kernel void scaledCopyLoop0(global outType *out, const dims_type odims,
+                            const dims_type ostrides, const int ooffset,
+                            global const inType *in, const dims_type idims,
+                            const dims_type istrides, const int ioffset,
+                            const outType default_value,
+                            const factorType factor) {
+    int id0              = get_global_id(0);
+    const int id0End_out = odims.dims[0];
+    if (id0 < id0End_out) {
+        const int ostrides0     = ostrides.dims[0];
+        const int id0Inc        = get_global_size(0);
+        int idx_out             = id0 * ostrides0 + ooffset;
+        const int idxID0Inc_out = id0Inc * ostrides0;
+        const int id0End_in     = idims.dims[0];
+        const int istrides0     = istrides.dims[0];
+        int idx_in              = id0 * istrides0 + ioffset;
+        const int idxID0Inc_in  = id0Inc * istrides0;
+
+        while (id0 < id0End_in) {
+            // inside input array, so convert
+            out[idx_out] = CONVERT(SCALE(in[idx_in], factor));
+            id0 += id0Inc;
+            idx_in += idxID0Inc_in;
+            idx_out += idxID0Inc_out;
+        }
+        if (!SAME_DIMS) {
+            while (id0 < id0End_out) {
+                // outside the input array, so copy default value
+                out[idx_out] = default_value;
+                id0 += id0Inc;
+                idx_out += idxID0Inc_out;
             }
-#endif
-            out[rep * ostride0] = temp;
         }
     }
 }
+
+// scaledCopy with looping over dims[1]
+// Conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] has no restrictions
+//      global dims[2] == dims[2]
+//      only dims[3] == 1 will be processed!!
+kernel void scaledCopyLoop1(global outType *out, const dims_type odims,
+                            const dims_type ostrides, const int ooffset,
+                            global const inType *in, const dims_type idims,
+                            const dims_type istrides, const int ioffset,
+                            const outType default_value,
+                            const factorType factor) {
+    const int id0        = get_global_id(0);
+    int id1              = get_global_id(1);
+    const int id1End_out = odims.dims[1];
+    if ((id0 < (int)odims.dims[0]) & (id1 < id1End_out)) {
+        const int id2       = get_global_id(2);
+        const int ostrides1 = ostrides.dims[1];
+        const int id1Inc    = get_global_size(1);
+        int idx_out         = id0 * (int)ostrides.dims[0] + id1 * ostrides1 +
+                      id2 * (int)ostrides.dims[2] + ooffset;
+        const int idxID1Inc_out = id1Inc * ostrides1;
+        const int id1End_in     = idims.dims[1];
+        const int istrides1     = istrides.dims[1];
+        int idx_in = id0 * (int)istrides.dims[0] + id1 * istrides1 +
+                     id2 * (int)istrides.dims[2] + ioffset;
+        const int idxID1Inc_in = id1Inc * istrides1;
+
+        if (SAME_DIMS | ((id0 < idims.dims[0]) & (id2 < idims.dims[2]))) {
+            while (id1 < id1End_in) {
+                // inside input array, so convert
+                out[idx_out] = CONVERT(SCALE(in[idx_in], factor));
+                id1 += id1Inc;
+                idx_in += idxID1Inc_in;
+                idx_out += idxID1Inc_out;
+            }
+        }
+        if (!SAME_DIMS) {
+            while (id1 < id1End_out) {
+                // outside the input array, so copy default value
+                out[idx_out] = default_value;
+                id1 += id1Inc;
+                idx_out += idxID1Inc_out;
+            }
+        }
+    }
+}
+
+// scaledCopy with looping over dims[1] and dims[3]
+// Conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] has no restrictions
+//      global dims[2] == dims[2]
+kernel void scaledCopyLoop13(global outType *out, const dims_type odims,
+                             const dims_type ostrides, const int ooffset,
+                             global const inType *in, const dims_type idims,
+                             const dims_type istrides, const int ioffset,
+                             const outType default_value,
+                             const factorType factor) {
+    const int id0        = get_global_id(0);
+    int id1              = get_global_id(1);
+    const int id1End_out = odims.dims[1];
+    if ((id0 < (int)odims.dims[0]) & (id1 < id1End_out)) {
+        const int id2               = get_global_id(2);
+        const int id1Inc            = get_global_size(1);
+        const int ostrides1         = ostrides.dims[1];
+        const int idxIncID3_out     = ostrides.dims[3];
+        const int idxBaseIncID1_out = id1Inc * ostrides1;
+        int idxBase_out             = id0 * ostrides.dims[0] + id1 * ostrides1 +
+                          id2 * ostrides.dims[2] + ooffset;
+        int idxEndID3_out = odims.dims[3] * idxIncID3_out + idxBase_out;
+
+        const int id0End_in        = idims.dims[0];
+        const int id1End_in        = idims.dims[1];
+        const int id2End_in        = idims.dims[2];
+        const int istrides1        = istrides.dims[1];
+        const int idxIncID3_in     = istrides.dims[3];
+        const int idxBaseIncID1_in = id1Inc * istrides1;
+        int idxBase_in             = id0 * istrides.dims[0] + id1 * istrides1 +
+                         id2 * istrides.dims[2] + ioffset;
+        int idxEndID3_in = idims.dims[3] * idxIncID3_in + idxBase_in;
+
+        do {
+            int idx_in  = idxBase_in;
+            int idx_out = idxBase_out;
+            if (SAME_DIMS |
+                ((id0 < id0End_in) & (id1 < id1End_in) & (id2 < id2End_in))) {
+                // inside input array, so convert
+                do {
+                    out[idx_out] = CONVERT(SCALE(in[idx_in], factor));
+                    idx_in += idxIncID3_in;
+                    idx_out += idxIncID3_out;
+                } while (idx_in != idxEndID3_in);
+            }
+            if (!SAME_DIMS) {
+                while (idx_out != idxEndID3_out) {
+                    // outside the input array, so copy default value
+                    out[idx_out] = default_value;
+                    idx_out += idxIncID3_out;
+                }
+            }
+            id1 += id1Inc;
+            if (id1 >= id1End_out) break;
+            idxBase_in += idxBaseIncID1_in;
+            idxEndID3_in += idxBaseIncID1_in;
+            idxBase_out += idxBaseIncID1_out;
+            idxEndID3_out += idxBaseIncID1_out;
+        } while (true);
+    }
+}
\ No newline at end of file
diff --git a/src/backend/opencl/kernel/memcopy.cl b/src/backend/opencl/kernel/memcopy.cl
index 912b5b028c..984ecf25f0 100644
--- a/src/backend/opencl/kernel/memcopy.cl
+++ b/src/backend/opencl/kernel/memcopy.cl
@@ -8,32 +8,168 @@
  ********************************************************/
 
 typedef struct {
-    dim_t dim[4];
+    int dims[4];
 } dims_t;
 
-kernel void memCopy(global T *out, dims_t ostrides, global const T *in,
-                    dims_t idims, dims_t istrides, int offset, int groups_0,
-                    int groups_1) {
-    const int lid0 = get_local_id(0);
-    const int lid1 = get_local_id(1);
-
-    const int id2        = get_group_id(0) / groups_0;
-    const int id3        = get_group_id(1) / groups_1;
-    const int group_id_0 = get_group_id(0) - groups_0 * id2;
-    const int group_id_1 = get_group_id(1) - groups_1 * id3;
-    const int id0        = group_id_0 * get_local_size(0) + lid0;
-    const int id1        = group_id_1 * get_local_size(1) + lid1;
-
-    in += offset;
-
-    // FIXME: Do more work per work group
-    out +=
-        id3 * ostrides.dim[3] + id2 * ostrides.dim[2] + id1 * ostrides.dim[1];
-    in += id3 * istrides.dim[3] + id2 * istrides.dim[2] + id1 * istrides.dim[1];
-
-    int istride0 = istrides.dim[0];
-    if (id0 < idims.dim[0] && id1 < idims.dim[1] && id2 < idims.dim[2] &&
-        id3 < idims.dim[3]) {
-        out[id0] = in[id0 * istride0];
+// memcopy without looping, so dim3 has to be 1.
+// conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] >= dims[1]
+//      global dims[2] == dims[2]
+//      only dims[3] == 1 will be processed!!
+kernel void memCopy(global T *d_out, const dims_t ostrides, const int ooffset,
+                    global const T *d_in, const dims_t idims,
+                    const dims_t istrides, const int ioffset) {
+    const int id0 = get_global_id(0);  // dim[0]
+    const int id1 = get_global_id(1);  // dim[1]
+    if ((id0 < idims.dims[0]) & (id1 < idims.dims[1])) {
+        const int id2 = get_global_id(2);  // dim[2] never overflows
+                                           // dim[3] is no processed
+        d_out[id0 * ostrides.dims[0] + id1 * ostrides.dims[1] +
+              id2 * ostrides.dims[2] + ooffset] =
+            d_in[id0 * istrides.dims[0] + id1 * istrides.dims[1] +
+                 id2 * istrides.dims[2] + ioffset];
+    }
+}
+
+// memcopy with looping over dims[0] -- VECTOR ONLY
+// Conditions:
+//      global dims[0] has no restrictions
+//      only dims[1] == 1 will be processed!!
+//      only dims[2] == 1 will be processed!!
+//      only dims[3] == 1 will be processed!!
+kernel void memCopyLoop0(global T *d_out, const dims_t ostrides,
+                         const int ooffset, global const T *d_in,
+                         const dims_t idims, const dims_t istrides,
+                         const int ioffset) {
+    int id0          = get_global_id(0);  // dim[0]
+    const int idims0 = idims.dims[0];
+    if (id0 < idims0) {
+        const int incID0        = get_global_size(0);
+        const int istrides0     = istrides.dims[0];
+        int idx_in              = id0 * istrides0 + ioffset;
+        const int idxIncID0_in  = incID0 * istrides0;
+        const int ostrides0     = ostrides.dims[0];
+        int idx_out             = id0 * ostrides0 + ooffset;
+        const int idxIncID0_out = incID0 * ostrides0;
+
+        do {
+            d_out[idx_out] = d_in[idx_in];
+            id0 += incID0;
+            if (id0 >= idims0) break;
+            idx_in += idxIncID0_in;
+            idx_out += idxIncID0_out;
+        } while (true);
+    }
+}
+
+// memcopy with looping over dims[1]
+// Conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] has no restrictions
+//      global dims[2] == dims[2]
+//      only dims[3] == 1 will be processed!!
+kernel void memCopyLoop1(global T *d_out, const dims_t ostrides,
+                         const int ooffset, global const T *d_in,
+                         const dims_t idims, const dims_t istrides,
+                         const int ioffset) {
+    const int id0    = get_global_id(0);  // dim[0]
+    int id1          = get_global_id(1);  // dim[1]
+    const int idims1 = idims.dims[1];
+    if ((id0 < idims.dims[0]) & (id1 < idims1)) {
+        const int id2 = get_global_id(2);  // dim[2] never overflows
+                                           // dim[3] is no processed
+        const int istrides1 = istrides.dims[1];
+        int idx_in          = id0 * istrides.dims[0] + id1 * istrides1 +
+                     id2 * istrides.dims[2] + ioffset;
+        const int incID1       = get_global_size(1);
+        const int idxIncID1_in = incID1 * istrides1;
+        const int ostrides1    = ostrides.dims[1];
+        int idx_out            = id0 * ostrides.dims[0] + id1 * ostrides1 +
+                      id2 * ostrides.dims[2] + ooffset;
+        const int idxIncID1_out = incID1 * ostrides1;
+
+        do {
+            d_out[idx_out] = d_in[idx_in];
+            id1 += incID1;
+            if (id1 >= idims1) break;
+            idx_in += idxIncID1_in;
+            idx_out += idxIncID1_out;
+        } while (true);
+    }
+}
+
+// memcopy with looping over dims[3]
+// Conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] >= dims[1]
+//      global dims[2] == dims[2]
+kernel void memCopyLoop3(global T *d_out, const dims_t ostrides,
+                         const int ooffset, global const T *d_in,
+                         const dims_t idims, const dims_t istrides,
+                         const int ioffset) {
+    const int id0 = get_global_id(0);  // dim[0]
+    const int id1 = get_global_id(1);  // dim[1]
+    if ((id0 < idims.dims[0]) & (id1 < idims.dims[1])) {
+        const int id2 = get_global_id(2);  // dim[2] never overflows
+                                           // dim[3] is no processed
+        int idx_in = id0 * istrides.dims[0] + id1 * istrides.dims[1] +
+                     id2 * istrides.dims[2] + ioffset;
+        const int idxIncID3_in = istrides.dims[3];
+        const int idxEnd_in    = idims.dims[3] * idxIncID3_in + idx_in;
+        int idx_out = id0 * ostrides.dims[0] + id1 * ostrides.dims[1] +
+                      id2 * ostrides.dims[2] + ooffset;
+        const int idxIncID3_out = ostrides.dims[3];
+
+        do {
+            d_out[idx_out] = d_in[idx_in];
+            idx_in += idxIncID3_in;
+            if (idx_in == idxEnd_in) break;
+            idx_out += idxIncID3_out;
+        } while (true);
+    }
+}
+
+// memcopy with looping over dims[1] and dims[3]
+// Conditions:
+//      global dims[0] >= dims[0]
+//      global dims[1] has no restrictions
+//      global dims[2] == dims[2]
+kernel void memCopyLoop13(global T *d_out, const dims_t ostrides,
+                          const int ooffset, global const T *d_in,
+                          const dims_t idims, const dims_t istrides,
+                          const int ioffset) {
+    const int id0    = get_global_id(0);  // dim[0]
+    int id1          = get_global_id(1);  // dim[1]
+    const int idims1 = idims.dims[1];
+    if ((id0 < idims.dims[0]) & (id1 < idims1)) {
+        const int id2       = get_global_id(2);  // dim[2] never overflows
+        const int istrides1 = istrides.dims[1];
+        int idxBase_in      = id0 * istrides.dims[0] + id1 * istrides1 +
+                         id2 * istrides.dims[2] + ioffset;
+        const int incID1           = get_global_size(1);
+        const int idxBaseIncID1_in = incID1 * istrides1;
+        const int idxIncID3_in     = istrides.dims[3];
+        int idxEndID3_in           = idims.dims[3] * idxIncID3_in + idxBase_in;
+        int idxBase_out = id0 * ostrides.dims[0] + id1 * ostrides.dims[1] +
+                          id2 * ostrides.dims[2] + ooffset;
+        const int idxBaseIncID1_out = incID1 * ostrides.dims[1];
+        const int idxIncID3_out     = ostrides.dims[3];
+
+        do {
+            int idx_in  = idxBase_in;
+            int idx_out = idxBase_out;
+            while (true) {
+                d_out[idx_out] = d_in[idx_in];
+                idx_in += idxIncID3_in;
+                if (idx_in == idxEndID3_in) break;
+                idx_out += idxIncID3_out;
+            }
+            id1 += incID1;
+            if (id1 >= idims1) break;
+            idxBase_in += idxBaseIncID1_in;
+            idxEndID3_in += idxBaseIncID1_in;
+            idxBase_out += idxBaseIncID1_out;
+        } while (true);
     }
 }
diff --git a/src/backend/opencl/kernel/memcopy.hpp b/src/backend/opencl/kernel/memcopy.hpp
index 115bc5178b..9358315cd5 100644
--- a/src/backend/opencl/kernel/memcopy.hpp
+++ b/src/backend/opencl/kernel/memcopy.hpp
@@ -10,113 +10,242 @@
 #pragma once
 
 #include <Param.hpp>
-#include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <common/traits.hpp>
 #include <debug_opencl.hpp>
 #include <kernel_headers/copy.hpp>
 #include <kernel_headers/memcopy.hpp>
+#include <threadsMgt.hpp>
 #include <traits.hpp>
 
 #include <algorithm>
+#include <iostream>
 #include <string>
 #include <vector>
 
+using std::string;
+using std::vector;
+
 namespace opencl {
 namespace kernel {
 typedef struct {
-    dim_t dim[4];
-} dims_t;
-
-constexpr uint DIM0 = 32;
-constexpr uint DIM1 = 8;
+    int dims[4];
+} dims_type;
+
+// Increase vectorization by increasing the used type up to maxVectorWidth.
+// Example:
+//  input array<int> with return value = 4, means that the array became
+//  array<int4>.
+//
+// Parameters
+//  - IN     maxVectorWidth: maximum vectorisation desired
+//  - IN/OUT dims[4]: dimensions of the array
+//  - IN/OUT istrides[4]: strides of the input array
+//  - IN/OUT indims: ndims of the input array.  Updates when dim[0] becomes 1
+//  - IN/OUT ioffset: offset of the input array
+//  - IN/OUT ostrides[4]: strides of the output array
+//  - IN/OUT ooffset: offset of the output array
+//
+// Returns
+//  - maximum obtained vectorization.
+//  - All the parameters are updated accordingly
+//
+static unsigned vectorizeShape(const unsigned maxVectorWidth, int dims[4],
+                               int istrides[4], int& indims, dim_t& ioffset,
+                               int ostrides[4], dim_t& ooffset) {
+    unsigned vectorWidth{1};
+    if ((maxVectorWidth != 1) & (istrides[0] == 1) & (ostrides[0] == 1)) {
+        // - Only adjacent items can be vectorized into a base vector type
+        // - global is the OR of the values to be checked.  When global is
+        // divisable by 2, than all source values are also
+        // - The buffers are always aligned at 128 Bytes, so the alignment is
+        // only dependable on the offsets
+        dim_t global{dims[0] | ioffset | ooffset};
+        for (int i{1}; i < indims; ++i) { global |= istrides[i] | ostrides[i]; }
+
+        // Determine the maximum vectorization possible
+        unsigned count{0};
+        while (((global & 1) == 0) & (vectorWidth < maxVectorWidth)) {
+            ++count;
+            vectorWidth <<= 1;
+            global >>= 1;
+        }
+        if (count != 0) {
+            // update the dimensions, to correspond with the new vectorization
+            dims[0] >>= count;
+            ioffset >>= count;
+            ooffset >>= count;
+            for (int i{1}; i < indims; ++i) {
+                istrides[i] >>= count;
+                ostrides[i] >>= count;
+            }
+            if (dims[0] == 1) {
+                // Vectorization has absorbed the full dim0, so eliminate
+                // the 1st dimension
+                --indims;
+                for (int i{0}; i < indims; ++i) {
+                    dims[i]     = dims[i + 1];
+                    istrides[i] = istrides[i + 1];
+                    ostrides[i] = ostrides[i + 1];
+                }
+                dims[indims] = 1;
+            }
+        }
+    }
+    return vectorWidth;
+}
 
 template<typename T>
-void memcopy(cl::Buffer out, const dim_t *ostrides, const cl::Buffer in,
-             const dim_t *idims, const dim_t *istrides, int offset,
-             uint ndims) {
-    std::vector<TemplateArg> targs = {
-        TemplateTypename<T>(),
-    };
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
-
-    auto memCopy =
-        common::getKernel("memCopy", {memcopy_cl_src}, targs, options);
-
-    dims_t _ostrides = {{ostrides[0], ostrides[1], ostrides[2], ostrides[3]}};
-    dims_t _istrides = {{istrides[0], istrides[1], istrides[2], istrides[3]}};
-    dims_t _idims    = {{idims[0], idims[1], idims[2], idims[3]}};
+void memcopy(const cl::Buffer& b_out, const dim4& ostrides,
+             const cl::Buffer& b_in, const dim4& idims, const dim4& istrides,
+             dim_t ioffset, const dim_t indims, dim_t ooffset = 0) {
+    dims_type idims_{
+        static_cast<int>(idims.dims[0]), static_cast<int>(idims.dims[1]),
+        static_cast<int>(idims.dims[2]), static_cast<int>(idims.dims[3])};
+    dims_type istrides_{
+        static_cast<int>(istrides.dims[0]), static_cast<int>(istrides.dims[1]),
+        static_cast<int>(istrides.dims[2]), static_cast<int>(istrides.dims[3])};
+    dims_type ostrides_{
+        static_cast<int>(ostrides.dims[0]), static_cast<int>(ostrides.dims[1]),
+        static_cast<int>(ostrides.dims[2]), static_cast<int>(ostrides.dims[3])};
+    int indims_{static_cast<int>(indims)};
+
+    const size_t totalSize{idims.elements() * sizeof(T) * 2};
+    removeEmptyColumns(idims_.dims, indims_, ostrides_.dims);
+    indims_ =
+        removeEmptyColumns(idims_.dims, indims_, idims_.dims, istrides_.dims);
+    indims_ =
+        combineColumns(idims_.dims, istrides_.dims, indims_, ostrides_.dims);
+
+    // Optimization memory access and caching.
+    // Best performance is achieved with the highest vectorization
+    // (<int> --> <int2>,<int4>, ...), since more data is processed per IO.
+    const cl::Device dev{opencl::getDevice()};
+    const unsigned DevicePreferredVectorWidthChar{
+        dev.getInfo<CL_DEVICE_PREFERRED_VECTOR_WIDTH_CHAR>()};
+    // When the architecture prefers some width's, it is certainly
+    // on char.  No preference means vector width 1 returned.
+    const bool DevicePreferredVectorWidth{DevicePreferredVectorWidthChar != 1};
+    unsigned maxVectorWidth{
+        DevicePreferredVectorWidth
+            ? sizeof(T) == 1 ? DevicePreferredVectorWidthChar
+              : sizeof(T) == 2
+                  ? dev.getInfo<CL_DEVICE_PREFERRED_VECTOR_WIDTH_SHORT>()
+              : sizeof(T) == 4
+                  ? dev.getInfo<CL_DEVICE_PREFERRED_VECTOR_WIDTH_INT>()
+              : sizeof(T) == 8
+                  ? dev.getInfo<CL_DEVICE_PREFERRED_VECTOR_WIDTH_DOUBLE>()
+                  : 1
+        : sizeof(T) > 8 ? 1
+                        : 16 / sizeof(T)};
+    const unsigned vectorWidth{vectorizeShape(maxVectorWidth, idims_.dims,
+                                              istrides_.dims, indims_, ioffset,
+                                              ostrides_.dims, ooffset)};
+    const dim_t sizeofNewT{sizeof(T) * vectorWidth};
+
+    threadsMgt<int> th(idims_.dims, indims_, 1, 1, totalSize, sizeofNewT);
+    const char* kernelName{
+        th.loop0   ? "memCopyLoop0"
+        : th.loop1 ? th.loop3 ? "memCopyLoop13" : "memCopyLoop1"
+        : th.loop3 ? "memCopyLoop3"
+                   : "memCopy"};  // Conversion to  base vector types.
+    const char* tArg{
+        sizeofNewT == 1   ? "char"
+        : sizeofNewT == 2 ? "short"
+        : sizeofNewT == 4 ? "float"
+        : sizeofNewT == 8 ? "float2"
+        : sizeofNewT == 16
+            ? "float4"
+            : "type is larger than 16 bytes, which is unsupported"};
+    auto memCopy{common::getKernel(kernelName, {memcopy_cl_src}, {tArg},
+                                   {DefineKeyValue(T, tArg)})};
+    const cl::NDRange local{th.genLocal(memCopy.get())};
+    const cl::NDRange global{th.genGlobal(local)};
+
+    memCopy(cl::EnqueueArgs(getQueue(), global, local), b_out, ostrides_,
+            static_cast<int>(ooffset), b_in, idims_, istrides_,
+            static_cast<int>(ioffset));
+    CL_DEBUG_FINISH(getQueue());
+}
 
-    size_t local_size[2] = {DIM0, DIM1};
-    if (ndims == 1) {
-        local_size[0] *= local_size[1];
-        local_size[1] = 1;
+template<typename inType, typename outType>
+void copy(const Param out, const Param in, dim_t ondims,
+          const outType default_value, const double factor) {
+    dims_type idims_{
+        static_cast<int>(in.info.dims[0]), static_cast<int>(in.info.dims[1]),
+        static_cast<int>(in.info.dims[2]), static_cast<int>(in.info.dims[3])};
+    dims_type istrides_{static_cast<int>(in.info.strides[0]),
+                        static_cast<int>(in.info.strides[1]),
+                        static_cast<int>(in.info.strides[2]),
+                        static_cast<int>(in.info.strides[3])};
+    dims_type odims_{
+        static_cast<int>(out.info.dims[0]), static_cast<int>(out.info.dims[1]),
+        static_cast<int>(out.info.dims[2]), static_cast<int>(out.info.dims[3])};
+    dims_type ostrides_{static_cast<int>(out.info.strides[0]),
+                        static_cast<int>(out.info.strides[1]),
+                        static_cast<int>(out.info.strides[2]),
+                        static_cast<int>(out.info.strides[3])};
+    int ondims_{static_cast<int>(ondims)};
+    const size_t totalSize{odims_.dims[0] * odims_.dims[1] * odims_.dims[2] *
+                               odims_.dims[3] * sizeof(outType) +
+                           idims_.dims[0] * idims_.dims[1] * idims_.dims[2] *
+                               idims_.dims[3] * sizeof(inType)};
+    bool same_dims{true};
+    for (int i{0}; i < ondims_; ++i) {
+        if (idims_.dims[i] > odims_.dims[i]) {
+            idims_.dims[i] = odims_.dims[i];
+        } else if (idims_.dims[i] != odims_.dims[i]) {
+            same_dims = false;
+        }
     }
 
-    int groups_0 = divup(idims[0], local_size[0]);
-    int groups_1 = divup(idims[1], local_size[1]);
+    removeEmptyColumns(odims_.dims, ondims_, idims_.dims, istrides_.dims);
+    ondims_ =
+        removeEmptyColumns(odims_.dims, ondims_, odims_.dims, ostrides_.dims);
+    ondims_ = combineColumns(odims_.dims, ostrides_.dims, ondims_, idims_.dims,
+                             istrides_.dims);
 
-    cl::NDRange local(local_size[0], local_size[1]);
-    cl::NDRange global(groups_0 * idims[2] * local_size[0],
-                       groups_1 * idims[3] * local_size[1]);
+    constexpr int factorTypeIdx{std::is_same<inType, double>::value ||
+                                std::is_same<inType, cdouble>::value};
+    const char* factorType[]{"float", "double"};
 
-    memCopy(cl::EnqueueArgs(getQueue(), global, local), out, _ostrides, in,
-            _idims, _istrides, offset, groups_0, groups_1);
-    CL_DEBUG_FINISH(getQueue());
-}
-
-template<typename inType, typename outType>
-void copy(Param dst, const Param src, const int ndims,
-          const outType default_value, const double factor,
-          const bool same_dims) {
-    using std::string;
-
-    std::vector<TemplateArg> targs = {
-        TemplateTypename<inType>(),
-        TemplateTypename<outType>(),
-        TemplateArg(same_dims),
+    const std::vector<TemplateArg> targs{
+        TemplateTypename<inType>(), TemplateTypename<outType>(),
+        TemplateArg(same_dims),     TemplateArg(factorType[factorTypeIdx]),
+        TemplateArg(factor != 1.0),
     };
-    std::vector<string> options = {
+    const std::vector<std::string> options{
         DefineKeyValue(inType, dtype_traits<inType>::getName()),
         DefineKeyValue(outType, dtype_traits<outType>::getName()),
-        string(" -D inType_" + string(dtype_traits<inType>::getName())),
-        string(" -D outType_" + string(dtype_traits<outType>::getName())),
+        std::string(" -D inType_") + dtype_traits<inType>::getName(),
+        std::string(" -D outType_") + dtype_traits<outType>::getName(),
         DefineKeyValue(SAME_DIMS, static_cast<int>(same_dims)),
+        std::string(" -D factorType=") + factorType[factorTypeIdx],
+        std::string((factor != 1.0) ? " -D FACTOR" : " -D NOFACTOR"),
+        {getTypeBuildDefinition<inType, outType>()},
     };
-    options.emplace_back(getTypeBuildDefinition<inType, outType>());
-
-    auto copy = common::getKernel("reshapeCopy", {copy_cl_src}, targs, options);
-
-    cl::NDRange local(DIM0, DIM1);
-    size_t local_size[] = {DIM0, DIM1};
-
-    local_size[0] *= local_size[1];
-    if (ndims == 1) { local_size[1] = 1; }
-
-    int blk_x = divup(dst.info.dims[0], local_size[0]);
-    int blk_y = divup(dst.info.dims[1], local_size[1]);
-
-    cl::NDRange global(blk_x * dst.info.dims[2] * DIM0,
-                       blk_y * dst.info.dims[3] * DIM1);
 
-    dims_t trgt_dims;
-    if (same_dims) {
-        trgt_dims = {{dst.info.dims[0], dst.info.dims[1], dst.info.dims[2],
-                      dst.info.dims[3]}};
+    threadsMgt<int> th(odims_.dims, ondims_, 1, 1, totalSize, sizeof(outType));
+    auto copy = common::getKernel(th.loop0   ? "scaledCopyLoop0"
+                                  : th.loop3 ? "scaledCopyLoop13"
+                                  : th.loop1 ? "scaledCopyLoop1"
+                                             : "scaledCopy",
+                                  {copy_cl_src}, targs, options);
+    const cl::NDRange local{th.genLocal(copy.get())};
+    const cl::NDRange global{th.genGlobal(local)};
+
+    if (factorTypeIdx == 0) {
+        copy(cl::EnqueueArgs(getQueue(), global, local), *out.data, odims_,
+             ostrides_, static_cast<uint>(out.info.offset), *in.data, idims_,
+             istrides_, static_cast<uint>(in.info.offset), default_value,
+             static_cast<float>(factor));
     } else {
-        dim_t trgt_l = std::min(dst.info.dims[3], src.info.dims[3]);
-        dim_t trgt_k = std::min(dst.info.dims[2], src.info.dims[2]);
-        dim_t trgt_j = std::min(dst.info.dims[1], src.info.dims[1]);
-        dim_t trgt_i = std::min(dst.info.dims[0], src.info.dims[0]);
-        trgt_dims    = {{trgt_i, trgt_j, trgt_k, trgt_l}};
+        copy(cl::EnqueueArgs(getQueue(), global, local), *out.data, odims_,
+             ostrides_, static_cast<uint>(out.info.offset), *in.data, idims_,
+             istrides_, static_cast<uint>(in.info.offset), default_value,
+             static_cast<double>(factor));
     }
 
-    copy(cl::EnqueueArgs(getQueue(), global, local), *dst.data, dst.info,
-         *src.data, src.info, default_value, (float)factor, trgt_dims, blk_x,
-         blk_y);
     CL_DEBUG_FINISH(getQueue());
 }
 }  // namespace kernel
diff --git a/src/backend/opencl/reshape.cpp b/src/backend/opencl/reshape.cpp
index 6eb8862e28..0ec77e27bc 100644
--- a/src/backend/opencl/reshape.cpp
+++ b/src/backend/opencl/reshape.cpp
@@ -21,8 +21,9 @@ template<typename inType, typename outType>
 Array<outType> reshape(const Array<inType> &in, const dim4 &outDims,
                        outType defaultValue, double scale) {
     Array<outType> out = createEmptyArray<outType>(outDims);
-    kernel::copy<inType, outType>(out, in, in.ndims(), defaultValue, scale,
-                                  in.dims() == outDims);
+    if (out.elements() > 0) {
+        kernel::copy<inType, outType>(out, in, in.ndims(), defaultValue, scale);
+    }
     return out;
 }
 

From 1dfeb9761287070ce282ed1dc9dfcf3d859ed0f1 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 4 Aug 2022 01:10:52 +0200
Subject: [PATCH 074/473] OPT: JIT

---
 src/backend/common/jit/Node.cpp              |  12 +-
 src/backend/common/jit/Node.hpp              |   6 +-
 src/backend/cuda/jit.cpp                     | 718 +++++++++++--------
 src/backend/cuda/jit/kernel_generators.hpp   |  46 +-
 src/backend/opencl/jit.cpp                   | 569 +++++++++------
 src/backend/opencl/jit/kernel_generators.hpp |  50 +-
 6 files changed, 843 insertions(+), 558 deletions(-)

diff --git a/src/backend/common/jit/Node.cpp b/src/backend/common/jit/Node.cpp
index 83767f502f..c637926d79 100644
--- a/src/backend/common/jit/Node.cpp
+++ b/src/backend/common/jit/Node.cpp
@@ -11,6 +11,7 @@
 #include <common/jit/Node.hpp>
 #include <common/util.hpp>
 
+#include <version.hpp>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -29,7 +30,7 @@ int Node::getNodesMap(Node_map_t &node_map, vector<Node *> &full_nodes,
             ids.child_ids[i] =
                 m_children[i]->getNodesMap(node_map, full_nodes, full_ids);
         }
-        ids.id         = node_map.size();
+        ids.id         = static_cast<int>(node_map.size());
         node_map[this] = ids.id;
         full_nodes.push_back(this);
         full_ids.push_back(ids);
@@ -40,10 +41,16 @@ int Node::getNodesMap(Node_map_t &node_map, vector<Node *> &full_nodes,
 
 std::string getFuncName(const vector<Node *> &output_nodes,
                         const vector<Node *> &full_nodes,
-                        const vector<Node_ids> &full_ids, bool is_linear) {
+                        const vector<Node_ids> &full_ids, const bool is_linear,
+                        const bool loop0, const bool loop1, const bool loop2,
+                        const bool loop3) {
     std::string funcName;
     funcName.reserve(512);
     funcName = (is_linear ? 'L' : 'G');
+    funcName += (loop0 ? '0' : 'X');
+    funcName += (loop1 ? '1' : 'X');
+    funcName += (loop2 ? '2' : 'X');
+    funcName += (loop3 ? '3' : 'X');
 
     for (const auto &node : output_nodes) {
         funcName += '_';
@@ -65,7 +72,6 @@ auto isBuffer(const Node &ptr) -> bool { return ptr.isBuffer(); }
 
 auto isScalar(const Node &ptr) -> bool { return ptr.isScalar(); }
 
-/// Returns true if the buffer is linear
 bool Node::isLinear(const dim_t dims[4]) const { return true; }
 
 }  // namespace common
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index ca557a50d6..bbe3fcb859 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -245,7 +245,7 @@ class Node {
     // Returns true if this node is a Buffer
     virtual bool isBuffer() const { return false; }
 
-    // Returns true if this node is a Buffer
+    // Returns true if this node is a Scalar
     virtual bool isScalar() const { return false; }
 
     /// Returns true if the buffer is linear
@@ -304,7 +304,9 @@ struct Node_ids {
 
 std::string getFuncName(const std::vector<Node *> &output_nodes,
                         const std::vector<Node *> &full_nodes,
-                        const std::vector<Node_ids> &full_ids, bool is_linear);
+                        const std::vector<Node_ids> &full_ids,
+                        const bool is_linear, const bool loop0,
+                        const bool loop1, const bool loop2, const bool loop3);
 
 auto isBuffer(const Node &ptr) -> bool;
 
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index c8612f1c19..262d5c8c45 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -9,7 +9,6 @@
 
 #include <Array.hpp>
 #include <Kernel.hpp>
-#include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <common/jit/ModdimNode.hpp>
 #include <common/jit/Node.hpp>
@@ -23,33 +22,46 @@
 #include <kernel_headers/jit_cuh.hpp>
 #include <math.hpp>
 #include <platform.hpp>
+#include <threadsMgt.hpp>
+#include <type_util.hpp>
 #include <af/dim4.hpp>
 
+#include <algorithm>
 #include <cstdlib>
 #include <sstream>
 #include <stdexcept>
 #include <string>
-#include <thread>
 #include <vector>
 
 using common::findModule;
 using common::getFuncName;
 using common::half;
+using common::ModdimNode;
 using common::Node;
 using common::Node_ids;
 using common::Node_map_t;
+using common::Node_ptr;
+using common::NodeIterator;
 
+using std::array;
+using std::equal;
+using std::for_each;
+using std::shared_ptr;
 using std::string;
 using std::stringstream;
 using std::to_string;
 using std::vector;
 
 namespace cuda {
-
-static string getKernelString(const string &funcName,
-                              const vector<Node *> &full_nodes,
-                              const vector<Node_ids> &full_ids,
-                              const vector<int> &output_ids, bool is_linear) {
+using jit::BufferNode;
+
+static string getKernelString(const string& funcName,
+                              const vector<Node*>& full_nodes,
+                              const vector<Node_ids>& full_ids,
+                              const vector<int>& output_ids,
+                              const bool is_linear, const bool loop0,
+                              const bool loop1, const bool loop2,
+                              const bool loop3) {
     const std::string includeFileStr(jit_cuh, jit_cuh_len);
 
     const std::string paramTStr = R"JIT(
@@ -61,144 +73,249 @@ struct Param {
 };
 )JIT";
 
-    std::string typedefStr = "typedef unsigned int uint;\n";
-    typedefStr += "typedef ";
+    std::string typedefStr{"typedef unsigned int uint;\ntypedef "};
     typedefStr += getFullName<dim_t>();
     typedefStr += " dim_t;\n";
 
     // Common CUDA code
     // This part of the code does not change with the kernel.
 
-    static const char *kernelVoid = "extern \"C\" __global__ void\n";
-    static const char *dimParams =
-        "uint blocks_x, uint blocks_y, uint blocks_x_total, uint num_odims";
-
-    static const char *loopStart = R"JIT(
-    for (int blockIdx_x = blockIdx.x; blockIdx_x < blocks_x_total; blockIdx_x += gridDim.x) {
-    )JIT";
-    static const char *loopEnd   = "}\n\n";
-
-    static const char *blockStart = "{\n\n";
-    static const char *blockEnd   = "\n\n}";
-
-    static const char *linearIndex = R"JIT(
-        uint threadId = threadIdx.x;
-        long long idx = blockIdx_x * blockDim.x * blockDim.y + threadId;
-        if (idx >= outref.dims[3] * outref.strides[3]) return;
-        )JIT";
-
-    static const char *generalIndex = R"JIT(
-        long long id0 = 0, id1 = 0, id2 = 0, id3 = 0;
-        long blockIdx_y = blockIdx.z * gridDim.y + blockIdx.y;
-        if (num_odims > 2) {
-            id2 = blockIdx_x / blocks_x;
-            id0 = blockIdx_x - id2 * blocks_x;
-            id0 = threadIdx.x + id0 * blockDim.x;
-            if (num_odims > 3) {
-                id3 = blockIdx_y / blocks_y;
-                id1 = blockIdx_y - id3 * blocks_y;
-                id1 = threadIdx.y + id1 * blockDim.y;
-            } else {
-                id1 = threadIdx.y + blockDim.y * blockIdx_y;
-            }
-        } else {
-            id3 = 0;
-            id2 = 0;
-            id1 = threadIdx.y + blockDim.y * blockIdx_y;
-            id0 = threadIdx.x + blockDim.x * blockIdx_x;
-        }
-
-        bool cond = id0 < outref.dims[0] &&
-                    id1 < outref.dims[1] &&
-                    id2 < outref.dims[2] &&
-                    id3 < outref.dims[3];
-
-        if (!cond) { continue; }
-
-        long long idx = outref.strides[3] * id3 +
-                        outref.strides[2] * id2 +
-                        outref.strides[1] * id1 + id0;
-        )JIT";
-
-    stringstream inParamStream;
-    stringstream outParamStream;
-    stringstream outWriteStream;
-    stringstream offsetsStream;
-    stringstream opsStream;
-    stringstream outrefstream;
-
-    for (int i = 0; i < static_cast<int>(full_nodes.size()); i++) {
-        const auto &node     = full_nodes[i];
-        const auto &ids_curr = full_ids[i];
+    static const char* kernelVoid = "extern \"C\" __global__ void\n";
+    static const char* dimParams  = "";
+
+    static const char* blockStart = "{";
+    static const char* blockEnd   = "\n}\n";
+
+    static const char* linearInit = R"JIT(
+    int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    const int idxEnd = outref.dims[0];
+    if (idx < idxEnd) {)JIT";
+    static const char* linearEnd  = R"JIT(
+    })JIT";
+
+    static const char* linearLoop0Start = R"JIT(
+        const int idxID0Inc = gridDim.x*blockDim.x;
+        do {)JIT";
+    static const char* linearLoop0End   = R"JIT(
+            idx += idxID0Inc;
+            if (idx >= idxEnd) break;
+        } while (true);)JIT";
+
+    // ///////////////////////////////////////////////
+    // oInfo = output optimized information (dims, strides, offset).
+    //         oInfo has removed dimensions, to optimized block scheduling
+    // iInfo = input internal information (dims, strides, offset)
+    //         iInfo has the original dimensions, auto generated code
+    //
+    // Loop3 is fastest and becomes inside loop, since
+    //      - #of loops is known upfront
+    // Loop1 is used for extra dynamic looping (writing into cache)
+    // Loop0 is used for extra dynamic looping (writing into cache),
+    //       VECTORS ONLY!!
+    // All loops are conditional and idependent Format Loop1 & Loop3
+    // ////////////////////////////
+    //  *stridedLoopNInit               // Always
+    //  *stridedLoop1Init               // Conditional
+    //  *stridedLoop2Init               // Conditional
+    //  *stridedLoop3Init               // Conditional
+    //  *stridedLoop1Start              // Conditional
+    //      *stridedLoop2Start          // Conditional
+    //          *stridedLoop3Start      // Conditional
+    //              auto generated code // Always
+    //          *stridedLoop3End        // Conditional
+    //      *stridedLoop2End            // Conditional
+    //  *stridedLoop1End                // Conditional
+    //  *stridedEnd                     // Always
+    //
+    // Format loop0 (Vector only)
+    // //////////////////////////
+    // *stridedLoop0Init                // Always
+    // *stridedLoop0Start               // Always
+    //      auto generated code         // Always
+    // *stridedLoop0End                 // Always
+    // *stridedEnd                      // Always
+
+    // -----
+    static const char* stridedLoop0Init  = R"JIT(
+    int id0 = blockIdx.x * blockDim.x + threadIdx.x;
+    const int id0End = outref.dims[0];
+    if (id0 < id0End) {
+#define id1 0
+#define id2 0
+#define id3 0
+        const int ostrides0 = outref.strides[0];
+        int idx = ostrides0*id0;)JIT";
+    static const char* stridedLoop0Start = R"JIT(
+        const int id0Inc = gridDim.x*blockDim.x;
+        const int idxID0Inc = ostrides0*id0Inc;
+        do {)JIT";
+    static const char* stridedLoop0End   = R"JIT(
+            id0 += id0Inc;
+            if (id0 >= id0End) break;
+            idx += idxID0Inc;
+        } while (true);)JIT";
+
+    static const char* stridedLoopNInit = R"JIT(
+    int id0 = blockIdx.x * blockDim.x + threadIdx.x;
+    int id1 = blockIdx.y * blockDim.y + threadIdx.y;
+    const int id0End = outref.dims[0];
+    const int id1End = outref.dims[1];
+    if ((id0 < id0End) & (id1 < id1End)) {
+        int id2 = blockIdx.z * blockDim.z + threadIdx.z;
+#define id3 0
+        const int ostrides1 = outref.strides[1];
+        int idx = (int)outref.strides[0]*id0 + ostrides1*id1 + (int)outref.strides[2]*id2;)JIT";
+    static const char* stridedEnd       = R"JIT(
+    })JIT";
+
+    static const char* stridedLoop3Init  = R"JIT(
+#undef id3
+        int id3 = 0;
+        const int id3End = outref.dims[3];
+        const int idxID3Inc = outref.strides[3];)JIT";
+    static const char* stridedLoop3Start = R"JIT(
+                    const int idxBaseID3 = idx;
+                    do {)JIT";
+    // Looping over outside dim3 means that all dimensions are present,
+    // so the internal id3 can be used directly
+    static const char* stridedLoop3End = R"JIT(
+                       ++id3;
+                       if (id3 == id3End) break;
+                       idx += idxID3Inc;
+                    } while (true);
+                    id3 = 0;
+                    idx = idxBaseID3;)JIT";
+
+    static const char* stridedLoop2Init  = R"JIT(
+        const int id2End = outref.dims[2];
+        const int id2Inc = gridDim.z*blockDim.z;
+        const int idxID2Inc = (int)outref.strides[2]*id2Inc;)JIT";
+    static const char* stridedLoop2Start = R"JIT(
+                const int idxBaseID2 = idx;
+                const int baseID2 = id2;
+                do {)JIT";
+    static const char* stridedLoop2End   = R"JIT(
+                    id2 += id2Inc;
+                    if (id2 >= id2End) break;
+                    idx += idxID2Inc;
+                } while (true);
+                id2 = baseID2;
+                idx = idxBaseID2;)JIT";
+
+    // No reset of od1/id[decode.dim1] is necessary since this is the overall
+    // loop
+    static const char* stridedLoop1Init  = R"JIT(
+        const int id1Inc = gridDim.y*blockDim.y;
+        const int idxID1Inc = ostrides1*id1Inc;)JIT";
+    static const char* stridedLoop1Start = R"JIT(
+            do {)JIT";
+    static const char* stridedLoop1End   = R"JIT(
+                id1 += id1Inc;
+                if (id1 >= id1End) break;
+                idx += idxID1Inc;
+            } while (true);)JIT";
+
+    // Reuse stringstreams, because they are very costly during initialization
+    thread_local stringstream inParamStream;
+    thread_local stringstream outParamStream;
+    thread_local stringstream inOffsetsStream;
+    thread_local stringstream opsStream;
+    thread_local stringstream outrefStream;
+
+    int oid{0};
+    for (size_t i{0}; i < full_nodes.size(); i++) {
+        const auto& node{full_nodes[i]};
+        const auto& ids_curr{full_ids[i]};
         // Generate input parameters, only needs current id
         node->genParams(inParamStream, ids_curr.id, is_linear);
         // Generate input offsets, only needs current id
-        node->genOffsets(offsetsStream, ids_curr.id, is_linear);
+        node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
         // Generate the core function body, needs children ids as well
         node->genFuncs(opsStream, ids_curr);
+        for (auto outIt{begin(output_ids)}, endIt{end(output_ids)};
+             (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) {
+            // Generate also output parameters
+            outParamStream << (oid == 0 ? "" : ",\n") << "Param<"
+                           << full_nodes[ids_curr.id]->getTypeStr() << "> out"
+                           << oid;
+            // Generate code to write the output (offset already in ptr)
+            opsStream << "out" << oid << ".ptr[idx] = val" << ids_curr.id
+                      << ";\n";
+            ++oid;
+        }
     }
 
-    outrefstream << "const Param<" << full_nodes[output_ids[0]]->getTypeStr()
-                 << "> &outref = out" << output_ids[0] << ";\n";
-
-    for (int id : output_ids) {
-        // Generate output parameters
-        outParamStream << "Param<" << full_nodes[id]->getTypeStr() << "> out"
-                       << id << ", \n";
-        // Generate code to write the output
-        outWriteStream << "out" << id << ".ptr[idx] = val" << id << ";\n";
-    }
+    outrefStream << "\n    const Param<"
+                 << full_nodes[output_ids[0]]->getTypeStr()
+                 << "> &outref = out0;";
 
     // Put various blocks into a single stream
-    stringstream kerStream;
-    kerStream << typedefStr;
-    kerStream << includeFileStr << "\n\n";
-    kerStream << paramTStr << "\n";
-    kerStream << kernelVoid;
-    kerStream << funcName;
-    kerStream << "(\n";
-    kerStream << inParamStream.str();
-    kerStream << outParamStream.str();
-    kerStream << dimParams;
-    kerStream << ")\n";
-    kerStream << blockStart;
-    kerStream << outrefstream.str();
-    kerStream << loopStart;
+    thread_local stringstream kerStream;
+    kerStream << typedefStr << includeFileStr << "\n\n"
+              << paramTStr << '\n'
+              << kernelVoid << funcName << "(\n"
+              << inParamStream.str() << outParamStream.str() << dimParams << ')'
+              << blockStart << outrefStream.str();
     if (is_linear) {
-        kerStream << linearIndex;
+        kerStream << linearInit;
+        if (loop0) kerStream << linearLoop0Start;
+        kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
+        if (loop0) kerStream << linearLoop0End;
+        kerStream << linearEnd;
     } else {
-        kerStream << generalIndex;
+        if (loop0) {
+            kerStream << stridedLoop0Init << stridedLoop0Start;
+        } else {
+            kerStream << stridedLoopNInit;
+            if (loop3) kerStream << stridedLoop3Init;
+            if (loop2) kerStream << stridedLoop2Init;
+            if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
+            if (loop2) kerStream << stridedLoop2Start;
+            if (loop3) kerStream << stridedLoop3Start;
+        }
+        kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
+        if (loop3) kerStream << stridedLoop3End;
+        if (loop2) kerStream << stridedLoop2End;
+        if (loop1) kerStream << stridedLoop1End;
+        if (loop0) kerStream << stridedLoop0End;
+        kerStream << stridedEnd;
     }
-    kerStream << offsetsStream.str();
-    kerStream << opsStream.str();
-    kerStream << outWriteStream.str();
-    kerStream << loopEnd;
     kerStream << blockEnd;
+    const string ret{kerStream.str()};
+
+    // Prepare for next round
+    inParamStream.str("");
+    outParamStream.str("");
+    inOffsetsStream.str("");
+    opsStream.str("");
+    outrefStream.str("");
+    kerStream.str("");
 
-    return kerStream.str();
+    return ret;
 }
 
-static CUfunction getKernel(const vector<Node *> &output_nodes,
-                            const vector<int> &output_ids,
-                            const vector<Node *> &full_nodes,
-                            const vector<Node_ids> &full_ids,
-                            const bool is_linear) {
-    const string funcName =
-        getFuncName(output_nodes, full_nodes, full_ids, is_linear);
-    const size_t moduleKey = deterministicHash(funcName);
-
-    // A forward lookup in module cache helps avoid recompiling the jit
-    // source generated from identical jit-trees. It also enables us
-    // with a way to save jit kernels to disk only once
-    auto entry = findModule(getActiveDeviceId(), moduleKey);
-
-    if (entry.get() == nullptr) {
-        const string jitKer = getKernelString(funcName, full_nodes, full_ids,
-                                              output_ids, is_linear);
+static CUfunction getKernel(const vector<Node*>& output_nodes,
+                            const vector<int>& output_ids,
+                            const vector<Node*>& full_nodes,
+                            const vector<Node_ids>& full_ids,
+                            const bool is_linear, const bool loop0,
+                            const bool loop1, const bool loop2,
+                            const bool loop3) {
+    const string funcName{getFuncName(output_nodes, full_nodes, full_ids,
+                                      is_linear, loop0, loop1, loop2, loop3)};
+    // A forward lookup in module cache helps avoid recompiling
+    // the JIT source generated from identical JIT-trees.
+    const auto entry{
+        findModule(getActiveDeviceId(), deterministicHash(funcName))};
+
+    if (!entry) {
+        const string jitKer{getKernelString(funcName, full_nodes, full_ids,
+                                            output_ids, is_linear, loop0, loop1,
+                                            loop2, loop3)};
         saveKernel(funcName, jitKer, ".cu");
 
-        common::Source jit_src{jitKer.c_str(), jitKer.size(),
-                               deterministicHash(jitKer)};
+        const common::Source jit_src{jitKer.c_str(), jitKer.size(),
+                                     deterministicHash(jitKer)};
 
         return common::getKernel(funcName, {jit_src}, {}, {}, true).get();
     }
@@ -206,158 +323,184 @@ static CUfunction getKernel(const vector<Node *> &output_nodes,
 }
 
 template<typename T>
-void evalNodes(vector<Param<T>> &outputs, const vector<Node *> &output_nodes) {
-    size_t num_outputs = outputs.size();
-    if (num_outputs == 0) { return; }
-
-    int device         = getActiveDeviceId();
-    dim_t *outDims     = outputs[0].dims;
-    size_t numOutElems = outDims[0] * outDims[1] * outDims[2] * outDims[3];
+void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
+    const unsigned nrOutputs{static_cast<unsigned>(output_nodes.size())};
+    if (nrOutputs == 0) { return; }
+    assert(outputs.size() == output_nodes.size());
+    dim_t* outDims{outputs[0].dims};
+    dim_t* outStrides{outputs[0].strides};
+    for_each(
+        begin(outputs)++, end(outputs),
+        [outDims, outStrides](Param<T>& output) {
+            assert(equal(output.dims, output.dims + AF_MAX_DIMS, outDims) &&
+                   equal(output.strides, output.strides + AF_MAX_DIMS,
+                         outStrides));
+        });
+
+    dim_t ndims{outDims[3] > 1   ? 4
+                : outDims[2] > 1 ? 3
+                : outDims[1] > 1 ? 2
+                : outDims[0] > 0 ? 1
+                                 : 0};
+    bool is_linear{true};
+    dim_t numOutElems{1};
+    for (dim_t dim{0}; dim < ndims; ++dim) {
+        is_linear &= (numOutElems == outStrides[dim]);
+        numOutElems *= outDims[dim];
+    }
     if (numOutElems == 0) { return; }
 
-    // Use thread local to reuse the memory every time you are here.
+    // Use thread local to reuse the memory every time you are
+    // here.
     thread_local Node_map_t nodes;
-    thread_local vector<Node *> full_nodes;
+    thread_local vector<Node*> full_nodes;
     thread_local vector<Node_ids> full_ids;
     thread_local vector<int> output_ids;
 
-    // Reserve some space to improve performance at smaller sizes
-    if (nodes.empty()) {
-        nodes.reserve(1024);
-        output_ids.reserve(output_nodes.size());
-        full_nodes.reserve(1024);
-        full_ids.reserve(1024);
+    // Reserve some space to improve performance at smaller
+    // sizes
+    constexpr size_t CAP{1024};
+    if (full_nodes.capacity() < CAP) {
+        nodes.reserve(CAP);
+        output_ids.reserve(10);
+        full_nodes.reserve(CAP);
+        full_ids.reserve(CAP);
     }
 
-    for (auto &node : output_nodes) {
-        int id = node->getNodesMap(nodes, full_nodes, full_ids);
+    const af::dtype outputType{output_nodes[0]->getType()};
+    const size_t outputSizeofType{size_of(outputType)};
+    for (Node* node : output_nodes) {
+        assert(node->getType() == outputType);
+        const int id = node->getNodesMap(nodes, full_nodes, full_ids);
         output_ids.push_back(id);
     }
 
-    using common::ModdimNode;
-    using common::NodeIterator;
-    using jit::BufferNode;
-
-    // find all moddims in the tree
-    vector<std::shared_ptr<Node>> node_clones;
-    for (auto *node : full_nodes) { node_clones.emplace_back(node->clone()); }
-
-    for (common::Node_ids ids : full_ids) {
-        auto &children = node_clones[ids.id]->m_children;
-        for (int i = 0; i < Node::kMaxChildren && children[i] != nullptr; i++) {
-            children[i] = node_clones[ids.child_ids[i]];
-        }
-    }
-
-    for (auto &node : node_clones) {
-        if (node->getOp() == af_moddims_t) {
-            ModdimNode *mn = static_cast<ModdimNode *>(node.get());
-            auto isBuffer  = [](const Node &ptr) { return ptr.isBuffer(); };
-
-            NodeIterator<> it(node.get());
-            auto new_strides = calcStrides(mn->m_new_shape);
-            while (it != NodeIterator<>()) {
-                it = find_if(it, NodeIterator<>(), isBuffer);
-                if (it == NodeIterator<>()) { break; }
-
-                BufferNode<T> *buf = static_cast<BufferNode<T> *>(&(*it));
-
-                buf->m_param.dims[0]    = mn->m_new_shape[0];
-                buf->m_param.dims[1]    = mn->m_new_shape[1];
-                buf->m_param.dims[2]    = mn->m_new_shape[2];
-                buf->m_param.dims[3]    = mn->m_new_shape[3];
-                buf->m_param.strides[0] = new_strides[0];
-                buf->m_param.strides[1] = new_strides[1];
-                buf->m_param.strides[2] = new_strides[2];
-                buf->m_param.strides[3] = new_strides[3];
-
-                ++it;
-            }
-        }
-    }
-
-    full_nodes.clear();
-    for (auto &node : node_clones) { full_nodes.push_back(node.get()); }
-
-    bool is_linear = true;
-    for (auto *node : full_nodes) {
-        is_linear &= node->isLinear(outputs[0].dims);
-    }
-
-    CUfunction ker =
-        getKernel(output_nodes, output_ids, full_nodes, full_ids, is_linear);
-
-    int threads_x = 1, threads_y = 1;
-    int blocks_x_ = 1, blocks_y_ = 1;
-    int blocks_x = 1, blocks_y = 1, blocks_z = 1, blocks_x_total;
-
-    cudaDeviceProp properties    = getDeviceProp(device);
-    const long long max_blocks_x = properties.maxGridSize[0];
-    const long long max_blocks_y = properties.maxGridSize[1];
-
-    int num_odims = 4;
-    while (num_odims >= 1) {
-        if (outDims[num_odims - 1] == 1) {
-            num_odims--;
-        } else {
-            break;
+    size_t inputSize{0};
+    unsigned nrInputs{0};
+    bool moddimsFound{false};
+    for (const Node* node : full_nodes) {
+        is_linear &= node->isLinear(outDims);
+        moddimsFound |= (node->getOp() == af_moddims_t);
+        if (node->isBuffer()) {
+            ++nrInputs;
+            inputSize += node->getBytes();
         }
     }
+    const size_t outputSize{numOutElems * outputSizeofType * nrOutputs};
+    const size_t totalSize{inputSize + outputSize};
 
+    bool emptyColumnsFound{false};
     if (is_linear) {
-        threads_x = 256;
-        threads_y = 1;
-
-        blocks_x_total = divup(
-            (outDims[0] * outDims[1] * outDims[2] * outDims[3]), threads_x);
-
-        int repeat_x = divup(blocks_x_total, max_blocks_x);
-        blocks_x     = divup(blocks_x_total, repeat_x);
+        outDims[0]    = numOutElems;
+        outDims[1]    = 1;
+        outDims[2]    = 1;
+        outDims[3]    = 1;
+        outStrides[0] = 1;
+        outStrides[1] = numOutElems;
+        outStrides[2] = numOutElems;
+        outStrides[3] = numOutElems;
+        ndims         = 1;
     } else {
-        threads_x = 32;
-        threads_y = 8;
+        emptyColumnsFound = ndims > (outDims[0] == 1   ? 1
+                                     : outDims[1] == 1 ? 2
+                                     : outDims[2] == 1 ? 3
+                                                       : 4);
+    }
 
-        blocks_x_ = divup(outDims[0], threads_x);
-        blocks_y_ = divup(outDims[1], threads_y);
+    // Keep node_clones in scope, so that the nodes remain active for later
+    // referral in case moddims or Column elimination operations have to take
+    // place
+    vector<Node_ptr> node_clones;
+    if (moddimsFound | emptyColumnsFound) {
+        node_clones.reserve(full_nodes.size());
+        for (Node* node : full_nodes) {
+            node_clones.emplace_back(node->clone());
+        }
 
-        blocks_x = blocks_x_ * outDims[2];
-        blocks_y = blocks_y_ * outDims[3];
+        for (const Node_ids& ids : full_ids) {
+            auto& children{node_clones[ids.id]->m_children};
+            for (int i{0}; i < Node::kMaxChildren && children[i] != nullptr;
+                 i++) {
+                children[i] = node_clones[ids.child_ids[i]];
+            }
+        }
 
-        blocks_z = divup(blocks_y, max_blocks_y);
-        blocks_y = divup(blocks_y, blocks_z);
+        if (moddimsFound) {
+            const auto isModdim{[](const Node_ptr& node) {
+                return node->getOp() == af_moddims_t;
+            }};
+            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
+                 (nodeIt = find_if(nodeIt, endIt, isModdim)) != endIt;
+                 ++nodeIt) {
+                const ModdimNode* mn{static_cast<ModdimNode*>(nodeIt->get())};
+
+                const auto new_strides{calcStrides(mn->m_new_shape)};
+                const auto isBuffer{
+                    [](const Node& ptr) { return ptr.isBuffer(); }};
+                for (NodeIterator<> it{nodeIt->get()}, end{NodeIterator<>()};
+                     (it = find_if(it, end, isBuffer)) != end; ++it) {
+                    BufferNode<T>* buf{static_cast<BufferNode<T>*>(&(*it))};
+                    buf->m_param.dims[0]    = mn->m_new_shape[0];
+                    buf->m_param.dims[1]    = mn->m_new_shape[1];
+                    buf->m_param.dims[2]    = mn->m_new_shape[2];
+                    buf->m_param.dims[3]    = mn->m_new_shape[3];
+                    buf->m_param.strides[0] = new_strides[0];
+                    buf->m_param.strides[1] = new_strides[1];
+                    buf->m_param.strides[2] = new_strides[2];
+                    buf->m_param.strides[3] = new_strides[3];
+                }
+            }
+        }
+        if (emptyColumnsFound) {
+            const auto isBuffer{
+                [](const Node_ptr& node) { return node->isBuffer(); }};
+            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
+                 (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt;
+                 ++nodeIt) {
+                BufferNode<T>* buf{static_cast<BufferNode<T>*>(nodeIt->get())};
+                removeEmptyColumns(outDims, ndims, buf->m_param.dims,
+                                   buf->m_param.strides);
+            }
+            for_each(++begin(outputs), end(outputs),
+                     [outDims, ndims](Param<T>& output) {
+                         removeEmptyColumns(outDims, ndims, output.dims,
+                                            output.strides);
+                     });
+            ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
+        }
 
-        blocks_x_total = blocks_x;
-        int repeat_x   = divup(blocks_x_total, max_blocks_x);
-        blocks_x       = divup(blocks_x_total, repeat_x);
+        full_nodes.clear();
+        for (Node_ptr& node : node_clones) { full_nodes.push_back(node.get()); }
     }
 
-    vector<void *> args;
+    threadsMgt<dim_t> th(outDims, ndims);
+    const dim3 threads{th.genThreads()};
+    const dim3 blocks{th.genBlocks(threads, nrInputs, nrOutputs, totalSize,
+                                   outputSizeofType)};
+    auto ker = getKernel(output_nodes, output_ids, full_nodes, full_ids,
+                         is_linear, th.loop0, th.loop1, th.loop2, th.loop3);
 
-    for (const auto &node : full_nodes) {
+    vector<void*> args;
+    for (const Node* node : full_nodes) {
         node->setArgs(0, is_linear,
-                      [&](int /*id*/, const void *ptr, size_t /*size*/) {
-                          args.push_back(const_cast<void *>(ptr));
+                      [&](int /*id*/, const void* ptr, size_t /*size*/) {
+                          args.push_back(const_cast<void*>(ptr));
                       });
     }
 
-    for (size_t i = 0; i < num_outputs; i++) {
-        args.push_back(static_cast<void *>(&outputs[i]));
-    }
-
-    args.push_back(static_cast<void *>(&blocks_x_));
-    args.push_back(static_cast<void *>(&blocks_y_));
-    args.push_back(static_cast<void *>(&blocks_x_total));
-    args.push_back(static_cast<void *>(&num_odims));
+    for (auto& out : outputs) { args.push_back(static_cast<void*>(&out)); }
 
     {
         using namespace cuda::kernel_logger;
-        AF_TRACE("Launching : Blocks: [{}] Threads: [{}] ",
-                 dim3(blocks_x, blocks_y, blocks_z),
-                 dim3(threads_x, threads_y));
+        AF_TRACE(
+            "Launching : Dims: [{},{},{},{}] Blocks: [{}] "
+            "Threads: [{}] threads: {}",
+            outDims[0], outDims[1], outDims[2], outDims[3], blocks, threads,
+            blocks.x * threads.x * blocks.y * threads.y * blocks.z * threads.z);
     }
-    CU_CHECK(cuLaunchKernel(ker, blocks_x, blocks_y, blocks_z, threads_x,
-                            threads_y, 1, 0, getActiveStream(), args.data(),
-                            NULL));
+    CU_CHECK(cuLaunchKernel(ker, blocks.x, blocks.y, blocks.z, threads.x,
+                            threads.y, threads.z, 0, getActiveStream(),
+                            args.data(), NULL));
 
     // Reset the thread local vectors
     nodes.clear();
@@ -367,53 +510,50 @@ void evalNodes(vector<Param<T>> &outputs, const vector<Node *> &output_nodes) {
 }
 
 template<typename T>
-void evalNodes(Param<T> out, Node *node) {
-    vector<Param<T>> outputs;
-    vector<Node *> output_nodes;
-
-    outputs.push_back(out);
-    output_nodes.push_back(node);
-    evalNodes(outputs, output_nodes);
+void evalNodes(Param<T> out, Node* node) {
+    vector<Param<T>> outputs{out};
+    vector<Node*> nodes{node};
+    evalNodes(outputs, nodes);
 }
 
-template void evalNodes<float>(Param<float> out, Node *node);
-template void evalNodes<double>(Param<double> out, Node *node);
-template void evalNodes<cfloat>(Param<cfloat> out, Node *node);
-template void evalNodes<cdouble>(Param<cdouble> out, Node *node);
-template void evalNodes<int>(Param<int> out, Node *node);
-template void evalNodes<uint>(Param<uint> out, Node *node);
-template void evalNodes<char>(Param<char> out, Node *node);
-template void evalNodes<uchar>(Param<uchar> out, Node *node);
-template void evalNodes<intl>(Param<intl> out, Node *node);
-template void evalNodes<uintl>(Param<uintl> out, Node *node);
-template void evalNodes<short>(Param<short> out, Node *node);
-template void evalNodes<ushort>(Param<ushort> out, Node *node);
-template void evalNodes<half>(Param<half> out, Node *node);
-
-template void evalNodes<float>(vector<Param<float>> &out,
-                               const vector<Node *> &node);
-template void evalNodes<double>(vector<Param<double>> &out,
-                                const vector<Node *> &node);
-template void evalNodes<cfloat>(vector<Param<cfloat>> &out,
-                                const vector<Node *> &node);
-template void evalNodes<cdouble>(vector<Param<cdouble>> &out,
-                                 const vector<Node *> &node);
-template void evalNodes<int>(vector<Param<int>> &out,
-                             const vector<Node *> &node);
-template void evalNodes<uint>(vector<Param<uint>> &out,
-                              const vector<Node *> &node);
-template void evalNodes<char>(vector<Param<char>> &out,
-                              const vector<Node *> &node);
-template void evalNodes<uchar>(vector<Param<uchar>> &out,
-                               const vector<Node *> &node);
-template void evalNodes<intl>(vector<Param<intl>> &out,
-                              const vector<Node *> &node);
-template void evalNodes<uintl>(vector<Param<uintl>> &out,
-                               const vector<Node *> &node);
-template void evalNodes<short>(vector<Param<short>> &out,
-                               const vector<Node *> &node);
-template void evalNodes<ushort>(vector<Param<ushort>> &out,
-                                const vector<Node *> &node);
-template void evalNodes<half>(vector<Param<half>> &out,
-                              const vector<Node *> &node);
+template void evalNodes<float>(Param<float> out, Node* node);
+template void evalNodes<double>(Param<double> out, Node* node);
+template void evalNodes<cfloat>(Param<cfloat> out, Node* node);
+template void evalNodes<cdouble>(Param<cdouble> out, Node* node);
+template void evalNodes<int>(Param<int> out, Node* node);
+template void evalNodes<uint>(Param<uint> out, Node* node);
+template void evalNodes<char>(Param<char> out, Node* node);
+template void evalNodes<uchar>(Param<uchar> out, Node* node);
+template void evalNodes<intl>(Param<intl> out, Node* node);
+template void evalNodes<uintl>(Param<uintl> out, Node* node);
+template void evalNodes<short>(Param<short> out, Node* node);
+template void evalNodes<ushort>(Param<ushort> out, Node* node);
+template void evalNodes<half>(Param<half> out, Node* node);
+
+template void evalNodes<float>(vector<Param<float>>& out,
+                               const vector<Node*>& node);
+template void evalNodes<double>(vector<Param<double>>& out,
+                                const vector<Node*>& node);
+template void evalNodes<cfloat>(vector<Param<cfloat>>& out,
+                                const vector<Node*>& node);
+template void evalNodes<cdouble>(vector<Param<cdouble>>& out,
+                                 const vector<Node*>& node);
+template void evalNodes<int>(vector<Param<int>>& out,
+                             const vector<Node*>& node);
+template void evalNodes<uint>(vector<Param<uint>>& out,
+                              const vector<Node*>& node);
+template void evalNodes<char>(vector<Param<char>>& out,
+                              const vector<Node*>& node);
+template void evalNodes<uchar>(vector<Param<uchar>>& out,
+                               const vector<Node*>& node);
+template void evalNodes<intl>(vector<Param<intl>>& out,
+                              const vector<Node*>& node);
+template void evalNodes<uintl>(vector<Param<uintl>>& out,
+                               const vector<Node*>& node);
+template void evalNodes<short>(vector<Param<short>>& out,
+                               const vector<Node*>& node);
+template void evalNodes<ushort>(vector<Param<ushort>>& out,
+                                const vector<Node*>& node);
+template void evalNodes<half>(vector<Param<half>>& out,
+                              const vector<Node*>& node);
 }  // namespace cuda
diff --git a/src/backend/cuda/jit/kernel_generators.hpp b/src/backend/cuda/jit/kernel_generators.hpp
index d048c0c7d0..cc67ac6996 100644
--- a/src/backend/cuda/jit/kernel_generators.hpp
+++ b/src/backend/cuda/jit/kernel_generators.hpp
@@ -48,18 +48,18 @@ int setKernelArguments(
 /// Generates the code to calculate the offsets for a buffer
 void generateBufferOffsets(std::stringstream& kerStream, int id, bool is_linear,
                            const std::string& type_str) {
-    std::string idx_str = std::string("int idx") + std::to_string(id);
+    const std::string idx_str  = std::string("idx") + std::to_string(id);
+    const std::string info_str = std::string("in") + std::to_string(id);
 
     if (is_linear) {
-        kerStream << idx_str << " = idx;\n";
+        kerStream << "#define " << idx_str << " idx\n";
     } else {
-        std::string info_str = std::string("in") + std::to_string(id);
-        kerStream << idx_str << " = (id3 < " << info_str << ".dims[3]) * "
-                  << info_str << ".strides[3] * id3 + (id2 < " << info_str
-                  << ".dims[2]) * " << info_str << ".strides[2] * id2 + (id1 < "
-                  << info_str << ".dims[1]) * " << info_str
-                  << ".strides[1] * id1 + (id0 < " << info_str
-                  << ".dims[0]) * id0;\n";
+        kerStream << "int " << idx_str << " = id0*(id0<" << info_str
+                  << ".dims[0])*" << info_str << ".strides[0] + id1*(id1<"
+                  << info_str << ".dims[1])*" << info_str
+                  << ".strides[1] + id2*(id2<" << info_str << ".dims[2])*"
+                  << info_str << ".strides[2] + id3*(id3<" << info_str
+                  << ".dims[3])*" << info_str << ".strides[3];\n";
         kerStream << type_str << " *in" << id << "_ptr = in" << id << ".ptr;\n";
     }
 }
@@ -75,28 +75,24 @@ inline void generateShiftNodeOffsets(std::stringstream& kerStream, int id,
                                      bool is_linear,
                                      const std::string& type_str) {
     UNUSED(is_linear);
-    std::string idx_str   = std::string("idx") + std::to_string(id);
-    std::string info_str  = std::string("in") + std::to_string(id);
-    std::string id_str    = std::string("sh_id_") + std::to_string(id) + "_";
-    std::string shift_str = std::string("shift") + std::to_string(id) + "_";
+    const std::string idx_str  = std::string("idx") + std::to_string(id);
+    const std::string info_str = std::string("in") + std::to_string(id);
+    const std::string id_str = std::string("sh_id_") + std::to_string(id) + '_';
+    const std::string shift_str =
+        std::string("shift") + std::to_string(id) + '_';
 
     for (int i = 0; i < 4; i++) {
         kerStream << "int " << id_str << i << " = __circular_mod(id" << i
                   << " + " << shift_str << i << ", " << info_str << ".dims["
                   << i << "]);\n";
     }
-
-    kerStream << "int " << idx_str << " = (" << id_str << "3 < " << info_str
-              << ".dims[3]) * " << info_str << ".strides[3] * " << id_str
-              << "3;\n";
-    kerStream << idx_str << " += (" << id_str << "2 < " << info_str
-              << ".dims[2]) * " << info_str << ".strides[2] * " << id_str
-              << "2;\n";
-    kerStream << idx_str << " += (" << id_str << "1 < " << info_str
-              << ".dims[1]) * " << info_str << ".strides[1] * " << id_str
-              << "1;\n";
-    kerStream << idx_str << " += (" << id_str << "0 < " << info_str
-              << ".dims[0]) * " << id_str << "0;\n";
+    kerStream << "int " << idx_str << " = " << id_str << "0*(" << id_str << "0<"
+              << info_str << ".dims[0])*" << info_str << ".strides[0] + "
+              << id_str << "1*(" << id_str << "1<" << info_str << ".dims[1])*"
+              << info_str << ".strides[1] + " << id_str << "2*(" << id_str
+              << "2<" << info_str << ".dims[2])*" << info_str
+              << ".strides[2] + " << id_str << "3*(" << id_str << "3<"
+              << info_str << ".dims[3])*" << info_str << ".strides[3];\n";
     kerStream << type_str << " *in" << id << "_ptr = in" << id << ".ptr;\n";
 }
 
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 06d2b41b08..8d717680d6 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -9,7 +9,6 @@
 
 #include <Array.hpp>
 #include <common/compile_module.hpp>
-#include <common/dispatch.hpp>
 #include <common/jit/ModdimNode.hpp>
 #include <common/jit/Node.hpp>
 #include <common/jit/NodeIterator.hpp>
@@ -18,12 +17,14 @@
 #include <copy.hpp>
 #include <device_manager.hpp>
 #include <err_opencl.hpp>
+#include <jit/BufferNode.hpp>
 #include <kernel_headers/jit.hpp>
+#include <threadsMgt.hpp>
+#include <type_util.hpp>
 #include <af/dim4.hpp>
 #include <af/opencl.h>
 
-#include <jit/BufferNode.hpp>
-
+#include <algorithm>
 #include <cstdio>
 #include <functional>
 #include <sstream>
@@ -31,139 +32,244 @@
 #include <string>
 #include <vector>
 
+using common::findModule;
 using common::getFuncName;
+using common::ModdimNode;
 using common::Node;
 using common::Node_ids;
 using common::Node_map_t;
+using common::Node_ptr;
+using common::NodeIterator;
 
 using cl::Kernel;
 using cl::NDRange;
 using cl::NullRange;
 
+using std::equal;
+using std::for_each;
+using std::shared_ptr;
 using std::string;
 using std::stringstream;
 using std::to_string;
 using std::vector;
 
 namespace opencl {
+using jit::BufferNode;
 
-string getKernelString(const string &funcName, const vector<Node *> &full_nodes,
-                       const vector<Node_ids> &full_ids,
-                       const vector<int> &output_ids, bool is_linear) {
+string getKernelString(const string& funcName, const vector<Node*>& full_nodes,
+                       const vector<Node_ids>& full_ids,
+                       const vector<int>& output_ids, const bool is_linear,
+                       const bool loop0, const bool loop1, const bool loop3) {
     // Common OpenCL code
     // This part of the code does not change with the kernel.
 
-    static const char *kernelVoid = "__kernel void\n";
-    static const char *dimParams =
-        "KParam oInfo, uint groups_0, uint groups_1, uint num_odims";
-    static const char *blockStart = "{\n";
-    static const char *blockEnd   = "\n}\n";
-
-    static const char *linearIndex = R"JIT(
-        uint groupId  = get_group_id(1) * get_num_groups(0) + get_group_id(0);
-        uint threadId = get_local_id(0);
-        int idx = groupId * get_local_size(0) * get_local_size(1) + threadId;
-        if (idx >= oInfo.dims[3] * oInfo.strides[3]) return;
-        )JIT";
-
-    static const char *generalIndex = R"JIT(
-        uint id0 = 0, id1 = 0, id2 = 0, id3 = 0;
-        if (num_odims > 2) {
-            id2 = get_group_id(0) / groups_0;
-            id0 = get_group_id(0) - id2 * groups_0;
-            id0 = get_local_id(0) + id0 * get_local_size(0);
-            if (num_odims > 3) {
-                id3 = get_group_id(1) / groups_1;
-                id1 = get_group_id(1) - id3 * groups_1;
-                id1 = get_local_id(1) + id1 * get_local_size(1);
-            } else {
-                id1 = get_global_id(1);
-            }
-        } else {
-            id3 = 0;
-            id2 = 0;
-            id1 = get_global_id(1);
-            id0 = get_global_id(0);
-        }
-        bool cond = id0 < oInfo.dims[0] &&
-                    id1 < oInfo.dims[1] &&
-                    id2 < oInfo.dims[2] &&
-                    id3 < oInfo.dims[3];
-        if (!cond) return;
-        int idx = oInfo.strides[3] * id3 +
-                  oInfo.strides[2] * id2 +
-                  oInfo.strides[1] * id1 +
-                  id0 + oInfo.offset;
-        )JIT";
-
-    stringstream inParamStream;
-    stringstream outParamStream;
-    stringstream outWriteStream;
-    stringstream offsetsStream;
-    stringstream opsStream;
-
-    for (size_t i = 0; i < full_nodes.size(); i++) {
-        const auto &node     = full_nodes[i];
-        const auto &ids_curr = full_ids[i];
+    static const char* kernelVoid = R"JIT(
+__kernel void )JIT";
+    static const char* dimParams  = "KParam oInfo";
+    static const char* blockStart = "{";
+    static const char* blockEnd   = "\n}\n";
+
+    static const char* linearInit = R"JIT(
+   int idx = get_global_id(0);
+   const int idxEnd = oInfo.dims[0];
+   if (idx < idxEnd) {
+)JIT";
+    static const char* linearEnd  = R"JIT(
+   })JIT";
+
+    static const char* linearLoop0Start = R"JIT(
+        const int idxID0Inc = get_global_size(0);
+        do {)JIT";
+    static const char* linearLoop0End   = R"JIT(
+            idx += idxID0Inc;
+            if (idx >= idxEnd) break;
+        } while (true);)JIT";
+
+    // ///////////////////////////////////////////////
+    // oInfo = output optimized information (dims, strides, offset).
+    //         oInfo has removed dimensions, to optimized block scheduling
+    // iInfo = input internal information (dims, strides, offset)
+    //         iInfo has the original dimensions, auto generated code
+    //
+    // Loop3 is fastest and becomes inside loop, since
+    //      - #of loops is known upfront
+    // Loop1 is used for extra dynamic looping (writing into cache)
+    // All loops are conditional and idependent
+    // Format Loop1 & Loop3
+    // ////////////////////////////
+    //  *stridedLoopNInit               // Always
+    //  *stridedLoop1Init               // Conditional
+    //  *stridedLoop2Init               // Conditional
+    //  *stridedLoop3Init               // Conditional
+    //  *stridedLoop1Start              // Conditional
+    //      *stridedLoop3Start          // Conditional
+    //          auto generated code     // Always
+    //      *stridedLoop3End            // Conditional
+    //  *stridedLoop1End                // Conditional
+    //  *StridedEnd                     // Always
+    //
+    // format loop0 (Vector only)
+    // //////////////////////////
+    // *stridedLoop0Init                // Always
+    // *stridedLoop0Start               // Always
+    //      auto generated code         // Always
+    // *stridedLoop0End                 // Always
+    // *stridedEnd                      // Always
+
+    static const char* stridedLoop0Init  = R"JIT(
+    int id0 = get_global_id(0);
+    const int id0End = oInfo.dims[0];
+    if (id0 < id0End) {
+#define id1 0
+#define id2 0
+#define id3 0
+        const int ostrides0 = oInfo.strides[0];
+        int idx = ostrides0*id0;)JIT";
+    static const char* stridedLoop0Start = R"JIT(
+        const int id0Inc = get_global_size(0);
+        const int idxID0Inc = ostrides0*id0Inc;
+        do {)JIT";
+    static const char* stridedLoop0End   = R"JIT(
+            id0 += id0Inc;
+            if (id0 >= id0End) break;
+            idx += idxID0Inc;
+        } while (true);)JIT";
+
+    // -------------
+    static const char* stridedLoopNInit = R"JIT(
+    int id0 = get_global_id(0);
+    int id1 = get_global_id(1);
+    const int id0End = oInfo.dims[0];
+    const int id1End = oInfo.dims[1];
+    if ((id0 < id0End) & (id1 < id1End)) {
+        const int id2 = get_global_id(2);
+#define id3 0
+        const int ostrides1 = oInfo.strides[1];
+        int idx = (int)oInfo.strides[0]*id0 + ostrides1*id1 + (int)oInfo.strides[2]*id2;)JIT";
+    static const char* stridedEnd       = R"JIT(
+    })JIT";
+
+    static const char* stridedLoop3Init  = R"JIT(
+#undef id3
+        int id3 = 0;
+        const int id3End = oInfo.dims[3];
+        const int idxID3Inc = oInfo.strides[3];)JIT";
+    static const char* stridedLoop3Start = R"JIT(
+                const int idxBaseID3 = idx;
+                do {)JIT";
+    static const char* stridedLoop3End   = R"JIT(
+                    ++id3;
+                    if (id3 == id3End) break;
+                    idx += idxID3Inc;
+                } while (true);
+                id3 = 0;
+                idx = idxBaseID3;)JIT";
+
+    static const char* stridedLoop1Init  = R"JIT(
+        const int id1Inc = get_global_size(1);
+        const int idxID1Inc = id1Inc * ostrides1;)JIT";
+    static const char* stridedLoop1Start = R"JIT(
+        do {)JIT";
+    static const char* stridedLoop1End   = R"JIT(
+            id1 += id1Inc;
+            if (id1 >= id1End) break;
+            idx += idxID1Inc;
+        } while (true);)JIT";
+
+    // Reuse stringstreams, because they are very costly during initilization
+    thread_local stringstream inParamStream;
+    thread_local stringstream outParamStream;
+    thread_local stringstream outOffsetStream;
+    thread_local stringstream inOffsetsStream;
+    thread_local stringstream opsStream;
+
+    int oid{0};
+    for (size_t i{0}; i < full_nodes.size(); i++) {
+        const auto& node{full_nodes[i]};
+        const auto& ids_curr{full_ids[i]};
         // Generate input parameters, only needs current id
         node->genParams(inParamStream, ids_curr.id, is_linear);
         // Generate input offsets, only needs current id
-        node->genOffsets(offsetsStream, ids_curr.id, is_linear);
+        node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
         // Generate the core function body, needs children ids as well
         node->genFuncs(opsStream, ids_curr);
+        for (auto outIt{begin(output_ids)}, endIt{end(output_ids)};
+             (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) {
+            // Generate also output parameters
+            outParamStream << "__global "
+                           << full_nodes[ids_curr.id]->getTypeStr() << " *out"
+                           << oid << ", int offset" << oid << ",\n";
+            // Apply output offset
+            outOffsetStream << "\nout" << oid << " += offset" << oid << ';';
+            // Generate code to write the output
+            opsStream << "out" << oid << "[idx] = val" << ids_curr.id << ";\n";
+            ++oid;
+        }
     }
 
-    for (int id : output_ids) {
-        // Generate output parameters
-        outParamStream << "__global " << full_nodes[id]->getTypeStr() << " *out"
-                       << id << ", \n";
-        // Generate code to write the output
-        outWriteStream << "out" << id << "[idx] = val" << id << ";\n";
-    }
-
-    // Put various blocks into a single stream
-    stringstream kerStream;
-    kerStream << kernelVoid;
-    kerStream << funcName;
-    kerStream << "(\n";
-    kerStream << inParamStream.str();
-    kerStream << outParamStream.str();
-    kerStream << dimParams;
-    kerStream << ")\n";
-    kerStream << blockStart;
+    thread_local stringstream kerStream;
+    kerStream << kernelVoid << funcName << "(\n"
+              << inParamStream.str() << outParamStream.str() << dimParams << ")"
+              << blockStart;
     if (is_linear) {
-        kerStream << linearIndex;
+        kerStream << linearInit << inOffsetsStream.str()
+                  << outOffsetStream.str() << '\n';
+        if (loop0) kerStream << linearLoop0Start;
+        kerStream << "\n\n" << opsStream.str();
+        if (loop0) kerStream << linearLoop0End;
+        kerStream << linearEnd;
     } else {
-        kerStream << generalIndex;
+        if (loop0) {
+            kerStream << stridedLoop0Init << outOffsetStream.str() << '\n'
+                      << stridedLoop0Start;
+        } else {
+            kerStream << stridedLoopNInit << outOffsetStream.str() << '\n';
+            if (loop3) kerStream << stridedLoop3Init;
+            if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
+            if (loop3) kerStream << stridedLoop3Start;
+        }
+        kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
+        if (loop3) kerStream << stridedLoop3End;
+        if (loop1) kerStream << stridedLoop1End;
+        if (loop0) kerStream << stridedLoop0End;
+        kerStream << stridedEnd;
     }
-    kerStream << offsetsStream.str();
-    kerStream << opsStream.str();
-    kerStream << outWriteStream.str();
     kerStream << blockEnd;
+    const string ret{kerStream.str()};
 
-    return kerStream.str();
-}
+    // Prepare for next round, limit memory
+    inParamStream.str("");
+    outParamStream.str("");
+    inOffsetsStream.str("");
+    outOffsetStream.str("");
+    opsStream.str("");
+    kerStream.str("");
 
-cl::Kernel getKernel(const vector<Node *> &output_nodes,
-                     const vector<int> &output_ids,
-                     const vector<Node *> &full_nodes,
-                     const vector<Node_ids> &full_ids, const bool is_linear) {
-    const string funcName =
-        getFuncName(output_nodes, full_nodes, full_ids, is_linear);
-    const size_t moduleKey = deterministicHash(funcName);
+    return ret;
+}
 
-    // A forward lookup in module cache helps avoid recompiling the jit
-    // source generated from identical jit-trees. It also enables us
-    // with a way to save jit kernels to disk only once
-    auto entry = common::findModule(getActiveDeviceId(), moduleKey);
+cl::Kernel getKernel(const vector<Node*>& output_nodes,
+                     const vector<int>& output_ids,
+                     const vector<Node*>& full_nodes,
+                     const vector<Node_ids>& full_ids, const bool is_linear,
+                     const bool loop0, const bool loop1, const bool loop3) {
+    const string funcName{getFuncName(output_nodes, full_nodes, full_ids,
+                                      is_linear, loop0, loop1, false, loop3)};
+    // A forward lookup in module cache helps avoid recompiling the JIT
+    // source generated from identical JIT-trees.
+    const auto entry{
+        findModule(getActiveDeviceId(), deterministicHash(funcName))};
 
     if (!entry) {
-        string jitKer = getKernelString(funcName, full_nodes, full_ids,
-                                        output_ids, is_linear);
-        common::Source jitKer_cl_src{
+        const string jitKer{getKernelString(funcName, full_nodes, full_ids,
+                                            output_ids, is_linear, loop0, loop1,
+                                            loop3)};
+        saveKernel(funcName, jitKer, ".cl");
+
+        const common::Source jitKer_cl_src{
             jitKer.data(), jitKer.size(),
             deterministicHash(jitKer.data(), jitKer.size())};
-        int device = getActiveDeviceId();
+        const cl::Device device{getDevice()};
         vector<string> options;
         if (isDoubleSupported(device)) {
             options.emplace_back(DefineKey(USE_DOUBLE));
@@ -171,9 +277,6 @@ cl::Kernel getKernel(const vector<Node *> &output_nodes,
         if (isHalfSupported(device)) {
             options.emplace_back(DefineKey(USE_HALF));
         }
-
-        saveKernel(funcName, jitKer, ".cl");
-
         return common::getKernel(funcName, {jit_cl_src, jitKer_cl_src}, {},
                                  options, true)
             .get();
@@ -181,152 +284,190 @@ cl::Kernel getKernel(const vector<Node *> &output_nodes,
     return common::getKernel(entry, funcName, true).get();
 }
 
-void evalNodes(vector<Param> &outputs, const vector<Node *> &output_nodes) {
-    if (outputs.empty()) { return; }
-
-    // Assume all ouputs are of same size
-    // FIXME: Add assert to check if all outputs are same size?
-    KParam out_info    = outputs[0].info;
-    dim_t *outDims     = out_info.dims;
-    size_t numOutElems = outDims[0] * outDims[1] * outDims[2] * outDims[3];
+void evalNodes(vector<Param>& outputs, const vector<Node*>& output_nodes) {
+    const unsigned nrOutputs{static_cast<unsigned>(outputs.size())};
+    if (nrOutputs == 0) { return; }
+    assert(outputs.size() == output_nodes.size());
+    KParam& out_info{outputs[0].info};
+    dim_t* outDims{out_info.dims};
+    dim_t* outStrides{out_info.strides};
+    for_each(begin(outputs)++, end(outputs),
+             [outDims, outStrides](Param& output) {
+                 assert(equal(output.info.dims, output.info.dims + AF_MAX_DIMS,
+                              outDims) &&
+                        equal(output.info.strides,
+                              output.info.strides + AF_MAX_DIMS, outStrides));
+             });
+
+    dim_t ndims{outDims[3] > 1   ? 4
+                : outDims[2] > 1 ? 3
+                : outDims[1] > 1 ? 2
+                : outDims[0] > 0 ? 1
+                                 : 0};
+    bool is_linear{true};
+    dim_t numOutElems{1};
+    for (dim_t dim{0}; dim < ndims; ++dim) {
+        is_linear &= (numOutElems == outStrides[dim]);
+        numOutElems *= outDims[dim];
+    }
     if (numOutElems == 0) { return; }
 
     // Use thread local to reuse the memory every time you are here.
     thread_local Node_map_t nodes;
-    thread_local vector<Node *> full_nodes;
+    thread_local vector<Node*> full_nodes;
     thread_local vector<Node_ids> full_ids;
     thread_local vector<int> output_ids;
 
     // Reserve some space to improve performance at smaller sizes
-    if (nodes.empty()) {
-        nodes.reserve(1024);
-        output_ids.reserve(output_nodes.size());
-        full_nodes.reserve(1024);
-        full_ids.reserve(1024);
+    constexpr size_t CAP{1024};
+    if (full_nodes.capacity() < CAP) {
+        nodes.reserve(CAP);
+        output_ids.reserve(10);
+        full_nodes.reserve(CAP);
+        full_ids.reserve(CAP);
     }
 
-    for (auto *node : output_nodes) {
-        int id = node->getNodesMap(nodes, full_nodes, full_ids);
+    const af::dtype outputType{output_nodes[0]->getType()};
+    const size_t outputSizeofType{size_of(outputType)};
+    for (Node* node : output_nodes) {
+        assert(node->getType() == outputType);
+        const int id{node->getNodesMap(nodes, full_nodes, full_ids)};
         output_ids.push_back(id);
     }
 
-    using common::ModdimNode;
-    using common::NodeIterator;
-    using jit::BufferNode;
-
-    // find all moddims in the tree
-    vector<std::shared_ptr<Node>> node_clones;
-    for (auto *node : full_nodes) { node_clones.emplace_back(node->clone()); }
-
-    for (common::Node_ids ids : full_ids) {
-        auto &children = node_clones[ids.id]->m_children;
-        for (int i = 0; i < Node::kMaxChildren && children[i] != nullptr; i++) {
-            children[i] = node_clones[ids.child_ids[i]];
+    const size_t outputSize{numOutElems * outputSizeofType * nrOutputs};
+    size_t inputSize{0};
+    unsigned nrInputs{0};
+    bool moddimsFound{false};
+    for (const Node* node : full_nodes) {
+        is_linear &= node->isLinear(outDims);
+        moddimsFound |= (node->getOp() == af_moddims_t);
+        if (node->isBuffer()) {
+            ++nrInputs;
+            inputSize += node->getBytes();
         }
     }
+    const size_t totalSize{inputSize + outputSize};
 
-    for (auto &node : node_clones) {
-        if (node->getOp() == af_moddims_t) {
-            ModdimNode *mn = static_cast<ModdimNode *>(node.get());
-            auto isBuffer  = [](const Node &ptr) { return ptr.isBuffer(); };
-
-            NodeIterator<> it(node.get());
-            auto new_strides = calcStrides(mn->m_new_shape);
-            while (it != NodeIterator<>()) {
-                it = find_if(it, NodeIterator<>(), isBuffer);
-                if (it == NodeIterator<>()) { break; }
-
-                BufferNode *buf = static_cast<BufferNode *>(&(*it));
-
-                buf->m_param.dims[0]    = mn->m_new_shape[0];
-                buf->m_param.dims[1]    = mn->m_new_shape[1];
-                buf->m_param.dims[2]    = mn->m_new_shape[2];
-                buf->m_param.dims[3]    = mn->m_new_shape[3];
-                buf->m_param.strides[0] = new_strides[0];
-                buf->m_param.strides[1] = new_strides[1];
-                buf->m_param.strides[2] = new_strides[2];
-                buf->m_param.strides[3] = new_strides[3];
-
-                ++it;
-            }
-        }
-    }
-
-    full_nodes.clear();
-    for (auto &node : node_clones) { full_nodes.push_back(node.get()); }
-
-    bool is_linear = true;
-    for (auto *node : full_nodes) {
-        is_linear &= node->isLinear(outputs[0].info.dims);
+    bool emptyColumnsFound{false};
+    if (is_linear) {
+        outDims[0]    = numOutElems;
+        outDims[1]    = 1;
+        outDims[2]    = 1;
+        outDims[3]    = 1;
+        outStrides[0] = 1;
+        outStrides[1] = numOutElems;
+        outStrides[2] = numOutElems;
+        outStrides[3] = numOutElems;
+        ndims         = 1;
+    } else {
+        emptyColumnsFound = ndims > (outDims[0] == 1   ? 1
+                                     : outDims[1] == 1 ? 2
+                                     : outDims[2] == 1 ? 3
+                                                       : 4);
     }
 
-    auto ker =
-        getKernel(output_nodes, output_ids, full_nodes, full_ids, is_linear);
-
-    uint local_0   = 1;
-    uint local_1   = 1;
-    uint global_0  = 1;
-    uint global_1  = 1;
-    uint groups_0  = 1;
-    uint groups_1  = 1;
-    uint num_odims = 4;
-
-    // CPUs seem to perform better with work group size 1024
-    const int work_group_size =
-        (getActiveDeviceType() == AFCL_DEVICE_TYPE_CPU) ? 1024 : 256;
-
-    while (num_odims >= 1) {
-        if (outDims[num_odims - 1] == 1) {
-            num_odims--;
-        } else {
-            break;
+    // Keep in global scope, so that the nodes remain active for later referral
+    // in case moddims operations or column elimination have to take place
+    vector<Node_ptr> node_clones;
+    // Avoid all cloning/copying when no moddims node is present (high chance)
+    if (moddimsFound | emptyColumnsFound) {
+        node_clones.reserve(full_nodes.size());
+        for (Node* node : full_nodes) {
+            node_clones.emplace_back(node->clone());
         }
-    }
 
-    if (is_linear) {
-        local_0           = work_group_size;
-        uint out_elements = outDims[3] * out_info.strides[3];
-        uint groups       = divup(out_elements, local_0);
-
-        global_1 = divup(groups, work_group_size) * local_1;
-        global_0 = divup(groups, global_1) * local_0;
-
-    } else {
-        local_1 = 4;
-        local_0 = work_group_size / local_1;
+        for (const Node_ids& ids : full_ids) {
+            auto& children{node_clones[ids.id]->m_children};
+            for (int i{0}; i < Node::kMaxChildren && children[i] != nullptr;
+                 i++) {
+                children[i] = node_clones[ids.child_ids[i]];
+            }
+        }
 
-        groups_0 = divup(outDims[0], local_0);
-        groups_1 = divup(outDims[1], local_1);
+        if (moddimsFound) {
+            const auto isModdim{[](const Node_ptr& ptr) {
+                return ptr->getOp() == af_moddims_t;
+            }};
+            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
+                 (nodeIt = find_if(nodeIt, endIt, isModdim)) != endIt;
+                 ++nodeIt) {
+                const ModdimNode* mn{static_cast<ModdimNode*>(nodeIt->get())};
+
+                const auto new_strides{calcStrides(mn->m_new_shape)};
+                const auto isBuffer{
+                    [](const Node& node) { return node.isBuffer(); }};
+                for (NodeIterator<> it{nodeIt->get()}, end{NodeIterator<>()};
+                     (it = find_if(it, end, isBuffer)) != end; ++it) {
+                    BufferNode* buf{static_cast<BufferNode*>(&(*it))};
+                    buf->m_param.dims[0]    = mn->m_new_shape[0];
+                    buf->m_param.dims[1]    = mn->m_new_shape[1];
+                    buf->m_param.dims[2]    = mn->m_new_shape[2];
+                    buf->m_param.dims[3]    = mn->m_new_shape[3];
+                    buf->m_param.strides[0] = new_strides[0];
+                    buf->m_param.strides[1] = new_strides[1];
+                    buf->m_param.strides[2] = new_strides[2];
+                    buf->m_param.strides[3] = new_strides[3];
+                }
+            }
+        }
+        if (emptyColumnsFound) {
+            const auto isBuffer{
+                [](const Node_ptr& ptr) { return ptr->isBuffer(); }};
+            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
+                 (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt;
+                 ++nodeIt) {
+                BufferNode* buf{static_cast<BufferNode*>(nodeIt->get())};
+                removeEmptyColumns(outDims, ndims, buf->m_param.dims,
+                                   buf->m_param.strides);
+            }
+            for_each(++begin(outputs), end(outputs),
+                     [outDims, ndims](Param& output) {
+                         removeEmptyColumns(outDims, ndims, output.info.dims,
+                                            output.info.strides);
+                     });
+            ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
+        }
 
-        global_0 = groups_0 * local_0 * outDims[2];
-        global_1 = groups_1 * local_1 * outDims[3];
+        full_nodes.clear();
+        for (Node_ptr& node : node_clones) { full_nodes.push_back(node.get()); }
     }
 
-    NDRange local(local_0, local_1);
-    NDRange global(global_0, global_1);
+    threadsMgt<dim_t> th(outDims, ndims, nrInputs, nrOutputs, totalSize,
+                         outputSizeofType);
+    auto ker = getKernel(output_nodes, output_ids, full_nodes, full_ids,
+                         is_linear, th.loop0, th.loop1, th.loop3);
+    const cl::NDRange local{th.genLocal(ker)};
+    const cl::NDRange global{th.genGlobal(local)};
 
-    int nargs = 0;
-    for (const auto &node : full_nodes) {
+    int nargs{0};
+    for (const Node* node : full_nodes) {
         nargs = node->setArgs(nargs, is_linear,
-                              [&ker](int id, const void *ptr, size_t arg_size) {
+                              [&ker](int id, const void* ptr, size_t arg_size) {
                                   ker.setArg(id, arg_size, ptr);
                               });
     }
 
     // Set output parameters
-    for (auto &output : outputs) {
-        ker.setArg(nargs, *(output.data));
-        ++nargs;
+    for (const auto& output : outputs) {
+        ker.setArg(nargs++, *(output.data));
+        ker.setArg(nargs++, static_cast<int>(output.info.offset));
     }
 
     // Set dimensions
     // All outputs are asserted to be of same size
     // Just use the size from the first output
-    ker.setArg(nargs + 0, out_info);
-    ker.setArg(nargs + 1, groups_0);
-    ker.setArg(nargs + 2, groups_1);
-    ker.setArg(nargs + 3, num_odims);
-
+    ker.setArg(nargs++, out_info);
+
+    {
+        using namespace opencl::kernel_logger;
+        AF_TRACE(
+            "Launching : Dims: [{},{},{},{}] Global: [{},{},{}] Local: "
+            "[{},{},{}] threads: {}",
+            outDims[0], outDims[1], outDims[2], outDims[3], global[0],
+            global[1], global[2], local[0], local[1], local[2],
+            global[0] * global[1] * global[2]);
+    }
     getQueue().enqueueNDRangeKernel(ker, NullRange, global, local);
 
     // Reset the thread local vectors
@@ -336,9 +477,9 @@ void evalNodes(vector<Param> &outputs, const vector<Node *> &output_nodes) {
     full_ids.clear();
 }
 
-void evalNodes(Param &out, Node *node) {
+void evalNodes(Param& out, Node* node) {
     vector<Param> outputs{out};
-    vector<Node *> nodes{node};
+    vector<Node*> nodes{node};
     return evalNodes(outputs, nodes);
 }
 
diff --git a/src/backend/opencl/jit/kernel_generators.hpp b/src/backend/opencl/jit/kernel_generators.hpp
index c2eb711c1b..fe87ebc21b 100644
--- a/src/backend/opencl/jit/kernel_generators.hpp
+++ b/src/backend/opencl/jit/kernel_generators.hpp
@@ -47,18 +47,21 @@ inline int setKernelArguments(
 inline void generateBufferOffsets(std::stringstream& kerStream, int id,
                                   bool is_linear, const std::string& type_str) {
     UNUSED(type_str);
-    std::string idx_str  = std::string("int idx") + std::to_string(id);
-    std::string info_str = std::string("iInfo") + std::to_string(id);
+    const std::string idx_str  = std::string("idx") + std::to_string(id);
+    const std::string info_str = std::string("iInfo") + std::to_string(id);
+    const std::string in_str   = std::string("in") + std::to_string(id);
 
     if (is_linear) {
-        kerStream << idx_str << " = idx + " << info_str << "_offset;\n";
+        kerStream << in_str << " += " << info_str << "_offset;\n"
+                  << "#define " << idx_str << " idx\n";
     } else {
-        kerStream << idx_str << " = (id3 < " << info_str << ".dims[3]) * "
-                  << info_str << ".strides[3] * id3 + (id2 < " << info_str
-                  << ".dims[2]) * " << info_str << ".strides[2] * id2 + (id1 < "
-                  << info_str << ".dims[1]) * " << info_str
-                  << ".strides[1] * id1 + (id0 < " << info_str
-                  << ".dims[0]) * id0 + " << info_str << ".offset;\n";
+        kerStream << "int " << idx_str << " = id0*(id0<" << info_str
+                  << ".dims[0])*" << info_str << ".strides[0] + id1*(id1<"
+                  << info_str << ".dims[1])*" << info_str
+                  << ".strides[1] + id2*(id2<" << info_str << ".dims[2])*"
+                  << info_str << ".strides[2] + id3*(id3<" << info_str
+                  << ".dims[3])*" << info_str << ".strides[3] + " << info_str
+                  << ".offset;\n";
     }
 }
 
@@ -74,28 +77,25 @@ inline void generateShiftNodeOffsets(std::stringstream& kerStream, int id,
                                      const std::string& type_str) {
     UNUSED(is_linear);
     UNUSED(type_str);
-    std::string idx_str   = std::string("idx") + std::to_string(id);
-    std::string info_str  = std::string("iInfo") + std::to_string(id);
-    std::string id_str    = std::string("sh_id_") + std::to_string(id) + "_";
-    std::string shift_str = std::string("shift") + std::to_string(id) + "_";
+    const std::string idx_str  = std::string("idx") + std::to_string(id);
+    const std::string info_str = std::string("iInfo") + std::to_string(id);
+    const std::string id_str = std::string("sh_id_") + std::to_string(id) + '_';
+    const std::string shift_str =
+        std::string("shift") + std::to_string(id) + '_';
 
     for (int i = 0; i < 4; i++) {
         kerStream << "int " << id_str << i << " = __circular_mod(id" << i
                   << " + " << shift_str << i << ", " << info_str << ".dims["
                   << i << "]);\n";
     }
-
-    kerStream << "int " << idx_str << " = (" << id_str << "3 < " << info_str
-              << ".dims[3]) * " << info_str << ".strides[3] * " << id_str
-              << "3;\n";
-    kerStream << idx_str << " += (" << id_str << "2 < " << info_str
-              << ".dims[2]) * " << info_str << ".strides[2] * " << id_str
-              << "2;\n";
-    kerStream << idx_str << " += (" << id_str << "1 < " << info_str
-              << ".dims[1]) * " << info_str << ".strides[1] * " << id_str
-              << "1;\n";
-    kerStream << idx_str << " += (" << id_str << "0 < " << info_str
-              << ".dims[0]) * " << id_str << "0 + " << info_str << ".offset;\n";
+    kerStream << "int " << idx_str << " = " << id_str << "0*(" << id_str << "0<"
+              << info_str << ".dims[0])*" << info_str << ".strides[0] + "
+              << id_str << "1*(" << id_str << "1<" << info_str << ".dims[1])*"
+              << info_str << ".strides[1] + " << id_str << "2*(" << id_str
+              << "2<" << info_str << ".dims[2])*" << info_str
+              << ".strides[2] + " << id_str << "3*(" << id_str << "3<"
+              << info_str << ".dims[3])*" << info_str << ".strides[3] + "
+              << info_str << ".offset;\n";
 }
 
 inline void generateShiftNodeRead(std::stringstream& kerStream, int id,

From bef4f10a9e31bb780bb89df675171fdee3207986 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 4 Aug 2022 01:11:18 +0200
Subject: [PATCH 075/473] OPT: join

---
 src/api/c/join.cpp                    | 113 ++++++-------
 src/backend/cuda/CMakeLists.txt       |   3 +-
 src/backend/cuda/join.cpp             | 213 ++++++++++++++++++------
 src/backend/cuda/kernel/join.cuh      |  50 ------
 src/backend/cuda/kernel/join.hpp      |  51 ------
 src/backend/cuda/platform.cpp         |   2 +-
 src/backend/opencl/CMakeLists.txt     |   2 +-
 src/backend/opencl/join.cpp           | 227 ++++++++++++++++++++------
 src/backend/opencl/kernel/join.cl     |  41 -----
 src/backend/opencl/kernel/join.hpp    |  55 -------
 src/backend/opencl/kernel/memcopy.hpp |  10 +-
 11 files changed, 397 insertions(+), 370 deletions(-)
 delete mode 100644 src/backend/cuda/kernel/join.cuh
 delete mode 100644 src/backend/cuda/kernel/join.hpp
 delete mode 100644 src/backend/opencl/kernel/join.cl
 delete mode 100644 src/backend/opencl/kernel/join.hpp

diff --git a/src/api/c/join.cpp b/src/api/c/join.cpp
index dad2bc1ffd..a31a728874 100644
--- a/src/api/c/join.cpp
+++ b/src/api/c/join.cpp
@@ -14,7 +14,9 @@
 #include <handle.hpp>
 #include <join.hpp>
 #include <af/data.h>
+
 #include <algorithm>
+#include <climits>
 #include <vector>
 
 using af::dim4;
@@ -43,30 +45,21 @@ static inline af_array join_many(const int dim, const unsigned n_arrays,
     vector<Array<T>> inputs_;
     inputs_.reserve(n_arrays);
 
-    for (unsigned i = 0; i < n_arrays; i++) {
-        inputs_.push_back(getArray<T>(inputs[i]));
-        if (inputs_.back().isEmpty()) { inputs_.pop_back(); }
+    dim_t dim_size{0};
+    for (unsigned i{0}; i < n_arrays; ++i) {
+        const Array<T> &iArray = getArray<T>(inputs[i]);
+        if (!iArray.isEmpty()) {
+            inputs_.push_back(iArray);
+            dim_size += iArray.dims().dims[dim];
+        }
     }
 
     // All dimensions except join dimension must be equal
     // calculate odims size
-    std::vector<af::dim4> idims(inputs_.size());
-    dim_t dim_size = 0;
-    for (unsigned i = 0; i < idims.size(); i++) {
-        idims[i] = inputs_[i].dims();
-        dim_size += idims[i][dim];
-    }
-
-    af::dim4 odims;
-    for (int i = 0; i < 4; i++) {
-        if (i == dim) {
-            odims[i] = dim_size;
-        } else {
-            odims[i] = idims[0][i];
-        }
-    }
+    af::dim4 odims{inputs_[0].dims()};
+    odims.dims[dim] = dim_size;
 
-    Array<T> out = createEmptyArray<T>(odims);
+    Array<T> out{createEmptyArray<T>(odims)};
     join<T>(out, dim, inputs_);
     return getHandle(out);
 }
@@ -74,24 +67,21 @@ static inline af_array join_many(const int dim, const unsigned n_arrays,
 af_err af_join(af_array *out, const int dim, const af_array first,
                const af_array second) {
     try {
-        const ArrayInfo &finfo = getInfo(first);
-        const ArrayInfo &sinfo = getInfo(second);
-        dim4 fdims             = finfo.dims();
-        dim4 sdims             = sinfo.dims();
+        const ArrayInfo &finfo{getInfo(first)};
+        const ArrayInfo &sinfo{getInfo(second)};
+        const dim4 &fdims{finfo.dims()};
+        const dim4 &sdims{sinfo.dims()};
 
         ARG_ASSERT(1, dim >= 0 && dim < 4);
         ARG_ASSERT(2, finfo.getType() == sinfo.getType());
         if (sinfo.elements() == 0) { return af_retain_array(out, first); }
-
         if (finfo.elements() == 0) { return af_retain_array(out, second); }
-
-        DIM_ASSERT(2, sinfo.elements() > 0);
-        DIM_ASSERT(3, finfo.elements() > 0);
+        DIM_ASSERT(2, finfo.elements() > 0);
+        DIM_ASSERT(3, sinfo.elements() > 0);
 
         // All dimensions except join dimension must be equal
-        // Compute output dims
-        for (int i = 0; i < 4; i++) {
-            if (i != dim) { DIM_ASSERT(2, fdims[i] == sdims[i]); }
+        for (int i{0}; i < AF_MAX_DIMS; i++) {
+            if (i != dim) { DIM_ASSERT(2, fdims.dims[i] == sdims.dims[i]); }
         }
 
         af_array output;
@@ -125,55 +115,46 @@ af_err af_join_many(af_array *out, const int dim, const unsigned n_arrays,
         ARG_ASSERT(3, inputs != nullptr);
 
         if (n_arrays == 1) {
-            af_array ret = nullptr;
-            AF_CHECK(af_retain_array(&ret, inputs[0]));
+            af_array ret{nullptr};
+            AF_CHECK(af_retain_array(&ret, *inputs));
             std::swap(*out, ret);
             return AF_SUCCESS;
         }
 
-        vector<ArrayInfo> info;
-        info.reserve(n_arrays);
-        vector<af::dim4> dims(n_arrays);
-        for (unsigned i = 0; i < n_arrays; i++) {
-            info.push_back(getInfo(inputs[i]));
-            dims[i] = info[i].dims();
-        }
+        ARG_ASSERT(1, dim >= 0 && dim < AF_MAX_DIMS);
+        ARG_ASSERT(2, n_arrays > 0);
 
-        ARG_ASSERT(1, dim >= 0 && dim < 4);
-
-        bool allEmpty = std::all_of(
-            info.begin(), info.end(),
-            [](const ArrayInfo &i) -> bool { return i.elements() <= 0; });
-        if (allEmpty) {
+        const af_array *inputIt{inputs};
+        const af_array *inputEnd{inputs + n_arrays};
+        while ((inputIt != inputEnd) && (getInfo(*inputIt).elements() == 0)) {
+            ++inputIt;
+        }
+        if (inputIt == inputEnd) {
+            // All arrays have 0 elements
             af_array ret = nullptr;
-            AF_CHECK(af_retain_array(&ret, inputs[0]));
+            AF_CHECK(af_retain_array(&ret, *inputs));
             std::swap(*out, ret);
             return AF_SUCCESS;
         }
 
-        auto first_valid_afinfo = std::find_if(
-            info.begin(), info.end(),
-            [](const ArrayInfo &i) -> bool { return i.elements() > 0; });
-
-        af_dtype assertType = first_valid_afinfo->getType();
-        for (unsigned i = 1; i < n_arrays; i++) {
-            if (info[i].elements() > 0) {
-                ARG_ASSERT(3, assertType == info[i].getType());
-            }
-        }
-
-        // All dimensions except join dimension must be equal
-        af::dim4 assertDims = first_valid_afinfo->dims();
-        for (int i = 0; i < 4; i++) {
-            if (i != dim) {
-                for (unsigned j = 0; j < n_arrays; j++) {
-                    if (info[j].elements() > 0) {
-                        DIM_ASSERT(3, assertDims[i] == dims[j][i]);
+        // inputIt points to first non empty array
+        const af_dtype assertType{getInfo(*inputIt).getType()};
+        const dim4 &assertDims{getInfo(*inputIt).dims()};
+
+        // Check all remaining arrays on assertType and assertDims
+        while (++inputIt != inputEnd) {
+            const ArrayInfo &info = getInfo(*inputIt);
+            if (info.elements() > 0) {
+                ARG_ASSERT(3, assertType == info.getType());
+                const dim4 &infoDims{getInfo(*inputIt).dims()};
+                // All dimensions except join dimension must be equal
+                for (int i{0}; i < AF_MAX_DIMS; i++) {
+                    if (i != dim) {
+                        DIM_ASSERT(3, assertDims.dims[i] == infoDims.dims[i]);
                     }
                 }
             }
         }
-
         af_array output;
 
         switch (assertType) {
@@ -190,7 +171,7 @@ af_err af_join_many(af_array *out, const int dim, const unsigned n_arrays,
             case u16: output = join_many<ushort>(dim, n_arrays, inputs); break;
             case u8: output = join_many<uchar>(dim, n_arrays, inputs); break;
             case f16: output = join_many<half>(dim, n_arrays, inputs); break;
-            default: TYPE_ERROR(1, info[0].getType());
+            default: TYPE_ERROR(1, assertType);
         }
         swap(*out, output);
     }
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 3fcf1d2259..a6a750f83f 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -208,7 +208,6 @@ set(nvrtc_src
   ${CMAKE_CURRENT_SOURCE_DIR}/kernel/index.cuh
   ${CMAKE_CURRENT_SOURCE_DIR}/kernel/iota.cuh
   ${CMAKE_CURRENT_SOURCE_DIR}/kernel/ireduce.cuh
-  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/join.cuh
   ${CMAKE_CURRENT_SOURCE_DIR}/kernel/lookup.cuh
   ${CMAKE_CURRENT_SOURCE_DIR}/kernel/lu_split.cuh
   ${CMAKE_CURRENT_SOURCE_DIR}/kernel/match_template.cuh
@@ -458,7 +457,6 @@ cuda_add_library(afcuda
     kernel/interp.hpp
     kernel/iota.hpp
     kernel/ireduce.hpp
-    kernel/join.hpp
     kernel/lookup.hpp
     kernel/lu_split.hpp
     kernel/match_template.hpp
@@ -659,6 +657,7 @@ cuda_add_library(afcuda
     svd.hpp
     tile.cpp
     tile.hpp
+    threadsMgt.hpp
     topk.hpp
     traits.hpp
     transform.hpp
diff --git a/src/backend/cuda/join.cpp b/src/backend/cuda/join.cpp
index 880716e22b..a605867863 100644
--- a/src/backend/cuda/join.cpp
+++ b/src/backend/cuda/join.cpp
@@ -11,76 +11,191 @@
 #include <common/half.hpp>
 #include <err_cuda.hpp>
 #include <join.hpp>
-#include <kernel/join.hpp>
+#include <kernel/memcopy.hpp>
 
 #include <algorithm>
+#include <map>
 #include <stdexcept>
+#include <vector>
 
+using af::dim4;
 using common::half;
+using common::Node;
+using common::Node_ptr;
+using std::vector;
 
 namespace cuda {
 
-af::dim4 calcOffset(const af::dim4 &dims, const int dim) {
-    af::dim4 offset;
-    offset[0] = (dim == 0) * dims[0];
-    offset[1] = (dim == 1) * dims[1];
-    offset[2] = (dim == 2) * dims[2];
-    offset[3] = (dim == 3) * dims[3];
-    return offset;
-}
-
 template<typename T>
-Array<T> join(const int dim, const Array<T> &first, const Array<T> &second) {
+Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     // All dimensions except join dimension must be equal
+    const dim4 &fdims{first.dims()};
+    const dim4 &sdims{second.dims()};
     // Compute output dims
-    af::dim4 odims;
-    af::dim4 fdims = first.dims();
-    af::dim4 sdims = second.dims();
+    dim4 odims(fdims);
+    odims.dims[jdim] += sdims.dims[jdim];
+    Array<T> out{createEmptyArray<T>(odims)};
+    const cudaStream_t activeStream{getActiveStream()};
+
+    // topspeed is achieved when byte size(in+out) ~= L2CacheSize
+    //
+    // 1 array: memcpy always copies 1 array.  topspeed
+    //      --> size(in) < L2CacheSize/2
+    // 2 arrays: topspeeds
+    //      - size(in) < L2CacheSize/2/2
+    //          --> JIT can copy 2 arrays in // and is fastest
+    //              (condition: array sizes have to be identical)
+    //      - size(in) < L2CacheSize/2
+    //          --> memcpy will achieve highest speed, although the kernel
+    //              has to be called twice
+    //      - size(in) >= L2CacheSize/2
+    //          --> memcpy will achieve veryLargeArray speed.  The kernel
+    //              will be called twice
+    if (fdims.dims[jdim] == sdims.dims[jdim]) {
+        const size_t L2CacheSize{getL2CacheSize(getActiveDeviceId())};
+        if (!(first.isReady() | second.isReady()) ||
+            (fdims.elements() * sizeof(T) * 2 * 2 < L2CacheSize)) {
+            // Both arrays have same size & everything fits into the cache,
+            // so treat in 1 JIT kernel, iso individual copies which is
+            // always slower
+            const dim_t *outStrides{out.strides().dims};
+            vector<Param<T>> outputs{
+                {out.get(), fdims.dims, outStrides},
+                {out.get() + fdims.dims[jdim] * outStrides[jdim], sdims.dims,
+                 outStrides}};
+            // Extend the life of the returned node, by saving the
+            // corresponding shared_ptr
+            const Node_ptr fNode{first.getNode()};
+            const Node_ptr sNode{second.getNode()};
+            vector<Node *> nodes{fNode.get(), sNode.get()};
+            evalNodes(outputs, nodes);
+            return out;
+        }
+        // continue because individually processing is faster
+    }
 
-    for (int i = 0; i < 4; i++) {
-        if (i == dim) {
-            odims[i] = fdims[i] + sdims[i];
+    // Handle each array individually
+    if (first.isReady()) {
+        if (1LL + jdim >= first.ndims() && first.isLinear()) {
+            // first & out are linear
+            CUDA_CHECK(cudaMemcpyAsync(out.get(), first.get(),
+                                       first.elements() * sizeof(T),
+                                       cudaMemcpyDeviceToDevice, activeStream));
         } else {
-            odims[i] = fdims[i];
+            kernel::memcopy<T>(out, first, first.ndims());
         }
+    } else {
+        // Write the result directly in the out array
+        const Param<T> output(out.get(), fdims.dims, out.strides().dims);
+        evalNodes(output, first.getNode().get());
     }
 
-    Array<T> out = createEmptyArray<T>(odims);
-
-    af::dim4 zero(0, 0, 0, 0);
-
-    kernel::join<T>(out, first, zero, dim);
-    kernel::join<T>(out, second, calcOffset(fdims, dim), dim);
+    if (second.isReady()) {
+        if (1LL + jdim >= second.ndims() && second.isLinear()) {
+            // second & out are linear
+            CUDA_CHECK(cudaMemcpyAsync(
+                out.get() + fdims.dims[jdim] * out.strides().dims[jdim],
+                second.get(), second.elements() * sizeof(T),
+                cudaMemcpyDeviceToDevice, activeStream));
+        } else {
+            Param<T> output(
+                out.get() + fdims.dims[jdim] * out.strides().dims[jdim],
+                sdims.dims, out.strides().dims);
+            kernel::memcopy<T>(output, second, second.ndims());
+        }
+    } else {
+        // Write the result directly in the out array
+        const Param<T> output(
+            out.get() + fdims.dims[jdim] * out.strides().dims[jdim], sdims.dims,
+            out.strides().dims);
+        evalNodes(output, second.getNode().get());
+    }
 
-    return out;
+    return (out);
 }
 
 template<typename T>
-void join_wrapper(const int dim, Array<T> &out,
-                  const std::vector<Array<T>> &inputs) {
-    af::dim4 zero(0, 0, 0, 0);
-    af::dim4 d = zero;
-
-    kernel::join<T>(out, inputs[0], zero, dim);
-    for (size_t i = 1; i < inputs.size(); i++) {
-        d += inputs[i - 1].dims();
-        kernel::join<T>(out, inputs[i], calcOffset(d, dim), dim);
+void join(Array<T> &out, const int jdim, const vector<Array<T>> &inputs) {
+    class eval {
+       public:
+        vector<Param<T>> outputs;
+        vector<Node_ptr> nodePtrs;
+        vector<Node *> nodes;
+        vector<const Array<T> *> ins;
+    };
+    std::map<dim_t, eval> evals;
+    const cudaStream_t activeStream{getActiveStream()};
+    const size_t L2CacheSize{getL2CacheSize(getActiveDeviceId())};
+
+    // topspeed is achieved when byte size(in+out) ~= L2CacheSize
+    //
+    // 1 array: memcpy always copies 1 array.  topspeed
+    //      --> size(in) <= L2CacheSize/2
+    // 2 arrays: topspeeds
+    //      - size(in) < L2CacheSize/2/2
+    //          --> JIT can copy 2 arrays in // and is fastest
+    //              (condition: array sizes have to be identical)
+    //      - else
+    //          --> memcpy will achieve highest speed, although the kernel
+    //              has to be called twice
+    // 3 arrays: topspeeds
+    //      - size(in) < L2CacheSize/2/3
+    //          --> JIT can copy 3 arrays in // and is fastest
+    //              (condition: array sizes have to be identical)
+    //      - else
+    //          --> memcpy will achieve highest speed, although the kernel
+    //              has to be called multiple times
+
+    // Group all arrays according to size
+    dim_t outOffset{0};
+    for (const Array<T> &iArray : inputs) {
+        const dim_t *idims{iArray.dims().dims};
+        eval &e{evals[idims[jdim]]};
+        e.outputs.emplace_back(out.get() + outOffset, idims,
+                               out.strides().dims);
+        // Extend life of the returned node by saving the corresponding
+        // shared_ptr
+        e.nodePtrs.emplace_back(iArray.getNode());
+        e.nodes.push_back(e.nodePtrs.back().get());
+        e.ins.push_back(&iArray);
+        outOffset += idims[jdim] * out.strides().dims[jdim];
     }
-}
 
-template<typename T>
-void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs) {
-    std::vector<Array<T> *> input_ptrs(inputs.size());
-    std::transform(
-        begin(inputs), end(inputs), begin(input_ptrs),
-        [](const Array<T> &input) { return const_cast<Array<T> *>(&input); });
-    evalMultiple(input_ptrs);
-
-    join_wrapper<T>(dim, out, inputs);
+    for (auto &eval : evals) {
+        auto &s{eval.second};
+        if (s.ins.size() == 1 ||
+            s.ins[0]->elements() * sizeof(T) * 2 * 2 > L2CacheSize) {
+            // Process (evaluated arrays) individually for
+            //  - single small array
+            //  - very large arrays
+            auto nodeIt{begin(s.nodes)};
+            auto outputIt{begin(s.outputs)};
+            for (const Array<T> *in : s.ins) {
+                if (in->isReady()) {
+                    if (1LL + jdim >= in->ndims() && in->isLinear()) {
+                        CUDA_CHECK(cudaMemcpyAsync(outputIt->ptr, in->get(),
+                                                   in->elements() * sizeof(T),
+                                                   cudaMemcpyHostToDevice,
+                                                   activeStream));
+                    } else {
+                        kernel::memcopy<T>(*outputIt, *in, in->ndims());
+                    }
+                    // eliminate this array from the list, so that it will
+                    // not be processed as bulk via JIT
+                    outputIt = s.outputs.erase(outputIt);
+                    nodeIt   = s.nodes.erase(nodeIt);
+                } else {
+                    ++outputIt;
+                    ++nodeIt;
+                }
+            }
+        }
+        evalNodes(s.outputs, s.nodes);
+    }
 }
 
-#define INSTANTIATE(T)                                              \
-    template Array<T> join<T>(const int dim, const Array<T> &first, \
+#define INSTANTIATE(T)                                               \
+    template Array<T> join<T>(const int jdim, const Array<T> &first, \
                               const Array<T> &second);
 
 INSTANTIATE(float)
@@ -99,9 +214,9 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 
-#define INSTANTIATE(T)                                   \
-    template void join<T>(Array<T> & out, const int dim, \
-                          const std::vector<Array<T>> &inputs);
+#define INSTANTIATE(T)                                    \
+    template void join<T>(Array<T> & out, const int jdim, \
+                          const vector<Array<T>> &inputs);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/cuda/kernel/join.cuh b/src/backend/cuda/kernel/join.cuh
deleted file mode 100644
index 666114e07b..0000000000
--- a/src/backend/cuda/kernel/join.cuh
+++ /dev/null
@@ -1,50 +0,0 @@
-/*******************************************************
- * Copyright (c) 2020, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-
-#include <Param.hpp>
-
-namespace cuda {
-
-template<typename T>
-__global__ void join(Param<T> out, CParam<T> in, const int o0, const int o1,
-                     const int o2, const int o3, const int blocksPerMatX,
-                     const int blocksPerMatY) {
-    const int incy = blocksPerMatY * blockDim.y;
-    const int incx = blocksPerMatX * blockDim.x;
-
-    const int iz         = blockIdx.x / blocksPerMatX;
-    const int blockIdx_x = blockIdx.x - iz * blocksPerMatX;
-    const int xx         = threadIdx.x + blockIdx_x * blockDim.x;
-
-    T *d_out      = out.ptr;
-    T const *d_in = in.ptr;
-
-    const int iw = (blockIdx.y + (blockIdx.z * gridDim.y)) / blocksPerMatY;
-    const int blockIdx_y =
-        (blockIdx.y + (blockIdx.z * gridDim.y)) - iw * blocksPerMatY;
-    const int yy = threadIdx.y + blockIdx_y * blockDim.y;
-
-    if (iz < in.dims[2] && iw < in.dims[3]) {
-        d_out = d_out + (iz + o2) * out.strides[2] + (iw + o3) * out.strides[3];
-        d_in  = d_in + iz * in.strides[2] + iw * in.strides[3];
-
-        for (int iy = yy; iy < in.dims[1]; iy += incy) {
-            T const *d_in_ = d_in + iy * in.strides[1];
-            T *d_out_      = d_out + (iy + o1) * out.strides[1];
-
-            for (int ix = xx; ix < in.dims[0]; ix += incx) {
-                d_out_[ix + o0] = d_in_[ix];
-            }
-        }
-    }
-}
-
-}  // namespace cuda
diff --git a/src/backend/cuda/kernel/join.hpp b/src/backend/cuda/kernel/join.hpp
deleted file mode 100644
index f404f7b8bf..0000000000
--- a/src/backend/cuda/kernel/join.hpp
+++ /dev/null
@@ -1,51 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-
-#include <Param.hpp>
-#include <common/dispatch.hpp>
-#include <common/kernel_cache.hpp>
-#include <debug_cuda.hpp>
-#include <nvrtc_kernel_headers/join_cuh.hpp>
-
-namespace cuda {
-namespace kernel {
-
-template<typename T>
-void join(Param<T> out, CParam<T> X, const af::dim4 &offset, int dim) {
-    constexpr unsigned TX    = 32;
-    constexpr unsigned TY    = 8;
-    constexpr unsigned TILEX = 256;
-    constexpr unsigned TILEY = 32;
-
-    auto join = common::getKernel("cuda::join", {join_cuh_src},
-                                  {TemplateTypename<T>()});
-
-    dim3 threads(TX, TY, 1);
-
-    int blocksPerMatX = divup(X.dims[0], TILEX);
-    int blocksPerMatY = divup(X.dims[1], TILEY);
-
-    dim3 blocks(blocksPerMatX * X.dims[2], blocksPerMatY * X.dims[3], 1);
-
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
-
-    EnqueueArgs qArgs(blocks, threads, getActiveStream());
-
-    join(qArgs, out, X, offset[0], offset[1], offset[2], offset[3],
-         blocksPerMatX, blocksPerMatY);
-    POST_LAUNCH_CHECK();
-}
-
-}  // namespace kernel
-}  // namespace cuda
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 520d4f90f5..fa412101f0 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -208,7 +208,7 @@ string getDeviceInfo(int device) noexcept {
     size_t mem_gpu_total = dev.totalGlobalMem;
     // double cc = double(dev.major) + double(dev.minor) / 10;
 
-    bool show_braces = getActiveDeviceId() == static_cast<unsigned>(device);
+    bool show_braces = getActiveDeviceId() == device;
 
     string id = (show_braces ? string("[") : "-") + to_string(device) +
                 (show_braces ? string("]") : "-");
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 506b9b3f55..024c92551a 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -227,6 +227,7 @@ target_sources(afopencl
     svd.hpp
     tile.cpp
     tile.hpp
+    threadsMgt.hpp
     topk.cpp
     topk.hpp
     traits.hpp
@@ -285,7 +286,6 @@ target_sources(afopencl
     kernel/interp.hpp
     kernel/iota.hpp
     kernel/ireduce.hpp
-    kernel/join.hpp
     kernel/laset.hpp
     #kernel/laset_band.hpp
     kernel/laswp.hpp
diff --git a/src/backend/opencl/join.cpp b/src/backend/opencl/join.cpp
index 0c7109a895..2d166b693e 100644
--- a/src/backend/opencl/join.cpp
+++ b/src/backend/opencl/join.cpp
@@ -11,80 +11,209 @@
 #include <common/half.hpp>
 #include <err_opencl.hpp>
 #include <join.hpp>
-#include <kernel/join.hpp>
+#include <kernel/memcopy.hpp>
 
 #include <algorithm>
+#include <map>
 #include <stdexcept>
 #include <vector>
 
 using af::dim4;
 using common::half;
-using std::transform;
+using common::Node;
+using common::Node_ptr;
 using std::vector;
 
 namespace opencl {
-dim4 calcOffset(const dim4 &dims, int dim) {
-    dim4 offset;
-    offset[0] = (dim == 0) ? dims[0] : 0;
-    offset[1] = (dim == 1) ? dims[1] : 0;
-    offset[2] = (dim == 2) ? dims[2] : 0;
-    offset[3] = (dim == 3) ? dims[3] : 0;
-    return offset;
-}
-
 template<typename T>
-Array<T> join(const int dim, const Array<T> &first, const Array<T> &second) {
+Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     // All dimensions except join dimension must be equal
+    const dim4 &fdims{first.dims()};
+    const dim4 &sdims{second.dims()};
     // Compute output dims
-    dim4 odims;
-    dim4 fdims = first.dims();
-    dim4 sdims = second.dims();
+    dim4 odims(fdims);
+    odims.dims[jdim] += sdims.dims[jdim];
+    Array<T> out = createEmptyArray<T>(odims);
 
-    for (int i = 0; i < 4; i++) {
-        if (i == dim) {
-            odims[i] = fdims[i] + sdims[i];
-        } else {
-            odims[i] = fdims[i];
+    // topspeed is achieved when byte size(in+out) ~= L2CacheSize
+    //
+    // 1 array: memcpy always copies 1 array.  topspeed
+    //      --> size(in) <= L2CacheSize/2
+    // 2 arrays: topspeeds
+    //      - size(in) < L2CacheSize/2/2
+    //          --> JIT can copy 2 arrays in // and is fastest
+    //              (condition: array sizes have to be identical)
+    //      - size(in) < L2CacheSize/2
+    //          --> memcpy will achieve highest speed, although the kernel
+    //              has to be called twice
+    //      - size(in) >= L2CacheSize/2
+    //          --> memcpy will achieve veryLargeArray speed.  The kernel
+    //              will be called twice
+    if (fdims.dims[jdim] == sdims.dims[jdim]) {
+        const size_t L2CacheSize{getL2CacheSize(opencl::getDevice())};
+        if (!(first.isReady() | second.isReady()) ||
+            (fdims.elements() * sizeof(T) * 2 * 2 < L2CacheSize)) {
+            // Both arrays have same size & everything fits into the cache,
+            // so thread in 1 JIT kernel, iso individual copies which is
+            // always slower
+            const dim_t *outStrides{out.strides().dims};
+            vector<Param> outputs{
+                {out.get(),
+                 {{fdims.dims[0], fdims.dims[1], fdims.dims[2], fdims.dims[3]},
+                  {outStrides[0], outStrides[1], outStrides[2], outStrides[3]},
+                  0}},
+                {out.get(),
+                 {{sdims.dims[0], sdims.dims[1], sdims.dims[2], sdims.dims[3]},
+                  {outStrides[0], outStrides[1], outStrides[2], outStrides[3]},
+                  fdims.dims[jdim] * outStrides[jdim]}}};
+            // Extend the life of the returned node, bij saving the
+            // corresponding shared_ptr
+            const Node_ptr fNode{first.getNode()};
+            const Node_ptr sNode{second.getNode()};
+            vector<Node *> nodes{fNode.get(), sNode.get()};
+            evalNodes(outputs, nodes);
+            return out;
         }
+        // continue because individually processing is faster
     }
 
-    Array<T> out = createEmptyArray<T>(odims);
-
-    dim4 zero(0, 0, 0, 0);
+    // Handle each array individually
+    if (first.isReady()) {
+        if (1LL + jdim >= first.ndims() && first.isLinear()) {
+            // first & out are linear
+            getQueue().enqueueCopyBuffer(
+                *first.get(), *out.get(), first.getOffset() * sizeof(T), 0,
+                first.elements() * sizeof(T), nullptr, nullptr);
+        } else {
+            kernel::memcopy<T>(*out.get(), out.strides(), *first.get(), fdims,
+                               first.strides(), first.getOffset(),
+                               first.ndims(), 0);
+        }
+    } else {
+        // Write the result directly in the out array
+        const dim_t *outStrides{out.strides().dims};
+        Param output{
+            out.get(),
+            {{fdims.dims[0], fdims.dims[1], fdims.dims[2], fdims.dims[3]},
+             {outStrides[0], outStrides[1], outStrides[2], outStrides[3]},
+             0}};
+        evalNodes(output, first.getNode().get());
+    }
 
-    kernel::join<T>(out, first, dim, zero);
-    kernel::join<T>(out, second, dim, calcOffset(fdims, dim));
+    if (second.isReady()) {
+        if (1LL + jdim >= second.ndims() && second.isLinear()) {
+            // second & out are linear
+            getQueue().enqueueCopyBuffer(
+                *second.get(), *out.get(), second.getOffset() * sizeof(T),
+                (fdims.dims[jdim] * out.strides().dims[jdim]) * sizeof(T),
+                second.elements() * sizeof(T), nullptr, nullptr);
+        } else {
+            kernel::memcopy<T>(*out.get(), out.strides(), *second.get(), sdims,
+                               second.strides(), second.getOffset(),
+                               second.ndims(),
+                               fdims.dims[jdim] * out.strides().dims[jdim]);
+        }
+    } else {
+        // Write the result directly in the out array
+        const dim_t *outStrides{out.strides().dims};
+        Param output{
+            out.get(),
+            {{sdims.dims[0], sdims.dims[1], sdims.dims[2], sdims.dims[3]},
+             {outStrides[0], outStrides[1], outStrides[2], outStrides[3]},
+             fdims.dims[jdim] * outStrides[jdim]}};
+        evalNodes(output, second.getNode().get());
+    }
 
     return out;
 }
 
 template<typename T>
-void join_wrapper(const int dim, Array<T> &out,
-                  const vector<Array<T>> &inputs) {
-    dim4 zero(0, 0, 0, 0);
-    dim4 d = zero;
-
-    kernel::join<T>(out, inputs[0], dim, zero);
-    for (size_t i = 1; i < inputs.size(); i++) {
-        d += inputs[i - 1].dims();
-        kernel::join<T>(out, inputs[i], dim, calcOffset(d, dim));
+void join(Array<T> &out, const int jdim, const vector<Array<T>> &inputs) {
+    class eval {
+       public:
+        vector<Param> outputs;
+        vector<Node_ptr> nodePtrs;
+        vector<Node *> nodes;
+        vector<const Array<T> *> ins;
+    };
+    std::map<dim_t, eval> evals;
+    const dim_t *ostrides{out.strides().dims};
+    const size_t L2CacheSize{getL2CacheSize(opencl::getDevice())};
+
+    // topspeed is achieved when byte size(in+out) ~= L2CacheSize
+    //
+    // 1 array: memcpy always copies 1 array.  topspeed
+    //      --> size(in) <= L2CacheSize/2
+    // 2 arrays: topspeeds
+    //      - size(in) < L2CacheSize/2/2
+    //          --> JIT can copy 2 arrays in // and is fastest
+    //              (condition: array sizes have to be identical)
+    //      - size(in) < L2CacheSize/2
+    //          --> memcpy will achieve highest speed, although the kernel
+    //              has to be called twice
+    //      - size(in) >= L2CacheSize/2
+    //          --> memcpy will achieve veryLargeArray speed.  The kernel
+    //              will be called twice
+
+    // Group all arrays according to size
+    dim_t outOffset{0};
+    for (const Array<T> &iArray : inputs) {
+        const dim_t *idims{iArray.dims().dims};
+        eval &e{evals[idims[jdim]]};
+        const Param output{
+            out.get(),
+            {{idims[0], idims[1], idims[2], idims[3]},
+             {ostrides[0], ostrides[1], ostrides[2], ostrides[3]},
+             outOffset}};
+        e.outputs.push_back(output);
+        // Extend life of the returned node by saving the corresponding
+        // shared_ptr
+        e.nodePtrs.emplace_back(iArray.getNode());
+        e.nodes.push_back(e.nodePtrs.back().get());
+        e.ins.push_back(&iArray);
+        outOffset += idims[jdim] * ostrides[jdim];
     }
-}
 
-template<typename T>
-void join(Array<T> &out, const int dim, const vector<Array<T>> &inputs) {
-    vector<Array<T> *> input_ptrs(inputs.size());
-    transform(
-        begin(inputs), end(inputs), begin(input_ptrs),
-        [](const Array<T> &input) { return const_cast<Array<T> *>(&input); });
-    evalMultiple(input_ptrs);
-    vector<Param> inputParams(inputs.begin(), inputs.end());
-
-    join_wrapper<T>(dim, out, inputs);
+    for (auto &eval : evals) {
+        auto &s{eval.second};
+        if (s.ins.size() == 1 ||
+            s.ins[0]->elements() * sizeof(T) * 2 * 2 > L2CacheSize) {
+            // Process (evaluate arrays) individually for
+            //  - single small array
+            //  - very large arrays
+            auto nodeIt{begin(s.nodes)};
+            auto outputIt{begin(s.outputs)};
+            for (const Array<T> *in : s.ins) {
+                if (in->isReady()) {
+                    if (1LL + jdim >= in->ndims() && in->isLinear()) {
+                        getQueue().enqueueCopyBuffer(
+                            *in->get(), *outputIt->data,
+                            in->getOffset() * sizeof(T),
+                            outputIt->info.offset * sizeof(T),
+                            in->elements() * sizeof(T), nullptr, nullptr);
+                    } else {
+                        kernel::memcopy<T>(*outputIt->data,
+                                           af::dim4(4, outputIt->info.strides),
+                                           *in->get(), in->dims(),
+                                           in->strides(), in->getOffset(),
+                                           in->ndims(), outputIt->info.offset);
+                    }
+                    // eliminate this array from the list, so that it will
+                    // not be processed in bulk via JIT
+                    outputIt = s.outputs.erase(outputIt);
+                    nodeIt   = s.nodes.erase(nodeIt);
+                } else {
+                    ++outputIt;
+                    ++nodeIt;
+                }
+            }
+        }
+        evalNodes(s.outputs, s.nodes);
+    }
 }
 
-#define INSTANTIATE(T)                                              \
-    template Array<T> join<T>(const int dim, const Array<T> &first, \
+#define INSTANTIATE(T)                                               \
+    template Array<T> join<T>(const int jdim, const Array<T> &first, \
                               const Array<T> &second);
 
 INSTANTIATE(float)
@@ -103,8 +232,8 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 
-#define INSTANTIATE(T)                                   \
-    template void join<T>(Array<T> & out, const int dim, \
+#define INSTANTIATE(T)                                    \
+    template void join<T>(Array<T> & out, const int jdim, \
                           const vector<Array<T>> &inputs);
 
 INSTANTIATE(float)
diff --git a/src/backend/opencl/kernel/join.cl b/src/backend/opencl/kernel/join.cl
deleted file mode 100644
index 884ec56d62..0000000000
--- a/src/backend/opencl/kernel/join.cl
+++ /dev/null
@@ -1,41 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-kernel void join_kernel(global T *d_out, const KParam out, global const T *d_in,
-                        const KParam in, const int o0, const int o1,
-                        const int o2, const int o3, const int blocksPerMatX,
-                        const int blocksPerMatY) {
-    const int iz = get_group_id(0) / blocksPerMatX;
-    const int iw = get_group_id(1) / blocksPerMatY;
-
-    const int blockIdx_x = get_group_id(0) - iz * blocksPerMatX;
-    const int blockIdx_y = get_group_id(1) - iw * blocksPerMatY;
-
-    const int xx = get_local_id(0) + blockIdx_x * get_local_size(0);
-    const int yy = get_local_id(1) + blockIdx_y * get_local_size(1);
-
-    const int incy = blocksPerMatY * get_local_size(1);
-    const int incx = blocksPerMatX * get_local_size(0);
-
-    d_in = d_in + in.offset;
-
-    if (iz < in.dims[2] && iw < in.dims[3]) {
-        d_out = d_out + (iz + o2) * out.strides[2] + (iw + o3) * out.strides[3];
-        d_in  = d_in + iz * in.strides[2] + iw * in.strides[3];
-
-        for (int iy = yy; iy < in.dims[1]; iy += incy) {
-            global T *d_in_  = d_in + iy * in.strides[1];
-            global T *d_out_ = d_out + (iy + o1) * out.strides[1];
-
-            for (int ix = xx; ix < in.dims[0]; ix += incx) {
-                d_out_[ix + o0] = d_in_[ix];
-            }
-        }
-    }
-}
diff --git a/src/backend/opencl/kernel/join.hpp b/src/backend/opencl/kernel/join.hpp
deleted file mode 100644
index 5a4016eee6..0000000000
--- a/src/backend/opencl/kernel/join.hpp
+++ /dev/null
@@ -1,55 +0,0 @@
-/*******************************************************
- * Copyright (c) 2014, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-
-#include <Param.hpp>
-#include <common/dispatch.hpp>
-#include <common/kernel_cache.hpp>
-#include <debug_opencl.hpp>
-#include <kernel_headers/join.hpp>
-#include <traits.hpp>
-
-#include <string>
-#include <vector>
-
-namespace opencl {
-namespace kernel {
-
-template<typename T>
-void join(Param out, const Param in, dim_t dim, const af::dim4 offset) {
-    constexpr int TX    = 32;
-    constexpr int TY    = 8;
-    constexpr int TILEX = 256;
-    constexpr int TILEY = 32;
-
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
-
-    auto join =
-        common::getKernel("join_kernel", {join_cl_src},
-                          {TemplateTypename<T>(), TemplateArg(dim)}, options);
-    cl::NDRange local(TX, TY, 1);
-
-    int blocksPerMatX = divup(in.info.dims[0], TILEX);
-    int blocksPerMatY = divup(in.info.dims[1], TILEY);
-    cl::NDRange global(local[0] * blocksPerMatX * in.info.dims[2],
-                       local[1] * blocksPerMatY * in.info.dims[3], 1);
-
-    join(cl::EnqueueArgs(getQueue(), global, local), *out.data, out.info,
-         *in.data, in.info, static_cast<int>(offset[0]),
-         static_cast<int>(offset[1]), static_cast<int>(offset[2]),
-         static_cast<int>(offset[3]), blocksPerMatX, blocksPerMatY);
-    CL_DEBUG_FINISH(getQueue());
-}
-
-}  // namespace kernel
-}  // namespace opencl
diff --git a/src/backend/opencl/kernel/memcopy.hpp b/src/backend/opencl/kernel/memcopy.hpp
index 9358315cd5..159fe4d35a 100644
--- a/src/backend/opencl/kernel/memcopy.hpp
+++ b/src/backend/opencl/kernel/memcopy.hpp
@@ -126,7 +126,7 @@ void memcopy(const cl::Buffer& b_out, const dim4& ostrides,
     // When the architecture prefers some width's, it is certainly
     // on char.  No preference means vector width 1 returned.
     const bool DevicePreferredVectorWidth{DevicePreferredVectorWidthChar != 1};
-    unsigned maxVectorWidth{
+    size_t maxVectorWidth{
         DevicePreferredVectorWidth
             ? sizeof(T) == 1 ? DevicePreferredVectorWidthChar
               : sizeof(T) == 2
@@ -138,10 +138,10 @@ void memcopy(const cl::Buffer& b_out, const dim4& ostrides,
                   : 1
         : sizeof(T) > 8 ? 1
                         : 16 / sizeof(T)};
-    const unsigned vectorWidth{vectorizeShape(maxVectorWidth, idims_.dims,
-                                              istrides_.dims, indims_, ioffset,
-                                              ostrides_.dims, ooffset)};
-    const dim_t sizeofNewT{sizeof(T) * vectorWidth};
+    const size_t vectorWidth{vectorizeShape(maxVectorWidth, idims_.dims,
+                                            istrides_.dims, indims_, ioffset,
+                                            ostrides_.dims, ooffset)};
+    const size_t sizeofNewT{sizeof(T) * vectorWidth};
 
     threadsMgt<int> th(idims_.dims, indims_, 1, 1, totalSize, sizeofNewT);
     const char* kernelName{

From 0842e286684dcca6785f0ae6f24d051fc5c7e6cb Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 26 Sep 2022 13:23:15 -0400
Subject: [PATCH 076/473] Update standard to C++17

---
 CMakeLists.txt                                |  6 +++
 CMakeModules/InternalUtils.cmake              | 13 +------
 CMakeModules/build_clFFT.cmake                |  8 ++++
 src/backend/common/ArrayInfo.cpp              |  3 +-
 src/backend/common/half.hpp                   | 38 +++++++++----------
 src/backend/common/jit/NodeIterator.hpp       |  9 +++--
 src/backend/cpu/Array.cpp                     |  2 +
 src/backend/cpu/jit/BinaryNode.hpp            |  2 +-
 src/backend/cpu/kernel/Array.hpp              |  2 +-
 src/backend/cpu/kernel/bilateral.hpp          | 10 +++--
 src/backend/cpu/kernel/fast.hpp               |  1 +
 .../cpu/kernel/sort_by_key/CMakeLists.txt     |  3 ++
 src/backend/cpu/math.hpp                      |  7 ----
 src/backend/cuda/jit.cpp                      |  1 +
 src/backend/cuda/math.hpp                     | 10 ++++-
 src/backend/opencl/jit.cpp                    |  1 +
 .../opencl/kernel/scan_by_key/CMakeLists.txt  |  3 ++
 .../opencl/kernel/sort_by_key/CMakeLists.txt  |  3 ++
 18 files changed, 74 insertions(+), 48 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c79cc691e5..73d8cbe9aa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -304,6 +304,9 @@ if(CMAKE_CROSSCOMPILING)
 else()
   add_executable(bin2cpp ${ArrayFire_SOURCE_DIR}/CMakeModules/bin2cpp.cpp
                          ${ArrayFire_SOURCE_DIR}/src/backend/common/util.cpp)
+  set_target_properties(bin2cpp
+    PROPERTIES
+      CXX_STANDARD 17)
 
   # NOSPDLOG is used to remove the spdlog dependency from bin2cpp
   target_compile_definitions(bin2cpp PRIVATE NOSPDLOG)
@@ -358,6 +361,9 @@ if(TARGET afopencl)
 endif()
 
 set_target_properties(${built_backends} PROPERTIES
+                      CXX_STANDARD 17
+                      CXX_EXTENSIONS OFF
+                      CXX_VISIBILITY_PRESET hidden
                       VERSION "${ArrayFire_VERSION}"
                       SOVERSION "${ArrayFire_VERSION_MAJOR}")
 
diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index f212c50750..3f0828ef3e 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -41,20 +41,15 @@ function(arrayfire_get_cuda_cxx_flags cuda_flags)
     endif()
     if(cplusplus_define)
       list(APPEND flags -Xcompiler /Zc:__cplusplus
-                        -Xcompiler /std:c++14)
+                        -Xcompiler /std:c++17)
     endif()
   else()
-    set(flags -std=c++14
+    set(flags -std=c++17
               -Xcompiler -fPIC
               -Xcompiler ${CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY}hidden
               --expt-relaxed-constexpr)
   endif()
 
-  if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND
-      CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "5.3.0" AND
-      ${CUDA_VERSION_MAJOR} LESS 8)
-    set(flags ${flags} -D_FORCE_INLINES -D_MWAITXINTRIN_H_INCLUDED)
-  endif()
   set(${cuda_flags} ${flags} PARENT_SCOPE)
 endfunction()
 
@@ -122,10 +117,6 @@ macro(arrayfire_set_cmake_default_variables)
   set(CMAKE_PREFIX_PATH "${ArrayFire_BINARY_DIR};${CMAKE_PREFIX_PATH}")
   set(BUILD_SHARED_LIBS ON)
 
-  set(CMAKE_CXX_STANDARD 14)
-  set(CMAKE_CXX_EXTENSIONS OFF)
-  set(CMAKE_CXX_VISIBILITY_PRESET hidden)
-
   set(CMAKE_CXX_FLAGS_COVERAGE
       "-g -O0"
       CACHE STRING "Flags used by the C++ compiler during coverage builds.")
diff --git a/CMakeModules/build_clFFT.cmake b/CMakeModules/build_clFFT.cmake
index dc29e22ced..d4f3081e63 100644
--- a/CMakeModules/build_clFFT.cmake
+++ b/CMakeModules/build_clFFT.cmake
@@ -13,6 +13,14 @@ af_dep_check_and_populate(${clfft_prefix}
 set(current_build_type ${BUILD_SHARED_LIBS})
 set(BUILD_SHARED_LIBS OFF)
 add_subdirectory(${${clfft_prefix}_SOURCE_DIR}/src ${${clfft_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
+
+# OpenCL targets need this flag to avoid ignored attribute warnings in the
+# OpenCL headers
+check_cxx_compiler_flag(-Wno-ignored-attributes has_ignored_attributes_flag)
+if(has_ignored_attributes_flag)
+  target_compile_options(clFFT
+    PRIVATE -Wno-ignored-attributes)
+endif()
 set(BUILD_SHARED_LIBS ${current_build_type})
 
 mark_as_advanced(
diff --git a/src/backend/common/ArrayInfo.cpp b/src/backend/common/ArrayInfo.cpp
index 585b48d403..6a0ca86086 100644
--- a/src/backend/common/ArrayInfo.cpp
+++ b/src/backend/common/ArrayInfo.cpp
@@ -188,7 +188,8 @@ const ArrayInfo &getInfo(const af_array arr, bool sparse_check,
     // are accepted Otherwise only regular Array<T> is accepted
     if (sparse_check) { ARG_ASSERT(0, info->isSparse() == false); }
 
-    if (device_check && info->getDevId() != detail::getActiveDeviceId()) {
+    if (device_check && info->getDevId() != static_cast<unsigned>(
+                                                detail::getActiveDeviceId())) {
         AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE);
     }
 
diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index fb25d0336d..a8737862f2 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -127,9 +127,9 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(T value) noexcept {
     if (S) value = -value;
     uint16_t bits = S << 15;
     if (value > 0xFFFF) {
-        if (R == std::round_toward_infinity)
+        if constexpr (R == std::round_toward_infinity)
             bits |= (0x7C00 - S);
-        else if (R == std::round_toward_neg_infinity)
+        else if constexpr (R == std::round_toward_neg_infinity)
             bits |= (0x7BFF + S);
         else
             bits |= (0x7BFF + (R != std::round_toward_zero));
@@ -141,15 +141,15 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(T value) noexcept {
             ;
         bits |= (exp << 10) + m;
         if (exp > 24) {
-            if (R == std::round_to_nearest)
+            if constexpr (R == std::round_to_nearest)
                 bits += (value >> (exp - 25)) & 1
 #if HALF_ROUND_TIES_TO_EVEN
                         & (((((1 << (exp - 25)) - 1) & value) != 0) | bits)
 #endif
                     ;
-            else if (R == std::round_toward_infinity)
+            else if constexpr (R == std::round_toward_infinity)
                 bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & !S;
-            else if (R == std::round_toward_neg_infinity)
+            else if constexpr (R == std::round_toward_neg_infinity)
                 bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & S;
         }
     }
@@ -277,7 +277,7 @@ __DH__ native_half_t float2half_impl(float value) noexcept {
     uint16_t hbits =
         base_table[bits >> 23] +
         static_cast<uint16_t>((bits & 0x7FFFFF) >> shift_table[bits >> 23]);
-    if (R == std::round_to_nearest)
+    if constexpr (R == std::round_to_nearest)
         hbits +=
             (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) |
              (((bits >> 23) & 0xFF) == 102)) &
@@ -289,16 +289,16 @@ __DH__ native_half_t float2half_impl(float value) noexcept {
              hbits)
 #endif
             ;
-    else if (R == std::round_toward_zero)
+    else if constexpr (R == std::round_toward_zero)
         hbits -= ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23];
-    else if (R == std::round_toward_infinity)
+    else if constexpr (R == std::round_toward_infinity)
         hbits += ((((bits & 0x7FFFFF &
                      ((static_cast<uint32_t>(1) << (shift_table[bits >> 23])) -
                       1)) != 0) |
                    (((bits >> 23) <= 102) & ((bits >> 23) != 0))) &
                   (hbits < 0x7C00)) -
                  ((hbits == 0xFC00) & ((bits >> 23) != 511));
-    else if (R == std::round_toward_neg_infinity)
+    else if constexpr (R == std::round_toward_neg_infinity)
         hbits += ((((bits & 0x7FFFFF &
                      ((static_cast<uint32_t>(1) << (shift_table[bits >> 23])) -
                       1)) != 0) |
@@ -328,9 +328,9 @@ __DH__ native_half_t float2half_impl(double value) {
         return hbits | 0x7C00 |
                (0x3FF & -static_cast<unsigned>((bits & 0xFFFFFFFFFFFFF) != 0));
     if (exp > 1038) {
-        if (R == std::round_toward_infinity)
+        if constexpr (R == std::round_toward_infinity)
             return hbits | (0x7C00 - (hbits >> 15));
-        if (R == std::round_toward_neg_infinity)
+        if constexpr (R == std::round_toward_neg_infinity)
             return hbits | (0x7BFF + (hbits >> 15));
         return hbits | (0x7BFF + (R != std::round_toward_zero));
     }
@@ -348,15 +348,15 @@ __DH__ native_half_t float2half_impl(double value) {
     } else {
         s |= hi != 0;
     }
-    if (R == std::round_to_nearest)
+    if constexpr (R == std::round_to_nearest)
 #if HALF_ROUND_TIES_TO_EVEN
         hbits += g & (s | hbits);
 #else
         hbits += g;
 #endif
-    else if (R == std::round_toward_infinity)
+    else if constexpr (R == std::round_toward_infinity)
         hbits += ~(hbits >> 15) & (s | g);
-    else if (R == std::round_toward_neg_infinity)
+    else if constexpr (R == std::round_toward_neg_infinity)
         hbits += (hbits >> 15) & (g | s);
     return hbits;
 }
@@ -773,20 +773,20 @@ AF_CONSTEXPR T half2int(native_half_t value) {
         return (value & 0x8000) ? std::numeric_limits<T>::min()
                                 : std::numeric_limits<T>::max();
     if (e < 0x3800) {
-        if (R == std::round_toward_infinity)
+        if constexpr (R == std::round_toward_infinity)
             return T(~(value >> 15) & (e != 0));
-        else if (R == std::round_toward_neg_infinity)
+        else if constexpr (R == std::round_toward_neg_infinity)
             return -T(value > 0x8000);
         return T();
     }
     unsigned int m = (value & 0x3FF) | 0x400;
     e >>= 10;
     if (e < 25) {
-        if (R == std::round_to_nearest)
+        if constexpr (R == std::round_to_nearest)
             m += (1 << (24 - e)) - (~(m >> (25 - e)) & E);
-        else if (R == std::round_toward_infinity)
+        else if constexpr (R == std::round_toward_infinity)
             m += ((value >> 15) - 1) & ((1 << (25 - e)) - 1U);
-        else if (R == std::round_toward_neg_infinity)
+        else if constexpr (R == std::round_toward_neg_infinity)
             m += -(value >> 15) & ((1 << (25 - e)) - 1U);
         m >>= 25 - e;
     } else
diff --git a/src/backend/common/jit/NodeIterator.hpp b/src/backend/common/jit/NodeIterator.hpp
index da01c0b5bb..e286f6359d 100644
--- a/src/backend/common/jit/NodeIterator.hpp
+++ b/src/backend/common/jit/NodeIterator.hpp
@@ -18,10 +18,13 @@ namespace common {
 
 /// A node iterator that performs a breadth first traversal of the node tree
 template<typename Node = common::Node>
-class NodeIterator : public std::iterator<std::input_iterator_tag, Node> {
+class NodeIterator {
    public:
-    using pointer   = Node*;
-    using reference = Node&;
+    using iterator_category = std::input_iterator_tag;
+    using value_type        = Node;
+    using difference_type   = std::ptrdiff_t;
+    using pointer           = Node*;
+    using reference         = Node&;
 
    private:
     std::vector<pointer> tree;
diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index dcd79dd9ed..159fd2aa7c 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -46,8 +46,10 @@ using common::NodeIterator;
 using cpu::jit::BufferNode;
 
 using nonstd::span;
+using std::accumulate;
 using std::adjacent_find;
 using std::copy;
+using std::find_if;
 using std::is_standard_layout;
 using std::make_shared;
 using std::move;
diff --git a/src/backend/cpu/jit/BinaryNode.hpp b/src/backend/cpu/jit/BinaryNode.hpp
index 2342bb30cb..0ce7e348f4 100644
--- a/src/backend/cpu/jit/BinaryNode.hpp
+++ b/src/backend/cpu/jit/BinaryNode.hpp
@@ -25,7 +25,7 @@ template<typename To, typename Ti, af_op_t op>
 class BinaryNode : public TNode<compute_t<To>> {
    protected:
     BinOp<compute_t<To>, compute_t<Ti>, op> m_op;
-    using common::Node::m_children;
+    using TNode<compute_t<To>>::m_children;
 
    public:
     BinaryNode(common::Node_ptr lhs, common::Node_ptr rhs)
diff --git a/src/backend/cpu/kernel/Array.hpp b/src/backend/cpu/kernel/Array.hpp
index 48987a5d4d..e13548aa60 100644
--- a/src/backend/cpu/kernel/Array.hpp
+++ b/src/backend/cpu/kernel/Array.hpp
@@ -53,7 +53,7 @@ void propagateModdimsShape(
 
             NodeIterator<> it(node.get());
             while (it != NodeIterator<>()) {
-                it = find_if(it, NodeIterator<>(), common::isBuffer);
+                it = std::find_if(it, NodeIterator<>(), common::isBuffer);
                 if (it == NodeIterator<>()) { break; }
 
                 it->setShape(mn->m_new_shape);
diff --git a/src/backend/cpu/kernel/bilateral.hpp b/src/backend/cpu/kernel/bilateral.hpp
index 343b83dd08..a2f316d15f 100644
--- a/src/backend/cpu/kernel/bilateral.hpp
+++ b/src/backend/cpu/kernel/bilateral.hpp
@@ -19,14 +19,18 @@ namespace kernel {
 template<typename OutT, typename InT>
 void bilateral(Param<OutT> out, CParam<InT> in, float const s_sigma,
                float const c_sigma) {
+    using std::clamp;
+    using std::max;
+    using std::min;
+
     af::dim4 const dims     = in.dims();
     af::dim4 const istrides = in.strides();
     af::dim4 const ostrides = out.strides();
 
     // clamp spatical and chromatic sigma's
-    float space_       = std::min(11.5f, std::max(s_sigma, 0.f));
-    float color_       = std::max(c_sigma, 0.f);
-    dim_t const radius = std::max((dim_t)(space_ * 1.5f), (dim_t)1);
+    float space_       = min(11.5f, max(s_sigma, 0.f));
+    float color_       = max(c_sigma, 0.f);
+    dim_t const radius = max((dim_t)(space_ * 1.5f), (dim_t)1);
     float const svar   = space_ * space_;
     float const cvar   = color_ * color_;
 
diff --git a/src/backend/cpu/kernel/fast.hpp b/src/backend/cpu/kernel/fast.hpp
index f2a3d148ee..b168021903 100644
--- a/src/backend/cpu/kernel/fast.hpp
+++ b/src/backend/cpu/kernel/fast.hpp
@@ -15,6 +15,7 @@ namespace cpu {
 namespace kernel {
 
 inline int idx_y(int i) {
+    using std::clamp;
     if (i >= 8) return clamp(-(i - 8 - 4), -3, 3);
 
     return clamp(i - 4, -3, 3);
diff --git a/src/backend/cpu/kernel/sort_by_key/CMakeLists.txt b/src/backend/cpu/kernel/sort_by_key/CMakeLists.txt
index 9abd9b3f84..3c894b37f5 100644
--- a/src/backend/cpu/kernel/sort_by_key/CMakeLists.txt
+++ b/src/backend/cpu/kernel/sort_by_key/CMakeLists.txt
@@ -23,6 +23,9 @@ foreach(SBK_TYPE ${SBK_TYPES})
   set_target_properties(cpu_sort_by_key_${SBK_TYPE}
     PROPERTIES
       COMPILE_DEFINITIONS "TYPE=${SBK_TYPE};AFDLL;$<TARGET_PROPERTY:Boost::boost,INTERFACE_COMPILE_DEFINITIONS>"
+      CXX_STANDARD 17
+      CXX_EXTENSIONS OFF
+      CXX_VISIBILITY_PRESET hidden
       FOLDER "Generated Targets")
 
   arrayfire_set_default_cxx_flags(cpu_sort_by_key_${SBK_TYPE})
diff --git a/src/backend/cpu/math.hpp b/src/backend/cpu/math.hpp
index 2142604095..b01a11bb04 100644
--- a/src/backend/cpu/math.hpp
+++ b/src/backend/cpu/math.hpp
@@ -107,13 +107,6 @@ cfloat scalar(float val);
 
 cdouble scalar(double val);
 
-#if __cplusplus < 201703L
-template<typename T>
-static inline T clamp(const T value, const T lo, const T hi) {
-    return (value < lo ? lo : (value > hi ? hi : value));
-}
-#endif
-
 inline double real(cdouble in) noexcept { return std::real(in); }
 inline float real(cfloat in) noexcept { return std::real(in); }
 inline double imag(cdouble in) noexcept { return std::imag(in); }
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 262d5c8c45..6904d0673d 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -45,6 +45,7 @@ using common::NodeIterator;
 
 using std::array;
 using std::equal;
+using std::find_if;
 using std::for_each;
 using std::shared_ptr;
 using std::string;
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index 7936ae8d57..a0b77265f4 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -388,9 +388,15 @@ static inline cdouble division(cdouble lhs, double rhs) {
     return retVal;
 }
 
+template<typename T, typename Compare>
+constexpr const __DH__ T clamp(const T value, const T lo, const T hi,
+                               Compare comp) {
+    return comp(value, lo) ? lo : comp(hi, value) ? hi : value;
+}
+
 template<typename T>
-static inline __DH__ T clamp(const T value, const T lo, const T hi) {
-    return max(lo, min(value, hi));
+constexpr const __DH__ T clamp(const T value, const T lo, const T hi) {
+    return clamp(value, lo, hi, [](auto lhs, auto rhs) { return lhs < rhs; });
 }
 
 }  // namespace cuda
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 8d717680d6..18a89e00a7 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -46,6 +46,7 @@ using cl::NDRange;
 using cl::NullRange;
 
 using std::equal;
+using std::find_if;
 using std::for_each;
 using std::shared_ptr;
 using std::string;
diff --git a/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt b/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
index 91f1cc9ffc..a59904cfe7 100644
--- a/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
+++ b/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
@@ -68,6 +68,9 @@ foreach(SBK_BINARY_OP ${SBK_BINARY_OPS})
 
     set_target_properties(opencl_scan_by_key_${SBK_BINARY_OP}
       PROPERTIES
+        CXX_STANDARD 17
+        CXX_EXTENSIONS False
+        CXX_VISIBILITY_PRESET hidden
         POSITION_INDEPENDENT_CODE ON
         FOLDER "Generated Targets")
 
diff --git a/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt b/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
index 0d55ffce4e..2853d75cd9 100644
--- a/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
+++ b/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
@@ -67,6 +67,9 @@ foreach(SBK_TYPE ${SBK_TYPES})
 
     set_target_properties(opencl_sort_by_key_${SBK_TYPE}
       PROPERTIES
+        CXX_STANDARD 17
+        CXX_EXTENSIONS False
+        CXX_VISIBILITY_PRESET hidden
         POSITION_INDEPENDENT_CODE ON
         FOLDER "Generated Targets")
 

From 0c391cc08cbc335a1a47b26ef16be407adac854e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 27 Sep 2022 01:55:58 -0400
Subject: [PATCH 077/473] Update select_compute_arch.cmake from version in 3.24

---
 CMakeModules/select_compute_arch.cmake | 134 ++++++++++---------------
 1 file changed, 55 insertions(+), 79 deletions(-)

diff --git a/CMakeModules/select_compute_arch.cmake b/CMakeModules/select_compute_arch.cmake
index 38180edeff..16abb8e6cd 100644
--- a/CMakeModules/select_compute_arch.cmake
+++ b/CMakeModules/select_compute_arch.cmake
@@ -7,7 +7,7 @@
 #      ARCH_AND_PTX : NAME | NUM.NUM | NUM.NUM(NUM.NUM) | NUM.NUM+PTX
 #      NAME: Fermi Kepler Maxwell Kepler+Tegra Kepler+Tesla Maxwell+Tegra Pascal Volta Turing Ampere
 #      NUM: Any number. Only those pairs are currently accepted by NVCC though:
-#            2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 7.0 7.2 7.5 8.0
+#            2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 7.0 7.2 7.5 8.0 8.6
 #      Returns LIST of flags to be added to CUDA_NVCC_FLAGS in ${out_variable}
 #      Additionally, sets ${out_variable}_readable to the resulting numeric list
 #      Example:
@@ -16,6 +16,7 @@
 #
 #      More info on CUDA architectures: https://en.wikipedia.org/wiki/CUDA
 #
+
 if(CMAKE_CUDA_COMPILER_LOADED) # CUDA as a language
   if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA"
       AND CMAKE_CUDA_COMPILER_VERSION MATCHES "^([0-9]+\\.[0-9]+)")
@@ -24,98 +25,85 @@ if(CMAKE_CUDA_COMPILER_LOADED) # CUDA as a language
 endif()
 
 # See: https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#gpu-feature-list
+# Additions, deprecations, and removals can be found in the release notes:
+# https://developer.nvidia.com/cuda-toolkit-archive
 
-# This list will be used for CUDA_ARCH_NAME = All option
-set(CUDA_KNOWN_GPU_ARCHITECTURES "Fermi" "Kepler" )
-
-# This list will be used for CUDA_ARCH_NAME = Common option (enabled by default)
-set(CUDA_COMMON_GPU_ARCHITECTURES "3.0" "3.5" "5.0")
-
-if(CUDA_VERSION VERSION_LESS "7.0")
-  set(CUDA_LIMIT_GPU_ARCHITECTURE "5.2")
-endif()
-
-# This list is used to filter CUDA archs when autodetecting
-set(CUDA_ALL_GPU_ARCHITECTURES "3.0" "3.2" "3.5" "5.0")
+# The initial status here is for CUDA 7.0
+set(CUDA_KNOWN_GPU_ARCHITECTURES  "Fermi" "Kepler" "Maxwell" "Kepler+Tegra" "Kepler+Tesla" "Maxwell+Tegra")
+set(CUDA_COMMON_GPU_ARCHITECTURES "2.0" "2.1" "3.0" "3.5" "5.0" "5.3")
+set(CUDA_LIMIT_GPU_ARCHITECTURE "6.0")
+set(CUDA_ALL_GPU_ARCHITECTURES "2.0" "2.1" "3.0" "3.2" "3.5" "3.7" "5.0" "5.2" "5.3")
+set(_CUDA_MAX_COMMON_ARCHITECTURE "5.2+PTX")
 
-if(CUDA_VERSION VERSION_GREATER "7.0" OR CUDA_VERSION VERSION_EQUAL "7.0" )
-  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Kepler+Tegra" "Kepler+Tesla" "Maxwell" "Maxwell+Tegra")
-  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "5.0" "5.2")
-  list(APPEND CUDA_ALL_GPU_ARCHITECTURES "5.0" "5.2" "5.3")
 
-  if(CUDA_VERSION VERSION_LESS "8.0")
-    list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "5.2+PTX")
-    set(CUDA_LIMIT_GPU_ARCHITECTURE "6.0")
-  endif()
-endif()
-
-if(CUDA_VERSION VERSION_GREATER "8.0" OR CUDA_VERSION VERSION_EQUAL "8.0" )
-  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Pascal" "Pascal+Tegra")
+if(CUDA_VERSION VERSION_GREATER_EQUAL "8.0")
+  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Pascal")
   list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "6.0" "6.1")
   list(APPEND CUDA_ALL_GPU_ARCHITECTURES "6.0" "6.1" "6.2")
 
-  if(CUDA_VERSION VERSION_LESS "9.0")
-    list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "6.2+PTX")
-    set(CUDA_LIMIT_GPU_ARCHITECTURE "7.0")
-  endif()
+  set(_CUDA_MAX_COMMON_ARCHITECTURE "6.2+PTX")
+  set(CUDA_LIMIT_GPU_ARCHITECTURE "7.0")
+
+  list(REMOVE_ITEM CUDA_COMMON_GPU_ARCHITECTURES "2.0" "2.1")
 endif ()
 
-if(CUDA_VERSION VERSION_GREATER "9.0" OR CUDA_VERSION VERSION_EQUAL "9.0")
+if(CUDA_VERSION VERSION_GREATER_EQUAL "9.0")
   list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Volta")
   list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "7.0")
-  list(APPEND CUDA_ALL_GPU_ARCHITECTURES "7.0")
-
-  if(CUDA_VERSION VERSION_GREATER "9.1" OR CUDA_VERSION VERSION_EQUAL "9.1")
-    list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Volta+Tegra")
-    list(APPEND CUDA_ALL_GPU_ARCHITECTURES "7.2")
-  endif()
-
-  list(REMOVE_ITEM CUDA_KNOWN_GPU_ARCHITECTURES "Fermi")
-  list(REMOVE_ITEM CUDA_COMMON_GPU_ARCHITECTURES "2.0")
-
-  if(CUDA_VERSION VERSION_GREATER "9.1" OR CUDA_VERSION VERSION_EQUAL "9.1"
-     AND CUDA_VERSION VERSION_LESS "10.0")
-    list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "7.0+PTX")
-  endif()
+  list(APPEND CUDA_ALL_GPU_ARCHITECTURES "7.0" "7.2")
 
+  set(_CUDA_MAX_COMMON_ARCHITECTURE "7.2+PTX")
   set(CUDA_LIMIT_GPU_ARCHITECTURE "8.0")
 
+  list(REMOVE_ITEM CUDA_KNOWN_GPU_ARCHITECTURES "Fermi")
+  list(REMOVE_ITEM CUDA_ALL_GPU_ARCHITECTURES "2.0" "2.1")
 endif()
 
-if(CUDA_VERSION VERSION_GREATER "10.0" OR CUDA_VERSION VERSION_EQUAL "10.0")
+if(CUDA_VERSION VERSION_GREATER_EQUAL "10.0")
   list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Turing")
   list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "7.5")
   list(APPEND CUDA_ALL_GPU_ARCHITECTURES "7.5")
 
-  if(CUDA_VERSION VERSION_LESS "11.0")
-    set(CUDA_LIMIT_GPU_ARCHITECTURE "8.0")
-    list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "7.5+PTX")
-  endif()
+  set(_CUDA_MAX_COMMON_ARCHITECTURE "7.5+PTX")
+  set(CUDA_LIMIT_GPU_ARCHITECTURE "8.0")
+
+  list(REMOVE_ITEM CUDA_COMMON_GPU_ARCHITECTURES "3.0")
 endif()
 
-if(CUDA_VERSION VERSION_GREATER "11.0" OR CUDA_VERSION VERSION_EQUAL "11.0")
+# https://docs.nvidia.com/cuda/archive/11.0/cuda-toolkit-release-notes/index.html#cuda-general-new-features
+# https://docs.nvidia.com/cuda/archive/11.0/cuda-toolkit-release-notes/index.html#deprecated-features
+if(CUDA_VERSION VERSION_GREATER_EQUAL "11.0")
   list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Ampere")
   list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "8.0")
   list(APPEND CUDA_ALL_GPU_ARCHITECTURES "8.0")
 
-  list(REMOVE_ITEM CUDA_KNOWN_GPU_ARCHITECTURES "Kepler+Tegra")
-  list(REMOVE_ITEM CUDA_KNOWN_GPU_ARCHITECTURES "Kepler")
-  list(REMOVE_ITEM CUDA_COMMON_GPU_ARCHITECTURES "3.0" "3.2")
+  set(_CUDA_MAX_COMMON_ARCHITECTURE "8.0+PTX")
+  set(CUDA_LIMIT_GPU_ARCHITECTURE "8.6")
 
-  if(CUDA_VERSION VERSION_GREATER "11.1" OR CUDA_VERSION VERSION_EQUAL "11.1")
-    list(APPEND CUDA_ALL_GPU_ARCHITECTURES "8.6")
-  endif()
+  list(REMOVE_ITEM CUDA_COMMON_GPU_ARCHITECTURES "3.5" "5.0")
+  list(REMOVE_ITEM CUDA_ALL_GPU_ARCHITECTURES "3.0" "3.2")
+endif()
 
-  if(CUDA_VERSION VERSION_GREATER "11.1" OR CUDA_VERSION VERSION_EQUAL "11.1"
-      AND CUDA_VERSION VERSION_LESS "12.0")
-    list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "8.0+PTX")
-  endif()
+if(CUDA_VERSION VERSION_GREATER_EQUAL "11.1")
+  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "8.6")
+  list(APPEND CUDA_ALL_GPU_ARCHITECTURES "8.6")
 
-  if(CUDA_VERSION VERSION_LESS "12.0")
-    set(CUDA_LIMIT_GPU_ARCHITECTURE "9.0")
-  endif()
+  set(_CUDA_MAX_COMMON_ARCHITECTURE "8.6+PTX")
+  set(CUDA_LIMIT_GPU_ARCHITECTURE "9.0")
+endif()
+
+list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "${_CUDA_MAX_COMMON_ARCHITECTURE}")
+
+# Check with: cmake -DCUDA_VERSION=7.0 -P select_compute_arch.cmake
+if(DEFINED CMAKE_SCRIPT_MODE_FILE)
+  include(CMakePrintHelpers)
+  cmake_print_variables(CUDA_KNOWN_GPU_ARCHITECTURES)
+  cmake_print_variables(CUDA_COMMON_GPU_ARCHITECTURES)
+  cmake_print_variables(CUDA_LIMIT_GPU_ARCHITECTURE)
+  cmake_print_variables(CUDA_ALL_GPU_ARCHITECTURES)
 endif()
 
+
 ################################################################################################
 # A function for automatic detection of GPUs installed  (if autodetection is enabled)
 # Usage:
@@ -174,8 +162,7 @@ function(CUDA_DETECT_INSTALLED_GPUS OUT_VARIABLE)
     set(CUDA_GPU_DETECT_OUTPUT_FILTERED "")
     separate_arguments(CUDA_GPU_DETECT_OUTPUT)
     foreach(ITEM IN ITEMS ${CUDA_GPU_DETECT_OUTPUT})
-      if(CUDA_LIMIT_GPU_ARCHITECTURE AND (ITEM VERSION_GREATER CUDA_LIMIT_GPU_ARCHITECTURE OR
-                                          ITEM VERSION_EQUAL CUDA_LIMIT_GPU_ARCHITECTURE))
+        if(CUDA_LIMIT_GPU_ARCHITECTURE AND ITEM VERSION_GREATER_EQUAL CUDA_LIMIT_GPU_ARCHITECTURE)
         list(GET CUDA_COMMON_GPU_ARCHITECTURES -1 NEWITEM)
         string(APPEND CUDA_GPU_DETECT_OUTPUT_FILTERED " ${NEWITEM}")
       else()
@@ -201,11 +188,9 @@ function(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable)
 
   set(cuda_arch_bin)
   set(cuda_arch_ptx)
-  set(cuda_arch_with_ptx false)
 
   if("${CUDA_ARCH_LIST}" STREQUAL "All")
     set(CUDA_ARCH_LIST ${CUDA_KNOWN_GPU_ARCHITECTURES})
-    set(cuda_arch_with_ptx true)
   elseif("${CUDA_ARCH_LIST}" STREQUAL "Common")
     set(CUDA_ARCH_LIST ${CUDA_COMMON_GPU_ARCHITECTURES})
   elseif("${CUDA_ARCH_LIST}" STREQUAL "Auto")
@@ -216,18 +201,10 @@ function(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable)
   # Now process the list and look for names
   string(REGEX REPLACE "[ \t]+" ";" CUDA_ARCH_LIST "${CUDA_ARCH_LIST}")
   list(REMOVE_DUPLICATES CUDA_ARCH_LIST)
-
-  list(GET CUDA_ARCH_LIST -1 latest_arch)
-
   foreach(arch_name ${CUDA_ARCH_LIST})
     set(arch_bin)
     set(arch_ptx)
     set(add_ptx FALSE)
-
-    if(${arch_name} STREQUAL ${latest_arch} AND cuda_arch_with_ptx)
-      set(add_ptx TRUE)
-    endif()
-
     # Check to see if we are compiling PTX
     if(arch_name MATCHES "(.*)\\+PTX$")
       set(add_ptx TRUE)
@@ -242,11 +219,10 @@ function(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable)
         set(arch_bin 2.0 "2.1(2.0)")
       elseif(${arch_name} STREQUAL "Kepler+Tegra")
         set(arch_bin 3.2)
-      elseif(${arch_name} STREQUAL "Kepler")
-        set(arch_bin 3.0)
-        set(arch_ptx 3.0)
       elseif(${arch_name} STREQUAL "Kepler+Tesla")
-        set(arch_bin 3.5 3.7)
+        set(arch_bin 3.7)
+      elseif(${arch_name} STREQUAL "Kepler")
+        set(arch_bin 3.0 3.5)
         set(arch_ptx 3.5)
       elseif(${arch_name} STREQUAL "Maxwell+Tegra")
         set(arch_bin 5.3)

From 51b0b36e1576c1609f8c9cf368b78e18f43ffa56 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 27 Sep 2022 01:59:40 -0400
Subject: [PATCH 078/473] use __NVCC__ definition instead of the NVCC macro

Looks like the NVCC macro is only used when compiling cuda with cmake.
this does not seem to be a standard definition
---
 src/backend/common/half.hpp | 6 +++---
 src/backend/cuda/types.hpp  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index a8737862f2..7904598eb8 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -9,7 +9,7 @@
 
 #pragma once
 
-#if defined(NVCC) || defined(__CUDACC_RTC__)
+#if defined(__NVCC__) || defined(__CUDACC_RTC__)
 
 // MSVC sets __cplusplus to 199711L for all versions unless you specify
 // the new \Zc:__cplusplus flag in Visual Studio 2017. This is not possible
@@ -824,7 +824,7 @@ AF_CONSTEXPR __DH__ static inline bool isnan(common::half val) noexcept;
 class alignas(2) half {
     native_half_t data_ = native_half_t();
 
-#if !defined(NVCC) && !defined(__CUDACC_RTC__)
+#if !defined(__NVCC__) && !defined(__CUDACC_RTC__)
     // NVCC on OSX performs a weird transformation where it removes the std::
     // namespace and complains that the std:: namespace is not there
     friend class std::numeric_limits<half>;
@@ -1054,7 +1054,7 @@ static inline std::string to_string(const half&& val) {
 
 }  // namespace common
 
-#if !defined(NVCC) && !defined(__CUDACC_RTC__)
+#if !defined(__NVCC__) && !defined(__CUDACC_RTC__)
 //#endif
 /// Extensions to the C++ standard library.
 namespace std {
diff --git a/src/backend/cuda/types.hpp b/src/backend/cuda/types.hpp
index de98d2b24f..c3897a3397 100644
--- a/src/backend/cuda/types.hpp
+++ b/src/backend/cuda/types.hpp
@@ -161,7 +161,7 @@ struct kernel_type<common::half> {
     // outside of a cuda kernel use float
     using compute = float;
 
-#if defined(NVCC) || defined(__CUDACC_RTC__)
+#if defined(__NVCC__) || defined(__CUDACC_RTC__)
     using native = __half;
 #else
     using native = common::half;

From c7555d2797170873333b85c42c23055bfccfaf84 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 27 Sep 2022 02:02:22 -0400
Subject: [PATCH 079/473] Use updated CUDA language support in CMake

This commit moves the CUDA code to use the new CUDA language support
in cmake. This allows us to remove the cuda_add_* functions in favor
of the normal CMake versions.
---
 CMakeLists.txt                    |  14 ++++
 CMakeModules/AFcuda_helpers.cmake |  70 +++++++---------
 CMakeModules/InternalUtils.cmake  |   5 --
 CMakeModules/config_ccache.cmake  |   2 +-
 src/backend/cuda/CMakeLists.txt   | 131 ++++++++++++++----------------
 test/CMakeLists.txt               |   9 +-
 6 files changed, 108 insertions(+), 123 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 73d8cbe9aa..091980b6e3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,6 +6,7 @@
 # http://arrayfire.com/licenses/BSD-3-Clause
 
 cmake_minimum_required(VERSION 3.10.2)
+include(CheckLanguage)
 
 include(CMakeModules/AF_vcpkg_options.cmake)
 
@@ -131,6 +132,19 @@ option(AF_WITH_STATIC_FREEIMAGE "Use Static FreeImage Lib" OFF)
 
 set(AF_WITH_CPUID ON CACHE BOOL "Build with CPUID integration")
 
+if(AF_BUILD_CUDA)
+  check_language(CUDA)
+  if(CMAKE_CUDA_COMPILER)
+    enable_language(CUDA)
+  elseif(CUDA_NVCC_EXECUTABLE)
+    message(STATUS "Using the FindCUDA script to search for the CUDA compiler")
+    set(CMAKE_CUDA_COMPILER ${CUDA_NVCC_EXECUTABLE} CACHE INTERNAL "CUDA compiler executable")
+    enable_language(CUDA)
+  else()
+    message(WARNING "No CUDA support")
+  endif()
+endif()
+
 af_deprecate(BUILD_CPU             AF_BUILD_CPU)
 af_deprecate(BUILD_CUDA            AF_BUILD_CUDA)
 af_deprecate(BUILD_OPENCL          AF_BUILD_OPENCL)
diff --git a/CMakeModules/AFcuda_helpers.cmake b/CMakeModules/AFcuda_helpers.cmake
index 59cfb2002a..a5d20c4a62 100644
--- a/CMakeModules/AFcuda_helpers.cmake
+++ b/CMakeModules/AFcuda_helpers.cmake
@@ -6,6 +6,34 @@
 # http://arrayfire.com/licenses/BSD-3-Clause
 
 find_program(NVPRUNE NAMES nvprune)
+cuda_select_nvcc_arch_flags(cuda_architecture_flags ${CUDA_architecture_build_targets})
+set(cuda_architecture_flags ${cuda_architecture_flags} CACHE INTERNAL "CUDA compute flags" FORCE)
+set(cuda_architecture_flags_readable ${cuda_architecture_flags_readable} CACHE INTERNAL "Readable CUDA compute flags" FORCE)
+
+function(af_detect_and_set_cuda_architectures target)
+  if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
+    string(REGEX REPLACE "sm_([0-9]+)[ ]*" "\\1-real|" cuda_build_targets ${cuda_architecture_flags_readable})
+    string(REGEX REPLACE "compute_([0-9]+)[ ]*" "\\1-virtual|" cuda_build_targets ${cuda_build_targets})
+    string(REPLACE "|" ";" cuda_build_targets ${cuda_build_targets})
+
+    set_target_properties(${target}
+      PROPERTIES
+        CUDA_ARCHITECTURES "${cuda_build_targets}")
+  else()
+    # CMake 3.12 adds deduplication of compile options. This breaks the way the
+    # gencode flags are passed into the compiler. these replace instructions add
+    # the SHELL: prefix to each of the gencode options so that it is not removed
+    # from the command
+    if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.12")
+      string(REPLACE ";" "|" cuda_architecture_flags "${cuda_architecture_flags}")
+      string(REGEX REPLACE "(-gencode)\\|" "SHELL:\\1 " cuda_architecture_flags2 "${cuda_architecture_flags}")
+      string(REPLACE "|" ";" cuda_architecture_flags ${cuda_architecture_flags2})
+    endif()
+    target_compile_options(${target}
+      PRIVATE
+        $<$<COMPILE_LANGUAGE:CUDA>:${cuda_architecture_flags}>)
+  endif()
+endfunction()
 
 # The following macro uses a macro defined by
 # FindCUDA module from cmake.
@@ -39,45 +67,3 @@ function(af_find_static_cuda_libs libname)
   mark_as_advanced(CUDA_${libname}_LIBRARY)
 endfunction()
 
-## Copied from FindCUDA.cmake
-## The target_link_library needs to link with the cuda libraries using
-## PRIVATE
-function(cuda_add_library cuda_target)
-  cuda_add_cuda_include_once()
-
-  # Separate the sources from the options
-  cuda_get_sources_and_options(_sources _cmake_options _options ${ARGN})
-  cuda_build_shared_library(_cuda_shared_flag ${ARGN})
-  # Create custom commands and targets for each file.
-  cuda_wrap_srcs( ${cuda_target} OBJ _generated_files ${_sources}
-    ${_cmake_options} ${_cuda_shared_flag}
-    OPTIONS ${_options} )
-
-  # Compute the file name of the intermedate link file used for separable
-  # compilation.
-  cuda_compute_separable_compilation_object_file_name(link_file ${cuda_target} "${${cuda_target}_SEPARABLE_COMPILATION_OBJECTS}")
-
-  # Add the library.
-  add_library(${cuda_target} ${_cmake_options}
-    ${_generated_files}
-    ${_sources}
-    ${link_file}
-    )
-
-  # Add a link phase for the separable compilation if it has been enabled.  If
-  # it has been enabled then the ${cuda_target}_SEPARABLE_COMPILATION_OBJECTS
-  # variable will have been defined.
-  cuda_link_separable_compilation_objects("${link_file}" ${cuda_target} "${_options}" "${${cuda_target}_SEPARABLE_COMPILATION_OBJECTS}")
-
-  target_link_libraries(${cuda_target}
-      PRIVATE ${CUDA_LIBRARIES}
-    )
-
-  # We need to set the linker language based on what the expected generated file
-  # would be. CUDA_C_OR_CXX is computed based on CUDA_HOST_COMPILATION_CPP.
-  set_target_properties(${cuda_target}
-    PROPERTIES
-    LINKER_LANGUAGE ${CUDA_C_OR_CXX}
-    POSITION_INDEPENDENT_CODE ON
-  )
-endfunction()
diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index 3f0828ef3e..5d02277b61 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -43,11 +43,6 @@ function(arrayfire_get_cuda_cxx_flags cuda_flags)
       list(APPEND flags -Xcompiler /Zc:__cplusplus
                         -Xcompiler /std:c++17)
     endif()
-  else()
-    set(flags -std=c++17
-              -Xcompiler -fPIC
-              -Xcompiler ${CMAKE_CXX_COMPILE_OPTIONS_VISIBILITY}hidden
-              --expt-relaxed-constexpr)
   endif()
 
   set(${cuda_flags} ${flags} PARENT_SCOPE)
diff --git a/CMakeModules/config_ccache.cmake b/CMakeModules/config_ccache.cmake
index 80783b06c1..04b3a97901 100644
--- a/CMakeModules/config_ccache.cmake
+++ b/CMakeModules/config_ccache.cmake
@@ -34,7 +34,7 @@ if(${AF_USE_CCACHE})
     # Support Unix Makefiles and Ninja
     set(CMAKE_C_COMPILER_LAUNCHER   "${CCACHE_PROGRAM}")
     set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
-    set(CUDA_NVCC_EXECUTABLE ${CCACHE_PROGRAM} "${CUDA_NVCC_EXECUTABLE}")
+    set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
   endif()
 endif()
 mark_as_advanced(CCACHE_PROGRAM)
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index a6a750f83f..67061740a0 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -48,16 +48,6 @@ endif()
 set(CUDA_architecture_build_targets "Auto" CACHE
   STRING "The compute architectures targeted by this build. (Options: Auto;3.0;Maxwell;All;Common)")
 
-cuda_select_nvcc_arch_flags(cuda_architecture_flags ${CUDA_architecture_build_targets})
-
-string(REGEX REPLACE "-gencodearch=compute_[0-9]+,code=sm_([0-9]+)" "\\1|" cuda_build_targets ${cuda_architecture_flags})
-string(REGEX REPLACE "-gencodearch=compute_[0-9]+,code=compute_([0-9]+)" "\\1+PTX|" cuda_build_targets ${cuda_build_targets})
-string(REGEX REPLACE "([0-9]+)([0-9])\\|" "\\1.\\2 " cuda_build_targets ${cuda_build_targets})
-string(REGEX REPLACE "([0-9]+)([0-9]\\+PTX)\\|" "\\1.\\2 " cuda_build_targets ${cuda_build_targets})
-message(STATUS "CUDA_architecture_build_targets: ${CUDA_architecture_build_targets} ( ${cuda_build_targets} )")
-
-set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};${cuda_architecture_flags})
-
 find_cuda_helper_libs(nvrtc)
 find_cuda_helper_libs(nvrtc-builtins)
 list(APPEND nvrtc_libs ${CUDA_nvrtc_LIBRARY})
@@ -123,20 +113,6 @@ mark_as_advanced(
     CUDA_LIBRARIES_PATH
     CUDA_architecture_build_targets)
 
-get_target_property(COMMON_INTERFACE_DIRS afcommon_interface INTERFACE_INCLUDE_DIRECTORIES)
-
-cuda_include_directories(
-  ${CMAKE_CURRENT_BINARY_DIR}
-  ${CMAKE_CURRENT_SOURCE_DIR}
-  ${ArrayFire_SOURCE_DIR}/include
-  ${ArrayFire_BINARY_DIR}/include
-  ${CMAKE_CURRENT_SOURCE_DIR}/kernel
-  ${CMAKE_CURRENT_SOURCE_DIR}/jit
-  ${ArrayFire_SOURCE_DIR}/src/api/c
-  ${ArrayFire_SOURCE_DIR}/src/backend
-  ${COMMON_INTERFACE_DIRS}
-  $<TARGET_PROPERTY:af_spdlog,INTERFACE_INCLUDE_DIRECTORIES>
-  )
 if(CUDA_VERSION_MAJOR VERSION_LESS 11)
   af_dep_check_and_populate(${cub_prefix}
     URI https://github.com/NVIDIA/cub.git
@@ -254,15 +230,6 @@ file_to_string(
 arrayfire_get_cuda_cxx_flags(cuda_cxx_flags)
 arrayfire_get_platform_definitions(platform_flags)
 
-get_property(boost_includes TARGET Boost::boost PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
-get_property(boost_definitions TARGET Boost::boost PROPERTY INTERFACE_COMPILE_DEFINITIONS)
-
-string(REPLACE ";" ";-I" boost_includes "-I${boost_includes}")
-string(REPLACE ";" ";-D" boost_definitions "-D${boost_definitions}")
-
-set(cuda_cxx_flags "${cuda_cxx_flags};${boost_includes}")
-set(cuda_cxx_flags "${cuda_cxx_flags};${boost_definitions}")
-
 # New API of cuSparse was introduced in 10.1.168 for Linux and the older
 # 10.1.105 fix version doesn't it. Unfortunately, the new API was introduced in
 # in a fix release of CUDA - unconventionally. As CMake's FindCUDA module
@@ -283,23 +250,8 @@ list(APPEND cuda_cxx_flags ${cxx_definitions})
 include(kernel/scan_by_key/CMakeLists.txt)
 include(kernel/thrust_sort_by_key/CMakeLists.txt)
 
-# CUDA static libraries require device linking to successfully link
-# against afcuda target. Device linking requires CUDA_SEPARABLE_COMPILATION
-# to be ON. Therefore, we turn on separable compilation for a subset of
-# source files while compiling af_cuda_static_cuda_library target. Once
-# this subset is compiled, separable compilation is reset to it's original
-# value.
-if(UNIX)
-  # Static linking cuda libs require device linking, which in turn
-  # requires separable compilation.
-  set(pior_val_CUDA_SEPARABLE_COMPILATION OFF)
-  if(DEFINED CUDA_SEPARABLE_COMPILATION)
-    set(pior_val_CUDA_SEPARABLE_COMPILATION ${CUDA_SEPARABLE_COMPILATION})
-  endif()
-  set(CUDA_SEPARABLE_COMPILATION ON)
-endif()
-
-cuda_add_library(af_cuda_static_cuda_library STATIC
+add_library(af_cuda_static_cuda_library
+  STATIC
     blas.cu
     blas.hpp
     cudaDataType.hpp
@@ -315,18 +267,46 @@ cuda_add_library(af_cuda_static_cuda_library STATIC
     sparse_blas.hpp
     solve.cu
     solve.hpp
-
-    OPTIONS
-    ${platform_flags} ${cuda_cxx_flags} ${af_cuda_static_flags}
-    -Xcudafe --display_error_number -Xcudafe \"--diag_suppress=1427\" -DAFDLL
 )
 
+af_detect_and_set_cuda_architectures(af_cuda_static_cuda_library)
+if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
+  set_target_properties(af_cuda_static_cuda_library
+    PROPERTIES
+      CUDA_STANDARD 17)
+else()
+  target_compile_options(af_cuda_static_cuda_library
+    PRIVATE
+      $<$<COMPILE_LANGUAGE:CUDA>:--std=c++17>)
+endif()
+
 set_target_properties(af_cuda_static_cuda_library
   PROPERTIES
-  LINKER_LANGUAGE CXX
-  FOLDER "Generated Targets"
+    POSITION_INDEPENDENT_CODE ON
+    LINKER_LANGUAGE CUDA
+    FOLDER "Generated Targets")
+
+target_include_directories(af_cuda_static_cuda_library
+  PRIVATE
+    ${CMAKE_CURRENT_BINARY_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${ArrayFire_SOURCE_DIR}/include
+    ${ArrayFire_BINARY_DIR}/include
+    ${ArrayFire_SOURCE_DIR}/src/api/c
+    ${ArrayFire_SOURCE_DIR}/src/backend
 )
 
+target_compile_definitions(af_cuda_static_cuda_library
+  PRIVATE
+    ${platform_flags}
+    AFDLL)
+
+target_compile_options(af_cuda_static_cuda_library
+  PRIVATE
+    $<$<COMPILE_LANGUAGE:CUDA>:-Xcudafe --diag_suppress=unrecognized_gcc_pragma>
+    $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
+    ${cuda_cxx_flags})
+
 if(CUDA_VERSION_MAJOR VERSION_GREATER 10 OR
    (UNIX AND
     CUDA_VERSION_MAJOR VERSION_EQUAL 10 AND CUDA_VERSION_MINOR VERSION_GREATER 1))
@@ -372,7 +352,6 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
         ${CUDA_lapack_static_LIBRARY})
   endif()
 
-  set(CUDA_SEPARABLE_COMPILATION ${pior_val_CUDA_SEPARABLE_COMPILATION})
 else()
   target_link_libraries(af_cuda_static_cuda_library
     PUBLIC
@@ -383,7 +362,7 @@ else()
   )
 endif()
 
-cuda_add_library(afcuda
+add_library(afcuda
     $<$<PLATFORM_ID:Windows>:${af_cuda_ver_res_file}>
     ${thrust_sort_sources}
 
@@ -681,13 +660,30 @@ cuda_add_library(afcuda
     jit/kernel_generators.hpp
 
     ${scan_by_key_sources}
+  )
 
-    OPTIONS
-    ${platform_flags}
+af_detect_and_set_cuda_architectures(afcuda)
+if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
+  set_target_properties(afcuda
+    PROPERTIES
+      CUDA_STANDARD 17)
+else()
+  target_compile_options(afcuda
+    PRIVATE
+      $<$<COMPILE_LANGUAGE:CUDA>:--std=c++17>)
+endif()
+
+target_compile_definitions(afcuda
+  PRIVATE
+    ${platform_flags})
+
+target_compile_options(afcuda
+  PRIVATE
     ${cuda_cxx_flags}
-    -Xcudafe --display_error_number
-    -Xcudafe \"--diag_suppress=1427\"
-  )
+    $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
+    $<$<COMPILE_LANGUAGE:CUDA>:-Xcudafe --diag_suppress=unrecognized_gcc_pragma>
+)
+
 
 if(AF_WITH_CUDNN)
   target_sources(afcuda PRIVATE
@@ -709,15 +705,6 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS AND AF_cusparse_LINK_LOADING STREQU
       AF_cusparse_STATIC_LINKING)
 endif()
 
-
-arrayfire_set_default_cxx_flags(afcuda)
-
-# NOTE: Do not add additional CUDA specific definitions here. Add it to the
-# cxx_definitions variable above. cxx_definitions is used to propigate
-# definitions to the scan_by_key and thrust_sort_by_key targets as well as the
-# cuda library above.
-target_compile_options(afcuda PRIVATE ${cxx_definitions})
-
 add_library(ArrayFire::afcuda ALIAS afcuda)
 
 add_dependencies(afcuda ${jit_kernel_targets} ${nvrtc_kernel_targets})
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index c7add80ca3..e6468848d6 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -333,6 +333,7 @@ if(OpenCL_FOUND)
 endif()
 
 if(CUDA_FOUND)
+  include(AFcuda_helpers)
   foreach(backend ${enabled_backends})
     set(cuda_test_backends "cuda" "unified")
     if(${backend} IN_LIST cuda_test_backends)
@@ -345,7 +346,7 @@ if(CUDA_FOUND)
           ${CMAKE_CURRENT_SOURCE_DIR}
         )
       endif()
-      cuda_add_executable(${target} cuda.cu  $<TARGET_OBJECTS:arrayfire_test>)
+      add_executable(${target} cuda.cu  $<TARGET_OBJECTS:arrayfire_test>)
       target_include_directories(${target} PRIVATE
         ${ArrayFire_SOURCE_DIR}/extern/half/include
         ${CMAKE_SOURCE_DIR}
@@ -369,10 +370,12 @@ if(CUDA_FOUND)
         target_link_libraries(${target} -pthread)
       endif()
 
+      af_detect_and_set_cuda_architectures(${target})
+
       set_target_properties(${target}
         PROPERTIES
-        FOLDER "Tests"
-        OUTPUT_NAME "cuda_${backend}")
+          FOLDER "Tests"
+          OUTPUT_NAME "cuda_${backend}")
 
       if(NOT ${backend} STREQUAL "unified")
         add_test(NAME ${target} COMMAND ${target})

From 1731fff184426112f6c7086ce083ff8401bd1448 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 27 Sep 2022 02:03:42 -0400
Subject: [PATCH 080/473] Remove cudaDeviceSynchronize from the
 ThrustArrayFirePolicy

---
 src/backend/cuda/ThrustArrayFirePolicy.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/cuda/ThrustArrayFirePolicy.hpp b/src/backend/cuda/ThrustArrayFirePolicy.hpp
index d58b508453..6787d405de 100644
--- a/src/backend/cuda/ThrustArrayFirePolicy.hpp
+++ b/src/backend/cuda/ThrustArrayFirePolicy.hpp
@@ -49,7 +49,7 @@ __DH__ inline cudaStream_t get_stream<::cuda::ThrustArrayFirePolicy>(
 __DH__
 inline cudaError_t synchronize_stream(const ::cuda::ThrustArrayFirePolicy &) {
 #if defined(__CUDA_ARCH__)
-    return cudaDeviceSynchronize();
+    return cudaSuccess;
 #else
     return cudaStreamSynchronize(::cuda::getActiveStream());
 #endif

From 274da93f474ffd0016ad31aa81648b9ee9bcb4f7 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 28 Sep 2022 13:20:24 -0400
Subject: [PATCH 081/473] Fix several CI issues due to changes in GitHub
 actions' environment

VCPKG_ROOT is now defined as an environment variable in GitHub actions. This
change causes some of our jobs to fail because our scripts detect the
environment variable to trigger some work. In this commit I remove
the VCPKG_ROOT environment variable from the ubuntu jobs and remove
the setting of the VCPKG_ROOT CMake variable on the windows job

Use clean-after-build flag instead of Remove-Item to clean vcpkg builds

Fix missing expat package in new macOS GitHub workflow
---
 .github/workflows/docs_build.yml     | 2 +-
 .github/workflows/unix_cpu_build.yml | 4 ++--
 .github/workflows/win_cpu_build.yml  | 4 +---
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml
index bf81164cdd..38091d113a 100644
--- a/.github/workflows/docs_build.yml
+++ b/.github/workflows/docs_build.yml
@@ -32,7 +32,7 @@ jobs:
 
             - name: Configure
               run: |
-                  mkdir build && cd build
+                  mkdir build && cd build && unset VCPKG_ROOT
                   cmake -DAF_BUILD_CPU:BOOL=OFF -DAF_BUILD_CUDA:BOOL=OFF \
                         -DAF_BUILD_OPENCL:BOOL=OFF -DAF_BUILD_UNIFIED:BOOL=OFF \
                         -DAF_BUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTING:BOOL=OFF \
diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 1962db4891..114799bbca 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -58,7 +58,7 @@ jobs:
             - name: Install Dependencies for Macos
               if: matrix.os == 'macos-latest'
               run: |
-                  brew install boost fontconfig glfw freeimage fftw lapack openblas
+                  brew install boost fontconfig glfw freeimage fftw lapack openblas expat
                   echo "CMAKE_PROGRAM=cmake" >> $GITHUB_ENV
 
             - name: Install Common Dependencies for Ubuntu
@@ -103,7 +103,7 @@ jobs:
                   backend=$(if [ "$USE_MKL" == 1 ]; then echo "Intel-MKL"; else echo "FFTW/LAPACK/BLAS"; fi)
                   buildname="$buildname-cpu-$BLAS_BACKEND"
                   cmake_rpath=$(if [ $OS_NAME == 'macos-latest' ]; then echo "-DCMAKE_INSTALL_RPATH=/opt/arrayfire/lib"; fi)
-                  mkdir build && cd build
+                  mkdir build && cd build && unset VCPKG_ROOT
                   ${CMAKE_PROGRAM} -G Ninja \
                       -DCMAKE_MAKE_PROGRAM:FILEPATH=${GITHUB_WORKSPACE}/ninja \
                       -DAF_BUILD_CUDA:BOOL=OFF -DAF_BUILD_OPENCL:BOOL=OFF \
diff --git a/.github/workflows/win_cpu_build.yml b/.github/workflows/win_cpu_build.yml
index 067f951fff..9d5419f7dd 100644
--- a/.github/workflows/win_cpu_build.yml
+++ b/.github/workflows/win_cpu_build.yml
@@ -36,8 +36,7 @@ jobs:
                 cd vcpkg
                 git checkout $env:VCPKG_HASH
                 .\bootstrap-vcpkg.bat
-                .\vcpkg.exe install boost-compute boost-math boost-stacktrace fftw3 freeimage freetype[core] forge glfw3 openblas
-                Remove-Item .\downloads,.\buildtrees,.\packages -Recurse -Force
+                .\vcpkg.exe install --clean-after-build boost-compute boost-math boost-stacktrace fftw3 freeimage freetype[core] forge glfw3 openblas
 
             - name: CMake Configure
               run: |
@@ -49,7 +48,6 @@ jobs:
                   $buildname = "$buildname-cpu-openblas"
                   mkdir build && cd build
                   cmake .. -G "Visual Studio 17 2022" -A x64 `
-                      -DVCPKG_ROOT:PATH="~/vcpkg" `
                       -DVCPKG_MANIFEST_MODE:BOOL=OFF `
                       -DAF_BUILD_CUDA:BOOL=OFF -DAF_BUILD_OPENCL:BOOL=OFF `
                       -DAF_BUILD_UNIFIED:BOOL=OFF -DAF_BUILD_FORGE:BOOL=ON `

From cdb6797f39c9efc50400152229331cedbc97d826 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 29 Sep 2022 15:56:11 -0400
Subject: [PATCH 082/473] CMake compiler flag refactor

---
 CMakeLists.txt                    |  13 +-
 CMakeModules/InternalUtils.cmake  | 100 ++++++-------
 CMakeModules/build_CLBlast.cmake  |   5 +-
 CMakeModules/platform.cmake       |  21 ---
 src/api/unified/CMakeLists.txt    |  92 ++++++------
 src/backend/cpu/CMakeLists.txt    |   2 -
 src/backend/cuda/CMakeLists.txt   | 226 ++++++++++++------------------
 src/backend/opencl/CMakeLists.txt |   2 -
 8 files changed, 176 insertions(+), 285 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 091980b6e3..08445a986f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -411,16 +411,11 @@ find_library(Backtrace_LIBRARY backtrace
 find_program(ADDR2LINE_PROGRAM addr2line
   DOC "The path to the addr2line program for informative stacktraces")
 
+check_cxx_compiler_flag(-Wno-ignored-attributes has_ignored_attributes_flag)
+check_cxx_compiler_flag(-Wall has_all_warnings_flag)
+
 foreach(backend ${built_backends})
-  target_compile_definitions(${backend} PRIVATE AFDLL)
-  if(AF_WITH_LOGGING)
-    target_compile_definitions(${backend}
-      PRIVATE AF_WITH_LOGGING)
-  endif()
-  if(AF_CACHE_KERNELS_TO_DISK)
-    target_compile_definitions(${backend}
-      PRIVATE AF_CACHE_KERNELS_TO_DISK)
-  endif()
+  arrayfire_set_default_cxx_flags(${backend})
 endforeach()
 
 if(AF_BUILD_FRAMEWORK)
diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index 5d02277b61..dde0756aaa 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -18,68 +18,54 @@ function(conditional_directory variable directory)
   endif()
 endfunction()
 
-function(arrayfire_get_platform_definitions variable)
+include(CheckCXXCompilerFlag)
+
 if(WIN32)
-  set(${variable} -DOS_WIN -DWIN32_LEAN_AND_MEAN -DNOMINMAX PARENT_SCOPE)
-elseif(APPLE)
-  set(${variable} -DOS_MAC PARENT_SCOPE)
-elseif(UNIX)
-  set(${variable} -DOS_LNX PARENT_SCOPE)
+  check_cxx_compiler_flag(/Zc:__cplusplus cplusplus_define)
+  check_cxx_compiler_flag(/permissive- cxx_compliance)
 endif()
-endfunction()
-
-function(arrayfire_get_cuda_cxx_flags cuda_flags)
-  if(MSVC)
-    set(flags -Xcompiler /wd4251
-              -Xcompiler /wd4068
-              -Xcompiler /wd4275
-              -Xcompiler /bigobj
-              -Xcompiler /EHsc
-              --expt-relaxed-constexpr)
-    if(CMAKE_GENERATOR MATCHES "Ninja")
-      set(flags ${flags} -Xcompiler /FS)
-    endif()
-    if(cplusplus_define)
-      list(APPEND flags -Xcompiler /Zc:__cplusplus
-                        -Xcompiler /std:c++17)
-    endif()
-  endif()
-
-  set(${cuda_flags} ${flags} PARENT_SCOPE)
-endfunction()
-
-include(CheckCXXCompilerFlag)
 
 function(arrayfire_set_default_cxx_flags target)
-  arrayfire_get_platform_definitions(defs)
-  target_compile_definitions(${target} PRIVATE ${defs})
-
-  if(MSVC)
-    target_compile_options(${target}
-      PRIVATE
-        /wd4251 /wd4068 /wd4275 /bigobj /EHsc)
-
-    if(CMAKE_GENERATOR MATCHES "Ninja")
-      target_compile_options(${target}
-        PRIVATE
-          /FS)
-    endif()
-  else()
-    check_cxx_compiler_flag(-Wno-ignored-attributes has_ignored_attributes_flag)
-
-    # OpenCL targets need this flag to avoid ignored attribute warnings in the
-    # OpenCL headers
-    if(has_ignored_attributes_flag)
-        target_compile_options(${target}
-          PRIVATE -Wno-ignored-attributes)
-    endif()
+  target_compile_options(${target}
+    PRIVATE
+      $<$<COMPILE_LANGUAGE:CXX>:
+              # C4068: Warnings about unknown pragmas
+              # C4668: Warnings about unknown defintions
+              # C4275: Warnings about using non-exported classes as base class of an
+              #        exported class
+              $<$<CXX_COMPILER_ID:MSVC>:  /wd4251
+                                          /wd4068
+                                          /wd4275
+                                          /wd4668
+                                          /wd4710
+                                          /wd4505
+                                          /bigobj
+                                          /EHsc
+                                          # MSVC incorrectly sets the cplusplus to 199711L even if the compiler supports
+                                          # c++11 features. This flag sets it to the correct standard supported by the
+                                          # compiler
+                                          $<$<BOOL:${cplusplus_define}>:/Zc:__cplusplus>
+                                          $<$<BOOL:${cxx_compliance}>:/permissive-> >
+
+              # OpenCL targets need this flag to avoid
+              # ignored attribute warnings in the OpenCL
+              # headers
+              $<$<BOOL:${has_ignored_attributes_flag}>:-Wno-ignored-attributes>
+              $<$<BOOL:${has_all_warnings_flag}>:-Wall>>
+    )
 
-    check_cxx_compiler_flag(-Wall has_all_warnings_flag)
-    if(has_all_warnings_flag)
-      target_compile_options(${target}
-        PRIVATE -Wall)
-    endif()
-  endif()
+  target_compile_definitions(${target}
+    PRIVATE
+      AFDLL
+      $<$<PLATFORM_ID:Windows>:             OS_WIN
+                                            WIN32_LEAN_AND_MEAN
+                                            NOMINMAX>
+      $<$<PLATFORM_ID:Darwin>:              OS_MAC>
+      $<$<PLATFORM_ID:Linux>:               OS_LNX>
+
+      $<$<BOOL:${AF_WITH_LOGGING}>:           AF_WITH_LOGGING>
+      $<$<BOOL:${AF_CACHE_KERNELS_TO_DISK}>:  AF_CACHE_KERNELS_TO_DISK>
+  )
 endfunction()
 
 function(__af_deprecate_var var access value)
diff --git a/CMakeModules/build_CLBlast.cmake b/CMakeModules/build_CLBlast.cmake
index 780cddbaaf..446ceb7e00 100644
--- a/CMakeModules/build_CLBlast.cmake
+++ b/CMakeModules/build_CLBlast.cmake
@@ -63,10 +63,11 @@ else()
       CONFIGURE_COMMAND ${CMAKE_COMMAND} ${extproj_gen_opts}
         -Wno-dev <SOURCE_DIR>
         -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}
-        "-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS} -w -fPIC"
+        "-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}"
         -DOVERRIDE_MSVC_FLAGS_TO_MT:BOOL=OFF
         -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}
-        "-DCMAKE_C_FLAGS:STRING=${CMAKE_C_FLAGS} -w -fPIC"
+        "-DCMAKE_C_FLAGS:STRING=${CMAKE_C_FLAGS}"
+        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
         ${extproj_build_type_option}
         -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
         -DCMAKE_INSTALL_LIBDIR:PATH=lib
diff --git a/CMakeModules/platform.cmake b/CMakeModules/platform.cmake
index cfaf92dd5d..cf0f72f8ed 100644
--- a/CMakeModules/platform.cmake
+++ b/CMakeModules/platform.cmake
@@ -19,24 +19,3 @@ if(UNIX AND NOT APPLE)
   set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};/opt/intel/mkl/lib/intel64")
 endif()
 
-if(WIN32)
-  # C4068: Warnings about unknown pragmas
-  # C4275: Warnings about using non-exported classes as base class of an
-  #        exported class
-  add_compile_options(/wd4068 /wd4275)
-
-  # MSVC incorrectly sets the cplusplus to 199711L even if the compiler supports
-  # c++11 features. This flag sets it to the correct standard supported by the
-  # compiler
-  check_cxx_compiler_flag(/Zc:__cplusplus cplusplus_define)
-  if(cplusplus_define)
-    add_compile_options(/Zc:__cplusplus)
-  endif()
-
-  # The "permissive-" option enforces strict(er?) standards compliance by
-  # MSVC
-  check_cxx_compiler_flag(/permissive- cxx_compliance)
-  if(cxx_compliance)
-    add_compile_options(/permissive-)
-  endif()
-endif()
diff --git a/src/api/unified/CMakeLists.txt b/src/api/unified/CMakeLists.txt
index 522a19ba2a..67b6b80dd2 100644
--- a/src/api/unified/CMakeLists.txt
+++ b/src/api/unified/CMakeLists.txt
@@ -1,3 +1,10 @@
+# Copyright (c) 2022, ArrayFire
+# All rights reserved.
+#
+# This file is distributed under 3-clause BSD license.
+# The complete license agreement can be obtained at:
+# http://arrayfire.com/licenses/BSD-3-Clause
+
 generate_product_version(af_unified_ver_res_file
   FILE_NAME "af"
   FILE_DESCRIPTION "Unified Backend Dynamic-link library"
@@ -9,58 +16,36 @@ add_library(ArrayFire::af ALIAS af)
 target_sources(af
   PRIVATE
     ${af_unified_ver_res_file}
-    ${CMAKE_CURRENT_SOURCE_DIR}/algorithm.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/arith.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/array.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/blas.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/data.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/error.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/event.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/features.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/graphics.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/image.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/index.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/internal.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/jit_test_api.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/lapack.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/memory.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/ml.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/moments.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/random.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/signal.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/sparse.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/statistics.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/symbol_manager.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/symbol_manager.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/util.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/vision.cpp
-  )
-
-if(OpenCL_FOUND)
-  target_sources(af
-    PRIVATE
-    ${CMAKE_CURRENT_SOURCE_DIR}/opencl.cpp
-  )
+    algorithm.cpp
+    arith.cpp
+    array.cpp
+    blas.cpp
+    data.cpp
+    device.cpp
+    error.cpp
+    event.cpp
+    features.cpp
+    graphics.cpp
+    image.cpp
+    index.cpp
+    internal.cpp
+    jit_test_api.cpp
+    lapack.cpp
+    memory.cpp
+    ml.cpp
+    moments.cpp
+    random.cpp
+    signal.cpp
+    sparse.cpp
+    statistics.cpp
+    symbol_manager.cpp
+    symbol_manager.hpp
+    util.cpp
+    vision.cpp
+
+    $<$<BOOL:${OpenCL_FOUND}>: ${CMAKE_CURRENT_SOURCE_DIR}/opencl.cpp>
+    $<$<BOOL:${CUDA_FOUND}>: ${CMAKE_CURRENT_SOURCE_DIR}/cuda.cpp>
 
-  target_include_directories(af
-    PRIVATE
-      $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>)
-
-endif()
-
-if(CUDA_FOUND)
-  target_sources(af
-    PRIVATE
-      ${CMAKE_CURRENT_SOURCE_DIR}/cuda.cpp)
-
-  target_include_directories(af
-    PRIVATE
-      ${CUDA_INCLUDE_DIRS})
-endif()
-
-target_sources(af
-  PRIVATE
     ${ArrayFire_SOURCE_DIR}/src/api/c/type_util.cpp
     ${ArrayFire_SOURCE_DIR}/src/api/c/version.cpp
     ${ArrayFire_SOURCE_DIR}/src/backend/common/Logger.cpp
@@ -73,7 +58,6 @@ target_sources(af
     ${ArrayFire_SOURCE_DIR}/src/backend/common/deprecated.hpp
   )
 
-arrayfire_set_default_cxx_flags(af)
 if(WIN32)
   target_sources(af
     PRIVATE
@@ -94,8 +78,10 @@ target_include_directories(af
   PRIVATE
     ${ArrayFire_SOURCE_DIR}/src/api/c
     ${ArrayFire_SOURCE_DIR}/src/api/unified
+    ${ArrayFire_BINARY_DIR}
     $<TARGET_PROPERTY:afcommon_interface,INTERFACE_INCLUDE_DIRECTORIES>
-    ${CMAKE_BINARY_DIR}
+    $<$<BOOL:${OpenCL_FOUND}>: $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>>
+    $<$<BOOL:${CUDA_FOUND}>:  ${CUDA_INCLUDE_DIRS}>
   )
 
 target_link_libraries(af
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index 7aa10bc529..04d0d3390b 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -283,8 +283,6 @@ target_sources(afcpu
     ${${threads_prefix}_SOURCE_DIR}/include/threads/event.hpp
   )
 
-arrayfire_set_default_cxx_flags(afcpu)
-
 include("${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/CMakeLists.txt")
 
 target_include_directories(afcpu
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 67061740a0..e1f47b2947 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -227,31 +227,14 @@ file_to_string(
     NULLTERM
     )
 
-arrayfire_get_cuda_cxx_flags(cuda_cxx_flags)
-arrayfire_get_platform_definitions(platform_flags)
-
-# New API of cuSparse was introduced in 10.1.168 for Linux and the older
-# 10.1.105 fix version doesn't it. Unfortunately, the new API was introduced in
-# in a fix release of CUDA - unconventionally. As CMake's FindCUDA module
-# doesn't provide patch/fix version number, we use 10.2 as the minimum
-# CUDA version to enable this new cuSparse API.
-if(CUDA_VERSION_MAJOR VERSION_GREATER 10 OR
-   (UNIX AND
-    CUDA_VERSION_MAJOR VERSION_EQUAL 10 AND CUDA_VERSION_MINOR VERSION_GREATER 1))
-  list(APPEND cxx_definitions -DAF_USE_NEW_CUSPARSE_API)
-endif()
-
-# CUDA_NO_HALF prevents the inclusion of the half class in the global namespace
-# which conflicts with the half class in ArrayFire's common namespace. prefer
-# using __half class instead for CUDA
-list(APPEND cxx_definitions -DAF_CUDA;-DCUDA_NO_HALF)
-list(APPEND cuda_cxx_flags ${cxx_definitions})
-
 include(kernel/scan_by_key/CMakeLists.txt)
 include(kernel/thrust_sort_by_key/CMakeLists.txt)
 
-add_library(af_cuda_static_cuda_library
-  STATIC
+
+add_library(afcuda
+    $<$<PLATFORM_ID:Windows>:${af_cuda_ver_res_file}>
+    ${thrust_sort_sources}
+
     blas.cu
     blas.hpp
     cudaDataType.hpp
@@ -267,104 +250,6 @@ add_library(af_cuda_static_cuda_library
     sparse_blas.hpp
     solve.cu
     solve.hpp
-)
-
-af_detect_and_set_cuda_architectures(af_cuda_static_cuda_library)
-if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
-  set_target_properties(af_cuda_static_cuda_library
-    PROPERTIES
-      CUDA_STANDARD 17)
-else()
-  target_compile_options(af_cuda_static_cuda_library
-    PRIVATE
-      $<$<COMPILE_LANGUAGE:CUDA>:--std=c++17>)
-endif()
-
-set_target_properties(af_cuda_static_cuda_library
-  PROPERTIES
-    POSITION_INDEPENDENT_CODE ON
-    LINKER_LANGUAGE CUDA
-    FOLDER "Generated Targets")
-
-target_include_directories(af_cuda_static_cuda_library
-  PRIVATE
-    ${CMAKE_CURRENT_BINARY_DIR}
-    ${CMAKE_CURRENT_SOURCE_DIR}
-    ${ArrayFire_SOURCE_DIR}/include
-    ${ArrayFire_BINARY_DIR}/include
-    ${ArrayFire_SOURCE_DIR}/src/api/c
-    ${ArrayFire_SOURCE_DIR}/src/backend
-)
-
-target_compile_definitions(af_cuda_static_cuda_library
-  PRIVATE
-    ${platform_flags}
-    AFDLL)
-
-target_compile_options(af_cuda_static_cuda_library
-  PRIVATE
-    $<$<COMPILE_LANGUAGE:CUDA>:-Xcudafe --diag_suppress=unrecognized_gcc_pragma>
-    $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
-    ${cuda_cxx_flags})
-
-if(CUDA_VERSION_MAJOR VERSION_GREATER 10 OR
-   (UNIX AND
-    CUDA_VERSION_MAJOR VERSION_EQUAL 10 AND CUDA_VERSION_MINOR VERSION_GREATER 1))
-  target_compile_definitions(af_cuda_static_cuda_library PRIVATE AF_USE_NEW_CUSPARSE_API)
-endif()
-
-target_link_libraries(af_cuda_static_cuda_library
-  PRIVATE
-    Boost::boost
-    af_spdlog
-    nonstd::span-lite)
-
-if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
-  check_cxx_compiler_flag("-Wl,--start-group -Werror" group_flags)
-  if(group_flags)
-    set(START_GROUP -Wl,--start-group)
-    set(END_GROUP -Wl,--end-group)
-  endif()
-
-  target_link_libraries(af_cuda_static_cuda_library
-    PRIVATE
-      ${CMAKE_DL_LIBS}
-      ${cusolver_lib}
-      ${START_GROUP}
-      ${CUDA_culibos_LIBRARY} #also a static libary
-      ${AF_CUDA_cublas_static_LIBRARY}
-      ${AF_CUDA_cublasLt_static_LIBRARY}
-      ${AF_CUDA_cufft_static_LIBRARY}
-      ${AF_CUDA_optionally_static_libraries}
-      ${nvrtc_libs}
-      ${cusolver_static_lib}
-      ${END_GROUP})
-
-  if(CUDA_VERSION VERSION_GREATER 10.0)
-    target_link_libraries(af_cuda_static_cuda_library
-      PRIVATE
-        ${AF_CUDA_cublasLt_static_LIBRARY})
-  endif()
-
-  if(CUDA_VERSION VERSION_GREATER 9.5)
-    target_link_libraries(af_cuda_static_cuda_library
-      PRIVATE
-        ${CUDA_lapack_static_LIBRARY})
-  endif()
-
-else()
-  target_link_libraries(af_cuda_static_cuda_library
-    PUBLIC
-      ${CUDA_CUBLAS_LIBRARIES}
-      ${CUDA_CUFFT_LIBRARIES}
-      ${CUDA_cusolver_LIBRARY}
-      ${nvrtc_libs}
-  )
-endif()
-
-add_library(afcuda
-    $<$<PLATFORM_ID:Windows>:${af_cuda_ver_res_file}>
-    ${thrust_sort_sources}
 
     EnqueueArgs.hpp
     all.cu
@@ -520,6 +405,12 @@ add_library(afcuda
     cu_check_macro.hpp
     cublas.cpp
     cublas.hpp
+
+    $<$<BOOL:${AF_WITH_CUDNN}>: cudnn.cpp
+                             cudnn.hpp
+                             cudnnModule.cpp
+                             cudnnModule.hpp>
+
     cufft.hpp
     cusolverDn.cpp
     cusolverDn.hpp
@@ -662,11 +553,56 @@ add_library(afcuda
     ${scan_by_key_sources}
   )
 
+
+if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
+  check_cxx_compiler_flag("-Wl,--start-group -Werror" group_flags)
+  if(group_flags)
+    set(START_GROUP -Wl,--start-group)
+    set(END_GROUP -Wl,--end-group)
+  endif()
+
+  target_link_libraries(afcuda
+    PRIVATE
+      ${cusolver_lib}
+      ${START_GROUP}
+      ${CUDA_culibos_LIBRARY} #also a static libary
+      ${AF_CUDA_cublas_static_LIBRARY}
+      ${AF_CUDA_cublasLt_static_LIBRARY}
+      ${AF_CUDA_cufft_static_LIBRARY}
+      ${AF_CUDA_optionally_static_libraries}
+      ${nvrtc_libs}
+      ${cusolver_static_lib}
+      ${END_GROUP})
+
+  if(CUDA_VERSION VERSION_GREATER 10.0)
+    target_link_libraries(afcuda
+      PRIVATE
+        ${AF_CUDA_cublasLt_static_LIBRARY})
+  endif()
+
+  if(CUDA_VERSION VERSION_GREATER 9.5)
+    target_link_libraries(afcuda
+      PRIVATE
+        ${CUDA_lapack_static_LIBRARY})
+  endif()
+
+else()
+  target_link_libraries(afcuda
+    PUBLIC
+      ${CUDA_CUBLAS_LIBRARIES}
+      ${CUDA_CUFFT_LIBRARIES}
+      ${CUDA_cusolver_LIBRARY}
+      ${nvrtc_libs}
+  )
+endif()
+
+
 af_detect_and_set_cuda_architectures(afcuda)
 if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
   set_target_properties(afcuda
     PROPERTIES
-      CUDA_STANDARD 17)
+      CUDA_STANDARD 17
+      CUDA_STANDARD_REQUIRED ON)
 else()
   target_compile_options(afcuda
     PRIVATE
@@ -675,30 +611,43 @@ endif()
 
 target_compile_definitions(afcuda
   PRIVATE
-    ${platform_flags})
+    AF_CUDA
+
+    # CUDA_NO_HALF prevents the inclusion of the half class in the global namespace
+    # which conflicts with the half class in ArrayFire's common namespace. prefer
+    # using __half class instead for CUDA
+    CUDA_NO_HALF
+
+    $<$<BOOL:${AF_WITH_CUDNN}>:WITH_CUDNN>
+)
+
+# New API of cuSparse was introduced in 10.1.168 for Linux and the older
+# 10.1.105 fix version doesn't it. Unfortunately, the new API was introduced in
+# in a fix release of CUDA - unconventionally. As CMake's FindCUDA module
+# doesn't provide patch/fix version number, we use 10.2 as the minimum
+# CUDA version to enable this new cuSparse API.
+if(CUDA_VERSION_MAJOR VERSION_GREATER 10 OR
+    (UNIX AND
+      CUDA_VERSION_MAJOR VERSION_EQUAL 10 AND CUDA_VERSION_MINOR VERSION_GREATER 1))
+  target_compile_definitions(afcuda
+    PRIVATE
+      AF_USE_NEW_CUSPARSE_API)
+endif()
 
 target_compile_options(afcuda
   PRIVATE
-    ${cuda_cxx_flags}
     $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
     $<$<COMPILE_LANGUAGE:CUDA>:-Xcudafe --diag_suppress=unrecognized_gcc_pragma>
+    $<$<COMPILE_LANGUAGE:CUDA>: $<$<CXX_COMPILER_ID:MSVC>:  -Xcompiler=/wd4251
+                                                            -Xcompiler=/wd4068
+                                                            -Xcompiler=/wd4275
+                                                            -Xcompiler=/wd4668
+                                                            -Xcompiler=/wd4710
+                                                            -Xcompiler=/wd4505
+                                                            -Xcompiler=/bigobj>>
 )
 
 
-if(AF_WITH_CUDNN)
-  target_sources(afcuda PRIVATE
-    cudnn.cpp
-    cudnn.hpp
-    cudnnModule.cpp
-    cudnnModule.hpp)
-  target_compile_definitions(afcuda PRIVATE WITH_CUDNN)
-
-  target_include_directories (afcuda
-    PRIVATE
-      ${cuDNN_INCLUDE_DIRS}
-    )
-endif()
-
 if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS AND AF_cusparse_LINK_LOADING STREQUAL "Static")
   target_compile_definitions(afcuda
     PRIVATE
@@ -708,7 +657,6 @@ endif()
 add_library(ArrayFire::afcuda ALIAS afcuda)
 
 add_dependencies(afcuda ${jit_kernel_targets} ${nvrtc_kernel_targets})
-add_dependencies(af_cuda_static_cuda_library ${nvrtc_kernel_targets})
 
 if(UNIX AND AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS)
   add_dependencies(afcuda ${cuda_pruned_library_targets})
@@ -720,6 +668,7 @@ target_include_directories (afcuda
     $<BUILD_INTERFACE:${ArrayFire_BINARY_DIR}/include>
     $<INSTALL_INTERFACE:${AF_INSTALL_INC_DIR}>
   PRIVATE
+    $<$<BOOL:${AF_WITH_CUDNN}>:${cuDNN_INCLUDE_DIRS}>
     ${CUDA_INCLUDE_DIRS}
     ${ArrayFire_SOURCE_DIR}/src/api/c
     ${CMAKE_CURRENT_SOURCE_DIR}
@@ -734,7 +683,6 @@ target_link_libraries(afcuda
     cpp_api_interface
     afcommon_interface
     ${CMAKE_DL_LIBS}
-    af_cuda_static_cuda_library
   )
 
 # If the driver is not found the cuda driver api need to be linked against the
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 024c92551a..32d3172a1a 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -399,8 +399,6 @@ target_include_directories(afopencl
     ../../../include
   )
 
-arrayfire_set_default_cxx_flags(afopencl)
-
 add_dependencies(afopencl ${cl_kernel_targets} CLBlast-ext)
 
 set_target_properties(afopencl PROPERTIES POSITION_INDEPENDENT_CODE ON)

From 83babaf91a42b442d2b2c77b0782fdee6e6b136f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 30 Sep 2022 12:16:34 -0400
Subject: [PATCH 083/473] Fix vcpkg support and improve external detection of
 packages

---
 CMakeModules/AF_vcpkg_options.cmake |  4 ++++
 CMakeModules/build_CLBlast.cmake    |  9 ++++++++-
 CMakeModules/build_cl2hpp.cmake     | 12 +++++++++++-
 src/backend/opencl/CMakeLists.txt   |  4 +++-
 test/CMakeLists.txt                 |  2 +-
 vcpkg.json                          | 13 ++++++++++---
 6 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/CMakeModules/AF_vcpkg_options.cmake b/CMakeModules/AF_vcpkg_options.cmake
index 75297a02b6..00745f846c 100644
--- a/CMakeModules/AF_vcpkg_options.cmake
+++ b/CMakeModules/AF_vcpkg_options.cmake
@@ -23,6 +23,10 @@ if(AF_BUILD_FORGE)
   list(APPEND VCPKG_MANIFEST_FEATURES "forge")
 endif()
 
+if(BUILD_TESTING)
+  list(APPEND VCPKG_MANIFEST_FEATURES "tests")
+endif()
+
 if(AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL")
   list(APPEND VCPKG_MANIFEST_FEATURES "mkl")
 endif()
diff --git a/CMakeModules/build_CLBlast.cmake b/CMakeModules/build_CLBlast.cmake
index 446ceb7e00..0f67d3fdee 100644
--- a/CMakeModules/build_CLBlast.cmake
+++ b/CMakeModules/build_CLBlast.cmake
@@ -12,7 +12,14 @@ if(TARGET clblast OR AF_WITH_EXTERNAL_PACKAGES_ONLY)
     # another package so we dont need this property to link against
     # CLBlast.
     set_target_properties(clblast PROPERTIES
-      IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE "")
+      IMPORTED_LINK_INTERFACE_LIBRARIES_RELEASE ""
+      IMPORTED_LINK_INTERFACE_LIBRARIES_DEBUG "")
+
+    if(WIN32 AND VCPKG_ROOT)
+      set_target_properties(clblast PROPERTIES
+        IMPORTED_LOCATION_RELEASE ""
+        IMPORTED_LOCATION_DEBUG "")
+    endif()
   else()
     message(ERROR "CLBlast now found")
   endif()
diff --git a/CMakeModules/build_cl2hpp.cmake b/CMakeModules/build_cl2hpp.cmake
index e090dd0800..14c2646c2e 100644
--- a/CMakeModules/build_cl2hpp.cmake
+++ b/CMakeModules/build_cl2hpp.cmake
@@ -13,7 +13,17 @@
 
 find_package(OpenCL)
 
-if (NOT TARGET OpenCL::cl2hpp OR NOT TARGET cl2hpp)
+find_path(cl2hpp_header_file_path
+  NAMES CL/cl2.hpp
+  PATHS ${OpenCL_INCLUDE_PATHS})
+
+if(cl2hpp_header_file_path)
+  add_library(cl2hpp IMPORTED INTERFACE GLOBAL)
+  add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
+
+  set_target_properties(cl2hpp OpenCL::cl2hpp PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES ${cl2hpp_header_file_path})
+elseif (NOT TARGET OpenCL::cl2hpp OR NOT TARGET cl2hpp)
   af_dep_check_and_populate(${cl2hpp_prefix}
     URI https://github.com/KhronosGroup/OpenCL-CLHPP.git
     REF v2.0.12)
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 32d3172a1a..a827c55193 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -399,7 +399,9 @@ target_include_directories(afopencl
     ../../../include
   )
 
-add_dependencies(afopencl ${cl_kernel_targets} CLBlast-ext)
+if(NOT TARGET clblast)
+  add_dependencies(afopencl ${cl_kernel_targets} CLBlast-ext)
+endif()
 
 set_target_properties(afopencl PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index e6468848d6..2a66ea8291 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -17,7 +17,7 @@ endif()
 
 if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
     dependency_check(GTest_FOUND)
-else()
+elseif(NOT TARGET GTest::gtest)
   af_dep_check_and_populate(${gtest_prefix}
     URI https://github.com/google/googletest.git
     REF release-1.8.1
diff --git a/vcpkg.json b/vcpkg.json
index 8986d52dbe..70aab906ed 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -8,19 +8,26 @@
         "boost-math",
         "boost-stacktrace",
         "spdlog",
-        "freeimage"
+        "freeimage",
+        "span-lite"
     ],
     "overrides": [
         {
             "name": "fmt",
-            "version": "7.1.3"
+            "version": "8.1.1"
         },
         {
             "name": "spdlog",
-            "version": "1.8.5"
+            "version": "1.9.2"
         }
     ],
     "features": {
+        "tests": {
+            "description": "Build with tests",
+            "dependencies": [
+                "gtest"
+            ]
+        },
         "forge": {
             "description": "Build Forge",
             "dependencies": [

From 83edd0983824549cfab33789a2039f8a205cc27e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 30 Sep 2022 14:15:59 -0400
Subject: [PATCH 084/473] Update deprecated macro from GTest. Add workaround
 for older versions

---
 test/anisotropic_diffusion.cpp |  2 +-
 test/approx1.cpp               |  6 ++--
 test/approx2.cpp               |  6 ++--
 test/array.cpp                 |  2 +-
 test/arrayio.cpp               |  2 +-
 test/assign.cpp                |  2 +-
 test/bilateral.cpp             |  2 +-
 test/binary.cpp                | 56 +++++++++++++++++-----------------
 test/blas.cpp                  | 14 ++++-----
 test/canny.cpp                 |  2 +-
 test/cholesky_dense.cpp        |  2 +-
 test/clamp.cpp                 |  2 +-
 test/compare.cpp               |  2 +-
 test/confidence_connected.cpp  |  4 +--
 test/constant.cpp              |  2 +-
 test/convolve.cpp              | 10 +++---
 test/corrcoef.cpp              |  2 +-
 test/covariance.cpp            |  2 +-
 test/diagonal.cpp              |  2 +-
 test/diff1.cpp                 |  2 +-
 test/diff2.cpp                 |  2 +-
 test/dog.cpp                   |  2 +-
 test/dot.cpp                   | 14 ++++-----
 test/fast.cpp                  |  4 +--
 test/fft.cpp                   | 24 +++++++--------
 test/fft_real.cpp              |  2 +-
 test/fftconvolve.cpp           |  4 +--
 test/gaussiankernel.cpp        |  2 +-
 test/gen_index.cpp             |  2 +-
 test/gloh.cpp                  |  2 +-
 test/gradient.cpp              |  2 +-
 test/half.cpp                  | 46 ++++++++++++++--------------
 test/hamming.cpp               |  4 +--
 test/harris.cpp                |  2 +-
 test/histogram.cpp             |  2 +-
 test/homography.cpp            |  2 +-
 test/iir.cpp                   |  2 +-
 test/imageio.cpp               |  2 +-
 test/index.cpp                 | 10 +++---
 test/inverse_deconv.cpp        |  2 +-
 test/inverse_dense.cpp         |  2 +-
 test/iota.cpp                  |  2 +-
 test/iterative_deconv.cpp      |  2 +-
 test/jit.cpp                   |  4 +--
 test/join.cpp                  |  2 +-
 test/lu_dense.cpp              |  2 +-
 test/match_template.cpp        |  2 +-
 test/mean.cpp                  |  4 +--
 test/meanshift.cpp             |  2 +-
 test/meanvar.cpp               | 14 ++++-----
 test/medfilt.cpp               |  4 +--
 test/memory.cpp                |  2 +-
 test/moddims.cpp               |  2 +-
 test/moments.cpp               |  2 +-
 test/morph.cpp                 |  2 +-
 test/nearest_neighbour.cpp     | 14 ++++-----
 test/orb.cpp                   |  2 +-
 test/pad_borders.cpp           |  2 +-
 test/pinverse.cpp              |  2 +-
 test/qr_dense.cpp              |  2 +-
 test/random.cpp                | 10 +++---
 test/range.cpp                 |  4 +--
 test/rank_dense.cpp            |  4 +--
 test/reduce.cpp                | 14 ++++-----
 test/regions.cpp               |  2 +-
 test/reorder.cpp               |  2 +-
 test/replace.cpp               |  2 +-
 test/resize.cpp                |  4 +--
 test/rng_match.cpp             |  2 +-
 test/rng_quality.cpp           |  2 +-
 test/rotate.cpp                |  2 +-
 test/rotate_linear.cpp         |  2 +-
 test/sat.cpp                   |  2 +-
 test/select.cpp                | 38 +++++++++++------------
 test/shift.cpp                 |  2 +-
 test/sift.cpp                  |  2 +-
 test/sobel.cpp                 |  4 +--
 test/solve_dense.cpp           |  2 +-
 test/sort.cpp                  |  2 +-
 test/sort_by_key.cpp           |  2 +-
 test/sort_index.cpp            |  2 +-
 test/sparse.cpp                |  2 +-
 test/stdev.cpp                 |  2 +-
 test/susan.cpp                 |  2 +-
 test/svd_dense.cpp             |  2 +-
 test/testHelpers.hpp           | 10 ++++++
 test/tile.cpp                  |  2 +-
 test/topk.cpp                  |  4 +--
 test/transform.cpp             |  8 ++---
 test/transform_coordinates.cpp |  2 +-
 test/translate.cpp             |  4 +--
 test/transpose.cpp             |  2 +-
 test/transpose_inplace.cpp     |  2 +-
 test/triangle.cpp              |  2 +-
 test/unwrap.cpp                |  2 +-
 test/var.cpp                   |  2 +-
 test/where.cpp                 |  2 +-
 test/wrap.cpp                  | 10 +++---
 test/write.cpp                 |  2 +-
 99 files changed, 252 insertions(+), 242 deletions(-)

diff --git a/test/anisotropic_diffusion.cpp b/test/anisotropic_diffusion.cpp
index f20f1f009c..f4d78382f3 100644
--- a/test/anisotropic_diffusion.cpp
+++ b/test/anisotropic_diffusion.cpp
@@ -32,7 +32,7 @@ class AnisotropicDiffusion : public ::testing::Test {};
 typedef ::testing::Types<float, double, int, uint, uchar, short, ushort>
     TestTypes;
 
-TYPED_TEST_CASE(AnisotropicDiffusion, TestTypes);
+TYPED_TEST_SUITE(AnisotropicDiffusion, TestTypes);
 
 template<typename T>
 array normalize(const array &p_in) {
diff --git a/test/approx1.cpp b/test/approx1.cpp
index a13c51c173..17d7579cec 100644
--- a/test/approx1.cpp
+++ b/test/approx1.cpp
@@ -63,7 +63,7 @@ class Approx1 : public ::testing::Test {
 typedef ::testing::Types<float, double, cfloat, cdouble> TestTypes;
 
 // Register the type list
-TYPED_TEST_CASE(Approx1, TestTypes);
+TYPED_TEST_SUITE(Approx1, TestTypes);
 
 template<typename T>
 void approx1Test(string pTestFile, const unsigned resultIdx,
@@ -926,7 +926,7 @@ class Approx1V2 : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Approx1V2, TestTypes);
+TYPED_TEST_SUITE(Approx1V2, TestTypes);
 
 class SimpleTestData {
    public:
@@ -969,7 +969,7 @@ class Approx1V2Simple : public Approx1V2<T> {
     }
 };
 
-TYPED_TEST_CASE(Approx1V2Simple, TestTypes);
+TYPED_TEST_SUITE(Approx1V2Simple, TestTypes);
 
 TYPED_TEST(Approx1V2Simple, UseNullOutputArray) {
     this->testSpclOutArray(NULL_ARRAY);
diff --git a/test/approx2.cpp b/test/approx2.cpp
index 8ea4f5b8a4..796c639fd0 100644
--- a/test/approx2.cpp
+++ b/test/approx2.cpp
@@ -56,7 +56,7 @@ class Approx2 : public ::testing::Test {
 typedef ::testing::Types<float, double, cfloat, cdouble> TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Approx2, TestTypes);
+TYPED_TEST_SUITE(Approx2, TestTypes);
 
 template<typename T>
 void approx2Test(string pTestFile, const unsigned resultIdx,
@@ -862,7 +862,7 @@ class Approx2V2 : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Approx2V2, TestTypes);
+TYPED_TEST_SUITE(Approx2V2, TestTypes);
 
 class SimpleTestData {
    public:
@@ -911,7 +911,7 @@ class Approx2V2Simple : public Approx2V2<T> {
     }
 };
 
-TYPED_TEST_CASE(Approx2V2Simple, TestTypes);
+TYPED_TEST_SUITE(Approx2V2Simple, TestTypes);
 
 TYPED_TEST(Approx2V2Simple, UseNullOutputArray) {
     this->testSpclOutArray(NULL_ARRAY);
diff --git a/test/array.cpp b/test/array.cpp
index 08b5a568d7..eeb7f2952b 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -26,7 +26,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, char, unsigned char,
                          half_float::half>
     TestTypes;
 
-TYPED_TEST_CASE(Array, TestTypes);
+TYPED_TEST_SUITE(Array, TestTypes);
 
 TEST(Array, ConstructorDefault) {
     array a;
diff --git a/test/arrayio.cpp b/test/arrayio.cpp
index fbbb9c5030..7a578b612a 100644
--- a/test/arrayio.cpp
+++ b/test/arrayio.cpp
@@ -42,7 +42,7 @@ string getTypeName(
     return info.param.name;
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Types, ArrayIOType,
     ::testing::Values(type_params("f32", f32, 3.14f, 0),
                       type_params("f64", f64, 3.14, 0),
diff --git a/test/assign.cpp b/test/assign.cpp
index 0e2aea05d7..7c32a2cc33 100644
--- a/test/assign.cpp
+++ b/test/assign.cpp
@@ -99,7 +99,7 @@ typedef ::testing::Types<float, cdouble, cfloat, double, int, uint, char, uchar,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(ArrayAssign, TestTypes);
+TYPED_TEST_SUITE(ArrayAssign, TestTypes);
 
 template<typename inType, typename outType>
 void assignTest(string pTestFile, const vector<af_seq> *seqv) {
diff --git a/test/bilateral.cpp b/test/bilateral.cpp
index 07d95debba..d4da723ddb 100644
--- a/test/bilateral.cpp
+++ b/test/bilateral.cpp
@@ -77,7 +77,7 @@ typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
     DataTestTypes;
 
 // register the type list
-TYPED_TEST_CASE(BilateralOnData, DataTestTypes);
+TYPED_TEST_SUITE(BilateralOnData, DataTestTypes);
 
 template<typename inType>
 void bilateralDataTest(string pTestFile) {
diff --git a/test/binary.cpp b/test/binary.cpp
index 06e720ed8e..f5fd0610e8 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -386,27 +386,27 @@ DEF_TEST(UChar, unsigned char)
 
 #undef DEF_TEST
 
-INSTANTIATE_TEST_CASE_P(PositiveValues, PowPrecisionTestULong,
-                        testing::Range<unsigned long long>(1, 1e7, 1e6));
-INSTANTIATE_TEST_CASE_P(PositiveValues, PowPrecisionTestLong,
-                        testing::Range<long long>(1, 1e7, 1e6));
-INSTANTIATE_TEST_CASE_P(PositiveValues, PowPrecisionTestUInt,
-                        testing::Range<unsigned int>(1, 65000, 15e3));
-INSTANTIATE_TEST_CASE_P(PositiveValues, PowPrecisionTestInt,
-                        testing::Range<int>(1, 46340, 10e3));
-INSTANTIATE_TEST_CASE_P(PositiveValues, PowPrecisionTestUShort,
-                        testing::Range<unsigned short>(1, 255, 100));
-INSTANTIATE_TEST_CASE_P(PositiveValues, PowPrecisionTestShort,
-                        testing::Range<short>(1, 180, 50));
-INSTANTIATE_TEST_CASE_P(PositiveValues, PowPrecisionTestUChar,
-                        testing::Range<unsigned char>(1, 12, 5));
-
-INSTANTIATE_TEST_CASE_P(NegativeValues, PowPrecisionTestLong,
-                        testing::Range<long long>(-1e7, 0, 1e6));
-INSTANTIATE_TEST_CASE_P(NegativeValues, PowPrecisionTestInt,
-                        testing::Range<int>(-46340, 0, 10e3));
-INSTANTIATE_TEST_CASE_P(NegativeValues, PowPrecisionTestShort,
-                        testing::Range<short>(-180, 0, 50));
+INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestULong,
+                         testing::Range<unsigned long long>(1, 1e7, 1e6));
+INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestLong,
+                         testing::Range<long long>(1, 1e7, 1e6));
+INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestUInt,
+                         testing::Range<unsigned int>(1, 65000, 15e3));
+INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestInt,
+                         testing::Range<int>(1, 46340, 10e3));
+INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestUShort,
+                         testing::Range<unsigned short>(1, 255, 100));
+INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestShort,
+                         testing::Range<short>(1, 180, 50));
+INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestUChar,
+                         testing::Range<unsigned char>(1, 12, 5));
+
+INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestLong,
+                         testing::Range<long long>(-1e7, 0, 1e6));
+INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestInt,
+                         testing::Range<int>(-46340, 0, 10e3));
+INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestShort,
+                         testing::Range<short>(-180, 0, 50));
 
 struct result_type_param {
     af_dtype result_;
@@ -453,7 +453,7 @@ std::string print_types(
     return ss.str();
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     SameTypes, ResultType,
     // clang-format off
     ::testing::Values(result_type_param(f32),
@@ -472,7 +472,7 @@ INSTANTIATE_TEST_CASE_P(
     // clang-format on
     print_types);
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Float, ResultType,
     // clang-format off
     ::testing::Values(result_type_param(f32),
@@ -491,7 +491,7 @@ INSTANTIATE_TEST_CASE_P(
     // clang-format on
     print_types);
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Double, ResultType,
     ::testing::Values(
         // clang-format off
@@ -540,7 +540,7 @@ class ResultTypeScalar : public ::testing::Test {
 typedef ::testing::Types<float, double, unsigned int, int, short,
                          unsigned short, char, unsigned char, half_float::half>
     TestTypes;
-TYPED_TEST_CASE(ResultTypeScalar, TestTypes);
+TYPED_TEST_SUITE(ResultTypeScalar, TestTypes);
 
 TYPED_TEST(ResultTypeScalar, HalfAddition) {
     SUPPORTED_TYPE_CHECK(half_float::half);
@@ -583,7 +583,7 @@ class Broadcast : public ::testing::TestWithParam<std::tuple<dim4, dim4> > {
 };
 /// clang-format off
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     CorrectCases, Broadcast,
     ::testing::Combine(
         ::testing::Values(dim4(1), dim4(10), dim4(1, 10), dim4(1, 1, 10),
@@ -715,7 +715,7 @@ TEST_P(Broadcast, AdditionLHSIndexed) {
 
     af::dim4 outdims       = broadcastOut(get<0>(params), rhs.dims());
     af::array indexedlhs   = lhs(seq(lhs_dims[0]), seq(lhs_dims[1]),
-                               seq(lhs_dims[2]), seq(lhs_dims[3]));
+                                 seq(lhs_dims[2]), seq(lhs_dims[3]));
     af::dim4 tilerepetions = tileRepeations(get<0>(params), rhs.dims());
     af::array tiledlhs     = tile(indexedlhs, tilerepetions);
 
@@ -760,7 +760,7 @@ TEST_P(Broadcast, AdditionBothIndexed) {
 
     af::dim4 outdims       = broadcastOut(lhs_dims, rhs_dims);
     af::array indexedlhs   = lhs(seq(lhs_dims[0]), seq(lhs_dims[1]),
-                               seq(lhs_dims[2]), seq(lhs_dims[3]));
+                                 seq(lhs_dims[2]), seq(lhs_dims[3]));
     af::dim4 tilerepetions = tileRepeations(get<0>(params), get<1>(params));
     af::array tiledlhs     = tile(indexedlhs, tilerepetions);
 
diff --git a/test/blas.cpp b/test/blas.cpp
index 612f6dd97f..62491a366f 100644
--- a/test/blas.cpp
+++ b/test/blas.cpp
@@ -45,7 +45,7 @@ template<typename T>
 class MatrixMultiply : public ::testing::Test {};
 
 typedef ::testing::Types<float, double, cdouble, cfloat> TestTypes;
-TYPED_TEST_CASE(MatrixMultiply, TestTypes);
+TYPED_TEST_SUITE(MatrixMultiply, TestTypes);
 
 template<typename T, bool isBVector>
 void MatMulCheck(string TestFile) {
@@ -339,7 +339,7 @@ std::string print_blas_params(
     return ss.str();
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     LHSBroadcast, MatrixMultiplyBatch,
     ::testing::Values(
 
@@ -365,7 +365,7 @@ INSTANTIATE_TEST_CASE_P(
         ),
     print_blas_params);
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     RHSBroadcast, MatrixMultiplyBatch,
     ::testing::Values(
         // clang-format off
@@ -389,7 +389,7 @@ INSTANTIATE_TEST_CASE_P(
         ),
     print_blas_params);
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     SameBatch, MatrixMultiplyBatch,
     ::testing::Values(
         // clang-format off
@@ -609,7 +609,7 @@ string out_info(const ::testing::TestParamInfo<Gemm::ParamType> info) {
 }
 
 // clang-format off
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Square, Gemm,
     ::testing::Values(
         //          lhs_opts     rhs_opts     alpha  lhs    rhs    gold    lhs_dims    rhs_dims    out_dims    beta  out_array_type
@@ -623,7 +623,7 @@ INSTANTIATE_TEST_CASE_P(
 // clang-format on
 
 // clang-format off
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Batched, Gemm,
     ::testing::Values(
         //          lhs_opts     rhs_opts     alpha  lhs          rhs    gold          lhs_dims       rhs_dims    out_dims       beta  out_array_type
@@ -637,7 +637,7 @@ INSTANTIATE_TEST_CASE_P(
 // clang-format on
 
 // clang-format off
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     NonSquare, Gemm,
     ::testing::Values(
         //          lhs_opts      rhs_opts      alpha  lhs         rhs         gold       lhs_dims    rhs_dims    out_dims    beta  out_array_type
diff --git a/test/canny.cpp b/test/canny.cpp
index e00e9b0c30..8e1cb9c2b6 100644
--- a/test/canny.cpp
+++ b/test/canny.cpp
@@ -32,7 +32,7 @@ typedef ::testing::Types<float, int, uint, short, ushort, uchar, double>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(CannyEdgeDetector, TestTypes);
+TYPED_TEST_SUITE(CannyEdgeDetector, TestTypes);
 
 template<typename T>
 void cannyTest(string pTestFile) {
diff --git a/test/cholesky_dense.cpp b/test/cholesky_dense.cpp
index 3800d0c0e1..0631ec2bad 100644
--- a/test/cholesky_dense.cpp
+++ b/test/cholesky_dense.cpp
@@ -78,7 +78,7 @@ template<typename T>
 class Cholesky : public ::testing::Test {};
 
 typedef ::testing::Types<float, cfloat, double, cdouble> TestTypes;
-TYPED_TEST_CASE(Cholesky, TestTypes);
+TYPED_TEST_SUITE(Cholesky, TestTypes);
 
 template<typename T>
 double eps();
diff --git a/test/clamp.cpp b/test/clamp.cpp
index eb0b46a187..7f888a56ac 100644
--- a/test/clamp.cpp
+++ b/test/clamp.cpp
@@ -104,7 +104,7 @@ string testNameGenerator(const ::testing::TestParamInfo<clamp_params> info) {
 typedef Clamp<double> ClampFloatingPoint;
 
 // clang-format off
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     SmallDims, ClampFloatingPoint,
     ::testing::Values(
                       clamp_params(dim4(10), f32, f32, f32, f32),
diff --git a/test/compare.cpp b/test/compare.cpp
index 576186d164..66d9778039 100644
--- a/test/compare.cpp
+++ b/test/compare.cpp
@@ -26,7 +26,7 @@ class Compare : public ::testing::Test {};
 typedef ::testing::Types<float, double, uint, int, intl, uintl, uchar, short,
                          ushort, half_float::half>
     TestTypes;
-TYPED_TEST_CASE(Compare, TestTypes);
+TYPED_TEST_SUITE(Compare, TestTypes);
 
 #define COMPARE(OP, Name)                                   \
     TYPED_TEST(Compare, Test_##Name) {                      \
diff --git a/test/confidence_connected.cpp b/test/confidence_connected.cpp
index 6963edcc1e..8ef707aca7 100644
--- a/test/confidence_connected.cpp
+++ b/test/confidence_connected.cpp
@@ -31,7 +31,7 @@ class ConfidenceConnectedImageTest : public testing::Test {
 
 typedef ::testing::Types<float, uint, ushort, uchar> TestTypes;
 
-TYPED_TEST_CASE(ConfidenceConnectedImageTest, TestTypes);
+TYPED_TEST_SUITE(ConfidenceConnectedImageTest, TestTypes);
 
 struct CCCTestParams {
     const char *prefix;
@@ -185,7 +185,7 @@ TEST_P(ConfidenceConnectedDataTest, SegmentARegion) {
     testData<unsigned char>(GetParam());
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     SingleSeed, ConfidenceConnectedDataTest,
     testing::Values(CCCTestParams{"core", 0u, 1u, 5u, 255.0},
                     CCCTestParams{"background", 0u, 1u, 5u, 255.0},
diff --git a/test/constant.cpp b/test/constant.cpp
index e54a3d01f7..0a75e3d974 100644
--- a/test/constant.cpp
+++ b/test/constant.cpp
@@ -33,7 +33,7 @@ class Constant : public ::testing::Test {};
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, char,
                          uchar, uintl, intl, short, ushort, half_float::half>
     TestTypes;
-TYPED_TEST_CASE(Constant, TestTypes);
+TYPED_TEST_SUITE(Constant, TestTypes);
 
 template<typename T>
 void ConstantCPPCheck(T value) {
diff --git a/test/convolve.cpp b/test/convolve.cpp
index c3abe056cd..7b31e532a3 100644
--- a/test/convolve.cpp
+++ b/test/convolve.cpp
@@ -38,7 +38,7 @@ typedef ::testing::Types<cdouble, cfloat, float, double, int, uint, char, uchar,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Convolve, TestTypes);
+TYPED_TEST_SUITE(Convolve, TestTypes);
 
 template<typename T>
 void convolveTest(string pTestFile, int baseDim, bool expand) {
@@ -877,9 +877,9 @@ vector<conv2_strided_params> genConsistencyTests() {
             conv2_consistency_data(dim4(257, 257), dim4(3, 3))};
 }
 
-INSTANTIATE_TEST_CASE_P(Conv2Consistency, Conv2ConsistencyTest,
-                        ::testing::ValuesIn(genConsistencyTests()),
-                        testNameGenerator<Conv2ConsistencyTest>);
+INSTANTIATE_TEST_SUITE_P(Conv2Consistency, Conv2ConsistencyTest,
+                         ::testing::ValuesIn(genConsistencyTests()),
+                         testNameGenerator<Conv2ConsistencyTest>);
 
 TEST_P(Conv2ConsistencyTest, RandomConvolutions) {
     conv2_strided_params params = GetParam();
@@ -1039,7 +1039,7 @@ typedef ::testing::Types<float, double, half_float::half>
     TestTypesStrided;  // TODO: integral types??
 
 // register the type list
-TYPED_TEST_CASE(ConvolveStrided, TestTypesStrided);
+TYPED_TEST_SUITE(ConvolveStrided, TestTypesStrided);
 
 TYPED_TEST(ConvolveStrided, Strided_sig1010_filt33_s11_p11_d11) {
     convolve2stridedTest<TypeParam>(
diff --git a/test/corrcoef.cpp b/test/corrcoef.cpp
index 7fa6e57ffa..1c7f378961 100644
--- a/test/corrcoef.cpp
+++ b/test/corrcoef.cpp
@@ -35,7 +35,7 @@ typedef ::testing::Types<float, double, int, uint, intl, uintl, char, uchar>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(CorrelationCoefficient, TestTypes);
+TYPED_TEST_SUITE(CorrelationCoefficient, TestTypes);
 
 template<typename T>
 struct f32HelperType {
diff --git a/test/covariance.cpp b/test/covariance.cpp
index 6eea33e224..aa06c58a10 100644
--- a/test/covariance.cpp
+++ b/test/covariance.cpp
@@ -39,7 +39,7 @@ typedef ::testing::Types<float, double, int, uint, intl, uintl, uchar, short,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Covariance, TestTypes);
+TYPED_TEST_SUITE(Covariance, TestTypes);
 
 template<typename T>
 struct f32HelperType {
diff --git a/test/diagonal.cpp b/test/diagonal.cpp
index a73a2096ff..1eecb883ae 100644
--- a/test/diagonal.cpp
+++ b/test/diagonal.cpp
@@ -34,7 +34,7 @@ class Diagonal : public ::testing::Test {};
 typedef ::testing::Types<float, double, int, uint, char, unsigned char,
                          half_float::half>
     TestTypes;
-TYPED_TEST_CASE(Diagonal, TestTypes);
+TYPED_TEST_SUITE(Diagonal, TestTypes);
 
 TYPED_TEST(Diagonal, Create) {
     SUPPORTED_TYPE_CHECK(TypeParam);
diff --git a/test/diff1.cpp b/test/diff1.cpp
index 510d9ce61b..605cd75fa9 100644
--- a/test/diff1.cpp
+++ b/test/diff1.cpp
@@ -50,7 +50,7 @@ typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, intl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Diff1, TestTypes);
+TYPED_TEST_SUITE(Diff1, TestTypes);
 
 template<typename T>
 void diff1Test(string pTestFile, unsigned dim, bool isSubRef = false,
diff --git a/test/diff2.cpp b/test/diff2.cpp
index c5ff4ce9f3..4a68627d7b 100644
--- a/test/diff2.cpp
+++ b/test/diff2.cpp
@@ -55,7 +55,7 @@ typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, intl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Diff2, TestTypes);
+TYPED_TEST_SUITE(Diff2, TestTypes);
 
 template<typename T>
 void diff2Test(string pTestFile, unsigned dim, bool isSubRef = false,
diff --git a/test/dog.cpp b/test/dog.cpp
index 9b8e952567..0b764f2c06 100644
--- a/test/dog.cpp
+++ b/test/dog.cpp
@@ -37,7 +37,7 @@ typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(DOG, TestTypes);
+TYPED_TEST_SUITE(DOG, TestTypes);
 
 TYPED_TEST(DOG, Basic) {
     SUPPORTED_TYPE_CHECK(TypeParam);
diff --git a/test/dot.cpp b/test/dot.cpp
index 37b84d2818..357e0784d4 100644
--- a/test/dot.cpp
+++ b/test/dot.cpp
@@ -44,8 +44,8 @@ typedef ::testing::Types<float, double> TestTypesF;
 typedef ::testing::Types<cfloat, cdouble> TestTypesC;
 
 // register the type list
-TYPED_TEST_CASE(DotF, TestTypesF);
-TYPED_TEST_CASE(DotC, TestTypesC);
+TYPED_TEST_SUITE(DotF, TestTypesF);
+TYPED_TEST_SUITE(DotC, TestTypesC);
 
 bool isinf(af::af_cfloat val) {
     using std::isinf;
@@ -301,11 +301,11 @@ std::string print_dot(const ::testing::TestParamInfo<Dot::ParamType> info) {
     return ss.str();
 }
 
-INSTANTIATE_TEST_CASE_P(Small, Dot,
-                        ::testing::Values(2, 4, 5, 10, 31, 32, 33, 100, 127,
-                                          128, 129, 200, 500, 511, 512, 513,
-                                          1000),
-                        print_dot);
+INSTANTIATE_TEST_SUITE_P(Small, Dot,
+                         ::testing::Values(2, 4, 5, 10, 31, 32, 33, 100, 127,
+                                           128, 129, 200, 500, 511, 512, 513,
+                                           1000),
+                         print_dot);
 
 TEST_P(Dot, Half) {
     SUPPORTED_TYPE_CHECK(half_float::half);
diff --git a/test/fast.cpp b/test/fast.cpp
index 4dc0c8896f..77281955a5 100644
--- a/test/fast.cpp
+++ b/test/fast.cpp
@@ -63,8 +63,8 @@ class FixedFAST : public ::testing::Test {
 typedef ::testing::Types<float, double> FloatTestTypes;
 typedef ::testing::Types<int, unsigned, short, ushort> FixedTestTypes;
 
-TYPED_TEST_CASE(FloatFAST, FloatTestTypes);
-TYPED_TEST_CASE(FixedFAST, FixedTestTypes);
+TYPED_TEST_SUITE(FloatFAST, FloatTestTypes);
+TYPED_TEST_SUITE(FixedFAST, FixedTestTypes);
 
 template<typename T>
 void fastTest(string pTestFile, bool nonmax) {
diff --git a/test/fft.cpp b/test/fft.cpp
index ce654d3c05..acd0ad7521 100644
--- a/test/fft.cpp
+++ b/test/fft.cpp
@@ -742,34 +742,34 @@ string to_test_params(const ::testing::TestParamInfo<FFTBase::ParamType> info) {
     return out.replace(out.find("."), 1, "_");
 }
 
-INSTANTIATE_TEST_CASE_P(
-    Inputs2D, FFTC2R2D,
-    ::testing::Values(fft_params(dim4(513, 512), false, 0.5),
-                      fft_params(dim4(1025, 1024), false, 0.5),
-                      fft_params(dim4(2049, 2048), false, 0.5)),
-    to_test_params);
-
-INSTANTIATE_TEST_CASE_P(
+// INSTANTIATE_TEST_SUITE_P(
+//     Inputs2D, FFTC2R2D,
+//     ::testing::Values(fft_params(dim4(513, 512), false, 0.5),
+//                       fft_params(dim4(1025, 1024), false, 0.5),
+//                       fft_params(dim4(2049, 2048), false, 0.5)),
+//     to_test_params);
+
+INSTANTIATE_TEST_SUITE_P(
     Inputs2D, FFT2D,
     ::testing::Values(fft_params(dim4(512, 512), false, 0.5),
                       fft_params(dim4(1024, 1024), false, 0.5),
                       fft_params(dim4(2048, 2048), false, 0.5)),
     to_test_params);
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Inputs3D, FFTC2R3D,
     ::testing::Values(fft_params(dim4(512, 512, 3), false, 0.5),
                       fft_params(dim4(1024, 1024, 3), false, 0.5),
                       fft_params(dim4(2048, 2048, 3), false, 0.5)),
     to_test_params);
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Inputs3D, FFT3D,
     ::testing::Values(fft_params(dim4(1024, 1024, 3), true, 0.5),
                       fft_params(dim4(1024, 1024, 3), false, 0.5)),
     to_test_params);
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     InputsND, FFTND,
     ::testing::Values(fft_params(dim4(512), false, 0.5),
                       fft_params(dim4(1024), false, 0.5),
@@ -777,7 +777,7 @@ INSTANTIATE_TEST_CASE_P(
                       fft_params(dim4(1024, 1024, 3), false, 0.5)),
     to_test_params);
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     InputsND, FFTC2R,
     ::testing::Values(fft_params(dim4(513), false, 0.5),
                       fft_params(dim4(1025), false, 0.5),
diff --git a/test/fft_real.cpp b/test/fft_real.cpp
index d0816d976c..863f66d74c 100644
--- a/test/fft_real.cpp
+++ b/test/fft_real.cpp
@@ -37,7 +37,7 @@ template<typename T>
 class FFT_REAL : public ::testing::Test {};
 
 typedef ::testing::Types<cfloat, cdouble> TestTypes;
-TYPED_TEST_CASE(FFT_REAL, TestTypes);
+TYPED_TEST_SUITE(FFT_REAL, TestTypes);
 
 template<int rank>
 array fft(const array &in, double norm) {
diff --git a/test/fftconvolve.cpp b/test/fftconvolve.cpp
index 98fa9c315c..7465891bde 100644
--- a/test/fftconvolve.cpp
+++ b/test/fftconvolve.cpp
@@ -45,8 +45,8 @@ typedef ::testing::Types<cfloat, cdouble, float, double, int, uint, char, uchar,
 typedef ::testing::Types<float, double> TestTypesLarge;
 
 // register the type list
-TYPED_TEST_CASE(FFTConvolve, TestTypes);
-TYPED_TEST_CASE(FFTConvolveLarge, TestTypesLarge);
+TYPED_TEST_SUITE(FFTConvolve, TestTypes);
+TYPED_TEST_SUITE(FFTConvolveLarge, TestTypesLarge);
 
 template<typename T, int baseDim>
 void fftconvolveTest(string pTestFile, bool expand) {
diff --git a/test/gaussiankernel.cpp b/test/gaussiankernel.cpp
index a6675720ef..3c4db5386f 100644
--- a/test/gaussiankernel.cpp
+++ b/test/gaussiankernel.cpp
@@ -30,7 +30,7 @@ class GaussianKernel : public ::testing::Test {
 typedef ::testing::Types<float> TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(GaussianKernel, TestTypes);
+TYPED_TEST_SUITE(GaussianKernel, TestTypes);
 
 template<typename T>
 void gaussianKernelTest(string pFileName, double sigma) {
diff --git a/test/gen_index.cpp b/test/gen_index.cpp
index b8f041d47b..b491a9ac4c 100644
--- a/test/gen_index.cpp
+++ b/test/gen_index.cpp
@@ -103,7 +103,7 @@ string testNameGenerator(
     return ss.str();
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Legacy, IndexGeneralizedLegacy,
     ::testing::Combine(
         ::testing::Values(index_test(
diff --git a/test/gloh.cpp b/test/gloh.cpp
index 004f00b7be..eb193e7ec4 100644
--- a/test/gloh.cpp
+++ b/test/gloh.cpp
@@ -132,7 +132,7 @@ class GLOH : public ::testing::Test {
 
 typedef ::testing::Types<float, double> TestTypes;
 
-TYPED_TEST_CASE(GLOH, TestTypes);
+TYPED_TEST_SUITE(GLOH, TestTypes);
 
 template<typename T>
 void glohTest(string pTestFile) {
diff --git a/test/gradient.cpp b/test/gradient.cpp
index 98df0830c5..b30e9bb649 100644
--- a/test/gradient.cpp
+++ b/test/gradient.cpp
@@ -41,7 +41,7 @@ class Grad : public ::testing::Test {
 typedef ::testing::Types<float, double, cfloat, cdouble> TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Grad, TestTypes);
+TYPED_TEST_SUITE(Grad, TestTypes);
 
 template<typename T>
 void gradTest(string pTestFile, const unsigned resultIdx0,
diff --git a/test/half.cpp b/test/half.cpp
index 541af826a9..18fcdb4077 100644
--- a/test/half.cpp
+++ b/test/half.cpp
@@ -36,29 +36,29 @@ struct convert_params {
 
 class HalfConvert : public ::testing::TestWithParam<convert_params> {};
 
-INSTANTIATE_TEST_CASE_P(ToF16, HalfConvert,
-                        ::testing::Values(convert_params(f32, f16, 10),
-                                          convert_params(f64, f16, 10),
-                                          convert_params(s32, f16, 10),
-                                          convert_params(u32, f16, 10),
-                                          convert_params(u8, f16, 10),
-                                          convert_params(s64, f16, 10),
-                                          convert_params(u64, f16, 10),
-                                          convert_params(s16, f16, 10),
-                                          convert_params(u16, f16, 10),
-                                          convert_params(f16, f16, 10)));
-
-INSTANTIATE_TEST_CASE_P(FromF16, HalfConvert,
-                        ::testing::Values(convert_params(f16, f32, 10),
-                                          convert_params(f16, f64, 10),
-                                          convert_params(f16, s32, 10),
-                                          convert_params(f16, u32, 10),
-                                          convert_params(f16, u8, 10),
-                                          convert_params(f16, s64, 10),
-                                          convert_params(f16, u64, 10),
-                                          convert_params(f16, s16, 10),
-                                          convert_params(f16, u16, 10),
-                                          convert_params(f16, f16, 10)));
+INSTANTIATE_TEST_SUITE_P(ToF16, HalfConvert,
+                         ::testing::Values(convert_params(f32, f16, 10),
+                                           convert_params(f64, f16, 10),
+                                           convert_params(s32, f16, 10),
+                                           convert_params(u32, f16, 10),
+                                           convert_params(u8, f16, 10),
+                                           convert_params(s64, f16, 10),
+                                           convert_params(u64, f16, 10),
+                                           convert_params(s16, f16, 10),
+                                           convert_params(u16, f16, 10),
+                                           convert_params(f16, f16, 10)));
+
+INSTANTIATE_TEST_SUITE_P(FromF16, HalfConvert,
+                         ::testing::Values(convert_params(f16, f32, 10),
+                                           convert_params(f16, f64, 10),
+                                           convert_params(f16, s32, 10),
+                                           convert_params(f16, u32, 10),
+                                           convert_params(f16, u8, 10),
+                                           convert_params(f16, s64, 10),
+                                           convert_params(f16, u64, 10),
+                                           convert_params(f16, s16, 10),
+                                           convert_params(f16, u16, 10),
+                                           convert_params(f16, f16, 10)));
 
 TEST_P(HalfConvert, convert) {
     SUPPORTED_TYPE_CHECK(af_half);
diff --git a/test/hamming.cpp b/test/hamming.cpp
index 6c0edd0618..8b3d9f85f7 100644
--- a/test/hamming.cpp
+++ b/test/hamming.cpp
@@ -39,8 +39,8 @@ typedef ::testing::Types<uchar, ushort> TestTypes8;
 typedef ::testing::Types<uint, uintl> TestTypes32;
 
 // register the type list
-TYPED_TEST_CASE(HammingMatcher8, TestTypes8);
-TYPED_TEST_CASE(HammingMatcher32, TestTypes32);
+TYPED_TEST_SUITE(HammingMatcher8, TestTypes8);
+TYPED_TEST_SUITE(HammingMatcher32, TestTypes32);
 
 template<typename T>
 void hammingMatcherTest(string pTestFile, int feat_dim) {
diff --git a/test/harris.cpp b/test/harris.cpp
index e4e832fc05..955c676251 100644
--- a/test/harris.cpp
+++ b/test/harris.cpp
@@ -56,7 +56,7 @@ class Harris : public ::testing::Test {
 
 typedef ::testing::Types<float, double> TestTypes;
 
-TYPED_TEST_CASE(Harris, TestTypes);
+TYPED_TEST_SUITE(Harris, TestTypes);
 
 template<typename T>
 void harrisTest(string pTestFile, float sigma, unsigned block_size) {
diff --git a/test/histogram.cpp b/test/histogram.cpp
index 826eebd506..ff2049b390 100644
--- a/test/histogram.cpp
+++ b/test/histogram.cpp
@@ -37,7 +37,7 @@ typedef ::testing::Types<half_float::half, float, double, int, uint, char,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Histogram, TestTypes);
+TYPED_TEST_SUITE(Histogram, TestTypes);
 
 template<typename inType, typename outType>
 void histTest(string pTestFile, unsigned nbins, double minval, double maxval) {
diff --git a/test/homography.cpp b/test/homography.cpp
index f305933396..6b0e620869 100644
--- a/test/homography.cpp
+++ b/test/homography.cpp
@@ -33,7 +33,7 @@ class Homography : public ::testing::Test {
 
 typedef ::testing::Types<float, double> TestTypes;
 
-TYPED_TEST_CASE(Homography, TestTypes);
+TYPED_TEST_SUITE(Homography, TestTypes);
 
 template<typename T>
 array perspectiveTransform(dim4 inDims, array H) {
diff --git a/test/iir.cpp b/test/iir.cpp
index dba2369061..fd03e7ccc6 100644
--- a/test/iir.cpp
+++ b/test/iir.cpp
@@ -37,7 +37,7 @@ class filter : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble> TestTypes;
-TYPED_TEST_CASE(filter, TestTypes);
+TYPED_TEST_SUITE(filter, TestTypes);
 
 template<typename T>
 void firTest(const int xrows, const int xcols, const int brows,
diff --git a/test/imageio.cpp b/test/imageio.cpp
index 9dc85a5865..a4e12e834e 100644
--- a/test/imageio.cpp
+++ b/test/imageio.cpp
@@ -33,7 +33,7 @@ class ImageIO : public ::testing::Test {
 typedef ::testing::Types<float> TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(ImageIO, TestTypes);
+TYPED_TEST_SUITE(ImageIO, TestTypes);
 
 void loadImageTest(string pTestFile, string pImageFile, const bool isColor) {
     if (noImageIOTests()) return;
diff --git a/test/index.cpp b/test/index.cpp
index aaac6f74f7..2f61d40adb 100644
--- a/test/index.cpp
+++ b/test/index.cpp
@@ -141,7 +141,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned,
                          unsigned char, intl, uintl, short, ushort,
                          half_float::half>
     AllTypes;
-TYPED_TEST_CASE(Indexing1D, AllTypes);
+TYPED_TEST_SUITE(Indexing1D, AllTypes);
 
 TYPED_TEST(Indexing1D, Continious) {
     DimCheck<TypeParam>(this->continuous_seqs);
@@ -373,7 +373,7 @@ void DimCheck2D(const vector<vector<af_seq> > &seqs, string TestFile,
     }
 }
 
-TYPED_TEST_CASE(Indexing2D, AllTypes);
+TYPED_TEST_SUITE(Indexing2D, AllTypes);
 
 TYPED_TEST(Indexing2D, ColumnContinious) {
     DimCheck2D<TypeParam>(this->column_continuous_seq,
@@ -548,7 +548,7 @@ void DimCheckND(const vector<vector<af_seq> > &seqs, string TestFile,
     DimCheck2D<T>(seqs, TestFile, NDims);
 }
 
-TYPED_TEST_CASE(Indexing, AllTypes);
+TYPED_TEST_SUITE(Indexing, AllTypes);
 
 TYPED_TEST(Indexing, 4D_to_4D) {
     DimCheckND<TypeParam>(this->continuous4d_to_4d,
@@ -710,7 +710,7 @@ class lookup : public ::testing::Test {
 typedef ::testing::Types<float, double, int, unsigned, unsigned char, short,
                          ushort, intl, uintl, half_float::half>
     ArrIdxTestTypes;
-TYPED_TEST_CASE(lookup, ArrIdxTestTypes);
+TYPED_TEST_SUITE(lookup, ArrIdxTestTypes);
 
 template<typename T>
 void arrayIndexTest(string pTestFile, int dim) {
@@ -1249,7 +1249,7 @@ class IndexedMembers : public ::testing::Test {
     virtual void SetUp() {}
 };
 
-TYPED_TEST_CASE(IndexedMembers, AllTypes);
+TYPED_TEST_SUITE(IndexedMembers, AllTypes);
 
 TYPED_TEST(IndexedMembers, MemFuncs) {
     SUPPORTED_TYPE_CHECK(TypeParam);
diff --git a/test/inverse_deconv.cpp b/test/inverse_deconv.cpp
index e811fe3f8b..9cce59ea62 100644
--- a/test/inverse_deconv.cpp
+++ b/test/inverse_deconv.cpp
@@ -28,7 +28,7 @@ class InverseDeconvolution : public ::testing::Test {};
 typedef ::testing::Types<float, uchar, short, ushort> TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(InverseDeconvolution, TestTypes);
+TYPED_TEST_SUITE(InverseDeconvolution, TestTypes);
 
 template<typename T, bool isColor>
 void invDeconvImageTest(string pTestFile, const float gamma,
diff --git a/test/inverse_dense.cpp b/test/inverse_dense.cpp
index cd39d0239e..a0bb6145d9 100644
--- a/test/inverse_dense.cpp
+++ b/test/inverse_dense.cpp
@@ -81,7 +81,7 @@ double eps<cdouble>() {
 }
 
 typedef ::testing::Types<float, cfloat, double, cdouble> TestTypes;
-TYPED_TEST_CASE(Inverse, TestTypes);
+TYPED_TEST_SUITE(Inverse, TestTypes);
 
 TYPED_TEST(Inverse, Square) {
     inverseTester<TypeParam>(1000, 1000, eps<TypeParam>());
diff --git a/test/iota.cpp b/test/iota.cpp
index 09cba79a94..c776d7628e 100644
--- a/test/iota.cpp
+++ b/test/iota.cpp
@@ -43,7 +43,7 @@ typedef ::testing::Types<float, double, int, unsigned int, intl, uintl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Iota, TestTypes);
+TYPED_TEST_SUITE(Iota, TestTypes);
 
 template<typename T>
 void iotaTest(const dim4 idims, const dim4 tdims) {
diff --git a/test/iterative_deconv.cpp b/test/iterative_deconv.cpp
index 80403786d5..59e6b4598b 100644
--- a/test/iterative_deconv.cpp
+++ b/test/iterative_deconv.cpp
@@ -28,7 +28,7 @@ class IterativeDeconvolution : public ::testing::Test {};
 typedef ::testing::Types<float, uchar, short, ushort> TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(IterativeDeconvolution, TestTypes);
+TYPED_TEST_SUITE(IterativeDeconvolution, TestTypes);
 
 template<typename T, bool isColor>
 void iterDeconvImageTest(string pTestFile, const unsigned iters, const float rf,
diff --git a/test/jit.cpp b/test/jit.cpp
index c1f0fbd2fa..64d72d25b7 100644
--- a/test/jit.cpp
+++ b/test/jit.cpp
@@ -534,7 +534,7 @@ std::string tile_info(const ::testing::TestParamInfo<JIT::ParamType> info) {
 }
 
 // clang-format off
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
                         JitTile, JIT,
                                                    //  input_dim            tile_dim             output_dim
                         ::testing::Values(
@@ -677,7 +677,7 @@ class JITSelect : public ::testing::TestWithParam<std::tuple<int, int, int> > {
 };
 
 // clang-format off
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
                         JitSelect, JITSelect,
                         testing::Combine(
                                          testing::Range(10, 22),
diff --git a/test/join.cpp b/test/join.cpp
index 4a98763b9b..de61bdf91e 100644
--- a/test/join.cpp
+++ b/test/join.cpp
@@ -52,7 +52,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Join, TestTypes);
+TYPED_TEST_SUITE(Join, TestTypes);
 
 template<typename T>
 void joinTest(string pTestFile, const unsigned dim, const unsigned in0,
diff --git a/test/lu_dense.cpp b/test/lu_dense.cpp
index 88ed274112..e5b4b8ac97 100644
--- a/test/lu_dense.cpp
+++ b/test/lu_dense.cpp
@@ -212,7 +212,7 @@ template<typename T>
 class LU : public ::testing::Test {};
 
 typedef ::testing::Types<float, cfloat, double, cdouble> TestTypes;
-TYPED_TEST_CASE(LU, TestTypes);
+TYPED_TEST_SUITE(LU, TestTypes);
 
 TYPED_TEST(LU, SquareLarge) { luTester<TypeParam>(500, 500, eps<TypeParam>()); }
 
diff --git a/test/match_template.cpp b/test/match_template.cpp
index a94ab94f15..90c199bd0a 100644
--- a/test/match_template.cpp
+++ b/test/match_template.cpp
@@ -35,7 +35,7 @@ typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(MatchTemplate, TestTypes);
+TYPED_TEST_SUITE(MatchTemplate, TestTypes);
 
 template<typename T>
 void matchTemplateTest(string pTestFile, af_match_type pMatchType) {
diff --git a/test/mean.cpp b/test/mean.cpp
index 9c4c8f7fb4..89a89efeb9 100644
--- a/test/mean.cpp
+++ b/test/mean.cpp
@@ -44,7 +44,7 @@ typedef ::testing::Types<cdouble, cfloat, float, double, int, uint, intl, uintl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Mean, TestTypes);
+TYPED_TEST_SUITE(Mean, TestTypes);
 
 template<typename T>
 struct f32HelperType {
@@ -270,7 +270,7 @@ class WeightedMean : public ::testing::Test {
 };
 
 // register the type list
-TYPED_TEST_CASE(WeightedMean, TestTypes);
+TYPED_TEST_SUITE(WeightedMean, TestTypes);
 
 template<typename T, typename wtsType>
 void weightedMeanAllTest(dim4 dims) {
diff --git a/test/meanshift.cpp b/test/meanshift.cpp
index 92d2408ef6..59f6bd2ee7 100644
--- a/test/meanshift.cpp
+++ b/test/meanshift.cpp
@@ -32,7 +32,7 @@ typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort,
                          intl, uintl>
     TestTypes;
 
-TYPED_TEST_CASE(Meanshift, TestTypes);
+TYPED_TEST_SUITE(Meanshift, TestTypes);
 
 TYPED_TEST(Meanshift, InvalidArgs) {
     SUPPORTED_TYPE_CHECK(TypeParam);
diff --git a/test/meanvar.cpp b/test/meanvar.cpp
index 059f694842..e9286027a2 100644
--- a/test/meanvar.cpp
+++ b/test/meanvar.cpp
@@ -73,10 +73,10 @@ struct meanvar_test {
         for (auto &v : mean) mean_.push_back((outType<T>)v);
         for (auto &v : variance) variance_.push_back((outType<T>)v);
     }
-    meanvar_test()                        = default;
-    meanvar_test(meanvar_test<T> &&other) = default;
+    meanvar_test()                                   = default;
+    meanvar_test(meanvar_test<T> &&other)            = default;
     meanvar_test &operator=(meanvar_test<T> &&other) = default;
-    meanvar_test &operator=(meanvar_test<T> &other) = delete;
+    meanvar_test &operator=(meanvar_test<T> &other)  = delete;
 
     meanvar_test(const meanvar_test<T> &other)
         : test_description_(other.test_description_)
@@ -279,12 +279,12 @@ vector<meanvar_test<T> > large_test_values() {
 
 #define MEANVAR_TEST(NAME, TYPE)                                              \
     using MeanVar##NAME = MeanVarTyped<TYPE>;                                 \
-    INSTANTIATE_TEST_CASE_P(                                                  \
+    INSTANTIATE_TEST_SUITE_P(                                                 \
         Small, MeanVar##NAME, ::testing::ValuesIn(small_test_values<TYPE>()), \
         [](const ::testing::TestParamInfo<MeanVar##NAME::ParamType> info) {   \
             return info.param.test_description_;                              \
         });                                                                   \
-    INSTANTIATE_TEST_CASE_P(                                                  \
+    INSTANTIATE_TEST_SUITE_P(                                                 \
         Large, MeanVar##NAME, ::testing::ValuesIn(large_test_values<TYPE>()), \
         [](const ::testing::TestParamInfo<MeanVar##NAME::ParamType> info) {   \
             return info.param.test_description_;                              \
@@ -313,7 +313,7 @@ MEANVAR_TEST(ComplexDouble, af::af_cdouble)
 #undef MEANVAR_TEST
 
 using MeanVarHalf = MeanVarTyped<half_float::half>;
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     Small, MeanVarHalf,
     ::testing::ValuesIn(small_test_values<half_float::half>()),
     [](const ::testing::TestParamInfo<MeanVarHalf::ParamType> info) {
@@ -330,7 +330,7 @@ TEST_P(MeanVarHalf, TestingCPP) {
 
 #define MEANVAR_TEST(NAME, TYPE)                                              \
     using MeanVar##NAME = MeanVarTyped<TYPE>;                                 \
-    INSTANTIATE_TEST_CASE_P(                                                  \
+    INSTANTIATE_TEST_SUITE_P(                                                 \
         Small, MeanVar##NAME, ::testing::ValuesIn(small_test_values<TYPE>()), \
         [](const ::testing::TestParamInfo<MeanVar##NAME::ParamType> &info) {  \
             return info.param.test_description_;                              \
diff --git a/test/medfilt.cpp b/test/medfilt.cpp
index 1e330d3702..4bc7e69924 100644
--- a/test/medfilt.cpp
+++ b/test/medfilt.cpp
@@ -39,8 +39,8 @@ typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(MedianFilter, TestTypes);
-TYPED_TEST_CASE(MedianFilter1d, TestTypes);
+TYPED_TEST_SUITE(MedianFilter, TestTypes);
+TYPED_TEST_SUITE(MedianFilter1d, TestTypes);
 
 template<typename T>
 void medfiltTest(string pTestFile, dim_t w_len, dim_t w_wid,
diff --git a/test/memory.cpp b/test/memory.cpp
index e67a7cfb69..37a1de87b1 100644
--- a/test/memory.cpp
+++ b/test/memory.cpp
@@ -78,7 +78,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(MemAlloc, TestTypes);
+TYPED_TEST_SUITE(MemAlloc, TestTypes);
 
 size_t roundUpToStep(size_t bytes) {
     if (step_bytes == 0) return bytes;
diff --git a/test/moddims.cpp b/test/moddims.cpp
index 630e4e6783..69af67860e 100644
--- a/test/moddims.cpp
+++ b/test/moddims.cpp
@@ -41,7 +41,7 @@ typedef ::testing::Types<float, double, int, unsigned, char, unsigned char,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Moddims, TestTypes);
+TYPED_TEST_SUITE(Moddims, TestTypes);
 
 template<typename T>
 void moddimsTest(string pTestFile, bool isSubRef = false,
diff --git a/test/moments.cpp b/test/moments.cpp
index f0ea3072de..5656a17ec5 100644
--- a/test/moments.cpp
+++ b/test/moments.cpp
@@ -39,7 +39,7 @@ class Image : public ::testing::Test {
 typedef ::testing::Types<float, double, int> TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Image, TestTypes);
+TYPED_TEST_SUITE(Image, TestTypes);
 
 template<typename T>
 void momentsTest(string pTestFile) {
diff --git a/test/morph.cpp b/test/morph.cpp
index ecce0738f8..220253c8c4 100644
--- a/test/morph.cpp
+++ b/test/morph.cpp
@@ -34,7 +34,7 @@ typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Morph, TestTypes);
+TYPED_TEST_SUITE(Morph, TestTypes);
 
 template<typename inType, bool isDilation, bool isVolume>
 void morphTest(string pTestFile) {
diff --git a/test/nearest_neighbour.cpp b/test/nearest_neighbour.cpp
index e2a09dc20d..5286923dd8 100644
--- a/test/nearest_neighbour.cpp
+++ b/test/nearest_neighbour.cpp
@@ -59,7 +59,7 @@ struct otype_t<uchar> {
 };
 
 // register the type list
-TYPED_TEST_CASE(NearestNeighbour, TestTypes);
+TYPED_TEST_SUITE(NearestNeighbour, TestTypes);
 
 template<typename T>
 void nearestNeighbourTest(string pTestFile, int feat_dim,
@@ -426,13 +426,13 @@ vector<nearest_neighbors_params> genKNNTests() {
             knn_data("1q1000t256k", 1, 1000, 1, 256, 0)};
 }
 
-INSTANTIATE_TEST_CASE_P(KNearestNeighborsSSD, NearestNeighborsTest,
-                        ::testing::ValuesIn(genNNTests()),
-                        testNameGenerator<NearestNeighborsTest>);
+INSTANTIATE_TEST_SUITE_P(KNearestNeighborsSSD, NearestNeighborsTest,
+                         ::testing::ValuesIn(genNNTests()),
+                         testNameGenerator<NearestNeighborsTest>);
 
-INSTANTIATE_TEST_CASE_P(KNearestNeighborsSSD, KNearestNeighborsTest,
-                        ::testing::ValuesIn(genKNNTests()),
-                        testNameGenerator<KNearestNeighborsTest>);
+INSTANTIATE_TEST_SUITE_P(KNearestNeighborsSSD, KNearestNeighborsTest,
+                         ::testing::ValuesIn(genKNNTests()),
+                         testNameGenerator<KNearestNeighborsTest>);
 
 TEST_P(NearestNeighborsTest, SingleQTests) {
     nearest_neighbors_params params = GetParam();
diff --git a/test/orb.cpp b/test/orb.cpp
index 846bb2146b..42df3ea2f5 100644
--- a/test/orb.cpp
+++ b/test/orb.cpp
@@ -125,7 +125,7 @@ class ORB : public ::testing::Test {
 
 typedef ::testing::Types<float, double> TestTypes;
 
-TYPED_TEST_CASE(ORB, TestTypes);
+TYPED_TEST_SUITE(ORB, TestTypes);
 
 template<typename T>
 void orbTest(string pTestFile) {
diff --git a/test/pad_borders.cpp b/test/pad_borders.cpp
index 33a977e03d..028c946719 100644
--- a/test/pad_borders.cpp
+++ b/test/pad_borders.cpp
@@ -29,7 +29,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, char, unsigned char,
                          ushort /*, half_float::half*/>
     TestTypes;
 
-TYPED_TEST_CASE(PadBorders, TestTypes);
+TYPED_TEST_SUITE(PadBorders, TestTypes);
 
 template<typename T>
 void testPad(const vector<T>& input, const dim4& inDims, const dim4& lbPadding,
diff --git a/test/pinverse.cpp b/test/pinverse.cpp
index 0e8575feca..44a0f884b0 100644
--- a/test/pinverse.cpp
+++ b/test/pinverse.cpp
@@ -119,7 +119,7 @@ double relEps(array in) {
 }
 
 typedef ::testing::Types<float, cfloat, double, cdouble> TestTypes;
-TYPED_TEST_CASE(Pinverse, TestTypes);
+TYPED_TEST_SUITE(Pinverse, TestTypes);
 
 // Test Moore-Penrose conditions in the following first 4 tests
 // See https://en.wikipedia.org/wiki/Moore%E2%80%93Penrose_inverse#Definition
diff --git a/test/qr_dense.cpp b/test/qr_dense.cpp
index 640171a754..09477dcbf5 100644
--- a/test/qr_dense.cpp
+++ b/test/qr_dense.cpp
@@ -162,7 +162,7 @@ template<typename T>
 class QR : public ::testing::Test {};
 
 typedef ::testing::Types<float, cfloat, double, cdouble> TestTypes;
-TYPED_TEST_CASE(QR, TestTypes);
+TYPED_TEST_SUITE(QR, TestTypes);
 
 TYPED_TEST(QR, RectangularLarge0) {
     qrTester<TypeParam>(1000, 500, eps<TypeParam>());
diff --git a/test/random.cpp b/test/random.cpp
index 4669b7515e..df65ac8006 100644
--- a/test/random.cpp
+++ b/test/random.cpp
@@ -40,7 +40,7 @@ typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, intl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Random, TestTypes);
+TYPED_TEST_SUITE(Random, TestTypes);
 
 template<typename T>
 class Random_norm : public ::testing::Test {
@@ -69,21 +69,21 @@ class RandomSeed : public ::testing::Test {
 // create a list of types to be tested
 typedef ::testing::Types<float, cfloat, double, cdouble, af_half> TestTypesNorm;
 // register the type list
-TYPED_TEST_CASE(Random_norm, TestTypesNorm);
+TYPED_TEST_SUITE(Random_norm, TestTypesNorm);
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double> TestTypesEngine;
 // register the type list
-TYPED_TEST_CASE(RandomEngine, TestTypesEngine);
+TYPED_TEST_SUITE(RandomEngine, TestTypesEngine);
 
 typedef ::testing::Types<unsigned> TestTypesEngineSeed;
 // register the type list
-TYPED_TEST_CASE(RandomEngineSeed, TestTypesEngineSeed);
+TYPED_TEST_SUITE(RandomEngineSeed, TestTypesEngineSeed);
 
 // create a list of types to be tested
 typedef ::testing::Types<unsigned> TestTypesSeed;
 // register the type list
-TYPED_TEST_CASE(RandomSeed, TestTypesSeed);
+TYPED_TEST_SUITE(RandomSeed, TestTypesSeed);
 
 template<typename T>
 void randuTest(dim4 &dims) {
diff --git a/test/range.cpp b/test/range.cpp
index 78e7782379..4d90b8a42f 100644
--- a/test/range.cpp
+++ b/test/range.cpp
@@ -55,8 +55,8 @@ typedef ::testing::Types<float, double, int, unsigned int, intl, uintl,
     RegularTypes;
 
 // register the type list
-TYPED_TEST_CASE(Range, AllTypes);
-TYPED_TEST_CASE(RangeMax, RegularTypes);
+TYPED_TEST_SUITE(Range, AllTypes);
+TYPED_TEST_SUITE(RangeMax, RegularTypes);
 
 template<typename T>
 void rangeTest(const uint x, const uint y, const uint z, const uint w,
diff --git a/test/rank_dense.cpp b/test/rank_dense.cpp
index 003979ad62..30c7ade1ca 100644
--- a/test/rank_dense.cpp
+++ b/test/rank_dense.cpp
@@ -40,8 +40,8 @@ template<typename T>
 class Det : public ::testing::Test {};
 
 typedef ::testing::Types<float, double, cfloat, cdouble> TestTypes;
-TYPED_TEST_CASE(Rank, TestTypes);
-TYPED_TEST_CASE(Det, TestTypes);
+TYPED_TEST_SUITE(Rank, TestTypes);
+TYPED_TEST_SUITE(Det, TestTypes);
 
 template<typename T>
 void rankSmall() {
diff --git a/test/reduce.cpp b/test/reduce.cpp
index 69e6573d3c..bfff42959f 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -39,7 +39,7 @@ class Reduce : public ::testing::Test {};
 typedef ::testing::Types<float, double, cfloat, cdouble, uint, int, intl, uintl,
                          uchar, short, ushort>
     TestTypes;
-TYPED_TEST_CASE(Reduce, TestTypes);
+TYPED_TEST_SUITE(Reduce, TestTypes);
 
 typedef af_err (*reduceFunc)(af_array *, const af_array, const int);
 
@@ -546,9 +546,9 @@ string testNameGenerator(
     return s.str();
 }
 
-INSTANTIATE_TEST_CASE_P(UniqueKeyTests, ReduceByKeyP,
-                        ::testing::ValuesIn(generateAllTypes()),
-                        testNameGenerator<ReduceByKeyP>);
+INSTANTIATE_TEST_SUITE_P(UniqueKeyTests, ReduceByKeyP,
+                         ::testing::ValuesIn(generateAllTypes()),
+                         testNameGenerator<ReduceByKeyP>);
 
 TEST_P(ReduceByKeyP, SumDim0) {
     if (noHalfTests(GetParam()->vType_)) { return; }
@@ -1307,7 +1307,7 @@ struct reduce_params {
 
 class ReduceHalf : public ::testing::TestWithParam<reduce_params> {};
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     SumFirstNonZeroDim, ReduceHalf,
     ::testing::Values(
         reduce_params(1, dim4(10), dim4(1), -1),
@@ -1330,7 +1330,7 @@ INSTANTIATE_TEST_CASE_P(
         reduce_params(1, dim4(8192, 10, 10), dim4(1, 10, 10), -1),
         reduce_params(1, dim4(8192, 10, 10, 10), dim4(1, 10, 10, 10), -1)));
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     SumNonZeroDim, ReduceHalf,
     ::testing::Values(
         reduce_params(1.25, dim4(10, 10), dim4(10), 1),
@@ -2031,7 +2031,7 @@ string testNameGeneratorRagged(
     return s.str();
 }
 
-INSTANTIATE_TEST_CASE_P(RaggedReduceTests, RaggedReduceMaxRangeP,
+INSTANTIATE_TEST_SUITE_P(RaggedReduceTests, RaggedReduceMaxRangeP,
                         ::testing::ValuesIn(generateAllTypesRagged()),
                         testNameGeneratorRagged<RaggedReduceMaxRangeP>);
 
diff --git a/test/regions.cpp b/test/regions.cpp
index 7deae9f5a5..4df7b90793 100644
--- a/test/regions.cpp
+++ b/test/regions.cpp
@@ -39,7 +39,7 @@ class Regions : public ::testing::Test {
 typedef ::testing::Types<float, double, int, unsigned, short, ushort> TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Regions, TestTypes);
+TYPED_TEST_SUITE(Regions, TestTypes);
 
 template<typename T>
 void regionsTest(string pTestFile, af_connectivity connectivity,
diff --git a/test/reorder.cpp b/test/reorder.cpp
index f835de8fea..6652f75210 100644
--- a/test/reorder.cpp
+++ b/test/reorder.cpp
@@ -48,7 +48,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Reorder, TestTypes);
+TYPED_TEST_SUITE(Reorder, TestTypes);
 
 template<typename T>
 void reorderTest(string pTestFile, const unsigned resultIdx, const uint x,
diff --git a/test/replace.cpp b/test/replace.cpp
index 26baf63a9d..1d0a758489 100644
--- a/test/replace.cpp
+++ b/test/replace.cpp
@@ -38,7 +38,7 @@ typedef ::testing::Types<half_float::half, float, double, cfloat, cdouble, uint,
                          int, intl, uintl, uchar, char, short, ushort>
     TestTypes;
 
-TYPED_TEST_CASE(Replace, TestTypes);
+TYPED_TEST_SUITE(Replace, TestTypes);
 
 template<typename T>
 void replaceTest(const dim4 &dims) {
diff --git a/test/resize.cpp b/test/resize.cpp
index ab53631fd4..816dd7cf9e 100644
--- a/test/resize.cpp
+++ b/test/resize.cpp
@@ -60,8 +60,8 @@ typedef ::testing::Types<int, unsigned, intl, uintl, unsigned char, char, short,
     TestTypesI;
 
 // register the type list
-TYPED_TEST_CASE(Resize, TestTypesF);
-TYPED_TEST_CASE(ResizeI, TestTypesI);
+TYPED_TEST_SUITE(Resize, TestTypesF);
+TYPED_TEST_SUITE(ResizeI, TestTypesI);
 
 TYPED_TEST(Resize, InvalidDims) {
     SUPPORTED_TYPE_CHECK(TypeParam);
diff --git a/test/rng_match.cpp b/test/rng_match.cpp
index 4e64ddf121..f13872889e 100644
--- a/test/rng_match.cpp
+++ b/test/rng_match.cpp
@@ -94,7 +94,7 @@ std::string rngmatch_info(
     return ss.str();
 }
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     PhiloxCPU_CUDA, RNGMatch,
     ::testing::Combine(
         ::testing::Values(AF_RANDOM_ENGINE_PHILOX),
diff --git a/test/rng_quality.cpp b/test/rng_quality.cpp
index 0c2ec5667e..8274b1dfa9 100644
--- a/test/rng_quality.cpp
+++ b/test/rng_quality.cpp
@@ -26,7 +26,7 @@ class RandomEngine : public ::testing::Test {
 // create a list of types to be tested
 typedef ::testing::Types<float, double> TestTypesEngine;
 // register the type list
-TYPED_TEST_CASE(RandomEngine, TestTypesEngine);
+TYPED_TEST_SUITE(RandomEngine, TestTypesEngine);
 
 template<typename T>
 void testRandomEnginePeriod(randomEngineType type) {
diff --git a/test/rotate.cpp b/test/rotate.cpp
index 7a576804ae..31019db269 100644
--- a/test/rotate.cpp
+++ b/test/rotate.cpp
@@ -38,7 +38,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, intl, char, short>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Rotate, TestTypes);
+TYPED_TEST_SUITE(Rotate, TestTypes);
 
 #define PI 3.1415926535897931f
 
diff --git a/test/rotate_linear.cpp b/test/rotate_linear.cpp
index 807859e91d..7d0dc8d5b7 100644
--- a/test/rotate_linear.cpp
+++ b/test/rotate_linear.cpp
@@ -43,7 +43,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, intl, char, short>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(RotateLinear, TestTypes);
+TYPED_TEST_SUITE(RotateLinear, TestTypes);
 
 #define PI 3.1415926535897931f
 
diff --git a/test/sat.cpp b/test/sat.cpp
index b4811bb8e5..892e2f8f4e 100644
--- a/test/sat.cpp
+++ b/test/sat.cpp
@@ -36,7 +36,7 @@ typedef ::testing::Types<float, double, int, uint, char, uchar, uintl, intl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(SAT, TestTypes);
+TYPED_TEST_SUITE(SAT, TestTypes);
 
 TYPED_TEST(SAT, IntegralImage) {
     SUPPORTED_TYPE_CHECK(TypeParam);
diff --git a/test/select.cpp b/test/select.cpp
index 7df6b6a862..a147bb3039 100644
--- a/test/select.cpp
+++ b/test/select.cpp
@@ -44,7 +44,7 @@ class Select : public ::testing::Test {};
 typedef ::testing::Types<float, double, cfloat, cdouble, uint, int, intl, uintl,
                          uchar, char, short, ushort, half_float::half>
     TestTypes;
-TYPED_TEST_CASE(Select, TestTypes);
+TYPED_TEST_SUITE(Select, TestTypes);
 
 template<typename T>
 void selectTest(const dim4& dims) {
@@ -337,17 +337,17 @@ vector<select_params> getSelectTestParams(int M, int N) {
     return vector<select_params>(_, _ + sizeof(_) / sizeof(_[0]));
 }
 
-INSTANTIATE_TEST_CASE_P(SmallDims, Select_,
-                        ::testing::ValuesIn(getSelectTestParams(10, 5)),
-                        testNameGenerator);
+INSTANTIATE_TEST_SUITE_P(SmallDims, Select_,
+                         ::testing::ValuesIn(getSelectTestParams(10, 5)),
+                         testNameGenerator);
 
-INSTANTIATE_TEST_CASE_P(Dims33_9, Select_,
-                        ::testing::ValuesIn(getSelectTestParams(33, 9)),
-                        testNameGenerator);
+INSTANTIATE_TEST_SUITE_P(Dims33_9, Select_,
+                         ::testing::ValuesIn(getSelectTestParams(33, 9)),
+                         testNameGenerator);
 
-INSTANTIATE_TEST_CASE_P(DimsLg, Select_,
-                        ::testing::ValuesIn(getSelectTestParams(512, 32)),
-                        testNameGenerator);
+INSTANTIATE_TEST_SUITE_P(DimsLg, Select_,
+                         ::testing::ValuesIn(getSelectTestParams(512, 32)),
+                         testNameGenerator);
 
 TEST_P(Select_, Batch) {
     select_params params = GetParam();
@@ -404,17 +404,17 @@ string testNameGeneratorLR(
     return ss.str();
 }
 
-INSTANTIATE_TEST_CASE_P(SmallDims, SelectLR_,
-                        ::testing::ValuesIn(getSelectLRTestParams(10, 5)),
-                        testNameGeneratorLR);
+INSTANTIATE_TEST_SUITE_P(SmallDims, SelectLR_,
+                         ::testing::ValuesIn(getSelectLRTestParams(10, 5)),
+                         testNameGeneratorLR);
 
-INSTANTIATE_TEST_CASE_P(Dims33_9, SelectLR_,
-                        ::testing::ValuesIn(getSelectLRTestParams(33, 9)),
-                        testNameGeneratorLR);
+INSTANTIATE_TEST_SUITE_P(Dims33_9, SelectLR_,
+                         ::testing::ValuesIn(getSelectLRTestParams(33, 9)),
+                         testNameGeneratorLR);
 
-INSTANTIATE_TEST_CASE_P(DimsLg, SelectLR_,
-                        ::testing::ValuesIn(getSelectLRTestParams(512, 32)),
-                        testNameGeneratorLR);
+INSTANTIATE_TEST_SUITE_P(DimsLg, SelectLR_,
+                         ::testing::ValuesIn(getSelectLRTestParams(512, 32)),
+                         testNameGeneratorLR);
 
 TEST_P(SelectLR_, BatchL) {
     selectlr_params params = GetParam();
diff --git a/test/shift.cpp b/test/shift.cpp
index 394a9cd8c2..91df07c39c 100644
--- a/test/shift.cpp
+++ b/test/shift.cpp
@@ -45,7 +45,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
                          intl, uintl, char, unsigned char, short, ushort>
     TestTypes;
 // register the type list
-TYPED_TEST_CASE(Shift, TestTypes);
+TYPED_TEST_SUITE(Shift, TestTypes);
 
 template<typename T>
 void shiftTest(string pTestFile, const unsigned resultIdx, const int x,
diff --git a/test/sift.cpp b/test/sift.cpp
index 616557f93a..90d3b40cdc 100644
--- a/test/sift.cpp
+++ b/test/sift.cpp
@@ -132,7 +132,7 @@ class SIFT : public ::testing::Test {
 
 typedef ::testing::Types<float, double> TestTypes;
 
-TYPED_TEST_CASE(SIFT, TestTypes);
+TYPED_TEST_SUITE(SIFT, TestTypes);
 
 template<typename T>
 void siftTest(string pTestFile, unsigned nLayers, float contrastThr,
diff --git a/test/sobel.cpp b/test/sobel.cpp
index c1e7306b48..449722af38 100644
--- a/test/sobel.cpp
+++ b/test/sobel.cpp
@@ -39,8 +39,8 @@ typedef ::testing::Types<int, unsigned, char, unsigned char, short, ushort>
     TestTypesInt;
 
 // register the type list
-TYPED_TEST_CASE(Sobel, TestTypes);
-TYPED_TEST_CASE(Sobel_Integer, TestTypesInt);
+TYPED_TEST_SUITE(Sobel, TestTypes);
+TYPED_TEST_SUITE(Sobel_Integer, TestTypesInt);
 
 template<typename Ti, typename To>
 void testSobelDerivatives(string pTestFile) {
diff --git a/test/solve_dense.cpp b/test/solve_dense.cpp
index a63a8eede1..b09c77645c 100644
--- a/test/solve_dense.cpp
+++ b/test/solve_dense.cpp
@@ -174,7 +174,7 @@ template<typename T>
 class Solve : public ::testing::Test {};
 
 typedef ::testing::Types<float, cfloat, double, cdouble> TestTypes;
-TYPED_TEST_CASE(Solve, TestTypes);
+TYPED_TEST_SUITE(Solve, TestTypes);
 
 template<typename T>
 double eps();
diff --git a/test/sort.cpp b/test/sort.cpp
index 86b03eb8b2..307573d7a0 100644
--- a/test/sort.cpp
+++ b/test/sort.cpp
@@ -45,7 +45,7 @@ typedef ::testing::Types<float, double, uint, int, uchar, short, ushort, intl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Sort, TestTypes);
+TYPED_TEST_SUITE(Sort, TestTypes);
 
 template<typename T>
 void sortTest(string pTestFile, const bool dir, const unsigned resultIdx0,
diff --git a/test/sort_by_key.cpp b/test/sort_by_key.cpp
index dc7382e159..b76e31ffbf 100644
--- a/test/sort_by_key.cpp
+++ b/test/sort_by_key.cpp
@@ -45,7 +45,7 @@ typedef ::testing::Types<float, double, uint, int, uchar, short, ushort, intl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(SortByKey, TestTypes);
+TYPED_TEST_SUITE(SortByKey, TestTypes);
 
 template<typename T>
 void sortTest(string pTestFile, const bool dir, const unsigned resultIdx0,
diff --git a/test/sort_index.cpp b/test/sort_index.cpp
index 9eee997b29..bfec5b429b 100644
--- a/test/sort_index.cpp
+++ b/test/sort_index.cpp
@@ -45,7 +45,7 @@ typedef ::testing::Types<float, double, uint, int, uchar, short, ushort, intl,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(SortIndex, TestTypes);
+TYPED_TEST_SUITE(SortIndex, TestTypes);
 
 template<typename T>
 void sortTest(string pTestFile, const bool dir, const unsigned resultIdx0,
diff --git a/test/sparse.cpp b/test/sparse.cpp
index 75a577de56..a130a6bb58 100644
--- a/test/sparse.cpp
+++ b/test/sparse.cpp
@@ -185,7 +185,7 @@ template<typename T>
 class Sparse : public ::testing::Test {};
 
 typedef ::testing::Types<float, cfloat, double, cdouble> SparseTypes;
-TYPED_TEST_CASE(Sparse, SparseTypes);
+TYPED_TEST_SUITE(Sparse, SparseTypes);
 
 TYPED_TEST(Sparse, DeepCopy) {
     SUPPORTED_TYPE_CHECK(TypeParam);
diff --git a/test/stdev.cpp b/test/stdev.cpp
index 20187f8655..85f3bf079d 100644
--- a/test/stdev.cpp
+++ b/test/stdev.cpp
@@ -41,7 +41,7 @@ typedef ::testing::Types<float, double, int, uint, intl, uintl, char, uchar>
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(StandardDev, TestTypes);
+TYPED_TEST_SUITE(StandardDev, TestTypes);
 
 template<typename T>
 struct f32HelperType {
diff --git a/test/susan.cpp b/test/susan.cpp
index 223704bb26..6d40177132 100644
--- a/test/susan.cpp
+++ b/test/susan.cpp
@@ -62,7 +62,7 @@ class Susan : public ::testing::Test {
 typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
     TestTypes;
 
-TYPED_TEST_CASE(Susan, TestTypes);
+TYPED_TEST_SUITE(Susan, TestTypes);
 
 template<typename T>
 void susanTest(string pTestFile, float t, float g) {
diff --git a/test/svd_dense.cpp b/test/svd_dense.cpp
index 18b0173957..e31603a84b 100644
--- a/test/svd_dense.cpp
+++ b/test/svd_dense.cpp
@@ -38,7 +38,7 @@ template<typename T>
 class svd : public ::testing::Test {};
 
 typedef ::testing::Types<float, double, cfloat, cdouble> TestTypes;
-TYPED_TEST_CASE(svd, TestTypes);
+TYPED_TEST_SUITE(svd, TestTypes);
 
 template<typename T>
 inline double get_val(T val) {
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 2e13ff9bbf..035c76991b 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -28,7 +28,17 @@
 #if defined(USE_MTX)
 #include <mmio.h>
 #include <cstdlib>
+#endif
 
+/// GTest deprecated the INSTANTIATED_TEST_CASE_P macro in favor of the
+/// INSTANTIATE_TEST_SUITE_P macro which has the same syntax but the older
+/// versions of gtest do not support this new macro adds the
+/// INSTANTIATE_TEST_SUITE_P macro and maps it to the old macro
+#ifndef INSTANTIATE_TEST_SUITE_P
+#define INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P
+#endif
+#ifndef TYPED_TEST_SUITE
+#define TYPED_TEST_SUITE TYPED_TEST_CASE
 #endif
 
 bool operator==(const af_half &lhs, const af_half &rhs);
diff --git a/test/tile.cpp b/test/tile.cpp
index 8127379e78..0a649d00ac 100644
--- a/test/tile.cpp
+++ b/test/tile.cpp
@@ -52,7 +52,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Tile, TestTypes);
+TYPED_TEST_SUITE(Tile, TestTypes);
 
 template<typename T>
 void tileTest(string pTestFile, const unsigned resultIdx, const uint x,
diff --git a/test/topk.cpp b/test/topk.cpp
index 46c4355d6a..86cf1287f9 100644
--- a/test/topk.cpp
+++ b/test/topk.cpp
@@ -49,7 +49,7 @@ class TopK : public ::testing::Test {};
 
 typedef ::testing::Types<float, double, int, uint, half_float::half> TestTypes;
 
-TYPED_TEST_CASE(TopK, TestTypes);
+TYPED_TEST_SUITE(TopK, TestTypes);
 
 template<typename T>
 void increment_next(T& val,
@@ -318,7 +318,7 @@ ostream& operator<<(ostream& os, const topk_params& param) {
 
 class TopKParams : public ::testing::TestWithParam<topk_params> {};
 
-INSTANTIATE_TEST_CASE_P(
+INSTANTIATE_TEST_SUITE_P(
     InstantiationName, TopKParams,
     ::testing::Values(topk_params{100, 10, 32, 0, AF_TOPK_MIN},
                       topk_params{100, 10, 64, 0, AF_TOPK_MIN},
diff --git a/test/transform.cpp b/test/transform.cpp
index b5bf76f2ec..77cdcfc881 100644
--- a/test/transform.cpp
+++ b/test/transform.cpp
@@ -41,8 +41,8 @@ typedef ::testing::Types<float, double> TestTypes;
 typedef ::testing::Types<int, intl, uint, uintl, short, ushort, uchar>
     TestTypesInt;
 
-TYPED_TEST_CASE(Transform, TestTypes);
-TYPED_TEST_CASE(TransformInt, TestTypesInt);
+TYPED_TEST_SUITE(Transform, TestTypes);
+TYPED_TEST_SUITE(TransformInt, TestTypesInt);
 
 template<typename T>
 void genTestData(af_array *gold, af_array *in, af_array *transform,
@@ -403,7 +403,7 @@ class TransformV2 : public Transform<T> {
     }
 };
 
-TYPED_TEST_CASE(TransformV2, TestTypes);
+TYPED_TEST_SUITE(TransformV2, TestTypes);
 
 template<typename T>
 class TransformV2TuxNearest : public TransformV2<T> {
@@ -416,7 +416,7 @@ class TransformV2TuxNearest : public TransformV2<T> {
     }
 };
 
-TYPED_TEST_CASE(TransformV2TuxNearest, TestTypes);
+TYPED_TEST_SUITE(TransformV2TuxNearest, TestTypes);
 
 TYPED_TEST(TransformV2TuxNearest, UseNullOutputArray) {
     this->testSpclOutArray(NULL_ARRAY);
diff --git a/test/transform_coordinates.cpp b/test/transform_coordinates.cpp
index 7d8805d043..01ab960e93 100644
--- a/test/transform_coordinates.cpp
+++ b/test/transform_coordinates.cpp
@@ -31,7 +31,7 @@ class TransformCoordinates : public ::testing::Test {
 
 typedef ::testing::Types<float, double> TestTypes;
 
-TYPED_TEST_CASE(TransformCoordinates, TestTypes);
+TYPED_TEST_SUITE(TransformCoordinates, TestTypes);
 
 template<typename T>
 void transformCoordinatesTest(string pTestFile) {
diff --git a/test/translate.cpp b/test/translate.cpp
index dcdb06953a..4c84b19009 100644
--- a/test/translate.cpp
+++ b/test/translate.cpp
@@ -42,8 +42,8 @@ typedef ::testing::Types<float, double, cfloat, cdouble> TestTypes;
 typedef ::testing::Types<int, intl, char, short> TestTypesInt;
 
 // register the type list
-TYPED_TEST_CASE(Translate, TestTypes);
-TYPED_TEST_CASE(TranslateInt, TestTypesInt);
+TYPED_TEST_SUITE(Translate, TestTypes);
+TYPED_TEST_SUITE(TranslateInt, TestTypesInt);
 
 template<typename T>
 void translateTest(string pTestFile, const unsigned resultIdx, dim4 odims,
diff --git a/test/transpose.cpp b/test/transpose.cpp
index 72543d2e7a..cb36640885 100644
--- a/test/transpose.cpp
+++ b/test/transpose.cpp
@@ -49,7 +49,7 @@ typedef ::testing::Types<float, cfloat, double, cdouble, int, uint, char, uchar,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Transpose, TestTypes);
+TYPED_TEST_SUITE(Transpose, TestTypes);
 
 template<typename T>
 void trsTest(string pTestFile, bool isSubRef = false,
diff --git a/test/transpose_inplace.cpp b/test/transpose_inplace.cpp
index 88d61cad16..82b071488a 100644
--- a/test/transpose_inplace.cpp
+++ b/test/transpose_inplace.cpp
@@ -35,7 +35,7 @@ typedef ::testing::Types<float, cfloat, double, cdouble, int, uint, char, uchar,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Transpose, TestTypes);
+TYPED_TEST_SUITE(Transpose, TestTypes);
 
 template<typename T>
 void transposeip_test(dim4 dims) {
diff --git a/test/triangle.cpp b/test/triangle.cpp
index c7b9c7b029..90b50bb6dc 100644
--- a/test/triangle.cpp
+++ b/test/triangle.cpp
@@ -37,7 +37,7 @@ class Triangle : public ::testing::Test {};
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, char,
                          uchar, uintl, intl, short, ushort, half_float::half>
     TestTypes;
-TYPED_TEST_CASE(Triangle, TestTypes);
+TYPED_TEST_SUITE(Triangle, TestTypes);
 
 template<typename T>
 void triangleTester(const dim4 dims, bool is_upper, bool is_unit_diag = false) {
diff --git a/test/unwrap.cpp b/test/unwrap.cpp
index 9224e90d8f..b33dc8c7d5 100644
--- a/test/unwrap.cpp
+++ b/test/unwrap.cpp
@@ -41,7 +41,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Unwrap, TestTypes);
+TYPED_TEST_SUITE(Unwrap, TestTypes);
 
 template<typename T>
 void unwrapTest(string pTestFile, const unsigned resultIdx, const dim_t wx,
diff --git a/test/var.cpp b/test/var.cpp
index b02442dba1..45c7b6847f 100644
--- a/test/var.cpp
+++ b/test/var.cpp
@@ -28,7 +28,7 @@ class Var : public ::testing::Test {};
 typedef ::testing::Types<float, double, cfloat, cdouble, uint, int, uintl, intl,
                          char, uchar, short, ushort, half_float::half>
     TestTypes;
-TYPED_TEST_CASE(Var, TestTypes);
+TYPED_TEST_SUITE(Var, TestTypes);
 
 template<typename T>
 struct elseType {
diff --git a/test/where.cpp b/test/where.cpp
index 20913845a3..746a9aa5b4 100644
--- a/test/where.cpp
+++ b/test/where.cpp
@@ -36,7 +36,7 @@ class Where : public ::testing::Test {};
 typedef ::testing::Types<float, double, cfloat, cdouble, int, uint, intl, uintl,
                          char, uchar, short, ushort>
     TestTypes;
-TYPED_TEST_CASE(Where, TestTypes);
+TYPED_TEST_SUITE(Where, TestTypes);
 
 template<typename T>
 void whereTest(string pTestFile, bool isSubRef = false,
diff --git a/test/wrap.cpp b/test/wrap.cpp
index 92193bc88d..91b57c4bc0 100644
--- a/test/wrap.cpp
+++ b/test/wrap.cpp
@@ -46,7 +46,7 @@ typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Wrap, TestTypes);
+TYPED_TEST_SUITE(Wrap, TestTypes);
 
 template<typename T>
 inline double get_val(T val) {
@@ -354,7 +354,7 @@ class WrapV2 : public WrapCommon {
     }
 };
 
-TYPED_TEST_CASE(WrapV2, TestTypes);
+TYPED_TEST_SUITE(WrapV2, TestTypes);
 
 template<typename T>
 class WrapV2Simple : public WrapV2<T> {
@@ -379,7 +379,7 @@ class WrapV2Simple : public WrapV2<T> {
     }
 };
 
-TYPED_TEST_CASE(WrapV2Simple, TestTypes);
+TYPED_TEST_SUITE(WrapV2Simple, TestTypes);
 
 TYPED_TEST(WrapV2Simple, UseNullOutputArray) {
     this->testSpclOutArray(NULL_ARRAY);
@@ -510,7 +510,7 @@ TEST_P(WrapAPITest, CheckDifferentWrapArgs) {
 
     af_array out_ = 0;
     af_err err    = af_wrap(&out_, in_, in_dims[0], in_dims[1], win_d0, win_d1,
-                         str_d0, str_d1, pad_d0, pad_d1, input.is_column);
+                            str_d0, str_d1, pad_d0, pad_d1, input.is_column);
 
     ASSERT_EQ(err, input.err);
     if (out_ != 0) af_release_array(out_);
@@ -537,4 +537,4 @@ WrapArgs args[] = {
     // clang-format on
 };
 
-INSTANTIATE_TEST_CASE_P(BulkTest, WrapAPITest, ::testing::ValuesIn(args));
+INSTANTIATE_TEST_SUITE_P(BulkTest, WrapAPITest, ::testing::ValuesIn(args));
diff --git a/test/write.cpp b/test/write.cpp
index 5a6d14c021..8f18f6e954 100644
--- a/test/write.cpp
+++ b/test/write.cpp
@@ -38,7 +38,7 @@ typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, char,
     TestTypes;
 
 // register the type list
-TYPED_TEST_CASE(Write, TestTypes);
+TYPED_TEST_SUITE(Write, TestTypes);
 
 template<typename T>
 void writeTest(dim4 dims) {

From e401fce849cdde11120847c7be475e0e99063af4 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 30 Sep 2022 14:24:04 -0400
Subject: [PATCH 085/473] Update clang-format version on github workflow

---
 .github/workflows/clang-format-lint.yml       | 12 +--
 examples/getting_started/convolve.cpp         |  2 +-
 examples/image_processing/morphing.cpp        |  2 +-
 examples/pde/swe.cpp                          |  2 +-
 src/api/c/pinverse.cpp                        |  4 +-
 src/api/c/ycbcr_rgb.cpp                       |  2 +-
 src/api/cpp/array.cpp                         |  6 +-
 src/api/unified/symbol_manager.hpp            |  2 +-
 src/backend/common/DefaultMemoryManager.hpp   |  6 +-
 src/backend/common/HandleBase.hpp             |  4 +-
 src/backend/common/MemoryManagerBase.hpp      |  2 +-
 src/backend/common/graphics_common.cpp        |  2 +-
 src/backend/common/graphics_common.hpp        |  4 +-
 src/backend/common/host_memory.cpp            | 12 +--
 src/backend/common/jit/NodeIterator.hpp       | 10 +--
 src/backend/common/unique_handle.hpp          |  2 +-
 src/backend/cpu/Param.hpp                     | 16 ++--
 src/backend/cpu/convolve.cpp                  |  8 +-
 src/backend/cpu/device_manager.hpp            |  2 +-
 src/backend/cpu/kernel/diff.hpp               |  2 +-
 src/backend/cpu/kernel/fftconvolve.hpp        |  5 +-
 src/backend/cpu/kernel/orb.hpp                |  4 +-
 src/backend/cuda/LookupTable1D.hpp            |  8 +-
 src/backend/cuda/Param.hpp                    | 12 +--
 src/backend/cuda/convolveNN.cpp               | 10 +--
 src/backend/cuda/kernel/fast.hpp              |  2 +-
 src/backend/cuda/kernel/harris.hpp            |  2 +-
 src/backend/cuda/kernel/homography.hpp        |  2 +-
 src/backend/cuda/kernel/orb.hpp               |  4 +-
 src/backend/cuda/kernel/random_engine.hpp     |  4 +-
 src/backend/cuda/kernel/shfl_intrinsics.hpp   |  4 +-
 src/backend/cuda/kernel/unwrap.hpp            |  2 +-
 src/backend/cuda/types.hpp                    |  2 +-
 src/backend/opencl/convolve.cpp               |  4 +-
 .../opencl/kernel/convolve/conv_common.hpp    |  4 +-
 src/backend/opencl/kernel/homography.hpp      |  2 +-
 src/backend/opencl/kernel/index.hpp           |  2 +-
 src/backend/opencl/kernel/orb.hpp             |  4 +-
 src/backend/opencl/magma/geqrf2.cpp           |  4 +-
 src/backend/opencl/magma/magma_data.h         | 12 +--
 src/backend/opencl/magma/magma_types.h        |  2 +-
 src/backend/opencl/memory.cpp                 |  6 +-
 src/backend/opencl/svd.cpp                    |  4 +-
 src/backend/opencl/topk.cpp                   |  6 +-
 test/.clang-format                            |  2 +-
 test/approx1.cpp                              | 20 ++---
 test/approx2.cpp                              | 20 ++---
 test/arrayfire_test.cpp                       | 36 ++++----
 test/assign.cpp                               | 12 +--
 test/bilateral.cpp                            |  8 +-
 test/binary.cpp                               |  2 +-
 test/blas.cpp                                 |  8 +-
 test/canny.cpp                                |  4 +-
 test/confidence_connected.cpp                 |  4 +-
 test/convolve.cpp                             | 36 ++++----
 test/corrcoef.cpp                             |  4 +-
 test/covariance.cpp                           |  4 +-
 test/diff1.cpp                                | 12 +--
 test/diff2.cpp                                | 12 +--
 test/dot.cpp                                  | 24 +++---
 test/fast.cpp                                 |  4 +-
 test/fft.cpp                                  | 16 ++--
 test/fftconvolve.cpp                          | 16 ++--
 test/gaussiankernel.cpp                       |  8 +-
 test/gen_assign.cpp                           | 12 +--
 test/gen_index.cpp                            | 12 +--
 test/gloh.cpp                                 | 10 +--
 test/gradient.cpp                             |  8 +-
 test/hamming.cpp                              | 10 +--
 test/harris.cpp                               |  4 +-
 test/histogram.cpp                            |  8 +-
 test/homography.cpp                           |  4 +-
 test/hsv_rgb.cpp                              | 16 ++--
 test/iir.cpp                                  |  4 +-
 test/imageio.cpp                              | 16 ++--
 test/index.cpp                                | 86 +++++++++----------
 test/internal.cpp                             |  2 +-
 test/ireduce.cpp                              |  8 +-
 test/jit.cpp                                  |  4 +-
 test/join.cpp                                 |  8 +-
 test/lu_dense.cpp                             |  8 +-
 test/match_template.cpp                       |  4 +-
 test/mean.cpp                                 |  4 +-
 test/meanvar.cpp                              | 36 ++++----
 test/medfilt.cpp                              | 16 ++--
 test/moddims.cpp                              | 16 ++--
 test/moments.cpp                              |  8 +-
 test/morph.cpp                                | 14 +--
 test/nearest_neighbour.cpp                    | 12 +--
 test/orb.cpp                                  | 11 ++-
 test/pinverse.cpp                             |  8 +-
 test/qr_dense.cpp                             |  4 +-
 test/rank_dense.cpp                           |  4 +-
 test/reduce.cpp                               | 18 ++--
 test/regions.cpp                              |  8 +-
 test/reorder.cpp                              |  8 +-
 test/resize.cpp                               | 20 ++---
 test/rotate.cpp                               |  8 +-
 test/rotate_linear.cpp                        |  8 +-
 test/scan.cpp                                 |  8 +-
 test/set.cpp                                  |  8 +-
 test/shift.cpp                                |  8 +-
 test/sift.cpp                                 | 10 +--
 test/sobel.cpp                                |  4 +-
 test/sort.cpp                                 | 16 ++--
 test/sort_by_key.cpp                          | 16 ++--
 test/sort_index.cpp                           | 16 ++--
 test/stdev.cpp                                | 12 +--
 test/susan.cpp                                |  2 +-
 test/testHelpers.hpp                          | 14 +--
 test/threading.cpp                            |  8 +-
 test/tile.cpp                                 |  8 +-
 test/transform.cpp                            | 12 +--
 test/transform_coordinates.cpp                |  8 +-
 test/translate.cpp                            |  4 +-
 test/transpose.cpp                            |  8 +-
 test/unwrap.cpp                               |  8 +-
 test/var.cpp                                  |  6 +-
 test/where.cpp                                |  8 +-
 test/ycbcr_rgb.cpp                            | 16 ++--
 120 files changed, 546 insertions(+), 546 deletions(-)

diff --git a/.github/workflows/clang-format-lint.yml b/.github/workflows/clang-format-lint.yml
index 9b1037d4ab..25e79545ac 100644
--- a/.github/workflows/clang-format-lint.yml
+++ b/.github/workflows/clang-format-lint.yml
@@ -17,22 +17,22 @@ jobs:
             uses: actions/checkout@master
 
           - name: Check Sources
-            uses: DoozyX/clang-format-lint-action@v0.11
+            uses: DoozyX/clang-format-lint-action@v0.14
             with:
               source: './src'
               extensions: 'h,cpp,hpp'
-              clangFormatVersion: 11
+              clangFormatVersion: 14
 
           - name: Check Tests
-            uses: DoozyX/clang-format-lint-action@v0.11
+            uses: DoozyX/clang-format-lint-action@v0.14
             with:
               source: './test'
               extensions: 'h,cpp,hpp'
-              clangFormatVersion: 11
+              clangFormatVersion: 14
 
           - name: Check Examples
-            uses: DoozyX/clang-format-lint-action@v0.11
+            uses: DoozyX/clang-format-lint-action@v0.14
             with:
               source: './examples'
               extensions: 'h,cpp,hpp'
-              clangFormatVersion: 11
+              clangFormatVersion: 14
diff --git a/examples/getting_started/convolve.cpp b/examples/getting_started/convolve.cpp
index c07cedfc3c..7c2d0626ca 100644
--- a/examples/getting_started/convolve.cpp
+++ b/examples/getting_started/convolve.cpp
@@ -20,7 +20,7 @@ static array img;
 
 // 5x5 derivative with separable kernels
 static float h_dx[]     = {1.f / 12, -8.f / 12, 0, 8.f / 12,
-                       -1.f / 12};  // five point stencil
+                           -1.f / 12};  // five point stencil
 static float h_spread[] = {1.f / 5, 1.f / 5, 1.f / 5, 1.f / 5, 1.f / 5};
 static array dx, spread, kernel;  // device kernels
 
diff --git a/examples/image_processing/morphing.cpp b/examples/image_processing/morphing.cpp
index 51108490c2..ad66b7ea2a 100644
--- a/examples/image_processing/morphing.cpp
+++ b/examples/image_processing/morphing.cpp
@@ -45,7 +45,7 @@ array border(const array& img, const int left, const int right, const int top,
     array ret    = constant(value, imgDims);
     ret(seq(top, imgDims[0] - bottom), seq(left, imgDims[1] - right), span,
         span)    = img(seq(top, imgDims[0] - bottom),
-                    seq(left, imgDims[1] - right), span, span);
+                       seq(left, imgDims[1] - right), span, span);
 
     return ret;
 }
diff --git a/examples/pde/swe.cpp b/examples/pde/swe.cpp
index c7f9d6ebda..7e5a9af017 100644
--- a/examples/pde/swe.cpp
+++ b/examples/pde/swe.cpp
@@ -54,7 +54,7 @@ static void swe(bool console) {
         if (iter > 2000) {
             // Initial condition
             etam  = 0.01f * exp((-((x - io) * (x - io) + (y - jo) * (y - jo))) /
-                               (k * k));
+                                (k * k));
             m_eta = max<float>(etam);
             eta   = etam;
             iter  = 0;
diff --git a/src/api/c/pinverse.cpp b/src/api/c/pinverse.cpp
index 49086043af..05d2d92fba 100644
--- a/src/api/c/pinverse.cpp
+++ b/src/api/c/pinverse.cpp
@@ -92,7 +92,7 @@ Array<T> pinverseSvd(const Array<T> &in, const double tol) {
             Array<Tr> sVecSlice = getSubArray(
                 sVec, false, 0, sVec.dims()[0] - 1, 0, 0, i, i, j, j);
             Array<T> uSlice  = getSubArray(u, false, 0, u.dims()[0] - 1, 0,
-                                          u.dims()[1] - 1, i, i, j, j);
+                                           u.dims()[1] - 1, i, i, j, j);
             Array<T> vTSlice = getSubArray(vT, false, 0, vT.dims()[0] - 1, 0,
                                            vT.dims()[1] - 1, i, i, j, j);
             svd<T, Tr>(sVecSlice, uSlice, vTSlice, inSlice);
@@ -131,7 +131,7 @@ Array<T> pinverseSvd(const Array<T> &in, const double tol) {
         dim4(sVecRecip.dims()[0], (sVecRecip.dims()[2] * sVecRecip.dims()[3])));
     Array<T> sPinv = diagCreate<T>(sVecRecipMod, 0);
     sPinv          = modDims<T>(sPinv, dim4(sPinv.dims()[0], sPinv.dims()[1],
-                                   sVecRecip.dims()[2], sVecRecip.dims()[3]));
+                                            sVecRecip.dims()[2], sVecRecip.dims()[3]));
 
     Array<T> uT = transpose(u, true);
 
diff --git a/src/api/c/ycbcr_rgb.cpp b/src/api/c/ycbcr_rgb.cpp
index d3c56a7117..a871618d28 100644
--- a/src/api/c/ycbcr_rgb.cpp
+++ b/src/api/c/ycbcr_rgb.cpp
@@ -69,7 +69,7 @@ static af_array convert(const af_array& in, const af_ycc_std standard) {
     static const float INV_219 = 0.004566210;
     static const float INV_112 = 0.008928571;
     const static float k[6]    = {0.1140f, 0.2990f, 0.0722f,
-                               0.2126f, 0.0593f, 0.2627f};
+                                  0.2126f, 0.0593f, 0.2627f};
     unsigned stdIdx            = 0;  // Default standard is AF_YCC_601
     switch (standard) {
         case AF_YCC_709: stdIdx = 2; break;
diff --git a/src/api/cpp/array.cpp b/src/api/cpp/array.cpp
index 3600f60e83..5889c0d99c 100644
--- a/src/api/cpp/array.cpp
+++ b/src/api/cpp/array.cpp
@@ -166,9 +166,9 @@ struct array::array_proxy::array_proxy_impl {
         if (delete_on_destruction_) { delete parent_; }
     }
 
-    array_proxy_impl(const array_proxy_impl &)  = delete;
-    array_proxy_impl(const array_proxy_impl &&) = delete;
-    array_proxy_impl operator=(const array_proxy_impl &) = delete;
+    array_proxy_impl(const array_proxy_impl &)            = delete;
+    array_proxy_impl(const array_proxy_impl &&)           = delete;
+    array_proxy_impl operator=(const array_proxy_impl &)  = delete;
     array_proxy_impl operator=(const array_proxy_impl &&) = delete;
 };
 
diff --git a/src/api/unified/symbol_manager.hpp b/src/api/unified/symbol_manager.hpp
index cbf6e76861..b77f7e9bbe 100644
--- a/src/api/unified/symbol_manager.hpp
+++ b/src/api/unified/symbol_manager.hpp
@@ -152,7 +152,7 @@ bool checkArrays(af_backend activeBackend, T a, Args... arg) {
         if (index_ != unified::getActiveBackend()) {                             \
             index_ = unified::getActiveBackend();                                \
             func   = (af_func)common::getFunctionPointer(                        \
-                unified::getActiveHandle(), __func__);                         \
+                  unified::getActiveHandle(), __func__);                         \
         }                                                                        \
         return func(__VA_ARGS__);                                                \
     } else {                                                                     \
diff --git a/src/backend/common/DefaultMemoryManager.hpp b/src/backend/common/DefaultMemoryManager.hpp
index 25eb4bd06a..0881f318a1 100644
--- a/src/backend/common/DefaultMemoryManager.hpp
+++ b/src/backend/common/DefaultMemoryManager.hpp
@@ -57,9 +57,9 @@ class DefaultMemoryManager final : public common::memory::MemoryManagerBase {
             , lock_bytes(0)
             , lock_buffers(0) {}
 
-        memory_info(memory_info &other)  = delete;
-        memory_info(memory_info &&other) = default;
-        memory_info &operator=(memory_info &other) = delete;
+        memory_info(memory_info &other)             = delete;
+        memory_info(memory_info &&other)            = default;
+        memory_info &operator=(memory_info &other)  = delete;
         memory_info &operator=(memory_info &&other) = default;
     };
 
diff --git a/src/backend/common/HandleBase.hpp b/src/backend/common/HandleBase.hpp
index bf7df20a20..4ffaf4dca1 100644
--- a/src/backend/common/HandleBase.hpp
+++ b/src/backend/common/HandleBase.hpp
@@ -21,10 +21,10 @@ class HandleBase {
     operator H() { return handle_; }
     H* get() { return &handle_; }
 
-    HandleBase(HandleBase const&) = delete;
+    HandleBase(HandleBase const&)     = delete;
     void operator=(HandleBase const&) = delete;
 
-    HandleBase(HandleBase&& h) = default;
+    HandleBase(HandleBase&& h)            = default;
     HandleBase& operator=(HandleBase&& h) = default;
 };
 }  // namespace common
diff --git a/src/backend/common/MemoryManagerBase.hpp b/src/backend/common/MemoryManagerBase.hpp
index 5ba3281294..c338db1020 100644
--- a/src/backend/common/MemoryManagerBase.hpp
+++ b/src/backend/common/MemoryManagerBase.hpp
@@ -29,7 +29,7 @@ namespace memory {
  */
 class MemoryManagerBase {
    public:
-    MemoryManagerBase()        = default;
+    MemoryManagerBase()                                     = default;
     MemoryManagerBase &operator=(const MemoryManagerBase &) = delete;
     MemoryManagerBase(const MemoryManagerBase &)            = delete;
     virtual ~MemoryManagerBase() {}
diff --git a/src/backend/common/graphics_common.cpp b/src/backend/common/graphics_common.cpp
index fc8256f999..d1a572a153 100644
--- a/src/backend/common/graphics_common.cpp
+++ b/src/backend/common/graphics_common.cpp
@@ -258,7 +258,7 @@ fg_window ForgeManager::getMainWindow() {
             }
             fg_window w = nullptr;
             forgeError  = this->mPlugin->fg_create_window(
-                &w, WIDTH, HEIGHT, "ArrayFire", NULL, true);
+                 &w, WIDTH, HEIGHT, "ArrayFire", NULL, true);
             if (forgeError != FG_ERR_NONE) { return; }
             this->setWindowChartGrid(w, 1, 1);
             this->mPlugin->fg_make_window_current(w);
diff --git a/src/backend/common/graphics_common.hpp b/src/backend/common/graphics_common.hpp
index 1f2b9f60b1..6db366f323 100644
--- a/src/backend/common/graphics_common.hpp
+++ b/src/backend/common/graphics_common.hpp
@@ -53,10 +53,10 @@ class ForgeManager {
     using WindowGridDims = std::pair<int, int>;
 
     ForgeManager();
-    ForgeManager(ForgeManager const&) = delete;
+    ForgeManager(ForgeManager const&)            = delete;
     ForgeManager& operator=(ForgeManager const&) = delete;
     ForgeManager(ForgeManager&&)                 = delete;
-    ForgeManager& operator=(ForgeManager&&) = delete;
+    ForgeManager& operator=(ForgeManager&&)      = delete;
 
     /// \brief Module used to invoke forge API calls
     ForgeModule& plugin();
diff --git a/src/backend/common/host_memory.cpp b/src/backend/common/host_memory.cpp
index a44a920db3..51a01e2164 100644
--- a/src/backend/common/host_memory.cpp
+++ b/src/backend/common/host_memory.cpp
@@ -63,13 +63,13 @@ size_t getHostMemorySize() {
 
 #if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64))
     int mib[2];
-    mib[0] = CTL_HW;
+    mib[0]       = CTL_HW;
 #if defined(HW_MEMSIZE)
-    mib[1] = HW_MEMSIZE; /* OSX. --------------------- */
+    mib[1]       = HW_MEMSIZE; /* OSX. --------------------- */
 #elif defined(HW_PHYSMEM64)
     mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */
 #endif
-    int64_t size = 0; /* 64-bit */
+    int64_t size = 0;          /* 64-bit */
     size_t len   = sizeof(size);
     if (sysctl(mib, 2, &size, &len, NULL, 0) == 0) return (size_t)size;
     return 0L; /* Failed? */
@@ -90,13 +90,13 @@ size_t getHostMemorySize() {
 #elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
     /* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
     int mib[2];
-    mib[0] = CTL_HW;
+    mib[0]            = CTL_HW;
 #if defined(HW_REALMEM)
-    mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */
+    mib[1]            = HW_REALMEM; /* FreeBSD. ----------------- */
 #elif defined(HW_PYSMEM)
     mib[1] = HW_PHYSMEM; /* Others. ------------------ */
 #endif
-    unsigned int size = 0; /* 32-bit */
+    unsigned int size = 0;          /* 32-bit */
     size_t len        = sizeof(size);
     if (sysctl(mib, 2, &size, &len, NULL, 0) == 0) return (size_t)size;
     return 0L; /* Failed? */
diff --git a/src/backend/common/jit/NodeIterator.hpp b/src/backend/common/jit/NodeIterator.hpp
index e286f6359d..e2883079a1 100644
--- a/src/backend/common/jit/NodeIterator.hpp
+++ b/src/backend/common/jit/NodeIterator.hpp
@@ -92,11 +92,11 @@ class NodeIterator {
     pointer operator->() const noexcept { return tree[index]; }
 
     /// Creates a sentinel iterator. This is equivalent to the end iterator
-    NodeIterator()                              = default;
-    NodeIterator(const NodeIterator& other)     = default;
-    NodeIterator(NodeIterator&& other) noexcept = default;
-    ~NodeIterator() noexcept                    = default;
-    NodeIterator& operator=(const NodeIterator& other) = default;
+    NodeIterator()                                         = default;
+    NodeIterator(const NodeIterator& other)                = default;
+    NodeIterator(NodeIterator&& other) noexcept            = default;
+    ~NodeIterator() noexcept                               = default;
+    NodeIterator& operator=(const NodeIterator& other)     = default;
     NodeIterator& operator=(NodeIterator&& other) noexcept = default;
 };
 
diff --git a/src/backend/common/unique_handle.hpp b/src/backend/common/unique_handle.hpp
index 52d0acfeda..0c3fe8fe6f 100644
--- a/src/backend/common/unique_handle.hpp
+++ b/src/backend/common/unique_handle.hpp
@@ -60,7 +60,7 @@ class unique_handle {
         }
     }
 
-    unique_handle(const unique_handle &other) noexcept = delete;
+    unique_handle(const unique_handle &other) noexcept      = delete;
     unique_handle &operator=(unique_handle &other) noexcept = delete;
 
     AF_CONSTEXPR unique_handle(unique_handle &&other) noexcept
diff --git a/src/backend/cpu/Param.hpp b/src/backend/cpu/Param.hpp
index ec3613e21f..20686c4430 100644
--- a/src/backend/cpu/Param.hpp
+++ b/src/backend/cpu/Param.hpp
@@ -53,10 +53,10 @@ class CParam {
     /// \param[in] i The dimension
     constexpr dim_t strides(int i) const noexcept { return m_strides[i]; }
 
-    constexpr CParam()                    = delete;
-    constexpr CParam(const CParam &other) = default;
-    constexpr CParam(CParam &&other)      = default;
-    CParam<T> &operator=(CParam &&other) noexcept = default;
+    constexpr CParam()                                 = delete;
+    constexpr CParam(const CParam &other)              = default;
+    constexpr CParam(CParam &&other)                   = default;
+    CParam<T> &operator=(CParam &&other) noexcept      = default;
     CParam<T> &operator=(const CParam &other) noexcept = default;
     ~CParam()                                          = default;
 };
@@ -108,10 +108,10 @@ class Param {
     /// \param[in] i The dimension
     constexpr dim_t strides(int i) const noexcept { return m_strides[i]; }
 
-    ~Param()                            = default;
-    constexpr Param(const Param &other) = default;
-    constexpr Param(Param &&other)      = default;
-    Param<T> &operator=(Param &&other) noexcept = default;
+    ~Param()                                         = default;
+    constexpr Param(const Param &other)              = default;
+    constexpr Param(Param &&other)                   = default;
+    Param<T> &operator=(Param &&other) noexcept      = default;
     Param<T> &operator=(const Param &other) noexcept = default;
 };
 
diff --git a/src/backend/cpu/convolve.cpp b/src/backend/cpu/convolve.cpp
index dc780c450e..d760b724b9 100644
--- a/src/backend/cpu/convolve.cpp
+++ b/src/backend/cpu/convolve.cpp
@@ -144,7 +144,7 @@ Array<T> convolve2_unwrap(const Array<T> &signal, const Array<T> &filter,
 
     Array<T> collapsedFilter = flip(filter, {1, 1, 0, 0});
     collapsedFilter          = modDims(collapsedFilter,
-                              dim4(fDims[0] * fDims[1] * fDims[2], fDims[3]));
+                                       dim4(fDims[0] * fDims[1] * fDims[2], fDims[3]));
 
     Array<T> res =
         matmul(unwrapped, collapsedFilter, AF_MAT_TRANS, AF_MAT_NONE);
@@ -187,12 +187,12 @@ Array<T> conv2DataGradient(const Array<T> &incoming_gradient,
 
     Array<T> collapsed_filter = flip(original_filter, {1, 1, 0, 0});
     collapsed_filter          = modDims(collapsed_filter,
-                               dim4(fDims[0] * fDims[1] * fDims[2], fDims[3]));
+                                        dim4(fDims[0] * fDims[1] * fDims[2], fDims[3]));
 
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     Array<T> res =
         matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE, AF_MAT_TRANS);
@@ -231,7 +231,7 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     Array<T> res =
         matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE);
diff --git a/src/backend/cpu/device_manager.hpp b/src/backend/cpu/device_manager.hpp
index 170f61df4b..3015ae05f6 100644
--- a/src/backend/cpu/device_manager.hpp
+++ b/src/backend/cpu/device_manager.hpp
@@ -131,7 +131,7 @@ class DeviceManager {
     // avoid copying accidental copy/assignment
     // of instance returned by getInstance to other
     // variables
-    DeviceManager(DeviceManager const&) = delete;
+    DeviceManager(DeviceManager const&)  = delete;
     void operator=(DeviceManager const&) = delete;
 
     // Attributes
diff --git a/src/backend/cpu/kernel/diff.hpp b/src/backend/cpu/kernel/diff.hpp
index 72283e7a7e..9e2e8a4e21 100644
--- a/src/backend/cpu/kernel/diff.hpp
+++ b/src/backend/cpu/kernel/diff.hpp
@@ -35,7 +35,7 @@ void diff1(Param<T> out, CParam<T> in, int const dim) {
                     // in[index]
                     int idx     = getIdx(in.strides(), i, j, k, l);
                     int jdx     = getIdx(in.strides(), i + is_dim0, j + is_dim1,
-                                     k + is_dim2, l + is_dim3);
+                                         k + is_dim2, l + is_dim3);
                     int odx     = getIdx(out.strides(), i, j, k, l);
                     outPtr[odx] = inPtr[jdx] - inPtr[idx];
                 }
diff --git a/src/backend/cpu/kernel/fftconvolve.hpp b/src/backend/cpu/kernel/fftconvolve.hpp
index e85bd4b2f6..d6c6f8493e 100644
--- a/src/backend/cpu/kernel/fftconvolve.hpp
+++ b/src/backend/cpu/kernel/fftconvolve.hpp
@@ -202,8 +202,9 @@ void reorderHelper(To* out_ptr, const af::dim4& od, const af::dim4& os,
                                 (float)((in_ptr[iidx1] + in_ptr[iidx2]) /
                                         fftScale));
                         else
-                            out_ptr[oidx] = (To)(
-                                (in_ptr[iidx1] + in_ptr[iidx2]) / fftScale);
+                            out_ptr[oidx] =
+                                (To)((in_ptr[iidx1] + in_ptr[iidx2]) /
+                                     fftScale);
                     } else {
                         // Copy bottom elements
                         const int iidx =
diff --git a/src/backend/cpu/kernel/orb.hpp b/src/backend/cpu/kernel/orb.hpp
index 33c642cd8d..df36f3655b 100644
--- a/src/backend/cpu/kernel/orb.hpp
+++ b/src/backend/cpu/kernel/orb.hpp
@@ -257,12 +257,12 @@ void extract_orb(unsigned* desc_out, const unsigned n_feat, float* x_in_out,
                 int dist_x = ref_pat[i * 32 * 4 + j * 4];
                 int dist_y = ref_pat[i * 32 * 4 + j * 4 + 1];
                 T p1       = get_pixel(x, y, ori, size, dist_x, dist_y, image,
-                                 patch_size);
+                                       patch_size);
 
                 dist_x = ref_pat[i * 32 * 4 + j * 4 + 2];
                 dist_y = ref_pat[i * 32 * 4 + j * 4 + 3];
                 T p2   = get_pixel(x, y, ori, size, dist_x, dist_y, image,
-                                 patch_size);
+                                   patch_size);
 
                 // Calculate bit based on p1 and p2 and shifts it to correct
                 // position
diff --git a/src/backend/cuda/LookupTable1D.hpp b/src/backend/cuda/LookupTable1D.hpp
index 746607d5d5..ffbfb0f4c8 100644
--- a/src/backend/cuda/LookupTable1D.hpp
+++ b/src/backend/cuda/LookupTable1D.hpp
@@ -19,10 +19,10 @@ namespace cuda {
 template<typename T>
 class LookupTable1D {
    public:
-    LookupTable1D()                          = delete;
-    LookupTable1D(const LookupTable1D& arg)  = delete;
-    LookupTable1D(const LookupTable1D&& arg) = delete;
-    LookupTable1D& operator=(const LookupTable1D& arg) = delete;
+    LookupTable1D()                                     = delete;
+    LookupTable1D(const LookupTable1D& arg)             = delete;
+    LookupTable1D(const LookupTable1D&& arg)            = delete;
+    LookupTable1D& operator=(const LookupTable1D& arg)  = delete;
     LookupTable1D& operator=(const LookupTable1D&& arg) = delete;
 
     LookupTable1D(const Array<T>& lutArray) : mTexture(0), mData(lutArray) {
diff --git a/src/backend/cuda/Param.hpp b/src/backend/cuda/Param.hpp
index 3b7476f7a5..cd1651cae5 100644
--- a/src/backend/cuda/Param.hpp
+++ b/src/backend/cuda/Param.hpp
@@ -34,10 +34,10 @@ class Param {
         return dims[0] * dims[1] * dims[2] * dims[3];
     }
 
-    Param(const Param<T> &other) noexcept = default;
-    Param(Param<T> &&other) noexcept      = default;
+    Param(const Param<T> &other) noexcept               = default;
+    Param(Param<T> &&other) noexcept                    = default;
     Param<T> &operator=(const Param<T> &other) noexcept = default;
-    Param<T> &operator=(Param<T> &&other) noexcept = default;
+    Param<T> &operator=(Param<T> &&other) noexcept      = default;
 };
 
 template<typename T>
@@ -70,10 +70,10 @@ class CParam {
         return dims[0] * dims[1] * dims[2] * dims[3];
     }
 
-    CParam(const CParam<T> &other) noexcept = default;
-    CParam(CParam<T> &&other) noexcept      = default;
+    CParam(const CParam<T> &other) noexcept               = default;
+    CParam(CParam<T> &&other) noexcept                    = default;
     CParam<T> &operator=(const CParam<T> &other) noexcept = default;
-    CParam<T> &operator=(CParam<T> &&other) noexcept = default;
+    CParam<T> &operator=(CParam<T> &&other) noexcept      = default;
 };
 
 }  // namespace cuda
diff --git a/src/backend/cuda/convolveNN.cpp b/src/backend/cuda/convolveNN.cpp
index 0a95a7c9ae..075817925e 100644
--- a/src/backend/cuda/convolveNN.cpp
+++ b/src/backend/cuda/convolveNN.cpp
@@ -207,7 +207,7 @@ Array<T> convolve2_base(const Array<T> &signal, const Array<T> &filter,
     const int Ndim = 1;
     Array<T> res   = createEmptyArray<T>(
         dim4(unwrapped.dims()[Mdim], collapsedFilter.dims()[Ndim],
-             unwrapped.dims()[2], unwrapped.dims()[3]));
+               unwrapped.dims()[2], unwrapped.dims()[3]));
     gemm(res, AF_MAT_TRANS, AF_MAT_NONE, &alpha, unwrapped, collapsedFilter,
          &beta);
     res = modDims(res, dim4(outputWidth, outputHeight, signal.dims()[3],
@@ -259,7 +259,7 @@ Array<T> data_gradient_base(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     T alpha        = scalar<T>(1.0);
     T beta         = scalar<T>(0.0);
@@ -267,7 +267,7 @@ Array<T> data_gradient_base(const Array<T> &incoming_gradient,
     const int Ndim = 0;
     Array<T> res   = createEmptyArray<T>(
         dim4(collapsed_gradient.dims()[Mdim], collapsed_filter.dims()[Ndim],
-             collapsed_gradient.dims()[3], collapsed_gradient.dims()[3]));
+               collapsed_gradient.dims()[3], collapsed_gradient.dims()[3]));
     gemm(res, AF_MAT_NONE, AF_MAT_TRANS, &alpha, collapsed_gradient,
          collapsed_filter, &beta);
     res = modDims(res, dim4(res.dims()[0] / sDims[3], sDims[3],
@@ -389,7 +389,7 @@ Array<T> filter_gradient_base(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     T alpha        = scalar<T>(1.0);
     T beta         = scalar<T>(0.0);
@@ -397,7 +397,7 @@ Array<T> filter_gradient_base(const Array<T> &incoming_gradient,
     const int Ndim = 1;
     Array<T> res   = createEmptyArray<T>(
         dim4(unwrapped.dims()[Mdim], collapsed_gradient.dims()[Ndim],
-             unwrapped.dims()[2], unwrapped.dims()[3]));
+               unwrapped.dims()[2], unwrapped.dims()[3]));
     gemm(res, AF_MAT_NONE, AF_MAT_NONE, &alpha, unwrapped, collapsed_gradient,
          &beta);
     res = modDims(res, dim4(fDims[0], fDims[1], fDims[2], fDims[3]));
diff --git a/src/backend/cuda/kernel/fast.hpp b/src/backend/cuda/kernel/fast.hpp
index e88722c7bc..3521f8cfcb 100644
--- a/src/backend/cuda/kernel/fast.hpp
+++ b/src/backend/cuda/kernel/fast.hpp
@@ -246,7 +246,7 @@ __global__ void non_max_counts(unsigned *d_counts, unsigned *d_offsets,
             if (nonmax) {
                 float max_v = v;
                 max_v       = max_val(score[x - 1 + idim0 * (y - 1)],
-                                score[x - 1 + idim0 * y]);
+                                      score[x - 1 + idim0 * y]);
                 max_v       = max_val(max_v, score[x - 1 + idim0 * (y + 1)]);
                 max_v       = max_val(max_v, score[x + idim0 * (y - 1)]);
                 max_v       = max_val(max_v, score[x + idim0 * (y + 1)]);
diff --git a/src/backend/cuda/kernel/harris.hpp b/src/backend/cuda/kernel/harris.hpp
index 7db3a1fc57..e8fe490b52 100644
--- a/src/backend/cuda/kernel/harris.hpp
+++ b/src/backend/cuda/kernel/harris.hpp
@@ -249,7 +249,7 @@ void harris(unsigned* corners_out, float** x_out, float** y_out,
     // Calculate Harris responses for all pixels
     threads = dim3(BLOCK_SIZE, BLOCK_SIZE);
     blocks  = dim3(divup(in.dims[1] - border_len * 2, threads.x),
-                  divup(in.dims[0] - border_len * 2, threads.y));
+                   divup(in.dims[0] - border_len * 2, threads.y));
     CUDA_LAUNCH((harris_responses<T>), blocks, threads, d_responses.get(),
                 in.dims[0], in.dims[1], ixx.ptr, ixy.ptr, iyy.ptr, k_thr,
                 border_len);
diff --git a/src/backend/cuda/kernel/homography.hpp b/src/backend/cuda/kernel/homography.hpp
index 7d3033f647..aaad7af358 100644
--- a/src/backend/cuda/kernel/homography.hpp
+++ b/src/backend/cuda/kernel/homography.hpp
@@ -157,7 +157,7 @@ __device__ bool computeMeanScale(
     CParam<float> x_dst, CParam<float> y_dst, CParam<float> rnd, int i) {
     const unsigned ridx = rnd.dims[0] * i;
     unsigned r[4]       = {(unsigned)rnd.ptr[ridx], (unsigned)rnd.ptr[ridx + 1],
-                     (unsigned)rnd.ptr[ridx + 2], (unsigned)rnd.ptr[ridx + 3]};
+                           (unsigned)rnd.ptr[ridx + 2], (unsigned)rnd.ptr[ridx + 3]};
 
     // If one of the points is repeated, it's a bad samples, will still
     // compute homography to ensure all threads pass __syncthreads()
diff --git a/src/backend/cuda/kernel/orb.hpp b/src/backend/cuda/kernel/orb.hpp
index 15ef584bb0..672da31fc3 100644
--- a/src/backend/cuda/kernel/orb.hpp
+++ b/src/backend/cuda/kernel/orb.hpp
@@ -246,12 +246,12 @@ __global__ void extract_orb(unsigned* desc_out, const unsigned n_feat,
                 int dist_x = lookup(i * 16 * 4 + j * 4, luTable);
                 int dist_y = lookup(i * 16 * 4 + j * 4 + 1, luTable);
                 T p1       = get_pixel(x, y, ori, size, dist_x, dist_y, image,
-                                 patch_size);
+                                       patch_size);
 
                 dist_x = lookup(i * 16 * 4 + j * 4 + 2, luTable);
                 dist_y = lookup(i * 16 * 4 + j * 4 + 3, luTable);
                 T p2   = get_pixel(x, y, ori, size, dist_x, dist_y, image,
-                                 patch_size);
+                                   patch_size);
 
                 // Calculate bit based on p1 and p2 and shifts it to correct
                 // position
diff --git a/src/backend/cuda/kernel/random_engine.hpp b/src/backend/cuda/kernel/random_engine.hpp
index 1f983a08eb..e52e78d354 100644
--- a/src/backend/cuda/kernel/random_engine.hpp
+++ b/src/backend/cuda/kernel/random_engine.hpp
@@ -213,8 +213,8 @@ __device__ void sincos(__half val, __half *sptr, __half *cptr) {
     float s, c;
     float fval = __half2float(val);
     sincos(fval, &s, &c);
-    *sptr = __float2half(s);
-    *cptr = __float2half(c);
+    *sptr      = __float2half(s);
+    *cptr      = __float2half(c);
 #endif
 }
 
diff --git a/src/backend/cuda/kernel/shfl_intrinsics.hpp b/src/backend/cuda/kernel/shfl_intrinsics.hpp
index 9a3f3cf2f3..ef12aafe29 100644
--- a/src/backend/cuda/kernel/shfl_intrinsics.hpp
+++ b/src/backend/cuda/kernel/shfl_intrinsics.hpp
@@ -57,7 +57,7 @@ inline __device__ cuda::cfloat shfl_down_sync(unsigned mask, cuda::cfloat var,
     cuda::cfloat res = {__shfl_down_sync(mask, var.x, delta),
                         __shfl_down_sync(mask, var.y, delta)};
 #else
-    cuda::cfloat res = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)};
+    cuda::cfloat res  = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)};
 #endif
     return res;
 }
@@ -91,7 +91,7 @@ inline __device__ cuda::cfloat shfl_up_sync(unsigned mask, cuda::cfloat var,
     cuda::cfloat res = {__shfl_up_sync(mask, var.x, delta),
                         __shfl_up_sync(mask, var.y, delta)};
 #else
-    cuda::cfloat res = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)};
+    cuda::cfloat res  = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)};
 #endif
     return res;
 }
diff --git a/src/backend/cuda/kernel/unwrap.hpp b/src/backend/cuda/kernel/unwrap.hpp
index d1d83efa60..8e171ac816 100644
--- a/src/backend/cuda/kernel/unwrap.hpp
+++ b/src/backend/cuda/kernel/unwrap.hpp
@@ -36,7 +36,7 @@ void unwrap(Param<T> out, CParam<T> in, const int wx, const int wy,
         threads = dim3(TX, THREADS_PER_BLOCK / TX);
         blocks = dim3(divup(out.dims[1], threads.y), out.dims[2] * out.dims[3]);
         reps   = divup((wx * wy),
-                     threads.x);  // is > 1 only when TX == 256 && wx * wy > 256
+                       threads.x);  // is > 1 only when TX == 256 && wx * wy > 256
     } else {
         threads = dim3(THREADS_X, THREADS_Y);
         blocks = dim3(divup(out.dims[0], threads.x), out.dims[2] * out.dims[3]);
diff --git a/src/backend/cuda/types.hpp b/src/backend/cuda/types.hpp
index c3897a3397..91bcdbbda7 100644
--- a/src/backend/cuda/types.hpp
+++ b/src/backend/cuda/types.hpp
@@ -162,7 +162,7 @@ struct kernel_type<common::half> {
     using compute = float;
 
 #if defined(__NVCC__) || defined(__CUDACC_RTC__)
-    using native = __half;
+    using native  = __half;
 #else
     using native = common::half;
 #endif
diff --git a/src/backend/opencl/convolve.cpp b/src/backend/opencl/convolve.cpp
index dd05838760..a4924303f3 100644
--- a/src/backend/opencl/convolve.cpp
+++ b/src/backend/opencl/convolve.cpp
@@ -184,7 +184,7 @@ Array<T> conv2DataGradient(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     Array<T> res =
         matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE, AF_MAT_TRANS);
@@ -223,7 +223,7 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     Array<T> res =
         matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE);
diff --git a/src/backend/opencl/kernel/convolve/conv_common.hpp b/src/backend/opencl/kernel/convolve/conv_common.hpp
index 9f160703ef..92cf5858e7 100644
--- a/src/backend/opencl/kernel/convolve/conv_common.hpp
+++ b/src/backend/opencl/kernel/convolve/conv_common.hpp
@@ -63,7 +63,7 @@ void prepareKernelArgs(conv_kparam_t& param, dim_t* oDims, const dim_t* fDims,
         param.nBBS0    = divup(oDims[0], THREADS);
         param.nBBS1    = batchDims[2];
         param.global   = NDRange(param.nBBS0 * THREADS * batchDims[1],
-                               param.nBBS1 * batchDims[3]);
+                                 param.nBBS1 * batchDims[3]);
         param.loc_size = (THREADS + 2 * (fDims[0] - 1)) * sizeof(T);
     } else if (rank == 2) {
         param.local  = NDRange(THREADS_X, THREADS_Y);
@@ -77,7 +77,7 @@ void prepareKernelArgs(conv_kparam_t& param, dim_t* oDims, const dim_t* fDims,
         param.nBBS1    = divup(oDims[1], CUBE_Y);
         int blk_z      = divup(oDims[2], CUBE_Z);
         param.global   = NDRange(param.nBBS0 * CUBE_X * batchDims[3],
-                               param.nBBS1 * CUBE_Y, blk_z * CUBE_Z);
+                                 param.nBBS1 * CUBE_Y, blk_z * CUBE_Z);
         param.loc_size = (CUBE_X + 2 * (fDims[0] - 1)) *
                          (CUBE_Y + 2 * (fDims[1] - 1)) *
                          (CUBE_Z + 2 * (fDims[2] - 1)) * sizeof(T);
diff --git a/src/backend/opencl/kernel/homography.hpp b/src/backend/opencl/kernel/homography.hpp
index 3293c06ea0..4585d7636e 100644
--- a/src/backend/opencl/kernel/homography.hpp
+++ b/src/backend/opencl/kernel/homography.hpp
@@ -32,7 +32,7 @@ constexpr int HG_THREADS   = 256;
 template<typename T>
 std::array<Kernel, 5> getHomographyKernels(const af_homography_type htype) {
     std::vector<TemplateArg> targs   = {TemplateTypename<T>(),
-                                      TemplateArg(htype)};
+                                        TemplateArg(htype)};
     std::vector<std::string> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
     };
diff --git a/src/backend/opencl/kernel/index.hpp b/src/backend/opencl/kernel/index.hpp
index abcd89715c..3215ee22b5 100644
--- a/src/backend/opencl/kernel/index.hpp
+++ b/src/backend/opencl/kernel/index.hpp
@@ -37,7 +37,7 @@ void index(Param out, const Param in, const IndexKernelParam_t& p,
     options.emplace_back(getTypeBuildDefinition<T>());
 
     auto index    = common::getKernel("indexKernel", {index_cl_src},
-                                   {TemplateTypename<T>()}, options);
+                                      {TemplateTypename<T>()}, options);
     int threads_x = 256;
     int threads_y = 1;
     cl::NDRange local(threads_x, threads_y);
diff --git a/src/backend/opencl/kernel/orb.hpp b/src/backend/opencl/kernel/orb.hpp
index 14f28e6fe5..b755644e37 100644
--- a/src/backend/opencl/kernel/orb.hpp
+++ b/src/backend/opencl/kernel/orb.hpp
@@ -174,7 +174,7 @@ void orb(unsigned* out_feat, Param& x_out, Param& y_out, Param& score_out,
 
             lvl_img.info.offset = 0;
             lvl_img.data        = bufferAlloc(lvl_img.info.dims[3] *
-                                       lvl_img.info.strides[3] * sizeof(T));
+                                              lvl_img.info.strides[3] * sizeof(T));
 
             resize<T>(lvl_img, prev_img, AF_INTERP_BILINEAR);
 
@@ -331,7 +331,7 @@ void orb(unsigned* out_feat, Param& x_out, Param& y_out, Param& score_out,
             lvl_filt.data = bufferAlloc(lvl_filt.info.dims[0] *
                                         lvl_filt.info.dims[1] * sizeof(T));
             lvl_tmp.data  = bufferAlloc(lvl_tmp.info.dims[0] *
-                                       lvl_tmp.info.dims[1] * sizeof(T));
+                                        lvl_tmp.info.dims[1] * sizeof(T));
 
             // Calculate a separable Gaussian kernel
             if (h_gauss == nullptr) {
diff --git a/src/backend/opencl/magma/geqrf2.cpp b/src/backend/opencl/magma/geqrf2.cpp
index 2d09f0ba60..bcb71ad51f 100644
--- a/src/backend/opencl/magma/geqrf2.cpp
+++ b/src/backend/opencl/magma/geqrf2.cpp
@@ -234,8 +234,8 @@ magma_int_t magma_geqrf2_gpu(magma_int_t m, magma_int_t n, cl_mem dA,
                                    CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR,
                                    sizeof(Ty) * lwork, NULL, NULL);
     work          = (Ty *)clEnqueueMapBuffer(queue[0], buffer, CL_TRUE,
-                                    CL_MAP_READ | CL_MAP_WRITE, 0,
-                                    lwork * sizeof(Ty), 0, NULL, NULL, NULL);
+                                             CL_MAP_READ | CL_MAP_WRITE, 0,
+                                             lwork * sizeof(Ty), 0, NULL, NULL, NULL);
 
     cpu_lapack_geqrf_work_func<Ty> cpu_lapack_geqrf;
     cpu_lapack_larft_func<Ty> cpu_lapack_larft;
diff --git a/src/backend/opencl/magma/magma_data.h b/src/backend/opencl/magma/magma_data.h
index 38470a5f76..4d6834b42e 100644
--- a/src/backend/opencl/magma/magma_data.h
+++ b/src/backend/opencl/magma/magma_data.h
@@ -321,9 +321,9 @@ static void magma_setmatrix_async(magma_int_t m, magma_int_t n, T const* hA_src,
     size_t host_orig[3]     = {0, 0, 0};
     size_t region[3]        = {m * sizeof(T), (size_t)n, 1};
     cl_int err              = clEnqueueWriteBufferRect(
-        queue, dB_dst, CL_FALSE,  // non-blocking
-        buffer_origin, host_orig, region, lddb * sizeof(T), 0, ldha * sizeof(T),
-        0, hA_src, 0, NULL, event);
+                     queue, dB_dst, CL_FALSE,  // non-blocking
+                     buffer_origin, host_orig, region, lddb * sizeof(T), 0, ldha * sizeof(T),
+                     0, hA_src, 0, NULL, event);
     clFlush(queue);
     check_error(err);
 }
@@ -357,9 +357,9 @@ static void magma_getmatrix_async(magma_int_t m, magma_int_t n, cl_mem dA_src,
     size_t host_orig[3]     = {0, 0, 0};
     size_t region[3]        = {m * sizeof(T), (size_t)n, 1};
     cl_int err              = clEnqueueReadBufferRect(
-        queue, dA_src, CL_FALSE,  // non-blocking
-        buffer_origin, host_orig, region, ldda * sizeof(T), 0, ldhb * sizeof(T),
-        0, hB_dst, 0, NULL, event);
+                     queue, dA_src, CL_FALSE,  // non-blocking
+                     buffer_origin, host_orig, region, ldda * sizeof(T), 0, ldhb * sizeof(T),
+                     0, hB_dst, 0, NULL, event);
     clFlush(queue);
     check_error(err);
 }
diff --git a/src/backend/opencl/magma/magma_types.h b/src/backend/opencl/magma/magma_types.h
index fe844e78d4..90dcc6ab8d 100644
--- a/src/backend/opencl/magma/magma_types.h
+++ b/src/backend/opencl/magma/magma_types.h
@@ -388,7 +388,7 @@ typedef enum {
 // 2b) update min & max here, which are used to check bounds for
 // magma2lapack_constants[] 2c) add lapack_xxxx_const() converter below and in
 // control/constants.cpp
-#define Magma2lapack_Min MagmaFalse  // 0
+#define Magma2lapack_Min MagmaFalse    // 0
 #define Magma2lapack_Max MagmaRowwise  // 402
 
 // ----------------------------------------
diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index 77e8224bbb..8dab1f428b 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -188,8 +188,8 @@ size_t Allocator::getMaxMemorySize(int id) {
 void *Allocator::nativeAlloc(const size_t bytes) {
     cl_int err = CL_SUCCESS;
     auto ptr   = static_cast<void *>(clCreateBuffer(
-        getContext()(), CL_MEM_READ_WRITE,  // NOLINT(hicpp-signed-bitwise)
-        bytes, nullptr, &err));
+          getContext()(), CL_MEM_READ_WRITE,  // NOLINT(hicpp-signed-bitwise)
+          bytes, nullptr, &err));
 
     if (err != CL_SUCCESS) {
         auto str = fmt::format("Failed to allocate device memory of size {}",
@@ -237,7 +237,7 @@ void *AllocatorPinned::nativeAlloc(const size_t bytes) {
 
     cl_int err = CL_SUCCESS;
     auto buf   = clCreateBuffer(getContext()(), CL_MEM_ALLOC_HOST_PTR, bytes,
-                              nullptr, &err);
+                                nullptr, &err);
     if (err != CL_SUCCESS) {
         AF_ERROR("Failed to allocate pinned memory.", AF_ERR_NO_MEM);
     }
diff --git a/src/backend/opencl/svd.cpp b/src/backend/opencl/svd.cpp
index 5aa6c0e1ed..5c7aed92c4 100644
--- a/src/backend/opencl/svd.cpp
+++ b/src/backend/opencl/svd.cpp
@@ -136,8 +136,8 @@ void svd(Array<T> &arrU, Array<Tr> &arrS, Array<T> &arrVT, Array<T> &arrA,
 
     if (want_vectors) {
         mappedU  = static_cast<T *>(getQueue().enqueueMapBuffer(
-            *arrU.get(), CL_FALSE, CL_MAP_WRITE, sizeof(T) * arrU.getOffset(),
-            sizeof(T) * arrU.elements()));
+             *arrU.get(), CL_FALSE, CL_MAP_WRITE, sizeof(T) * arrU.getOffset(),
+             sizeof(T) * arrU.elements()));
         mappedVT = static_cast<T *>(getQueue().enqueueMapBuffer(
             *arrVT.get(), CL_TRUE, CL_MAP_WRITE, sizeof(T) * arrVT.getOffset(),
             sizeof(T) * arrVT.elements()));
diff --git a/src/backend/opencl/topk.cpp b/src/backend/opencl/topk.cpp
index 08155b9d8a..5fcf157946 100644
--- a/src/backend/opencl/topk.cpp
+++ b/src/backend/opencl/topk.cpp
@@ -75,13 +75,13 @@ void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
         cl::Event ev_in, ev_val, ev_ind;
 
         T* ptr     = static_cast<T*>(getQueue().enqueueMapBuffer(
-            *in_buf, CL_FALSE, CL_MAP_READ, 0, in.elements() * sizeof(T),
-            nullptr, &ev_in));
+                *in_buf, CL_FALSE, CL_MAP_READ, 0, in.elements() * sizeof(T),
+                nullptr, &ev_in));
         uint* iptr = static_cast<uint*>(getQueue().enqueueMapBuffer(
             *ibuf, CL_FALSE, CL_MAP_READ | CL_MAP_WRITE, 0, k * sizeof(uint),
             nullptr, &ev_ind));
         T* vptr    = static_cast<T*>(getQueue().enqueueMapBuffer(
-            *vbuf, CL_FALSE, CL_MAP_WRITE, 0, k * sizeof(T), nullptr, &ev_val));
+               *vbuf, CL_FALSE, CL_MAP_WRITE, 0, k * sizeof(T), nullptr, &ev_val));
 
         vector<uint> idx(in.elements());
 
diff --git a/test/.clang-format b/test/.clang-format
index 692cbc2f40..47afdf3208 100644
--- a/test/.clang-format
+++ b/test/.clang-format
@@ -138,7 +138,7 @@ SpacesInContainerLiterals: true
 SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
-Standard:        Cpp03
+Standard:        Cpp11
 TabWidth:        4
 UseTab:          Never
 
diff --git a/test/approx1.cpp b/test/approx1.cpp
index 17d7579cec..ed7bf83066 100644
--- a/test/approx1.cpp
+++ b/test/approx1.cpp
@@ -73,8 +73,8 @@ void approx1Test(string pTestFile, const unsigned resultIdx,
 
     typedef typename dtype_traits<T>::base_type BT;
     vector<dim4> numDims;
-    vector<vector<BT> > in;
-    vector<vector<T> > tests;
+    vector<vector<BT>> in;
+    vector<vector<T>> tests;
     readTests<BT, T, float>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -146,8 +146,8 @@ void approx1CubicTest(string pTestFile, const unsigned resultIdx,
 
     typedef typename dtype_traits<T>::base_type BT;
     vector<dim4> numDims;
-    vector<vector<BT> > in;
-    vector<vector<T> > tests;
+    vector<vector<BT>> in;
+    vector<vector<T>> tests;
     readTests<BT, T, float>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -233,8 +233,8 @@ void approx1ArgsTest(string pTestFile, const af_interp_type method,
     SUPPORTED_TYPE_CHECK(T);
     typedef typename dtype_traits<T>::base_type BT;
     vector<dim4> numDims;
-    vector<vector<BT> > in;
-    vector<vector<T> > tests;
+    vector<vector<BT>> in;
+    vector<vector<T>> tests;
     readTests<BT, T, float>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -279,8 +279,8 @@ void approx1ArgsTestPrecision(string pTestFile, const unsigned,
                               const af_interp_type method) {
     SUPPORTED_TYPE_CHECK(T);
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -335,8 +335,8 @@ TEST(Approx1, CPP) {
     const unsigned resultIdx = 1;
 #define BT dtype_traits<float>::base_type
     vector<dim4> numDims;
-    vector<vector<BT> > in;
-    vector<vector<float> > tests;
+    vector<vector<BT>> in;
+    vector<vector<float>> tests;
     readTests<BT, float, float>(string(TEST_DIR "/approx/approx1.test"),
                                 numDims, in, tests);
 
diff --git a/test/approx2.cpp b/test/approx2.cpp
index 796c639fd0..1b7901bf8d 100644
--- a/test/approx2.cpp
+++ b/test/approx2.cpp
@@ -65,8 +65,8 @@ void approx2Test(string pTestFile, const unsigned resultIdx,
     SUPPORTED_TYPE_CHECK(T);
     typedef typename dtype_traits<T>::base_type BT;
     vector<dim4> numDims;
-    vector<vector<BT> > in;
-    vector<vector<T> > tests;
+    vector<vector<BT>> in;
+    vector<vector<T>> tests;
     readTests<BT, T, float>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -152,8 +152,8 @@ void approx2ArgsTest(string pTestFile, const af_interp_type method,
     SUPPORTED_TYPE_CHECK(T);
     typedef typename dtype_traits<T>::base_type BT;
     vector<dim4> numDims;
-    vector<vector<BT> > in;
-    vector<vector<T> > tests;
+    vector<vector<BT>> in;
+    vector<vector<T>> tests;
     readTests<BT, T, float>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -208,8 +208,8 @@ void approx2ArgsTestPrecision(string pTestFile, const unsigned resultIdx,
     UNUSED(resultIdx);
     SUPPORTED_TYPE_CHECK(T);
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -264,8 +264,8 @@ TEST(Approx2, CPP) {
     const unsigned resultIdx = 1;
 #define BT dtype_traits<float>::base_type
     vector<dim4> numDims;
-    vector<vector<BT> > in;
-    vector<vector<float> > tests;
+    vector<vector<BT>> in;
+    vector<vector<float>> tests;
     readTests<BT, float, float>(string(TEST_DIR "/approx/approx2.test"),
                                 numDims, in, tests);
 
@@ -301,8 +301,8 @@ TEST(Approx2Cubic, CPP) {
     const unsigned resultIdx = 0;
 #define BT dtype_traits<float>::base_type
     vector<dim4> numDims;
-    vector<vector<BT> > in;
-    vector<vector<float> > tests;
+    vector<vector<BT>> in;
+    vector<vector<float>> tests;
     readTests<BT, float, float>(string(TEST_DIR "/approx/approx2_cubic.test"),
                                 numDims, in, tests);
 
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index a7d823e040..6a7f6e7000 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -280,8 +280,8 @@ af_half convert(int in) {
 
 template<typename inType, typename outType, typename FileElementType>
 void readTests(const std::string &FileName, std::vector<af::dim4> &inputDims,
-               std::vector<std::vector<inType> > &testInputs,
-               std::vector<std::vector<outType> > &testOutputs) {
+               std::vector<std::vector<inType>> &testInputs,
+               std::vector<std::vector<outType>> &testOutputs) {
     using std::vector;
 
     std::ifstream testFile(FileName.c_str());
@@ -326,8 +326,8 @@ void readTests(const std::string &FileName, std::vector<af::dim4> &inputDims,
 #define INSTANTIATE(Tin, Tout, Tfile)                                  \
     template void readTests<Tin, Tout, Tfile>(                         \
         const std::string &FileName, std::vector<af::dim4> &inputDims, \
-        std::vector<std::vector<Tin> > &testInputs,                    \
-        std::vector<std::vector<Tout> > &testOutputs)
+        std::vector<std::vector<Tin>> &testInputs,                     \
+        std::vector<std::vector<Tout>> &testOutputs)
 
 INSTANTIATE(float, float, int);
 INSTANTIATE(double, float, int);
@@ -814,8 +814,8 @@ bool noLAPACKTests() {
 template<typename inType, typename outType>
 void readTestsFromFile(const std::string &FileName,
                        std::vector<af::dim4> &inputDims,
-                       std::vector<std::vector<inType> > &testInputs,
-                       std::vector<std::vector<outType> > &testOutputs) {
+                       std::vector<std::vector<inType>> &testInputs,
+                       std::vector<std::vector<outType>> &testOutputs) {
     using std::vector;
 
     std::ifstream testFile(FileName.c_str());
@@ -863,8 +863,8 @@ void readTestsFromFile(const std::string &FileName,
 #define INSTANTIATE(Ti, To)                                            \
     template void readTestsFromFile<Ti, To>(                           \
         const std::string &FileName, std::vector<af::dim4> &inputDims, \
-        std::vector<std::vector<Ti> > &testInputs,                     \
-        std::vector<std::vector<To> > &testOutputs)
+        std::vector<std::vector<Ti>> &testInputs,                      \
+        std::vector<std::vector<To>> &testOutputs)
 
 INSTANTIATE(float, float);
 INSTANTIATE(float, af_cfloat);
@@ -880,7 +880,7 @@ template<typename outType>
 void readImageTests(const std::string &pFileName,
                     std::vector<af::dim4> &pInputDims,
                     std::vector<std::string> &pTestInputs,
-                    std::vector<std::vector<outType> > &pTestOutputs) {
+                    std::vector<std::vector<outType>> &pTestOutputs) {
     using std::vector;
 
     std::ifstream testFile(pFileName.c_str());
@@ -923,7 +923,7 @@ void readImageTests(const std::string &pFileName,
     template void readImageTests<To>(                                    \
         const std::string &pFileName, std::vector<af::dim4> &pInputDims, \
         std::vector<std::string> &pTestInputs,                           \
-        std::vector<std::vector<To> > &pTestOutputs)
+        std::vector<std::vector<To>> &pTestOutputs)
 
 INSTANTIATE(float);
 #undef INSTANTIATE
@@ -972,8 +972,8 @@ template<typename descType>
 void readImageFeaturesDescriptors(
     const std::string &pFileName, std::vector<af::dim4> &pInputDims,
     std::vector<std::string> &pTestInputs,
-    std::vector<std::vector<float> > &pTestFeats,
-    std::vector<std::vector<descType> > &pTestDescs) {
+    std::vector<std::vector<float>> &pTestFeats,
+    std::vector<std::vector<descType>> &pTestDescs) {
     using std::vector;
 
     std::ifstream testFile(pFileName.c_str());
@@ -1025,8 +1025,8 @@ void readImageFeaturesDescriptors(
     template void readImageFeaturesDescriptors<TYPE>(                    \
         const std::string &pFileName, std::vector<af::dim4> &pInputDims, \
         std::vector<std::string> &pTestInputs,                           \
-        std::vector<std::vector<float> > &pTestFeats,                    \
-        std::vector<std::vector<TYPE> > &pTestDescs)
+        std::vector<std::vector<float>> &pTestFeats,                     \
+        std::vector<std::vector<TYPE>> &pTestDescs)
 
 INSTANTIATE(float);
 INSTANTIATE(double);
@@ -1547,14 +1547,14 @@ bool absMatch::operator()<af::af_cdouble>(af::af_cdouble lhs,
 }
 
 template<>
-bool absMatch::operator()<std::complex<float> >(std::complex<float> lhs,
-                                                std::complex<float> rhs) {
+bool absMatch::operator()<std::complex<float>>(std::complex<float> lhs,
+                                               std::complex<float> rhs) {
     return std::abs(rhs - lhs) <= diff_;
 }
 
 template<>
-bool absMatch::operator()<std::complex<double> >(std::complex<double> lhs,
-                                                 std::complex<double> rhs) {
+bool absMatch::operator()<std::complex<double>>(std::complex<double> lhs,
+                                                std::complex<double> rhs) {
     return std::abs(rhs - lhs) <= diff_;
 }
 
diff --git a/test/assign.cpp b/test/assign.cpp
index 7c32a2cc33..cbfe6359b1 100644
--- a/test/assign.cpp
+++ b/test/assign.cpp
@@ -107,8 +107,8 @@ void assignTest(string pTestFile, const vector<af_seq> *seqv) {
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<inType> > in;
-    vector<vector<outType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<outType>> tests;
 
     readTests<inType, outType, int>(pTestFile, numDims, in, tests);
 
@@ -151,8 +151,8 @@ void assignTestCPP(string pTestFile, const vector<af_seq> &seqv) {
     SUPPORTED_TYPE_CHECK(T);
     try {
         vector<dim4> numDims;
-        vector<vector<T> > in;
-        vector<vector<T> > tests;
+        vector<vector<T>> in;
+        vector<vector<T>> tests;
 
         readTests<T, T, int>(pTestFile, numDims, in, tests);
 
@@ -290,8 +290,8 @@ void assignScalarCPP(string pTestFile, const vector<af_seq> &seqv) {
     SUPPORTED_TYPE_CHECK(T);
     try {
         vector<dim4> numDims;
-        vector<vector<T> > in;
-        vector<vector<T> > tests;
+        vector<vector<T>> in;
+        vector<vector<T>> tests;
 
         readTests<T, T, int>(pTestFile, numDims, in, tests);
 
diff --git a/test/bilateral.cpp b/test/bilateral.cpp
index d4da723ddb..8d83d2798b 100644
--- a/test/bilateral.cpp
+++ b/test/bilateral.cpp
@@ -87,8 +87,8 @@ void bilateralDataTest(string pTestFile) {
                                float>::type outType;
 
     vector<dim4> numDims;
-    vector<vector<inType> > in;
-    vector<vector<outType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<outType>> tests;
 
     readTests<inType, outType, float>(pTestFile, numDims, in, tests);
 
@@ -152,8 +152,8 @@ using af::bilateral;
 
 TEST(Bilateral, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, float>(string(TEST_DIR "/bilateral/rectangle.test"),
                                    numDims, in, tests);
diff --git a/test/binary.cpp b/test/binary.cpp
index f5fd0610e8..b0c04a4c30 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -578,7 +578,7 @@ TYPED_TEST(ResultTypeScalar, FloatDivision) {
     ASSERT_EQ(f32, (af::array(10, f32) / this->scalar).type());
 }
 
-class Broadcast : public ::testing::TestWithParam<std::tuple<dim4, dim4> > {
+class Broadcast : public ::testing::TestWithParam<std::tuple<dim4, dim4>> {
     void SetUp() override {}
 };
 /// clang-format off
diff --git a/test/blas.cpp b/test/blas.cpp
index 62491a366f..6b0590d73b 100644
--- a/test/blas.cpp
+++ b/test/blas.cpp
@@ -53,8 +53,8 @@ void MatMulCheck(string TestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<T> > hData;
-    vector<vector<T> > tests;
+    vector<vector<T>> hData;
+    vector<vector<T>> tests;
     readTests<T, T, int>(TestFile, numDims, hData, tests);
 
     af_array a, aT, b, bT;
@@ -132,8 +132,8 @@ void cppMatMulCheck(string TestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<T> > hData;
-    vector<vector<T> > tests;
+    vector<vector<T>> hData;
+    vector<vector<T>> tests;
     readTests<T, T, int>(TestFile, numDims, hData, tests);
 
     array a(numDims[0], &hData[0].front());
diff --git a/test/canny.cpp b/test/canny.cpp
index 8e1cb9c2b6..7e72d4e356 100644
--- a/test/canny.cpp
+++ b/test/canny.cpp
@@ -39,8 +39,8 @@ void cannyTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<char> > tests;
+    vector<vector<T>> in;
+    vector<vector<char>> tests;
 
     readTests<T, char, int>(pTestFile, numDims, in, tests);
 
diff --git a/test/confidence_connected.cpp b/test/confidence_connected.cpp
index 8ef707aca7..9d081f068d 100644
--- a/test/confidence_connected.cpp
+++ b/test/confidence_connected.cpp
@@ -122,8 +122,8 @@ void testData(CCCTestParams params) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     string file = string(TEST_DIR) + "/confidence_cc/" + string(params.prefix) +
                   "_" + to_string(params.radius) + "_" +
diff --git a/test/convolve.cpp b/test/convolve.cpp
index 7b31e532a3..5fb61e7ee0 100644
--- a/test/convolve.cpp
+++ b/test/convolve.cpp
@@ -45,8 +45,8 @@ void convolveTest(string pTestFile, int baseDim, bool expand) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
@@ -218,8 +218,8 @@ void sepConvolveTest(string pTestFile, bool expand) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
@@ -378,8 +378,8 @@ using af::sum;
 
 TEST(Convolve1, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(string(TEST_DIR "/convolve/vector_same.test"),
                                  numDims, in, tests);
@@ -411,8 +411,8 @@ TEST(Convolve1, CPP) {
 
 TEST(Convolve2, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(
         string(TEST_DIR "/convolve/rectangle_same_one2many.test"), numDims, in,
@@ -447,8 +447,8 @@ TEST(Convolve2, CPP) {
 
 TEST(Convolve3, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(
         string(TEST_DIR "/convolve/cuboid_same_many2many.test"), numDims, in,
@@ -482,8 +482,8 @@ TEST(Convolve3, CPP) {
 
 TEST(Convolve, separable_CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(
         string(TEST_DIR "/convolve/separable_conv2d_same_rectangle_batch.test"),
@@ -809,8 +809,8 @@ TEST(Convolve, CuboidBatchLaunchBugFix) {
     std::string testFile(TEST_DIR "/convolve/conv3d_launch_bug.test");
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, float>(testFile, numDims, in, tests);
 
@@ -917,8 +917,8 @@ void convolve2stridedTest(string pTestFile, dim4 stride, dim4 padding,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
@@ -962,8 +962,8 @@ void convolve2GradientTest(string pTestFile, dim4 stride, dim4 padding,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
diff --git a/test/corrcoef.cpp b/test/corrcoef.cpp
index 1c7f378961..213a8de092 100644
--- a/test/corrcoef.cpp
+++ b/test/corrcoef.cpp
@@ -73,8 +73,8 @@ TYPED_TEST(CorrelationCoefficient, All) {
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<int> > in;
-    vector<vector<float> > tests;
+    vector<vector<int>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<int, float>(
         string(TEST_DIR "/corrcoef/mat_10x10_scalar.test"), numDims, in, tests);
diff --git a/test/covariance.cpp b/test/covariance.cpp
index aa06c58a10..4d4e4877f1 100644
--- a/test/covariance.cpp
+++ b/test/covariance.cpp
@@ -79,8 +79,8 @@ void covTest(string pFileName, bool isbiased = true,
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<int> > in;
-    vector<vector<float> > tests;
+    vector<vector<int>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<int, float>(pFileName, numDims, in, tests);
 
diff --git a/test/diff1.cpp b/test/diff1.cpp
index 605cd75fa9..a7456fd0a2 100644
--- a/test/diff1.cpp
+++ b/test/diff1.cpp
@@ -59,8 +59,8 @@ void diff1Test(string pTestFile, unsigned dim, bool isSubRef = false,
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -151,8 +151,8 @@ void diff1ArgsTest(string pTestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -214,8 +214,8 @@ TEST(Diff1, CPP) {
     const unsigned dim = 0;
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/diff1/matrix0.test"),
                                  numDims, in, tests);
     dim4 dims = numDims[0];
diff --git a/test/diff2.cpp b/test/diff2.cpp
index 4a68627d7b..c7c17f333f 100644
--- a/test/diff2.cpp
+++ b/test/diff2.cpp
@@ -64,8 +64,8 @@ void diff2Test(string pTestFile, unsigned dim, bool isSubRef = false,
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -153,8 +153,8 @@ void diff2ArgsTest(string pTestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -209,8 +209,8 @@ TEST(Diff2, CPP) {
     const unsigned dim = 1;
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/diff2/matrix1.test"),
                                  numDims, in, tests);
     dim4 dims = numDims[0];
diff --git a/test/dot.cpp b/test/dot.cpp
index 357e0784d4..834260af44 100644
--- a/test/dot.cpp
+++ b/test/dot.cpp
@@ -63,8 +63,8 @@ void dotTest(string pTestFile, const int resultIdx,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, T>(pTestFile, numDims, in, tests);
 
@@ -118,8 +118,8 @@ void dotAllTest(string pTestFile, const int resultIdx,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, T>(pTestFile, numDims, in, tests);
 
@@ -194,8 +194,8 @@ INSTANTIATEC(25600, dot_c_25600);
 //
 TEST(DotF, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, float>(TEST_DIR "/blas/dot_f_1000.test", numDims,
                                    in, tests);
@@ -215,8 +215,8 @@ TEST(DotF, CPP) {
 
 TEST(DotCCU, CPP) {
     vector<dim4> numDims;
-    vector<vector<cfloat> > in;
-    vector<vector<cfloat> > tests;
+    vector<vector<cfloat>> in;
+    vector<vector<cfloat>> tests;
 
     readTests<cfloat, cfloat, cfloat>(TEST_DIR "/blas/dot_c_1000.test", numDims,
                                       in, tests);
@@ -236,8 +236,8 @@ TEST(DotCCU, CPP) {
 
 TEST(DotAllF, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, float>(TEST_DIR "/blas/dot_f_1000.test", numDims,
                                    in, tests);
@@ -257,8 +257,8 @@ TEST(DotAllF, CPP) {
 
 TEST(DotAllCCU, CPP) {
     vector<dim4> numDims;
-    vector<vector<cfloat> > in;
-    vector<vector<cfloat> > tests;
+    vector<vector<cfloat>> in;
+    vector<vector<cfloat>> tests;
 
     readTests<cfloat, cfloat, cfloat>(TEST_DIR "/blas/dot_c_1000.test", numDims,
                                       in, tests);
diff --git a/test/fast.cpp b/test/fast.cpp
index 77281955a5..316fe57ad6 100644
--- a/test/fast.cpp
+++ b/test/fast.cpp
@@ -73,7 +73,7 @@ void fastTest(string pTestFile, bool nonmax) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > gold;
+    vector<vector<float>> gold;
 
     readImageTests(pTestFile, inDims, inFiles, gold);
 
@@ -184,7 +184,7 @@ TEST(FloatFAST, CPP) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > gold;
+    vector<vector<float>> gold;
 
     readImageTests(string(TEST_DIR "/fast/square_nonmax_float.test"), inDims,
                    inFiles, gold);
diff --git a/test/fft.cpp b/test/fft.cpp
index acd0ad7521..49176ca522 100644
--- a/test/fft.cpp
+++ b/test/fft.cpp
@@ -127,8 +127,8 @@ void fftTest(string pTestFile, dim_t pad0 = 0, dim_t pad1 = 0, dim_t pad2 = 0) {
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<inType> > in;
-    vector<vector<outType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<outType>> tests;
 
     readTestsFromFile<inType, outType>(pTestFile, numDims, in, tests);
 
@@ -293,8 +293,8 @@ void fftBatchTest(string pTestFile, dim_t pad0 = 0, dim_t pad1 = 0,
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<inType> > in;
-    vector<vector<outType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<outType>> tests;
 
     readTestsFromFile<inType, outType>(pTestFile, numDims, in, tests);
 
@@ -430,8 +430,8 @@ void cppFFTTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<inType> > in;
-    vector<vector<outType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<outType>> tests;
 
     readTestsFromFile<inType, outType>(pTestFile, numDims, in, tests);
 
@@ -476,8 +476,8 @@ void cppDFTTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<inType> > in;
-    vector<vector<outType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<outType>> tests;
 
     readTestsFromFile<inType, outType>(pTestFile, numDims, in, tests);
 
diff --git a/test/fftconvolve.cpp b/test/fftconvolve.cpp
index 7465891bde..57d9398a04 100644
--- a/test/fftconvolve.cpp
+++ b/test/fftconvolve.cpp
@@ -53,8 +53,8 @@ void fftconvolveTest(string pTestFile, bool expand) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
@@ -345,8 +345,8 @@ TYPED_TEST(FFTConvolve, Same_Cuboid_One2Many) {
 
 TEST(FFTConvolve1, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(string(TEST_DIR "/convolve/vector.test"),
                                  numDims, in, tests);
@@ -378,8 +378,8 @@ TEST(FFTConvolve1, CPP) {
 
 TEST(FFTConvolve2, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(
         string(TEST_DIR "/convolve/rectangle_one2many.test"), numDims, in,
@@ -414,8 +414,8 @@ TEST(FFTConvolve2, CPP) {
 
 TEST(FFTConvolve3, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(
         string(TEST_DIR "/convolve/cuboid_many2many.test"), numDims, in, tests);
diff --git a/test/gaussiankernel.cpp b/test/gaussiankernel.cpp
index 3c4db5386f..3fc8de1c23 100644
--- a/test/gaussiankernel.cpp
+++ b/test/gaussiankernel.cpp
@@ -37,8 +37,8 @@ void gaussianKernelTest(string pFileName, double sigma) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<int> > in;
-    vector<vector<T> > tests;
+    vector<vector<int>> in;
+    vector<vector<T>> tests;
 
     readTestsFromFile<int, T>(pFileName, numDims, in, tests);
 
@@ -114,8 +114,8 @@ using af::gaussianKernel;
 
 void gaussianKernelTestCPP(string pFileName, double sigma) {
     vector<dim4> numDims;
-    vector<vector<int> > in;
-    vector<vector<float> > tests;
+    vector<vector<int>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<int, float>(pFileName, numDims, in, tests);
 
diff --git a/test/gen_assign.cpp b/test/gen_assign.cpp
index 716735740a..7cfd78ae62 100644
--- a/test/gen_assign.cpp
+++ b/test/gen_assign.cpp
@@ -38,8 +38,8 @@ using std::vector;
 void testGeneralAssignOneArray(string pTestFile, const dim_t ndims,
                                af_index_t *indexs, int arrayDim) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(pTestFile, numDims, in, tests);
 
@@ -105,8 +105,8 @@ TEST(GeneralAssign, SASS) {
 
 TEST(GeneralAssign, SSSS) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(
         string(TEST_DIR "/gen_assign/s10_14s0_9s0_ns0_n.test"), numDims, in,
@@ -152,8 +152,8 @@ TEST(GeneralAssign, SSSS) {
 
 TEST(GeneralAssign, AAAA) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(string(TEST_DIR "/gen_assign/aaaa.test"),
                                     numDims, in, tests);
diff --git a/test/gen_index.cpp b/test/gen_index.cpp
index b491a9ac4c..e65d4e48e5 100644
--- a/test/gen_index.cpp
+++ b/test/gen_index.cpp
@@ -47,8 +47,8 @@ class IndexGeneralizedLegacy : public ::testing::TestWithParam<index_params> {
     void SetUp() {
         index_params params = GetParam();
         vector<dim4> numDims;
-        vector<vector<float> > in;
-        vector<vector<float> > tests;
+        vector<vector<float>> in;
+        vector<vector<float>> tests;
 
         if (noDoubleTests(get<1>(params))) return;
         if (noHalfTests(get<1>(params))) return;
@@ -138,8 +138,8 @@ TEST_P(IndexGeneralizedLegacy, SSSA) {
 void testGeneralIndexOneArray(string pTestFile, const dim_t ndims,
                               af_index_t *indexs, int arrayDim) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(pTestFile, numDims, in, tests);
 
@@ -202,8 +202,8 @@ TEST(GeneralIndex, SASS) {
 
 TEST(GeneralIndex, AASS) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(
         string(TEST_DIR "/gen_index/aas0_ns0_n.test"), numDims, in, tests);
diff --git a/test/gloh.cpp b/test/gloh.cpp
index eb193e7ec4..e370984fbf 100644
--- a/test/gloh.cpp
+++ b/test/gloh.cpp
@@ -65,7 +65,7 @@ static void array_to_feat_desc(vector<feat_desc_t>& feat, float* x, float* y,
 
 static void array_to_feat_desc(vector<feat_desc_t>& feat, float* x, float* y,
                                float* score, float* ori, float* size,
-                               vector<vector<float> >& desc, unsigned nfeat) {
+                               vector<vector<float>>& desc, unsigned nfeat) {
     feat.resize(nfeat);
     for (size_t i = 0; i < feat.size(); i++) {
         feat[i].f[0] = x[i];
@@ -141,8 +141,8 @@ void glohTest(string pTestFile) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > goldFeat;
-    vector<vector<float> > goldDesc;
+    vector<vector<float>> goldFeat;
+    vector<vector<float>> goldDesc;
 
     readImageFeaturesDescriptors<float>(pTestFile, inDims, inFiles, goldFeat,
                                         goldDesc);
@@ -265,8 +265,8 @@ TEST(GLOH, CPP) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > goldFeat;
-    vector<vector<float> > goldDesc;
+    vector<vector<float>> goldFeat;
+    vector<vector<float>> goldDesc;
 
     readImageFeaturesDescriptors<float>(string(TEST_DIR "/gloh/man.test"),
                                         inDims, inFiles, goldFeat, goldDesc);
diff --git a/test/gradient.cpp b/test/gradient.cpp
index b30e9bb649..5d04d3dd98 100644
--- a/test/gradient.cpp
+++ b/test/gradient.cpp
@@ -50,8 +50,8 @@ void gradTest(string pTestFile, const unsigned resultIdx0,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -128,8 +128,8 @@ TEST(Grad, CPP) {
     const unsigned resultIdx1 = 1;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/grad/grad3D.test"),
                                    numDims, in, tests);
 
diff --git a/test/hamming.cpp b/test/hamming.cpp
index 8b3d9f85f7..763e0f7774 100644
--- a/test/hamming.cpp
+++ b/test/hamming.cpp
@@ -47,12 +47,12 @@ void hammingMatcherTest(string pTestFile, int feat_dim) {
     using af::dim4;
 
     vector<dim4> numDims;
-    vector<vector<uint> > in32;
-    vector<vector<uint> > tests;
+    vector<vector<uint>> in32;
+    vector<vector<uint>> tests;
 
     readTests<uint, uint, int>(pTestFile, numDims, in32, tests);
 
-    vector<vector<T> > in(in32.size());
+    vector<vector<T>> in(in32.size());
     for (size_t i = 0; i < in32[0].size(); i++) in[0].push_back((T)in32[0][i]);
     for (size_t i = 0; i < in32[1].size(); i++) in[1].push_back((T)in32[1][i]);
 
@@ -121,8 +121,8 @@ TEST(HammingMatcher, CPP) {
     using af::dim4;
 
     vector<dim4> numDims;
-    vector<vector<uint> > in;
-    vector<vector<uint> > tests;
+    vector<vector<uint>> in;
+    vector<vector<uint>> tests;
 
     readTests<uint, uint, int>(
         TEST_DIR "/hamming/hamming_500_5000_dim0_u32.test", numDims, in, tests);
diff --git a/test/harris.cpp b/test/harris.cpp
index 955c676251..ec6a1fa626 100644
--- a/test/harris.cpp
+++ b/test/harris.cpp
@@ -65,7 +65,7 @@ void harrisTest(string pTestFile, float sigma, unsigned block_size) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > gold;
+    vector<vector<float>> gold;
 
     readImageTests(pTestFile, inDims, inFiles, gold);
 
@@ -171,7 +171,7 @@ TEST(FloatHarris, CPP) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > gold;
+    vector<vector<float>> gold;
 
     readImageTests(string(TEST_DIR "/harris/square_0_3.test"), inDims, inFiles,
                    gold);
diff --git a/test/histogram.cpp b/test/histogram.cpp
index ff2049b390..ca3df72f74 100644
--- a/test/histogram.cpp
+++ b/test/histogram.cpp
@@ -46,8 +46,8 @@ void histTest(string pTestFile, unsigned nbins, double minval, double maxval) {
 
     vector<dim4> numDims;
 
-    vector<vector<inType> > in;
-    vector<vector<outType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<outType>> tests;
     readTests<inType, uint, uint>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -120,8 +120,8 @@ TEST(Histogram, CPP) {
 
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<uint> > tests;
+    vector<vector<float>> in;
+    vector<vector<uint>> tests;
     readTests<float, uint, int>(
         string(TEST_DIR "/histogram/100bin0min99max.test"), numDims, in, tests);
 
diff --git a/test/homography.cpp b/test/homography.cpp
index 6b0e620869..c6a6e43450 100644
--- a/test/homography.cpp
+++ b/test/homography.cpp
@@ -53,7 +53,7 @@ void homographyTest(string pTestFile, const af_homography_type htype,
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > gold;
+    vector<vector<float>> gold;
 
     readImageTests(pTestFile, inDims, inFiles, gold);
 
@@ -224,7 +224,7 @@ TEST(Homography, CPP) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > gold;
+    vector<vector<float>> gold;
 
     readImageTests(string(TEST_DIR "/homography/tux.test"), inDims, inFiles,
                    gold);
diff --git a/test/hsv_rgb.cpp b/test/hsv_rgb.cpp
index f00f5ab7f1..423fc5fad5 100644
--- a/test/hsv_rgb.cpp
+++ b/test/hsv_rgb.cpp
@@ -39,8 +39,8 @@ TEST(hsv_rgb, InvalidArray) {
 
 TEST(hsv2rgb, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(string(TEST_DIR "/hsv_rgb/hsv2rgb.test"),
                                     numDims, in, tests);
@@ -55,8 +55,8 @@ TEST(hsv2rgb, CPP) {
 
 TEST(rgb2hsv, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(string(TEST_DIR "/hsv_rgb/rgb2hsv.test"),
                                     numDims, in, tests);
@@ -71,8 +71,8 @@ TEST(rgb2hsv, CPP) {
 
 TEST(rgb2hsv, MaxDim) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(string(TEST_DIR "/hsv_rgb/rgb2hsv.test"),
                                     numDims, in, tests);
@@ -109,8 +109,8 @@ TEST(rgb2hsv, MaxDim) {
 
 TEST(hsv2rgb, MaxDim) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(string(TEST_DIR "/hsv_rgb/hsv2rgb.test"),
                                     numDims, in, tests);
diff --git a/test/iir.cpp b/test/iir.cpp
index fd03e7ccc6..85fda2a959 100644
--- a/test/iir.cpp
+++ b/test/iir.cpp
@@ -124,8 +124,8 @@ void iirTest(const char *testFile) {
     SUPPORTED_TYPE_CHECK(T);
     vector<dim4> inDims;
 
-    vector<vector<T> > inputs;
-    vector<vector<T> > outputs;
+    vector<vector<T>> inputs;
+    vector<vector<T>> outputs;
     readTests<T, T, float>(testFile, inDims, inputs, outputs);
 
     try {
diff --git a/test/imageio.cpp b/test/imageio.cpp
index a4e12e834e..6d3de9f45b 100644
--- a/test/imageio.cpp
+++ b/test/imageio.cpp
@@ -40,8 +40,8 @@ void loadImageTest(string pTestFile, string pImageFile, const bool isColor) {
 
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -126,8 +126,8 @@ TEST(ImageIO, CPP) {
 
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/imageio/color_small.test"),
                                    numDims, in, tests);
 
@@ -258,8 +258,8 @@ TEST(ImageIO, LoadImage16CPP) {
 
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(
         string(TEST_DIR "/imageio/color_seq_16.test"), numDims, in, tests);
 
@@ -316,8 +316,8 @@ void loadImageNativeCPPTest(string pTestFile, string pImageFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(pTestFile, numDims, in, tests);
 
     dim4 dims = numDims[0];
diff --git a/test/index.cpp b/test/index.cpp
index 2f61d40adb..a593348773 100644
--- a/test/index.cpp
+++ b/test/index.cpp
@@ -300,39 +300,39 @@ class Indexing2D : public ::testing::Test {
             make_vec(af_make_seq(3, 6, 4), af_make_seq(1, 9, 4)));
     }
 
-    vector<vector<af_seq> > column_continuous_seq;
-    vector<vector<af_seq> > column_continuous_reverse_seq;
-    vector<vector<af_seq> > column_strided_seq;
-    vector<vector<af_seq> > column_strided_reverse_seq;
-
-    vector<vector<af_seq> > row_continuous_seq;
-    vector<vector<af_seq> > row_continuous_reverse_seq;
-    vector<vector<af_seq> > row_strided_seq;
-    vector<vector<af_seq> > row_strided_reverse_seq;
-
-    vector<vector<af_seq> > continuous_continuous_seq;
-    vector<vector<af_seq> > continuous_strided_seq;
-    vector<vector<af_seq> > continuous_reverse_seq;
-    vector<vector<af_seq> > continuous_strided_reverse_seq;
-
-    vector<vector<af_seq> > reverse_continuous_seq;
-    vector<vector<af_seq> > reverse_reverse_seq;
-    vector<vector<af_seq> > reverse_strided_seq;
-    vector<vector<af_seq> > reverse_strided_reverse_seq;
-
-    vector<vector<af_seq> > strided_continuous_seq;
-    vector<vector<af_seq> > strided_strided_seq;
+    vector<vector<af_seq>> column_continuous_seq;
+    vector<vector<af_seq>> column_continuous_reverse_seq;
+    vector<vector<af_seq>> column_strided_seq;
+    vector<vector<af_seq>> column_strided_reverse_seq;
+
+    vector<vector<af_seq>> row_continuous_seq;
+    vector<vector<af_seq>> row_continuous_reverse_seq;
+    vector<vector<af_seq>> row_strided_seq;
+    vector<vector<af_seq>> row_strided_reverse_seq;
+
+    vector<vector<af_seq>> continuous_continuous_seq;
+    vector<vector<af_seq>> continuous_strided_seq;
+    vector<vector<af_seq>> continuous_reverse_seq;
+    vector<vector<af_seq>> continuous_strided_reverse_seq;
+
+    vector<vector<af_seq>> reverse_continuous_seq;
+    vector<vector<af_seq>> reverse_reverse_seq;
+    vector<vector<af_seq>> reverse_strided_seq;
+    vector<vector<af_seq>> reverse_strided_reverse_seq;
+
+    vector<vector<af_seq>> strided_continuous_seq;
+    vector<vector<af_seq>> strided_strided_seq;
 };
 
 template<typename T>
-void DimCheck2D(const vector<vector<af_seq> > &seqs, string TestFile,
+void DimCheck2D(const vector<vector<af_seq>> &seqs, string TestFile,
                 size_t NDims) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
 
-    vector<vector<T> > hData;
-    vector<vector<T> > tests;
+    vector<vector<T>> hData;
+    vector<vector<T>> tests;
     readTests<T, T, int>(TestFile, numDims, hData, tests);
     dim4 dimensions = numDims[0];
 
@@ -528,18 +528,18 @@ class Indexing : public ::testing::Test {
                       af_make_seq(0, 0, 1), af_make_seq(0, 0, 1)));
     }
 
-    vector<vector<af_seq> > continuous3d_to_3d;
-    vector<vector<af_seq> > continuous3d_to_2d;
-    vector<vector<af_seq> > continuous3d_to_1d;
+    vector<vector<af_seq>> continuous3d_to_3d;
+    vector<vector<af_seq>> continuous3d_to_2d;
+    vector<vector<af_seq>> continuous3d_to_1d;
 
-    vector<vector<af_seq> > continuous4d_to_4d;
-    vector<vector<af_seq> > continuous4d_to_3d;
-    vector<vector<af_seq> > continuous4d_to_2d;
-    vector<vector<af_seq> > continuous4d_to_1d;
+    vector<vector<af_seq>> continuous4d_to_4d;
+    vector<vector<af_seq>> continuous4d_to_3d;
+    vector<vector<af_seq>> continuous4d_to_2d;
+    vector<vector<af_seq>> continuous4d_to_1d;
 };
 
 template<typename T>
-void DimCheckND(const vector<vector<af_seq> > &seqs, string TestFile,
+void DimCheckND(const vector<vector<af_seq>> &seqs, string TestFile,
                 size_t NDims) {
     SUPPORTED_TYPE_CHECK(T);
 
@@ -589,7 +589,7 @@ TEST(Index, Docs_Util_C_API) {
     //![ex_index_util_0]
     af_index_t *indexers = 0;
     af_err err           = af_create_indexers(
-        &indexers);  // Memory is allocated on heap by the callee
+                  &indexers);  // Memory is allocated on heap by the callee
     // by default all the indexers span all the elements along the given
     // dimension
 
@@ -658,7 +658,7 @@ using af::span;
 using af::where;
 
 TEST(Indexing2D, ColumnContiniousCPP) {
-    vector<vector<af_seq> > seqs;
+    vector<vector<af_seq>> seqs;
 
     seqs.push_back(make_vec(af_span, af_make_seq(0, 6, 1)));
     // seqs.push_back(make_vec(span, af_make_seq(  4,  9,  1)));
@@ -666,8 +666,8 @@ TEST(Indexing2D, ColumnContiniousCPP) {
 
     vector<dim4> numDims;
 
-    vector<vector<float> > hData;
-    vector<vector<float> > tests;
+    vector<vector<float>> hData;
+    vector<vector<float>> tests;
     readTests<float, float, int>(TEST_DIR "/index/ColumnContinious.test",
                                  numDims, hData, tests);
     dim4 dimensions = numDims[0];
@@ -717,8 +717,8 @@ void arrayIndexTest(string pTestFile, int dim) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
@@ -767,8 +767,8 @@ TYPED_TEST(lookup, Dim3) {
 
 TEST(lookup, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(string(TEST_DIR "/arrayindex/dim0.test"),
                                  numDims, in, tests);
@@ -978,8 +978,8 @@ TEST(SeqIndex, CPP_SCOPE_ARR) {
 
 TEST(SeqIndex, CPPLarge) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(string(TEST_DIR "/arrayindex/dim0Large.test"),
                                  numDims, in, tests);
diff --git a/test/internal.cpp b/test/internal.cpp
index 3540ff0ee0..ede8e697a7 100644
--- a/test/internal.cpp
+++ b/test/internal.cpp
@@ -36,7 +36,7 @@ TEST(Internal, CreateStrided) {
     dim_t dims[]    = {3, 3, 2};
     dim_t strides[] = {1, 5, 20};
     array a         = createStridedArray((void *)ha, offset, dim4(ndims, dims),
-                                 dim4(ndims, strides), f32, afHost);
+                                         dim4(ndims, strides), f32, afHost);
 
     dim4 astrides = getStrides(a);
     dim4 adims    = a.dims();
diff --git a/test/ireduce.cpp b/test/ireduce.cpp
index 5c49e8c3e8..92596528d4 100644
--- a/test/ireduce.cpp
+++ b/test/ireduce.cpp
@@ -261,7 +261,7 @@ TEST(IndexedReduce, MinCplxNaN) {
     array min_idx;
     af::min(min_val, min_idx, a);
 
-    vector<complex<float> > h_min_val(cols);
+    vector<complex<float>> h_min_val(cols);
     min_val.host(&h_min_val[0]);
 
     vector<int> h_min_idx(cols);
@@ -296,7 +296,7 @@ TEST(IndexedReduce, MaxCplxNaN) {
     array max_idx;
     af::max(max_val, max_idx, a);
 
-    vector<complex<float> > h_max_val(cols);
+    vector<complex<float>> h_max_val(cols);
     max_val.host(&h_max_val[0]);
 
     vector<int> h_max_idx(cols);
@@ -371,7 +371,7 @@ TEST(IndexedReduce, MinCplxPreferLargerIdxIfEqual) {
     array min_idx;
     min(min_val, min_idx, a);
 
-    vector<complex<float> > h_min_val(1);
+    vector<complex<float>> h_min_val(1);
     min_val.host(&h_min_val[0]);
 
     vector<int> h_min_idx(1);
@@ -400,7 +400,7 @@ TEST(IndexedReduce, MaxCplxPreferSmallerIdxIfEqual) {
     array max_idx;
     max(max_val, max_idx, a);
 
-    vector<complex<float> > h_max_val(1);
+    vector<complex<float>> h_max_val(1);
     max_val.host(&h_max_val[0]);
 
     vector<int> h_max_idx(1);
diff --git a/test/jit.cpp b/test/jit.cpp
index 64d72d25b7..101580a488 100644
--- a/test/jit.cpp
+++ b/test/jit.cpp
@@ -665,13 +665,13 @@ TEST(JIT, TwoLargeNonLinearHalf) {
 }
 
 std::string select_info(
-    const ::testing::TestParamInfo<std::tuple<int, int, int> > info) {
+    const ::testing::TestParamInfo<std::tuple<int, int, int>> info) {
     return "a_" + to_string(get<0>(info.param)) + "_b_" +
            to_string(get<1>(info.param)) + "_cond_" +
            to_string(get<2>(info.param));
 }
 
-class JITSelect : public ::testing::TestWithParam<std::tuple<int, int, int> > {
+class JITSelect : public ::testing::TestWithParam<std::tuple<int, int, int>> {
    protected:
     void SetUp() {}
 };
diff --git a/test/join.cpp b/test/join.cpp
index de61bdf91e..cf33fccb67 100644
--- a/test/join.cpp
+++ b/test/join.cpp
@@ -61,8 +61,8 @@ void joinTest(string pTestFile, const unsigned dim, const unsigned in0,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
     dim4 i0dims = numDims[in0];
@@ -161,8 +161,8 @@ TEST(Join, CPP) {
     const unsigned dim       = 2;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/join/join_big.test"),
                                  numDims, in, tests);
 
diff --git a/test/lu_dense.cpp b/test/lu_dense.cpp
index e5b4b8ac97..ec69e1ccd9 100644
--- a/test/lu_dense.cpp
+++ b/test/lu_dense.cpp
@@ -42,8 +42,8 @@ TEST(LU, InPlaceSmall) {
     int resultIdx = 0;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/lapack/lu.test"), numDims,
                                    in, tests);
 
@@ -80,8 +80,8 @@ TEST(LU, SplitSmall) {
     int resultIdx = 0;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/lapack/lufactorized.test"),
                                    numDims, in, tests);
 
diff --git a/test/match_template.cpp b/test/match_template.cpp
index 90c199bd0a..33b6096815 100644
--- a/test/match_template.cpp
+++ b/test/match_template.cpp
@@ -45,8 +45,8 @@ void matchTemplateTest(string pTestFile, af_match_type pMatchType) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<outType> > tests;
+    vector<vector<T>> in;
+    vector<vector<outType>> tests;
 
     readTests<T, outType, float>(pTestFile, numDims, in, tests);
 
diff --git a/test/mean.cpp b/test/mean.cpp
index 89a89efeb9..c9c6eb567b 100644
--- a/test/mean.cpp
+++ b/test/mean.cpp
@@ -85,8 +85,8 @@ void meanDimTest(string pFileName, dim_t dim, bool isWeighted = false) {
     double tol = 1.0e-3;
     if ((af_dtype)af::dtype_traits<T>::af_type == f16) tol = 4.e-3;
     vector<dim4> numDims;
-    vector<vector<int> > in;
-    vector<vector<float> > tests;
+    vector<vector<int>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<int, float>(pFileName, numDims, in, tests);
 
diff --git a/test/meanvar.cpp b/test/meanvar.cpp
index e9286027a2..81f3fb8099 100644
--- a/test/meanvar.cpp
+++ b/test/meanvar.cpp
@@ -55,8 +55,8 @@ struct meanvar_test {
     af_array weights_;
     af_var_bias bias_;
     int dim_;
-    vector<outType<T> > mean_;
-    vector<outType<T> > variance_;
+    vector<outType<T>> mean_;
+    vector<outType<T>> variance_;
 
     meanvar_test(string description, af_array in, af_array weights,
                  af_var_bias bias, int dim, vector<double> &&mean,
@@ -105,7 +105,7 @@ template<typename T>
 af_dtype meanvar_test<T>::af_type = dtype_traits<T>::af_type;
 
 template<typename T>
-class MeanVarTyped : public ::testing::TestWithParam<meanvar_test<T> > {
+class MeanVarTyped : public ::testing::TestWithParam<meanvar_test<T>> {
    public:
     void meanvar_test_function(const meanvar_test<T> &test) {
         SUPPORTED_TYPE_CHECK(T);
@@ -119,18 +119,18 @@ class MeanVarTyped : public ::testing::TestWithParam<meanvar_test<T> > {
         EXPECT_EQ(AF_SUCCESS, af_meanvar(&mean, &var, in, test.weights_,
                                          test.bias_, test.dim_));
 
-        vector<outType<T> > h_mean(test.mean_.size()),
+        vector<outType<T>> h_mean(test.mean_.size()),
             h_var(test.variance_.size());
 
         dim4 outDim(1);
         af_get_dims(&outDim[0], &outDim[1], &outDim[2], &outDim[3], in);
         outDim[test.dim_] = 1;
 
-        if (is_same_type<half_float::half, outType<T> >::value) {
+        if (is_same_type<half_float::half, outType<T>>::value) {
             ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 1.f);
             ASSERT_VEC_ARRAY_NEAR(test.variance_, outDim, var, 0.5f);
-        } else if (is_same_type<float, outType<T> >::value ||
-                   is_same_type<cfloat, outType<T> >::value) {
+        } else if (is_same_type<float, outType<T>>::value ||
+                   is_same_type<cfloat, outType<T>>::value) {
             ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 0.001f);
             ASSERT_VEC_ARRAY_NEAR(test.variance_, outDim, var, 0.2f);
         } else {
@@ -160,17 +160,17 @@ class MeanVarTyped : public ::testing::TestWithParam<meanvar_test<T> > {
         array weights(weights_tmp);
         meanvar(mean, var, in, weights, test.bias_, test.dim_);
 
-        vector<outType<T> > h_mean(test.mean_.size()),
+        vector<outType<T>> h_mean(test.mean_.size()),
             h_var(test.variance_.size());
 
         dim4 outDim       = in.dims();
         outDim[test.dim_] = 1;
 
-        if (is_same_type<half_float::half, outType<T> >::value) {
+        if (is_same_type<half_float::half, outType<T>>::value) {
             ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 1.f);
             ASSERT_VEC_ARRAY_NEAR(test.variance_, outDim, var, 0.5f);
-        } else if (is_same_type<float, outType<T> >::value ||
-                   is_same_type<cfloat, outType<T> >::value) {
+        } else if (is_same_type<float, outType<T>>::value ||
+                   is_same_type<cfloat, outType<T>>::value) {
             ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 0.001f);
             ASSERT_VEC_ARRAY_NEAR(test.variance_, outDim, var, 0.2f);
         } else {
@@ -189,11 +189,11 @@ meanvar_test<T> meanvar_test_gen(string name, int in_index, int weight_index,
                                  af_var_bias bias, int dim, int mean_index,
                                  int var_index, test_size size) {
     vector<af_array> inputs;
-    vector<vector<double> > outputs;
+    vector<vector<double>> outputs;
     if (size == MEANVAR_SMALL) {
         vector<af::dim4> numDims_;
-        vector<vector<double> > in_;
-        vector<vector<double> > tests_;
+        vector<vector<double>> in_;
+        vector<vector<double>> tests_;
         readTests<double, typename varOutType<double>::type, double>(
             TEST_DIR "/meanvar/meanvar.data", numDims_, in_, tests_);
 
@@ -208,8 +208,8 @@ meanvar_test<T> meanvar_test_gen(string name, int in_index, int weight_index,
             copy(tests_[i].begin(), tests_[i].end(), back_inserter(outputs[i]));
         }
     } else {
-        dim_t full_array_size             = 2000;
-        vector<vector<dim_t> > dimensions = {
+        dim_t full_array_size            = 2000;
+        vector<vector<dim_t>> dimensions = {
             {2000, 1, 1, 1},  // 0
             {1, 2000, 1, 1},  // 1
             {1, 1, 2000, 1},  // 2
@@ -245,7 +245,7 @@ meanvar_test<T> meanvar_test_gen(string name, int in_index, int weight_index,
 }
 
 template<typename T>
-vector<meanvar_test<T> > small_test_values() {
+vector<meanvar_test<T>> small_test_values() {
     // clang-format off
     return {
         //                  |           Name |   in_index | weight_index |                  bias |  dim | mean_index | var_index |
@@ -262,7 +262,7 @@ vector<meanvar_test<T> > small_test_values() {
 }
 
 template<typename T>
-vector<meanvar_test<T> > large_test_values() {
+vector<meanvar_test<T>> large_test_values() {
     return {
         // clang-format off
         //                  |       Name |      in_index | weight_index |                  bias |  dim | mean_index | var_index |
diff --git a/test/medfilt.cpp b/test/medfilt.cpp
index 4bc7e69924..2120da8e4c 100644
--- a/test/medfilt.cpp
+++ b/test/medfilt.cpp
@@ -48,8 +48,8 @@ void medfiltTest(string pTestFile, dim_t w_len, dim_t w_wid,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
@@ -108,8 +108,8 @@ void medfilt1_Test(string pTestFile, dim_t w_wid, af_border_type pad) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
 
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
@@ -342,8 +342,8 @@ TEST(MedianFilter, CPP) {
     const dim_t w_wid = 3;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(
         string(TEST_DIR "/medianfilter/batch_symmetric_pad_3x3_window.test"),
@@ -368,8 +368,8 @@ TEST(MedianFilter1d, CPP) {
     const dim_t w_wid = 3;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTests<float, float, int>(
         string(TEST_DIR "/medianfilter/batch_symmetric_pad_3x1_window.test"),
diff --git a/test/moddims.cpp b/test/moddims.cpp
index 69af67860e..9674c5a4f1 100644
--- a/test/moddims.cpp
+++ b/test/moddims.cpp
@@ -50,8 +50,8 @@ void moddimsTest(string pTestFile, bool isSubRef = false,
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -131,8 +131,8 @@ void moddimsArgsTest(string pTestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -164,8 +164,8 @@ void moddimsMismatchTest(string pTestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -200,8 +200,8 @@ void cppModdimsTest(string pTestFile, bool isSubRef = false,
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
diff --git a/test/moments.cpp b/test/moments.cpp
index 5656a17ec5..d7a396ea95 100644
--- a/test/moments.cpp
+++ b/test/moments.cpp
@@ -47,8 +47,8 @@ void momentsTest(string pTestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<float> > tests;
+    vector<vector<T>> in;
+    vector<vector<float>> tests;
     readTests<T, float, float>(pTestFile, numDims, in, tests);
 
     array imgArray(numDims.front(), &in.front()[0]);
@@ -101,8 +101,8 @@ void momentsOnImageTest(string pTestFile, string pImageFile, bool isColor) {
     if (noImageIOTests()) return;
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(pTestFile, numDims, in, tests);
 
     array imgArray = loadImage(pImageFile.c_str(), isColor);
diff --git a/test/morph.cpp b/test/morph.cpp
index 220253c8c4..b24106b88b 100644
--- a/test/morph.cpp
+++ b/test/morph.cpp
@@ -41,8 +41,8 @@ void morphTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(inType);
 
     vector<dim4> numDims;
-    vector<vector<inType> > in;
-    vector<vector<inType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<inType>> tests;
 
     readTests<inType, inType, int>(pTestFile, numDims, in, tests);
 
@@ -458,11 +458,11 @@ TEST(Morph, EdgeIssue1564) {
                               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0,
                               0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1};
     int goldData[10 * 10]  = {0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0,
-                             0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-                             0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0,
-                             0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1,
-                             1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1,
-                             1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1};
+                              0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                              0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+                              0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1,
+                              1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1,
+                              1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1};
     array input(10, 10, inputData);
     int maskData[3 * 3] = {1, 1, 1, 1, 0, 1, 1, 1, 1};
     array mask(3, 3, maskData);
diff --git a/test/nearest_neighbour.cpp b/test/nearest_neighbour.cpp
index 5286923dd8..01847aea65 100644
--- a/test/nearest_neighbour.cpp
+++ b/test/nearest_neighbour.cpp
@@ -69,8 +69,8 @@ void nearestNeighbourTest(string pTestFile, int feat_dim,
     typedef typename otype_t<T>::otype To;
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<uint> > tests;
+    vector<vector<T>> in;
+    vector<vector<uint>> tests;
 
     readTests<T, uint, uint>(pTestFile, numDims, in, tests);
 
@@ -171,8 +171,8 @@ TYPED_TEST(NearestNeighbour, NN_SAD_500_5000_Dim1) {
 //
 TEST(NearestNeighbourSSD, CPP) {
     vector<dim4> numDims;
-    vector<vector<uint> > in;
-    vector<vector<uint> > tests;
+    vector<vector<uint>> in;
+    vector<vector<uint>> tests;
 
     readTests<uint, uint, uint>(TEST_DIR
                                 "/nearest_neighbour/ssd_500_5000_dim0.test",
@@ -207,8 +207,8 @@ TEST(NearestNeighbourSSD, CPP) {
 
 TEST(NearestNeighbourSAD, CPP) {
     vector<dim4> numDims;
-    vector<vector<uint> > in;
-    vector<vector<uint> > tests;
+    vector<vector<uint>> in;
+    vector<vector<uint>> tests;
 
     readTests<uint, uint, uint>(TEST_DIR
                                 "/nearest_neighbour/sad_100_1000_dim1.test",
diff --git a/test/orb.cpp b/test/orb.cpp
index 42df3ea2f5..b29c7021ba 100644
--- a/test/orb.cpp
+++ b/test/orb.cpp
@@ -64,8 +64,7 @@ static void array_to_feat_desc(vector<feat_desc_t>& feat, float* x, float* y,
 
 static void array_to_feat_desc(vector<feat_desc_t>& feat, float* x, float* y,
                                float* score, float* ori, float* size,
-                               vector<vector<unsigned> >& desc,
-                               unsigned nfeat) {
+                               vector<vector<unsigned>>& desc, unsigned nfeat) {
     feat.resize(nfeat);
     for (size_t i = 0; i < feat.size(); i++) {
         feat[i].f[0] = x[i];
@@ -134,8 +133,8 @@ void orbTest(string pTestFile) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > goldFeat;
-    vector<vector<unsigned> > goldDesc;
+    vector<vector<float>> goldFeat;
+    vector<vector<unsigned>> goldDesc;
 
     readImageFeaturesDescriptors<unsigned>(pTestFile, inDims, inFiles, goldFeat,
                                            goldDesc);
@@ -251,8 +250,8 @@ TEST(ORB, CPP) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > goldFeat;
-    vector<vector<unsigned> > goldDesc;
+    vector<vector<float>> goldFeat;
+    vector<vector<unsigned>> goldDesc;
 
     readImageFeaturesDescriptors<unsigned>(string(TEST_DIR "/orb/square.test"),
                                            inDims, inFiles, goldFeat, goldDesc);
diff --git a/test/pinverse.cpp b/test/pinverse.cpp
index 44a0f884b0..7258558bc2 100644
--- a/test/pinverse.cpp
+++ b/test/pinverse.cpp
@@ -48,8 +48,8 @@ array readTestInput(string testFilePath) {
     dtype outAfType = (dtype)dtype_traits<T>::af_type;
 
     vector<dim4> dimsVec;
-    vector<vector<InBaseType> > inVec;
-    vector<vector<InBaseType> > goldVec;
+    vector<vector<InBaseType>> inVec;
+    vector<vector<InBaseType>> goldVec;
     readTestsFromFile<InBaseType, InBaseType>(testFilePath, dimsVec, inVec,
                                               goldVec);
     dim4 inDims = dimsVec[0];
@@ -67,8 +67,8 @@ array readTestGold(string testFilePath) {
     dtype outAfType = (dtype)dtype_traits<T>::af_type;
 
     vector<dim4> dimsVec;
-    vector<vector<InBaseType> > inVec;
-    vector<vector<InBaseType> > goldVec;
+    vector<vector<InBaseType>> inVec;
+    vector<vector<InBaseType>> goldVec;
     readTestsFromFile<InBaseType, InBaseType>(testFilePath, dimsVec, inVec,
                                               goldVec);
     dim4 goldDims(dimsVec[0][1], dimsVec[0][0]);
diff --git a/test/qr_dense.cpp b/test/qr_dense.cpp
index 09477dcbf5..9d5f3f1c78 100644
--- a/test/qr_dense.cpp
+++ b/test/qr_dense.cpp
@@ -39,8 +39,8 @@ TEST(QRFactorized, CPP) {
     int resultIdx = 0;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/lapack/qrfactorized.test"),
                                    numDims, in, tests);
 
diff --git a/test/rank_dense.cpp b/test/rank_dense.cpp
index 30c7ade1ca..bb838686f5 100644
--- a/test/rank_dense.cpp
+++ b/test/rank_dense.cpp
@@ -99,8 +99,8 @@ void detTest() {
 
     vector<dim4> numDims;
 
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/lapack/detSmall.test"),
                                    numDims, in, tests);
     dim4 dims = numDims[0];
diff --git a/test/reduce.cpp b/test/reduce.cpp
index bfff42959f..31845b8d0c 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -51,8 +51,8 @@ void reduceTest(string pTestFile, int off = 0, bool isSubRef = false,
 
     vector<dim4> numDims;
 
-    vector<vector<int> > data;
-    vector<vector<int> > tests;
+    vector<vector<int>> data;
+    vector<vector<int>> tests;
     readTests<int, int, int>(pTestFile, numDims, data, tests);
     dim4 dims = numDims[0];
 
@@ -217,8 +217,8 @@ void cppReduceTest(string pTestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<int> > data;
-    vector<vector<int> > tests;
+    vector<vector<int>> data;
+    vector<vector<int>> tests;
     readTests<int, int, int>(pTestFile, numDims, data, tests);
     dim4 dims = numDims[0];
 
@@ -507,7 +507,7 @@ vector<reduce_by_key_params*> genSingleKeyTests() {
 
 vector<reduce_by_key_params *> generateAllTypes() {
     vector<reduce_by_key_params *> out;
-    vector<vector<reduce_by_key_params *> > tmp{
+    vector<vector<reduce_by_key_params *>> tmp{
         genUniqueKeyTests<int, float, float>(),
         genSingleKeyTests<int, float, float>(),
         genUniqueKeyTests<unsigned, float, float>(),
@@ -593,8 +593,8 @@ TEST(ReduceByKey, MultiBlockReduceSingleval) {
 
 void reduce_by_key_test(std::string test_fn) {
     vector<dim4> numDims;
-    vector<vector<float> > data;
-    vector<vector<float> > tests;
+    vector<vector<float>> data;
+    vector<vector<float>> tests;
     readTests<float, float, float>(test_fn, numDims, data, tests);
 
     for (size_t t = 0; t < numDims.size() / 2; ++t) {
@@ -1112,7 +1112,7 @@ TEST(MinMax, MinCplxNaN) {
 
     array min_val = af::min(a);
 
-    vector<complex<float> > h_min_val(cols);
+    vector<complex<float>> h_min_val(cols);
     min_val.host(&h_min_val[0]);
 
     for (int i = 0; i < cols; i++) {
@@ -1148,7 +1148,7 @@ TEST(MinMax, MaxCplxNaN) {
 
     array max_val = af::max(a);
 
-    vector<complex<float> > h_max_val(cols);
+    vector<complex<float>> h_max_val(cols);
     max_val.host(&h_max_val[0]);
 
     for (int i = 0; i < cols; i++) {
diff --git a/test/regions.cpp b/test/regions.cpp
index 4df7b90793..182a22e9b5 100644
--- a/test/regions.cpp
+++ b/test/regions.cpp
@@ -47,8 +47,8 @@ void regionsTest(string pTestFile, af_connectivity connectivity,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<uchar> > in;
-    vector<vector<T> > tests;
+    vector<vector<uchar>> in;
+    vector<vector<T>> tests;
     readTests<uchar, T, int>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -110,8 +110,8 @@ REGIONS_INIT(Regions3, regions_128x128, 8, AF_CONNECTIVITY_8);
 //
 TEST(Regions, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/regions/regions_8x8_4.test"),
                                  numDims, in, tests);
 
diff --git a/test/reorder.cpp b/test/reorder.cpp
index 6652f75210..b06f72cdda 100644
--- a/test/reorder.cpp
+++ b/test/reorder.cpp
@@ -57,8 +57,8 @@ void reorderTest(string pTestFile, const unsigned resultIdx, const uint x,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -141,8 +141,8 @@ TEST(Reorder, CPP) {
     const unsigned w         = 3;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/reorder/reorder4d.test"),
                                  numDims, in, tests);
 
diff --git a/test/resize.cpp b/test/resize.cpp
index 816dd7cf9e..423bb55416 100644
--- a/test/resize.cpp
+++ b/test/resize.cpp
@@ -119,8 +119,8 @@ void resizeTest(string pTestFile, const unsigned resultIdx, const dim_t odim0,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
     dim4 dims = numDims[0];
@@ -320,8 +320,8 @@ void resizeArgsTest(af_err err, string pTestFile, const dim4 odims,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
     dim4 dims = numDims[0];
@@ -363,8 +363,8 @@ using af::span;
 
 TEST(Resize, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/resize/square.test"),
                                    numDims, in, tests);
 
@@ -378,8 +378,8 @@ TEST(Resize, CPP) {
 
 TEST(ResizeScale1, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/resize/square.test"),
                                    numDims, in, tests);
 
@@ -393,8 +393,8 @@ TEST(ResizeScale1, CPP) {
 
 TEST(ResizeScale2, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/resize/square.test"),
                                    numDims, in, tests);
 
diff --git a/test/rotate.cpp b/test/rotate.cpp
index 31019db269..01675fa1d7 100644
--- a/test/rotate.cpp
+++ b/test/rotate.cpp
@@ -48,8 +48,8 @@ void rotateTest(string pTestFile, const unsigned resultIdx, const float angle,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
     dim4 dims = numDims[0];
@@ -164,8 +164,8 @@ TEST(Rotate, CPP) {
     const bool crop          = false;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(string(TEST_DIR "/rotate/rotate1.test"),
                                    numDims, in, tests);
 
diff --git a/test/rotate_linear.cpp b/test/rotate_linear.cpp
index 7d0dc8d5b7..ea19f217e7 100644
--- a/test/rotate_linear.cpp
+++ b/test/rotate_linear.cpp
@@ -54,8 +54,8 @@ void rotateTest(string pTestFile, const unsigned resultIdx, const float angle,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, float>(pTestFile, numDims, in, tests);
 
     dim4 dims = numDims[0];
@@ -182,8 +182,8 @@ TEST(RotateLinear, CPP) {
     const bool crop          = false;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, float>(
         string(TEST_DIR "/rotate/rotatelinear1.test"), numDims, in, tests);
 
diff --git a/test/scan.cpp b/test/scan.cpp
index cc42624ba9..a29c6e0e52 100644
--- a/test/scan.cpp
+++ b/test/scan.cpp
@@ -48,8 +48,8 @@ void scanTest(string pTestFile, int off = 0, bool isSubRef = false,
 
     vector<dim4> numDims;
 
-    vector<vector<int> > data;
-    vector<vector<int> > tests;
+    vector<vector<int>> data;
+    vector<vector<int>> tests;
     readTests<int, int, int>(pTestFile, numDims, data, tests);
     dim4 dims = numDims[0];
 
@@ -129,8 +129,8 @@ TEST(Scan, Test_Scan_Big1) {
 TEST(Accum, CPP) {
     vector<dim4> numDims;
 
-    vector<vector<int> > data;
-    vector<vector<int> > tests;
+    vector<vector<int>> data;
+    vector<vector<int>> tests;
     readTests<int, int, int>(string(TEST_DIR "/scan/accum.test"), numDims, data,
                              tests);
     dim4 dims = numDims[0];
diff --git a/test/set.cpp b/test/set.cpp
index f085da33b3..97e05d484b 100644
--- a/test/set.cpp
+++ b/test/set.cpp
@@ -32,8 +32,8 @@ void uniqueTest(string pTestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<int> > data;
-    vector<vector<int> > tests;
+    vector<vector<int>> data;
+    vector<vector<int>> tests;
     readTests<int, int, int>(pTestFile, numDims, data, tests);
 
     // Compare result
@@ -92,8 +92,8 @@ void setTest(string pTestFile) {
 
     vector<dim4> numDims;
 
-    vector<vector<int> > data;
-    vector<vector<int> > tests;
+    vector<vector<int>> data;
+    vector<vector<int>> tests;
     readTests<int, int, int>(pTestFile, numDims, data, tests);
 
     // Compare result
diff --git a/test/shift.cpp b/test/shift.cpp
index 91df07c39c..b37385a6f8 100644
--- a/test/shift.cpp
+++ b/test/shift.cpp
@@ -54,8 +54,8 @@ void shiftTest(string pTestFile, const unsigned resultIdx, const int x,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -118,8 +118,8 @@ TEST(Shift, CPP) {
     const unsigned w         = 0;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/shift/shift4d.test"),
                                  numDims, in, tests);
 
diff --git a/test/sift.cpp b/test/sift.cpp
index 90d3b40cdc..2410472b53 100644
--- a/test/sift.cpp
+++ b/test/sift.cpp
@@ -65,7 +65,7 @@ static void array_to_feat_desc(vector<feat_desc_t>& feat, float* x, float* y,
 
 static void array_to_feat_desc(vector<feat_desc_t>& feat, float* x, float* y,
                                float* score, float* ori, float* size,
-                               vector<vector<float> >& desc, unsigned nfeat) {
+                               vector<vector<float>>& desc, unsigned nfeat) {
     feat.resize(nfeat);
     for (size_t i = 0; i < feat.size(); i++) {
         feat[i].f[0] = x[i];
@@ -142,8 +142,8 @@ void siftTest(string pTestFile, unsigned nLayers, float contrastThr,
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > goldFeat;
-    vector<vector<float> > goldDesc;
+    vector<vector<float>> goldFeat;
+    vector<vector<float>> goldDesc;
 
     readImageFeaturesDescriptors<float>(pTestFile, inDims, inFiles, goldFeat,
                                         goldDesc);
@@ -276,8 +276,8 @@ TEST(SIFT, CPP) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > goldFeat;
-    vector<vector<float> > goldDesc;
+    vector<vector<float>> goldFeat;
+    vector<vector<float>> goldDesc;
 
     readImageFeaturesDescriptors<float>(string(TEST_DIR "/sift/man.test"),
                                         inDims, inFiles, goldFeat, goldDesc);
diff --git a/test/sobel.cpp b/test/sobel.cpp
index 449722af38..298d36d299 100644
--- a/test/sobel.cpp
+++ b/test/sobel.cpp
@@ -47,8 +47,8 @@ void testSobelDerivatives(string pTestFile) {
     SUPPORTED_TYPE_CHECK(Ti);
 
     vector<dim4> numDims;
-    vector<vector<Ti> > in;
-    vector<vector<To> > tests;
+    vector<vector<Ti>> in;
+    vector<vector<To>> tests;
 
     readTests<Ti, To, int>(pTestFile, numDims, in, tests);
 
diff --git a/test/sort.cpp b/test/sort.cpp
index 307573d7a0..c9da609f93 100644
--- a/test/sort.cpp
+++ b/test/sort.cpp
@@ -53,8 +53,8 @@ void sortTest(string pTestFile, const bool dir, const unsigned resultIdx0,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<float> > tests;
+    vector<vector<T>> in;
+    vector<vector<float>> tests;
     readTests<T, float, int>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -129,8 +129,8 @@ TEST(Sort, CPPDim0) {
     const unsigned resultIdx0 = 0;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/sort/sort_10x10.test"),
                                  numDims, in, tests);
 
@@ -160,8 +160,8 @@ TEST(Sort, CPPDim1) {
     const unsigned resultIdx0 = 0;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/sort/sort_10x10.test"),
                                  numDims, in, tests);
 
@@ -196,8 +196,8 @@ TEST(Sort, CPPDim2) {
     const unsigned resultIdx0 = 2;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/sort/sort_med.test"),
                                  numDims, in, tests);
 
diff --git a/test/sort_by_key.cpp b/test/sort_by_key.cpp
index b76e31ffbf..afd7908660 100644
--- a/test/sort_by_key.cpp
+++ b/test/sort_by_key.cpp
@@ -53,8 +53,8 @@ void sortTest(string pTestFile, const bool dir, const unsigned resultIdx0,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -126,8 +126,8 @@ TEST(SortByKey, CPPDim0) {
     const unsigned resultIdx1 = 1;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/sort/sort_by_key_tiny.test"),
                                  numDims, in, tests);
 
@@ -147,8 +147,8 @@ TEST(SortByKey, CPPDim1) {
     const unsigned resultIdx1 = 1;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(
         string(TEST_DIR "/sort/sort_by_key_large.test"), numDims, in, tests);
 
@@ -175,8 +175,8 @@ TEST(SortByKey, CPPDim2) {
     const unsigned resultIdx1 = 3;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(
         string(TEST_DIR "/sort/sort_by_key_large.test"), numDims, in, tests);
 
diff --git a/test/sort_index.cpp b/test/sort_index.cpp
index bfec5b429b..f3a10b9084 100644
--- a/test/sort_index.cpp
+++ b/test/sort_index.cpp
@@ -54,8 +54,8 @@ void sortTest(string pTestFile, const bool dir, const unsigned resultIdx0,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<float> > tests;
+    vector<vector<T>> in;
+    vector<vector<float>> tests;
     readTests<T, float, int>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -130,8 +130,8 @@ TEST(SortIndex, CPPDim0) {
     const unsigned resultIdx1 = 1;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/sort/sort_10x10.test"),
                                  numDims, in, tests);
 
@@ -155,8 +155,8 @@ TEST(SortIndex, CPPDim1) {
     const unsigned resultIdx1 = 1;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/sort/sort_10x10.test"),
                                  numDims, in, tests);
 
@@ -182,8 +182,8 @@ TEST(SortIndex, CPPDim2) {
     const unsigned resultIdx1 = 3;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/sort/sort_med.test"),
                                  numDims, in, tests);
 
diff --git a/test/stdev.cpp b/test/stdev.cpp
index 85f3bf079d..4b93f5b220 100644
--- a/test/stdev.cpp
+++ b/test/stdev.cpp
@@ -81,8 +81,8 @@ void stdevDimTest(string pFileName, dim_t dim,
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<int> > in;
-    vector<vector<float> > tests;
+    vector<vector<int>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<int, float>(pFileName, numDims, in, tests);
 
@@ -157,8 +157,8 @@ void stdevDimIndexTest(string pFileName, dim_t dim,
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<int> > in;
-    vector<vector<float> > tests;
+    vector<vector<int>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<int, float>(pFileName, numDims, in, tests);
 
@@ -212,8 +212,8 @@ void stdevAllTest(string pFileName, const bool useDeprecatedAPI = false) {
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<int> > in;
-    vector<vector<float> > tests;
+    vector<vector<int>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<int, float>(pFileName, numDims, in, tests);
 
diff --git a/test/susan.cpp b/test/susan.cpp
index 6d40177132..9bdc16d3d9 100644
--- a/test/susan.cpp
+++ b/test/susan.cpp
@@ -71,7 +71,7 @@ void susanTest(string pTestFile, float t, float g) {
 
     vector<dim4> inDims;
     vector<string> inFiles;
-    vector<vector<float> > gold;
+    vector<vector<float>> gold;
 
     readImageTests(pTestFile, inDims, inFiles, gold);
 
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 035c76991b..faf7162a3b 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -100,14 +100,14 @@ extern template af_half convert(int in);
 
 template<typename inType, typename outType, typename FileElementType>
 void readTests(const std::string &FileName, std::vector<af::dim4> &inputDims,
-               std::vector<std::vector<inType> > &testInputs,
-               std::vector<std::vector<outType> > &testOutputs);
+               std::vector<std::vector<inType>> &testInputs,
+               std::vector<std::vector<outType>> &testOutputs);
 
 template<typename inType, typename outType>
 void readTestsFromFile(const std::string &FileName,
                        std::vector<af::dim4> &inputDims,
-                       std::vector<std::vector<inType> > &testInputs,
-                       std::vector<std::vector<outType> > &testOutputs);
+                       std::vector<std::vector<inType>> &testInputs,
+                       std::vector<std::vector<outType>> &testOutputs);
 
 void readImageTests(const std::string &pFileName,
                     std::vector<af::dim4> &pInputDims,
@@ -119,14 +119,14 @@ template<typename outType>
 void readImageTests(const std::string &pFileName,
                     std::vector<af::dim4> &pInputDims,
                     std::vector<std::string> &pTestInputs,
-                    std::vector<std::vector<outType> > &pTestOutputs);
+                    std::vector<std::vector<outType>> &pTestOutputs);
 
 template<typename descType>
 void readImageFeaturesDescriptors(
     const std::string &pFileName, std::vector<af::dim4> &pInputDims,
     std::vector<std::string> &pTestInputs,
-    std::vector<std::vector<float> > &pTestFeats,
-    std::vector<std::vector<descType> > &pTestDescs);
+    std::vector<std::vector<float>> &pTestFeats,
+    std::vector<std::vector<descType>> &pTestDescs);
 
 /**
  * Below is not a pair wise comparition method, rather
diff --git a/test/threading.cpp b/test/threading.cpp
index daf613070e..f26047ce95 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -257,8 +257,8 @@ void fftTest(int targetDevice, string pTestFile, dim_t pad0 = 0, dim_t pad1 = 0,
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<inType> > in;
-    vector<vector<outType> > tests;
+    vector<vector<inType>> in;
+    vector<vector<outType>> tests;
 
     readTestsFromFile<inType, outType>(pTestFile, numDims, in, tests);
 
@@ -580,8 +580,8 @@ void cppMatMulCheck(int targetDevice, string TestFile) {
     using std::vector;
     vector<dim4> numDims;
 
-    vector<vector<T> > hData;
-    vector<vector<T> > tests;
+    vector<vector<T>> hData;
+    vector<vector<T>> tests;
     readTests<T, T, int>(TestFile, numDims, hData, tests);
 
     setDevice(targetDevice);
diff --git a/test/tile.cpp b/test/tile.cpp
index 0a649d00ac..bc0cdddba7 100644
--- a/test/tile.cpp
+++ b/test/tile.cpp
@@ -61,8 +61,8 @@ void tileTest(string pTestFile, const unsigned resultIdx, const uint x,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -128,8 +128,8 @@ TEST(Tile, CPP) {
     const unsigned w         = 1;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/tile/tile_large3D.test"),
                                  numDims, in, tests);
 
diff --git a/test/transform.cpp b/test/transform.cpp
index 77cdcfc881..b7719d46fc 100644
--- a/test/transform.cpp
+++ b/test/transform.cpp
@@ -62,8 +62,8 @@ void genTestData(af_array *gold, af_array *in, af_array *transform,
     dim4 objDims = inNumDims[0];
 
     vector<dim4> HNumDims;
-    vector<vector<float> > HIn;
-    vector<vector<float> > HTests;
+    vector<vector<float>> HIn;
+    vector<vector<float>> HTests;
     readTests<float, float, float>(pHomographyFile, HNumDims, HIn, HTests);
 
     dim4 HDims = HNumDims[0];
@@ -489,8 +489,8 @@ TEST(Transform, CPP) {
     vector<string> goldFiles;
 
     vector<dim4> HDims;
-    vector<vector<float> > HIn;
-    vector<vector<float> > HTests;
+    vector<vector<float>> HIn;
+    vector<vector<float>> HTests;
     readTests<float, float, float>(TEST_DIR "/transform/tux_tmat.test", HDims,
                                    HIn, HTests);
 
@@ -543,8 +543,8 @@ TEST(Transform, CPP) {
 // This test simply makes sure the batching is working correctly
 TEST(TransformBatching, CPP) {
     vector<dim4> vDims;
-    vector<vector<float> > in;
-    vector<vector<float> > gold;
+    vector<vector<float>> in;
+    vector<vector<float>> gold;
 
     readTests<float, float, int>(
         string(TEST_DIR "/transform/transform_batching.test"), vDims, in, gold);
diff --git a/test/transform_coordinates.cpp b/test/transform_coordinates.cpp
index 01ab960e93..2875f18c1a 100644
--- a/test/transform_coordinates.cpp
+++ b/test/transform_coordinates.cpp
@@ -38,8 +38,8 @@ void transformCoordinatesTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> inDims;
-    vector<vector<T> > in;
-    vector<vector<float> > gold;
+    vector<vector<T>> in;
+    vector<vector<float>> gold;
 
     readTests<T, float, float>(pTestFile, inDims, in, gold);
 
@@ -89,8 +89,8 @@ TYPED_TEST(TransformCoordinates, 3DMatrix) {
 //
 TEST(TransformCoordinates, CPP) {
     vector<dim4> inDims;
-    vector<vector<float> > in;
-    vector<vector<float> > gold;
+    vector<vector<float>> in;
+    vector<vector<float>> gold;
 
     readTests<float, float, float>(
         TEST_DIR "/transformCoordinates/3d_matrix.test", inDims, in, gold);
diff --git a/test/translate.cpp b/test/translate.cpp
index 4c84b19009..55fd570ffb 100644
--- a/test/translate.cpp
+++ b/test/translate.cpp
@@ -52,8 +52,8 @@ void translateTest(string pTestFile, const unsigned resultIdx, dim4 odims,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<float> > tests;
+    vector<vector<T>> in;
+    vector<vector<float>> tests;
     readTests<T, float, float>(pTestFile, numDims, in, tests);
 
     af_array inArray  = 0;
diff --git a/test/transpose.cpp b/test/transpose.cpp
index cb36640885..8bc0c1c6e9 100644
--- a/test/transpose.cpp
+++ b/test/transpose.cpp
@@ -58,8 +58,8 @@ void trsTest(string pTestFile, bool isSubRef = false,
 
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
     dim4 dims = numDims[0];
 
@@ -157,8 +157,8 @@ template<typename T>
 void trsCPPTest(string pFileName) {
     vector<dim4> numDims;
 
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pFileName, numDims, in, tests);
     dim4 dims = numDims[0];
 
diff --git a/test/unwrap.cpp b/test/unwrap.cpp
index b33dc8c7d5..f43b73e7f4 100644
--- a/test/unwrap.cpp
+++ b/test/unwrap.cpp
@@ -50,8 +50,8 @@ void unwrapTest(string pTestFile, const unsigned resultIdx, const dim_t wx,
     SUPPORTED_TYPE_CHECK(T);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<T> > tests;
+    vector<vector<T>> in;
+    vector<vector<T>> tests;
     readTests<T, T, int>(pTestFile, numDims, in, tests);
 
     dim4 idims = numDims[0];
@@ -161,8 +161,8 @@ TEST(Unwrap, CPP) {
     const unsigned py        = 3;
 
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
     readTests<float, float, int>(string(TEST_DIR "/unwrap/unwrap_small.test"),
                                  numDims, in, tests);
 
diff --git a/test/var.cpp b/test/var.cpp
index 45c7b6847f..db846f5d57 100644
--- a/test/var.cpp
+++ b/test/var.cpp
@@ -126,8 +126,8 @@ void dimCppSmallTest(const string pFileName,
     SUPPORTED_TYPE_CHECK(outType);
 
     vector<dim4> numDims;
-    vector<vector<T> > in;
-    vector<vector<outType> > tests;
+    vector<vector<T>> in;
+    vector<vector<outType>> tests;
 
     readTests<T, outType, float>(pFileName, numDims, in, tests);
 
@@ -148,7 +148,7 @@ void dimCppSmallTest(const string pFileName,
                               : var(input, AF_VARIANCE_POPULATION, 1));
 #pragma GCC diagnostic pop
 
-        vector<vector<outType> > h_out(4);
+        vector<vector<outType>> h_out(4);
 
         h_out[0].resize(bout.elements());
         h_out[1].resize(nbout.elements());
diff --git a/test/where.cpp b/test/where.cpp
index 746a9aa5b4..bb5375822c 100644
--- a/test/where.cpp
+++ b/test/where.cpp
@@ -45,8 +45,8 @@ void whereTest(string pTestFile, bool isSubRef = false,
 
     vector<dim4> numDims;
 
-    vector<vector<int> > data;
-    vector<vector<int> > tests;
+    vector<vector<int>> data;
+    vector<vector<int>> tests;
     readTests<int, int, int>(pTestFile, numDims, data, tests);
     dim4 dims = numDims[0];
 
@@ -99,8 +99,8 @@ TYPED_TEST(Where, CPP) {
 
     vector<dim4> numDims;
 
-    vector<vector<int> > data;
-    vector<vector<int> > tests;
+    vector<vector<int>> data;
+    vector<vector<int>> tests;
     readTests<int, int, int>(string(TEST_DIR "/where/where.test"), numDims,
                              data, tests);
     dim4 dims = numDims[0];
diff --git a/test/ycbcr_rgb.cpp b/test/ycbcr_rgb.cpp
index e137e1ede0..ec365db9a4 100644
--- a/test/ycbcr_rgb.cpp
+++ b/test/ycbcr_rgb.cpp
@@ -37,8 +37,8 @@ TEST(ycbcr_rgb, InvalidArray) {
 
 TEST(ycbcr2rgb, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(
         string(TEST_DIR "/ycbcr_rgb/ycbcr2rgb.test"), numDims, in, tests);
@@ -60,8 +60,8 @@ TEST(ycbcr2rgb, CPP) {
 
 TEST(ycbcr2rgb, MaxDim) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(
         string(TEST_DIR "/ycbcr_rgb/ycbcr2rgb.test"), numDims, in, tests);
@@ -98,8 +98,8 @@ TEST(ycbcr2rgb, MaxDim) {
 
 TEST(rgb2ycbcr, CPP) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(
         string(TEST_DIR "/ycbcr_rgb/rgb2ycbcr.test"), numDims, in, tests);
@@ -121,8 +121,8 @@ TEST(rgb2ycbcr, CPP) {
 
 TEST(rgb2ycbcr, MaxDim) {
     vector<dim4> numDims;
-    vector<vector<float> > in;
-    vector<vector<float> > tests;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;
 
     readTestsFromFile<float, float>(
         string(TEST_DIR "/ycbcr_rgb/rgb2ycbcr.test"), numDims, in, tests);

From 4a96346298ba6ad136ba306cfc984f6114ed7a8b Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 30 Sep 2022 18:06:06 -0400
Subject: [PATCH 086/473] Fix issue with multiple definition of symbols in
 tests on Windows

---
 test/CMakeLists.txt | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 2a66ea8291..d1bbebbdeb 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -100,7 +100,7 @@ if(AF_BUILD_UNIFIED)
 endif(AF_BUILD_UNIFIED)
 
 
-add_library(arrayfire_test OBJECT
+add_library(arrayfire_test STATIC
   testHelpers.hpp
   arrayfire_test.cpp)
 
@@ -110,9 +110,7 @@ target_include_directories(arrayfire_test
     ${ArrayFire_SOURCE_DIR}/include
     ${ArrayFire_BINARY_DIR}/include
     ${ArrayFire_SOURCE_DIR}/extern/half/include
-    mmio
-    $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
-    $<TARGET_PROPERTY:GTest::gtest,INTERFACE_INCLUDE_DIRECTORIES>)
+  )
 
 if(WIN32)
   target_compile_options(arrayfire_test
@@ -130,6 +128,14 @@ target_compile_definitions(arrayfire_test
     TEST_RESULT_IMAGE_DIR="${CMAKE_BINARY_DIR}/test/"
     USE_MTX)
 
+target_link_libraries(arrayfire_test
+  PRIVATE
+    mmio
+  PUBLIC
+    GTest::gtest
+    Boost::boost
+  )
+
 # Creates tests for all backends
 #
 # Creates a standard test for all backends. Most of the time you only need to
@@ -158,11 +164,7 @@ function(make_test)
     endif()
     set(target "test_${src_name}_${backend}")
 
-    if (${mt_args_NO_ARRAYFIRE_TEST})
-      add_executable(${target} ${mt_args_SRC})
-    else()
-      add_executable(${target} ${mt_args_SRC} $<TARGET_OBJECTS:arrayfire_test>)
-    endif()
+    add_executable(${target} ${mt_args_SRC})
     target_include_directories(${target}
       PRIVATE
         ${ArrayFire_SOURCE_DIR}/extern/half/include
@@ -172,7 +174,7 @@ function(make_test)
     target_link_libraries(${target}
       PRIVATE
         ${mt_args_LIBRARIES}
-	      GTest::gtest
+        arrayfire_test
       )
 
     if(${backend} STREQUAL "unified")
@@ -346,7 +348,7 @@ if(CUDA_FOUND)
           ${CMAKE_CURRENT_SOURCE_DIR}
         )
       endif()
-      add_executable(${target} cuda.cu  $<TARGET_OBJECTS:arrayfire_test>)
+      add_executable(${target} cuda.cu)
       target_include_directories(${target} PRIVATE
         ${ArrayFire_SOURCE_DIR}/extern/half/include
         ${CMAKE_SOURCE_DIR}
@@ -360,7 +362,7 @@ if(CUDA_FOUND)
       endif()
       target_link_libraries(${target}
         mmio
-        GTest::gtest)
+        arrayfire_test)
 
       # Couldn't get Threads::Threads to work with this cuda binary. The import
       # target would not add the -pthread flag which is required for this

From 93017c6f3a1fc269ee7860e529cf24e50fc50e36 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 1 Oct 2022 15:23:15 -0400
Subject: [PATCH 087/473] Reorder Error classes' members to reduce padding

---
 src/backend/common/err_common.cpp | 20 ++++++++++----------
 src/backend/common/err_common.hpp | 12 ++++++------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index 21e7b7212b..7a19bcb941 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -38,9 +38,9 @@ AfError::AfError(const char *const func, const char *const file, const int line,
     : logic_error(message)
     , functionName(func)
     , fileName(file)
+    , st_(move(st))
     , lineNumber(line)
-    , error(err)
-    , st_(move(st)) {}
+    , error(err) {}
 
 AfError::AfError(string func, string file, const int line,
                  const string &message, af_err err,
@@ -48,9 +48,9 @@ AfError::AfError(string func, string file, const int line,
     : logic_error(message)
     , functionName(move(func))
     , fileName(move(file))
+    , st_(move(st))
     , lineNumber(line)
-    , error(err)
-    , st_(move(st)) {}
+    , error(err) {}
 
 const string &AfError::getFunctionName() const noexcept { return functionName; }
 
@@ -66,8 +66,8 @@ TypeError::TypeError(const char *const func, const char *const file,
                      const int line, const int index, const af_dtype type,
                      boost::stacktrace::stacktrace st)
     : AfError(func, file, line, "Invalid data type", AF_ERR_TYPE, move(st))
-    , argIndex(index)
-    , errTypeName(getName(type)) {}
+    , errTypeName(getName(type))
+    , argIndex(index) {}
 
 const string &TypeError::getTypeName() const noexcept { return errTypeName; }
 
@@ -78,8 +78,8 @@ ArgumentError::ArgumentError(const char *const func, const char *const file,
                              const char *const expectString,
                              boost::stacktrace::stacktrace st)
     : AfError(func, file, line, "Invalid argument", AF_ERR_ARG, move(st))
-    , argIndex(index)
-    , expected(expectString) {}
+    , expected(expectString)
+    , argIndex(index) {}
 
 const string &ArgumentError::getExpectedCondition() const noexcept {
     return expected;
@@ -101,8 +101,8 @@ DimensionError::DimensionError(const char *const func, const char *const file,
                                const char *const expectString,
                                const boost::stacktrace::stacktrace &st)
     : AfError(func, file, line, "Invalid size", AF_ERR_SIZE, st)
-    , argIndex(index)
-    , expected(expectString) {}
+    , expected(expectString)
+    , argIndex(index) {}
 
 const string &DimensionError::getExpectedCondition() const noexcept {
     return expected;
diff --git a/src/backend/common/err_common.hpp b/src/backend/common/err_common.hpp
index 65e25bb0c8..6adf600cf6 100644
--- a/src/backend/common/err_common.hpp
+++ b/src/backend/common/err_common.hpp
@@ -26,9 +26,9 @@
 class AfError : public std::logic_error {
     std::string functionName;
     std::string fileName;
+    boost::stacktrace::stacktrace st_;
     int lineNumber;
     af_err error;
-    boost::stacktrace::stacktrace st_;
     AfError();
 
    public:
@@ -49,9 +49,9 @@ class AfError : public std::logic_error {
         : std::logic_error(std::forward<std::logic_error>(other))
         , functionName(std::forward<std::string>(other.functionName))
         , fileName(std::forward<std::string>(other.fileName))
+        , st_(std::forward<boost::stacktrace::stacktrace>(other.st_))
         , lineNumber(std::forward<int>(other.lineNumber))
-        , error(std::forward<af_err>(other.error))
-        , st_(std::forward<boost::stacktrace::stacktrace>(other.st_)) {}
+        , error(std::forward<af_err>(other.error)) {}
 
     const std::string& getFunctionName() const noexcept;
 
@@ -70,8 +70,8 @@ class AfError : public std::logic_error {
 
 // TODO: Perhaps add a way to return supported types
 class TypeError : public AfError {
-    int argIndex;
     std::string errTypeName;
+    int argIndex;
     TypeError();
 
    public:
@@ -89,8 +89,8 @@ class TypeError : public AfError {
 };
 
 class ArgumentError : public AfError {
-    int argIndex;
     std::string expected;
+    int argIndex;
     ArgumentError();
 
    public:
@@ -123,8 +123,8 @@ class SupportError : public AfError {
 };
 
 class DimensionError : public AfError {
-    int argIndex;
     std::string expected;
+    int argIndex;
     DimensionError();
 
    public:

From 9848c9348bd4622fdcf2211ed875174564b451e6 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 1 Oct 2022 15:24:08 -0400
Subject: [PATCH 088/473] Update CTestCustom to show more error contexts

---
 CMakeModules/CTestCustom.cmake | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/CMakeModules/CTestCustom.cmake b/CMakeModules/CTestCustom.cmake
index 514a5ee4d8..604f697465 100644
--- a/CMakeModules/CTestCustom.cmake
+++ b/CMakeModules/CTestCustom.cmake
@@ -5,8 +5,11 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-set(CTEST_CUSTOM_ERROR_POST_CONTEXT 50)
-set(CTEST_CUSTOM_ERROR_PRE_CONTEXT 50)
+set(CTEST_CUSTOM_ERROR_POST_CONTEXT 200)
+set(CTEST_CUSTOM_ERROR_PRE_CONTEXT 200)
+set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS 300)
+set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 300)
+
 if(WIN32)
   if(CMAKE_GENERATOR MATCHES "Ninja")
     set(CTEST_CUSTOM_POST_TEST ./bin/print_info.exe)

From 3996a4a6ad08f0d895f8bc57d813e51f46826968 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 1 Oct 2022 16:14:34 -0400
Subject: [PATCH 089/473] Remove STATIC_ definition with the inline keyword

---
 src/backend/common/defines.hpp      |  2 -
 src/backend/cpu/binary.hpp          |  8 ++--
 src/backend/cpu/math.hpp            | 20 ++++-----
 src/backend/cuda/complex.hpp        |  8 ++--
 src/backend/cuda/kernel/regions.hpp |  6 +--
 src/backend/cuda/math.hpp           | 70 ++++++++++++++---------------
 src/backend/cuda/unary.hpp          |  8 ++--
 src/backend/opencl/complex.hpp      |  8 ++--
 src/backend/opencl/kernel/names.hpp | 14 +++---
 src/backend/opencl/math.hpp         | 28 ++++++------
 src/backend/opencl/traits.hpp       | 10 ++---
 src/backend/opencl/unary.hpp        |  8 ++--
 12 files changed, 92 insertions(+), 98 deletions(-)

diff --git a/src/backend/common/defines.hpp b/src/backend/common/defines.hpp
index 79f39c5061..c72c7b1b32 100644
--- a/src/backend/common/defines.hpp
+++ b/src/backend/common/defines.hpp
@@ -33,10 +33,8 @@ inline std::string clipFilePath(std::string path, std::string str) {
 #if _MSC_VER < 1900
 #define snprintf sprintf_s
 #endif
-#define STATIC_ static
 #define __AF_FILENAME__ (clipFilePath(__FILE__, "src\\").c_str())
 #else
-#define STATIC_ inline
 #define __AF_FILENAME__ (clipFilePath(__FILE__, "src/").c_str())
 #endif
 
diff --git a/src/backend/cpu/binary.hpp b/src/backend/cpu/binary.hpp
index 635b082d99..1af89bd3a6 100644
--- a/src/backend/cpu/binary.hpp
+++ b/src/backend/cpu/binary.hpp
@@ -98,19 +98,19 @@ static T __rem(T lhs, T rhs) {
 }
 
 template<>
-STATIC_ float __mod<float>(float lhs, float rhs) {
+inline float __mod<float>(float lhs, float rhs) {
     return fmod(lhs, rhs);
 }
 template<>
-STATIC_ double __mod<double>(double lhs, double rhs) {
+inline double __mod<double>(double lhs, double rhs) {
     return fmod(lhs, rhs);
 }
 template<>
-STATIC_ float __rem<float>(float lhs, float rhs) {
+inline float __rem<float>(float lhs, float rhs) {
     return remainder(lhs, rhs);
 }
 template<>
-STATIC_ double __rem<double>(double lhs, double rhs) {
+inline double __rem<double>(double lhs, double rhs) {
     return remainder(lhs, rhs);
 }
 
diff --git a/src/backend/cpu/math.hpp b/src/backend/cpu/math.hpp
index b01a11bb04..f55530f531 100644
--- a/src/backend/cpu/math.hpp
+++ b/src/backend/cpu/math.hpp
@@ -47,48 +47,48 @@ static inline T division(T lhs, double rhs) {
 }
 
 template<>
-STATIC_ cfloat division<cfloat>(cfloat lhs, double rhs) {
+inline cfloat division<cfloat>(cfloat lhs, double rhs) {
     cfloat retVal(real(lhs) / static_cast<float>(rhs),
                   imag(lhs) / static_cast<float>(rhs));
     return retVal;
 }
 
 template<>
-STATIC_ cdouble division<cdouble>(cdouble lhs, double rhs) {
+inline cdouble division<cdouble>(cdouble lhs, double rhs) {
     cdouble retVal(real(lhs) / rhs, imag(lhs) / rhs);
     return retVal;
 }
 
 template<typename T>
-STATIC_ T maxval() {
+inline T maxval() {
     return std::numeric_limits<T>::max();
 }
 template<typename T>
-STATIC_ T minval() {
+inline T minval() {
     return std::numeric_limits<T>::lowest();
 }
 template<>
-STATIC_ float maxval() {
+inline float maxval() {
     return std::numeric_limits<float>::infinity();
 }
 template<>
-STATIC_ double maxval() {
+inline double maxval() {
     return std::numeric_limits<double>::infinity();
 }
 template<>
-STATIC_ common::half maxval() {
+inline common::half maxval() {
     return std::numeric_limits<common::half>::infinity();
 }
 template<>
-STATIC_ float minval() {
+inline float minval() {
     return -std::numeric_limits<float>::infinity();
 }
 template<>
-STATIC_ double minval() {
+inline double minval() {
     return -std::numeric_limits<double>::infinity();
 }
 template<>
-STATIC_ common::half minval() {
+inline common::half minval() {
     return -std::numeric_limits<common::half>::infinity();
 }
 
diff --git a/src/backend/cuda/complex.hpp b/src/backend/cuda/complex.hpp
index 605ac51ccd..68b5313150 100644
--- a/src/backend/cuda/complex.hpp
+++ b/src/backend/cuda/complex.hpp
@@ -46,11 +46,11 @@ static const char *abs_name() {
     return "fabs";
 }
 template<>
-STATIC_ const char *abs_name<cfloat>() {
+inline const char *abs_name<cfloat>() {
     return "__cabsf";
 }
 template<>
-STATIC_ const char *abs_name<cdouble>() {
+inline const char *abs_name<cdouble>() {
     return "__cabs";
 }
 
@@ -69,11 +69,11 @@ static const char *conj_name() {
     return "__noop";
 }
 template<>
-STATIC_ const char *conj_name<cfloat>() {
+inline const char *conj_name<cfloat>() {
     return "__cconjf";
 }
 template<>
-STATIC_ const char *conj_name<cdouble>() {
+inline const char *conj_name<cdouble>() {
     return "__cconj";
 }
 
diff --git a/src/backend/cuda/kernel/regions.hpp b/src/backend/cuda/kernel/regions.hpp
index 4a9547ef35..7a459a6fb9 100644
--- a/src/backend/cuda/kernel/regions.hpp
+++ b/src/backend/cuda/kernel/regions.hpp
@@ -40,9 +40,9 @@ static inline __device__ T fetch(const int n, cuda::Param<T> equiv_map,
 }
 
 template<>
-__device__ STATIC_ double fetch<double>(const int n,
-                                        cuda::Param<double> equiv_map,
-                                        cudaTextureObject_t tex) {
+__device__ inline double fetch<double>(const int n,
+                                       cuda::Param<double> equiv_map,
+                                       cudaTextureObject_t tex) {
     return equiv_map.ptr[n];
 }
 
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index a0b77265f4..5987017fa7 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -9,11 +9,7 @@
 
 #pragma once
 
-#ifdef __CUDACC_RTC__
-
-#define STATIC_ inline
-
-#else  //__CUDACC_RTC__
+#ifndef __CUDACC_RTC__
 
 #include <common/defines.hpp>
 
@@ -99,22 +95,22 @@ static inline __DH__ T max(T lhs, T rhs) {
 #endif
 
 template<>
-__DH__ STATIC_ cfloat max<cfloat>(cfloat lhs, cfloat rhs) {
+__DH__ inline cfloat max<cfloat>(cfloat lhs, cfloat rhs) {
     return abs(lhs) > abs(rhs) ? lhs : rhs;
 }
 
 template<>
-__DH__ STATIC_ cdouble max<cdouble>(cdouble lhs, cdouble rhs) {
+__DH__ inline cdouble max<cdouble>(cdouble lhs, cdouble rhs) {
     return abs(lhs) > abs(rhs) ? lhs : rhs;
 }
 
 template<>
-__DH__ STATIC_ cfloat min<cfloat>(cfloat lhs, cfloat rhs) {
+__DH__ inline cfloat min<cfloat>(cfloat lhs, cfloat rhs) {
     return abs(lhs) < abs(rhs) ? lhs : rhs;
 }
 
 template<>
-__DH__ STATIC_ cdouble min<cdouble>(cdouble lhs, cdouble rhs) {
+__DH__ inline cdouble min<cdouble>(cdouble lhs, cdouble rhs) {
     return abs(lhs) < abs(rhs) ? lhs : rhs;
 }
 
@@ -124,13 +120,13 @@ __DH__ static T scalar(double val) {
 }
 
 template<>
-__DH__ STATIC_ cfloat scalar<cfloat>(double val) {
+__DH__ inline cfloat scalar<cfloat>(double val) {
     cfloat cval = {(float)val, 0};
     return cval;
 }
 
 template<>
-__DH__ STATIC_ cdouble scalar<cdouble>(double val) {
+__DH__ inline cdouble scalar<cdouble>(double val) {
     cdouble cval = {val, 0};
     return cval;
 }
@@ -143,109 +139,109 @@ __DH__ static To scalar(Ti real, Ti imag) {
 
 #ifndef __CUDA_ARCH__
 template<typename T>
-STATIC_ T maxval() {
+inline T maxval() {
     return std::numeric_limits<T>::max();
 }
 template<typename T>
-STATIC_ T minval() {
+inline T minval() {
     return std::numeric_limits<T>::min();
 }
 template<>
-STATIC_ float maxval() {
+inline float maxval() {
     return std::numeric_limits<float>::infinity();
 }
 template<>
-STATIC_ double maxval() {
+inline double maxval() {
     return std::numeric_limits<double>::infinity();
 }
 template<>
-STATIC_ float minval() {
+inline float minval() {
     return -std::numeric_limits<float>::infinity();
 }
 template<>
-STATIC_ double minval() {
+inline double minval() {
     return -std::numeric_limits<double>::infinity();
 }
 #else
 template<typename T>
-STATIC_ __device__ T maxval() {
+inline __device__ T maxval() {
     return 1u << (8 * sizeof(T) - 1);
 }
 template<typename T>
-STATIC_ __device__ T minval() {
+inline __device__ T minval() {
     return scalar<T>(0);
 }
 
 template<>
-STATIC_ __device__ int maxval<int>() {
+inline __device__ int maxval<int>() {
     return 0x7fffffff;
 }
 template<>
-STATIC_ __device__ int minval<int>() {
+inline __device__ int minval<int>() {
     return 0x80000000;
 }
 template<>
-STATIC_ __device__ intl maxval<intl>() {
+inline __device__ intl maxval<intl>() {
     return 0x7fffffffffffffff;
 }
 template<>
-STATIC_ __device__ intl minval<intl>() {
+inline __device__ intl minval<intl>() {
     return 0x8000000000000000;
 }
 template<>
-STATIC_ __device__ uintl maxval<uintl>() {
+inline __device__ uintl maxval<uintl>() {
     return 1ULL << (8 * sizeof(uintl) - 1);
 }
 template<>
-STATIC_ __device__ char maxval<char>() {
+inline __device__ char maxval<char>() {
     return 0x7f;
 }
 template<>
-STATIC_ __device__ char minval<char>() {
+inline __device__ char minval<char>() {
     return 0x80;
 }
 template<>
-STATIC_ __device__ float maxval<float>() {
+inline __device__ float maxval<float>() {
     return CUDART_INF_F;
 }
 template<>
-STATIC_ __device__ float minval<float>() {
+inline __device__ float minval<float>() {
     return -CUDART_INF_F;
 }
 template<>
-STATIC_ __device__ double maxval<double>() {
+inline __device__ double maxval<double>() {
     return CUDART_INF;
 }
 template<>
-STATIC_ __device__ double minval<double>() {
+inline __device__ double minval<double>() {
     return -CUDART_INF;
 }
 template<>
-STATIC_ __device__ short maxval<short>() {
+inline __device__ short maxval<short>() {
     return 0x7fff;
 }
 template<>
-STATIC_ __device__ short minval<short>() {
+inline __device__ short minval<short>() {
     return 0x8000;
 }
 template<>
-STATIC_ __device__ ushort maxval<ushort>() {
+inline __device__ ushort maxval<ushort>() {
     return ((ushort)1) << (8 * sizeof(ushort) - 1);
 }
 template<>
-STATIC_ __device__ common::half maxval<common::half>() {
+inline __device__ common::half maxval<common::half>() {
     return common::half(65537.f);
 }
 template<>
-STATIC_ __device__ common::half minval<common::half>() {
+inline __device__ common::half minval<common::half>() {
     return common::half(-65537.f);
 }
 template<>
-STATIC_ __device__ __half maxval<__half>() {
+inline __device__ __half maxval<__half>() {
     return __float2half(CUDART_INF);
 }
 template<>
-STATIC_ __device__ __half minval<__half>() {
+inline __device__ __half minval<__half>() {
     return __float2half(-CUDART_INF);
 }
 #endif
diff --git a/src/backend/cuda/unary.hpp b/src/backend/cuda/unary.hpp
index f060fd8190..a94c84dfa2 100644
--- a/src/backend/cuda/unary.hpp
+++ b/src/backend/cuda/unary.hpp
@@ -19,10 +19,10 @@ namespace cuda {
 template<af_op_t op>
 static const char *unaryName();
 
-#define UNARY_DECL(OP, FNAME)                      \
-    template<>                                     \
-    STATIC_ const char *unaryName<af_##OP##_t>() { \
-        return FNAME;                              \
+#define UNARY_DECL(OP, FNAME)                     \
+    template<>                                    \
+    inline const char *unaryName<af_##OP##_t>() { \
+        return FNAME;                             \
     }
 
 #define UNARY_FN(OP) UNARY_DECL(OP, #OP)
diff --git a/src/backend/opencl/complex.hpp b/src/backend/opencl/complex.hpp
index 3facc57090..124d3b49ca 100644
--- a/src/backend/opencl/complex.hpp
+++ b/src/backend/opencl/complex.hpp
@@ -47,11 +47,11 @@ static const char *abs_name() {
     return "fabs";
 }
 template<>
-STATIC_ const char *abs_name<cfloat>() {
+inline const char *abs_name<cfloat>() {
     return "__cabsf";
 }
 template<>
-STATIC_ const char *abs_name<cdouble>() {
+inline const char *abs_name<cdouble>() {
     return "__cabs";
 }
 
@@ -70,11 +70,11 @@ static const char *conj_name() {
     return "__noop";
 }
 template<>
-STATIC_ const char *conj_name<cfloat>() {
+inline const char *conj_name<cfloat>() {
     return "__cconjf";
 }
 template<>
-STATIC_ const char *conj_name<cdouble>() {
+inline const char *conj_name<cdouble>() {
     return "__cconj";
 }
 
diff --git a/src/backend/opencl/kernel/names.hpp b/src/backend/opencl/kernel/names.hpp
index 73489b1e10..2dc4e63254 100644
--- a/src/backend/opencl/kernel/names.hpp
+++ b/src/backend/opencl/kernel/names.hpp
@@ -17,30 +17,30 @@ static const char *binOpName() {
 }
 
 template<>
-STATIC_ const char *binOpName<af_add_t>() {
+inline const char *binOpName<af_add_t>() {
     return "ADD_OP";
 }
 template<>
-STATIC_ const char *binOpName<af_mul_t>() {
+inline const char *binOpName<af_mul_t>() {
     return "MUL_OP";
 }
 template<>
-STATIC_ const char *binOpName<af_and_t>() {
+inline const char *binOpName<af_and_t>() {
     return "AND_OP";
 }
 template<>
-STATIC_ const char *binOpName<af_or_t>() {
+inline const char *binOpName<af_or_t>() {
     return "OR_OP";
 }
 template<>
-STATIC_ const char *binOpName<af_min_t>() {
+inline const char *binOpName<af_min_t>() {
     return "MIN_OP";
 }
 template<>
-STATIC_ const char *binOpName<af_max_t>() {
+inline const char *binOpName<af_max_t>() {
     return "MAX_OP";
 }
 template<>
-STATIC_ const char *binOpName<af_notzero_t>() {
+inline const char *binOpName<af_notzero_t>() {
     return "NOTZERO_OP";
 }
diff --git a/src/backend/opencl/math.hpp b/src/backend/opencl/math.hpp
index 86ee50556d..e1e9c28f12 100644
--- a/src/backend/opencl/math.hpp
+++ b/src/backend/opencl/math.hpp
@@ -58,22 +58,22 @@ cfloat division(cfloat lhs, double rhs);
 cdouble division(cdouble lhs, double rhs);
 
 template<>
-STATIC_ cfloat max<cfloat>(cfloat lhs, cfloat rhs) {
+inline cfloat max<cfloat>(cfloat lhs, cfloat rhs) {
     return abs(lhs) > abs(rhs) ? lhs : rhs;
 }
 
 template<>
-STATIC_ cdouble max<cdouble>(cdouble lhs, cdouble rhs) {
+inline cdouble max<cdouble>(cdouble lhs, cdouble rhs) {
     return abs(lhs) > abs(rhs) ? lhs : rhs;
 }
 
 template<>
-STATIC_ cfloat min<cfloat>(cfloat lhs, cfloat rhs) {
+inline cfloat min<cfloat>(cfloat lhs, cfloat rhs) {
     return abs(lhs) < abs(rhs) ? lhs : rhs;
 }
 
 template<>
-STATIC_ cdouble min<cdouble>(cdouble lhs, cdouble rhs) {
+inline cdouble min<cdouble>(cdouble lhs, cdouble rhs) {
     return abs(lhs) < abs(rhs) ? lhs : rhs;
 }
 
@@ -83,7 +83,7 @@ static T scalar(double val) {
 }
 
 template<>
-STATIC_ cfloat scalar<cfloat>(double val) {
+inline cfloat scalar<cfloat>(double val) {
     cfloat cval;
     cval.s[0] = (float)val;
     cval.s[1] = 0;
@@ -91,7 +91,7 @@ STATIC_ cfloat scalar<cfloat>(double val) {
 }
 
 template<>
-STATIC_ cdouble scalar<cdouble>(double val) {
+inline cdouble scalar<cdouble>(double val) {
     cdouble cval;
     cval.s[0] = val;
     cval.s[1] = 0;
@@ -107,38 +107,38 @@ static To scalar(Ti real, Ti imag) {
 }
 
 template<typename T>
-STATIC_ T maxval() {
+inline T maxval() {
     return std::numeric_limits<T>::max();
 }
 template<typename T>
-STATIC_ T minval() {
+inline T minval() {
     return std::numeric_limits<T>::min();
 }
 template<>
-STATIC_ float maxval() {
+inline float maxval() {
     return std::numeric_limits<float>::infinity();
 }
 template<>
-STATIC_ double maxval() {
+inline double maxval() {
     return std::numeric_limits<double>::infinity();
 }
 
 template<>
-STATIC_ common::half maxval() {
+inline common::half maxval() {
     return std::numeric_limits<common::half>::infinity();
 }
 
 template<>
-STATIC_ float minval() {
+inline float minval() {
     return -std::numeric_limits<float>::infinity();
 }
 
 template<>
-STATIC_ double minval() {
+inline double minval() {
     return -std::numeric_limits<double>::infinity();
 }
 template<>
-STATIC_ common::half minval() {
+inline common::half minval() {
     return -std::numeric_limits<common::half>::infinity();
 }
 
diff --git a/src/backend/opencl/traits.hpp b/src/backend/opencl/traits.hpp
index 60a08831e7..6610c7aee1 100644
--- a/src/backend/opencl/traits.hpp
+++ b/src/backend/opencl/traits.hpp
@@ -37,30 +37,30 @@ static bool iscplx() {
     return false;
 }
 template<>
-STATIC_ bool iscplx<opencl::cfloat>() {
+inline bool iscplx<opencl::cfloat>() {
     return true;
 }
 template<>
-STATIC_ bool iscplx<opencl::cdouble>() {
+inline bool iscplx<opencl::cdouble>() {
     return true;
 }
 
 template<typename T>
-STATIC_ std::string scalar_to_option(const T &val) {
+inline std::string scalar_to_option(const T &val) {
     using namespace common;
     using namespace std;
     return to_string(+val);
 }
 
 template<>
-STATIC_ std::string scalar_to_option<cl_float2>(const cl_float2 &val) {
+inline std::string scalar_to_option<cl_float2>(const cl_float2 &val) {
     std::ostringstream ss;
     ss << val.s[0] << "," << val.s[1];
     return ss.str();
 }
 
 template<>
-STATIC_ std::string scalar_to_option<cl_double2>(const cl_double2 &val) {
+inline std::string scalar_to_option<cl_double2>(const cl_double2 &val) {
     std::ostringstream ss;
     ss << val.s[0] << "," << val.s[1];
     return ss.str();
diff --git a/src/backend/opencl/unary.hpp b/src/backend/opencl/unary.hpp
index f4a81ab29f..65da1b690b 100644
--- a/src/backend/opencl/unary.hpp
+++ b/src/backend/opencl/unary.hpp
@@ -18,10 +18,10 @@ namespace opencl {
 template<af_op_t op>
 static const char *unaryName();
 
-#define UNARY_DECL(OP, FNAME)                      \
-    template<>                                     \
-    STATIC_ const char *unaryName<af_##OP##_t>() { \
-        return FNAME;                              \
+#define UNARY_DECL(OP, FNAME)                     \
+    template<>                                    \
+    inline const char *unaryName<af_##OP##_t>() { \
+        return FNAME;                             \
     }
 
 #define UNARY_FN(OP) UNARY_DECL(OP, #OP)

From cf314568c6b18a489a93cf6f48a345dfdd7c3930 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 3 Oct 2022 12:35:32 -0400
Subject: [PATCH 090/473] Update vcpkg baseline hash and update vcpkg caching
 in GitHub actions

---
 .github/workflows/win_cpu_build.yml | 27 ++++++++++++++++++---------
 CMakeModules/AF_vcpkg_options.cmake |  2 ++
 vcpkg.json                          | 10 +++++++++-
 3 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/win_cpu_build.yml b/.github/workflows/win_cpu_build.yml
index 9d5419f7dd..dc73cf7c28 100644
--- a/.github/workflows/win_cpu_build.yml
+++ b/.github/workflows/win_cpu_build.yml
@@ -13,16 +13,14 @@ jobs:
         name: CPU (fftw, OpenBLAS, windows-latest)
         runs-on: windows-latest
         env:
-
-          VCPKG_HASH: 14e7bb4ae24616ec54ff6b2f6ef4e8659434ea44
-
+          VCPKG_HASH: 6ca56aeb457f033d344a7106cb3f9f1abf8f4e98
           VCPKG_DEFAULT_TRIPLET: x64-windows
         steps:
             - name: Checkout Repository
               uses: actions/checkout@master
 
             - name: VCPKG Cache
-              uses: actions/cache@v2
+              uses: actions/cache@v3
               id: vcpkg-cache
               with:
                 path: ~/vcpkg
@@ -31,12 +29,20 @@ jobs:
             - name: Install VCPKG Dependencies
               if: steps.vcpkg-cache.outputs.cache-hit != 'true'
               run: |
+                pushd .
                 cd ~
                 git clone --quiet --recursive https://github.com/microsoft/vcpkg.git
                 cd vcpkg
                 git checkout $env:VCPKG_HASH
                 .\bootstrap-vcpkg.bat
-                .\vcpkg.exe install --clean-after-build boost-compute boost-math boost-stacktrace fftw3 freeimage freetype[core] forge glfw3 openblas
+                popd
+                mkdir build && cd build && set VCPKG_ROOT=
+                cmake .. -G "Visual Studio 17 2022" -A x64 `
+                      -DVCPKG_ROOT:PATH=~/vcpkg `
+                      -DAF_BUILD_CUDA:BOOL=OFF -DAF_BUILD_OPENCL:BOOL=OFF `
+                      -DAF_BUILD_UNIFIED:BOOL=OFF -DAF_BUILD_FORGE:BOOL=ON `
+                      -DBUILDNAME:STRING="$buildname" `
+                      -DAF_COMPUTE_LIBRARY:STRING="FFTW/LAPACK/BLAS"
 
             - name: CMake Configure
               run: |
@@ -46,9 +52,12 @@ jobs:
                   $buildname = if($prnum -eq $null) { $branch } else { "PR-$prnum" }
                   $dashboard = if($prnum -eq $null) { "Continuous" } else { "Experimental" }
                   $buildname = "$buildname-cpu-openblas"
-                  mkdir build && cd build
+                  if((Test-Path build) -eq 0) {
+                      mkdir build
+                  }
+                  cd build && set VCPKG_ROOT=
                   cmake .. -G "Visual Studio 17 2022" -A x64 `
-                      -DVCPKG_MANIFEST_MODE:BOOL=OFF `
+                      -DVCPKG_ROOT:PATH=~/vcpkg `
                       -DAF_BUILD_CUDA:BOOL=OFF -DAF_BUILD_OPENCL:BOOL=OFF `
                       -DAF_BUILD_UNIFIED:BOOL=OFF -DAF_BUILD_FORGE:BOOL=ON `
                       -DBUILDNAME:STRING="$buildname" `
@@ -58,6 +67,6 @@ jobs:
             - name: Build and Test
               run: |
                   cd build
-                  $vcpkg_path = (Resolve-Path ~).Path
-                  $Env:PATH += ";${vcpkg_path}/vcpkg/installed/x64-windows/bin"
+                  $build_path = (pwd).Path
+                  $Env:PATH += ";$build_path/vcpkg_installed/x64-windows/bin"
                   ctest -D Experimental --track ${CTEST_DASHBOARD} -T Test -T Submit -C RelWithDebInfo -R cpu -E pinverse -j2
diff --git a/CMakeModules/AF_vcpkg_options.cmake b/CMakeModules/AF_vcpkg_options.cmake
index 00745f846c..59cdeb8fbf 100644
--- a/CMakeModules/AF_vcpkg_options.cmake
+++ b/CMakeModules/AF_vcpkg_options.cmake
@@ -29,6 +29,8 @@ endif()
 
 if(AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL")
   list(APPEND VCPKG_MANIFEST_FEATURES "mkl")
+else()
+  list(APPEND VCPKG_MANIFEST_FEATURES "openblasfftw")
 endif()
 
 if(DEFINED VCPKG_ROOT AND NOT DEFINED CMAKE_TOOLCHAIN_FILE)
diff --git a/vcpkg.json b/vcpkg.json
index 70aab906ed..4562e14f80 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -43,6 +43,14 @@
                 "glad"
             ]
         },
+        "openblasfftw": {
+            "description": "Build with OpenBLAS/FFTW",
+            "dependencies": [
+                "fftw3",
+                "openblas",
+                "lapack"
+            ]
+        },
         "cuda": {
             "description": "Build CUDA backend",
             "dependencies": [
@@ -69,5 +77,5 @@
             ]
         }
     },
-    "builtin-baseline": "14e7bb4ae24616ec54ff6b2f6ef4e8659434ea44"
+    "builtin-baseline": "6ca56aeb457f033d344a7106cb3f9f1abf8f4e98"
 }

From 7e02b8cebd82ed9afecc1d5a1bea03a3b5390b6c Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 4 Oct 2022 03:17:12 -0400
Subject: [PATCH 091/473] Make a lapack overlay to build lapacke library and
 headers

---
 CMakeModules/AF_vcpkg_options.cmake           |   3 +-
 .../ports/lapack-reference/FindLAPACK.cmake   | 559 ++++++++++++++++++
 .../ports/lapack-reference/lapacke.patch      |  16 +
 .../ports/lapack-reference/portfile.cmake     | 164 +++++
 .../vcpkg-cmake-wrapper.cmake                 |  11 +
 .../vcpkg/ports/lapack-reference/vcpkg.json   |  48 ++
 .../vcpkg-triplets/x64-windows.cmake          |   0
 7 files changed, 800 insertions(+), 1 deletion(-)
 create mode 100644 CMakeModules/vcpkg/ports/lapack-reference/FindLAPACK.cmake
 create mode 100644 CMakeModules/vcpkg/ports/lapack-reference/lapacke.patch
 create mode 100644 CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake
 create mode 100644 CMakeModules/vcpkg/ports/lapack-reference/vcpkg-cmake-wrapper.cmake
 create mode 100644 CMakeModules/vcpkg/ports/lapack-reference/vcpkg.json
 rename CMakeModules/{ => vcpkg}/vcpkg-triplets/x64-windows.cmake (100%)

diff --git a/CMakeModules/AF_vcpkg_options.cmake b/CMakeModules/AF_vcpkg_options.cmake
index 59cdeb8fbf..09701af274 100644
--- a/CMakeModules/AF_vcpkg_options.cmake
+++ b/CMakeModules/AF_vcpkg_options.cmake
@@ -9,7 +9,8 @@ set(ENV{VCPKG_FEATURE_FLAGS} "versions")
 set(ENV{VCPKG_KEEP_ENV_VARS} "MKLROOT")
 set(VCPKG_MANIFEST_NO_DEFAULT_FEATURES ON)
 
-set(VCPKG_OVERLAY_TRIPLETS ${ArrayFire_SOURCE_DIR}/CMakeModules/vcpkg-triplets)
+set(VCPKG_OVERLAY_TRIPLETS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/vcpkg/vcpkg-triplets)
+set(VCPKG_OVERLAY_PORTS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/vcpkg/ports)
 
 if(AF_BUILD_CUDA)
   list(APPEND VCPKG_MANIFEST_FEATURES "cuda")
diff --git a/CMakeModules/vcpkg/ports/lapack-reference/FindLAPACK.cmake b/CMakeModules/vcpkg/ports/lapack-reference/FindLAPACK.cmake
new file mode 100644
index 0000000000..f4d25477d8
--- /dev/null
+++ b/CMakeModules/vcpkg/ports/lapack-reference/FindLAPACK.cmake
@@ -0,0 +1,559 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+FindLAPACK
+----------
+
+Find Linear Algebra PACKage (LAPACK) library
+
+This module finds an installed Fortran library that implements the
+LAPACK linear-algebra interface (see http://www.netlib.org/lapack/).
+
+The approach follows that taken for the ``autoconf`` macro file,
+``acx_lapack.m4`` (distributed at
+http://ac-archive.sourceforge.net/ac-archive/acx_lapack.html).
+
+Input Variables
+^^^^^^^^^^^^^^^
+
+The following variables may be set to influence this module's behavior:
+
+``BLA_STATIC``
+  if ``ON`` use static linkage
+
+``BLA_VENDOR``
+  If set, checks only the specified vendor, if not set checks all the
+  possibilities.  List of vendors valid in this module:
+
+  * ``OpenBLAS``
+  * ``FLAME``
+  * ``Intel10_32`` (intel mkl v10 32 bit)
+  * ``Intel10_64lp`` (intel mkl v10+ 64 bit, threaded code, lp64 model)
+  * ``Intel10_64lp_seq`` (intel mkl v10+ 64 bit, sequential code, lp64 model)
+  * ``Intel10_64ilp`` (intel mkl v10+ 64 bit, threaded code, ilp64 model)
+  * ``Intel10_64ilp_seq`` (intel mkl v10+ 64 bit, sequential code, ilp64 model)
+  * ``Intel10_64_dyn`` (intel mkl v10+ 64 bit, single dynamic library)
+  * ``Intel`` (obsolete versions of mkl 32 and 64 bit)
+  * ``ACML``
+  * ``Apple``
+  * ``NAS``
+  * ``Arm``
+  * ``Arm_mp``
+  * ``Arm_ilp64``
+  * ``Arm_ilp64_mp``
+  * ``Generic``
+
+``BLA_F95``
+  if ``ON`` tries to find the BLAS95/LAPACK95 interfaces
+
+Imported targets
+^^^^^^^^^^^^^^^^
+
+This module defines the following :prop_tgt:`IMPORTED` target:
+
+``LAPACK::LAPACK``
+  The libraries to use for LAPACK, if found.
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+This module defines the following variables:
+
+``LAPACK_FOUND``
+  library implementing the LAPACK interface is found
+``LAPACK_LINKER_FLAGS``
+  uncached list of required linker flags (excluding ``-l`` and ``-L``).
+``LAPACK_LIBRARIES``
+  uncached list of libraries (using full path name) to link against
+  to use LAPACK
+``LAPACK95_LIBRARIES``
+  uncached list of libraries (using full path name) to link against
+  to use LAPACK95
+``LAPACK95_FOUND``
+  library implementing the LAPACK95 interface is found
+
+.. note::
+
+  C, CXX or Fortran must be enabled to detect a BLAS/LAPACK library.
+  C or CXX must be enabled to use Intel Math Kernel Library (MKL).
+
+  For example, to use Intel MKL libraries and/or Intel compiler:
+
+  .. code-block:: cmake
+
+    set(BLA_VENDOR Intel10_64lp)
+    find_package(LAPACK)
+#]=======================================================================]
+
+enable_language(C)
+# Check the language being used
+if(NOT (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED OR CMAKE_Fortran_COMPILER_LOADED))
+  if(LAPACK_FIND_REQUIRED)
+    message(FATAL_ERROR "FindLAPACK requires Fortran, C, or C++ to be enabled.")
+  else()
+    message(STATUS "Looking for LAPACK... - NOT found (Unsupported languages)")
+    return()
+  endif()
+endif()
+
+if(CMAKE_Fortran_COMPILER_LOADED)
+  include(${CMAKE_ROOT}/Modules/CheckFortranFunctionExists.cmake)
+else()
+  include(${CMAKE_ROOT}/Modules/CheckFunctionExists.cmake)
+endif()
+include(${CMAKE_ROOT}/Modules/CMakePushCheckState.cmake)
+
+cmake_push_check_state()
+set(CMAKE_REQUIRED_QUIET ${LAPACK_FIND_QUIETLY})
+
+set(LAPACK_FOUND FALSE)
+set(LAPACK95_FOUND FALSE)
+
+# store original values for CMAKE_FIND_LIBRARY_SUFFIXES
+set(_lapack_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.3gfs .so.3 .so.4 .so.5)
+endif()
+
+# TODO: move this stuff to a separate module
+
+macro(CHECK_LAPACK_LIBRARIES LIBRARIES _prefix _name _flags _list _threadlibs _addlibdir _subdirs _blas)
+  # This macro checks for the existence of the combination of fortran libraries
+  # given by _list.  If the combination is found, this macro checks (using the
+  # Check_Fortran_Function_Exists macro) whether can link against that library
+  # combination using the name of a routine given by _name using the linker
+  # flags given by _flags.  If the combination of libraries is found and passes
+  # the link test, LIBRARIES is set to the list of complete library paths that
+  # have been found.  Otherwise, LIBRARIES is set to FALSE.
+
+  # N.B. _prefix is the prefix applied to the names of all cached variables that
+  # are generated internally and marked advanced by this macro.
+  # _addlibdir is a list of additional search paths. _subdirs is a list of path
+  # suffixes to be used by find_library().
+
+  set(_libraries_work TRUE)
+  set(${LIBRARIES})
+  set(_combined_name)
+
+  set(_extaddlibdir "${_addlibdir}")
+  if(WIN32)
+    list(APPEND _extaddlibdir ENV LIB)
+  elseif(APPLE)
+    list(APPEND _extaddlibdir ENV DYLD_LIBRARY_PATH)
+  else()
+    list(APPEND _extaddlibdir ENV LD_LIBRARY_PATH)
+  endif()
+  list(APPEND _extaddlibdir "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+
+  foreach(_library ${_list})
+    if(_library MATCHES "^-Wl,--(start|end)-group$")
+      # Respect linker flags like --start/end-group (required by MKL)
+      set(${LIBRARIES} ${${LIBRARIES}} "${_library}")
+    else()
+      set(_combined_name ${_combined_name}_${_library})
+      if(_libraries_work)
+        find_library(${_prefix}_${_library}_LIBRARY
+          NAMES ${_library}
+          PATHS ${_extaddlibdir}
+          PATH_SUFFIXES ${_subdirs}
+        )
+        #message("DEBUG: find_library(${_library}) got ${${_prefix}_${_library}_LIBRARY}")
+        mark_as_advanced(${_prefix}_${_library}_LIBRARY)
+        set(${LIBRARIES} ${${LIBRARIES}} ${${_prefix}_${_library}_LIBRARY})
+        set(_libraries_work ${${_prefix}_${_library}_LIBRARY})
+      endif()
+    endif()
+  endforeach()
+
+  if(_libraries_work)
+    # Test this combination of libraries.
+    set(CMAKE_REQUIRED_LIBRARIES ${_flags} ${${LIBRARIES}} ${_blas} ${_threadlibs})
+    #message("DEBUG: CMAKE_REQUIRED_LIBRARIES = ${CMAKE_REQUIRED_LIBRARIES}")
+    if(CMAKE_Fortran_COMPILER_LOADED)
+      check_fortran_function_exists("${_name}" ${_prefix}${_combined_name}_WORKS)
+    else()
+      check_function_exists("${_name}_" ${_prefix}${_combined_name}_WORKS)
+    endif()
+    set(CMAKE_REQUIRED_LIBRARIES)
+    set(_libraries_work ${${_prefix}${_combined_name}_WORKS})
+  endif()
+
+  if(_libraries_work)
+    if("${_list}${_blas}" STREQUAL "")
+      set(${LIBRARIES} "${LIBRARIES}-PLACEHOLDER-FOR-EMPTY-LIBRARIES")
+    else()
+      set(${LIBRARIES} ${${LIBRARIES}} ${_blas} ${_threadlibs})
+    endif()
+  else()
+    set(${LIBRARIES} FALSE)
+  endif()
+  #message("DEBUG: ${LIBRARIES} = ${${LIBRARIES}}")
+endmacro()
+
+set(LAPACK_LINKER_FLAGS)
+set(LAPACK_LIBRARIES)
+set(LAPACK95_LIBRARIES)
+
+include(CMakeFindDependencyMacro)
+find_dependency(BLAS)
+
+if(BLAS_FOUND)
+  set(LAPACK_LINKER_FLAGS ${BLAS_LINKER_FLAGS})
+  if(NOT $ENV{BLA_VENDOR} STREQUAL "")
+    set(BLA_VENDOR $ENV{BLA_VENDOR})
+  else()
+    if(NOT BLA_VENDOR)
+      set(BLA_VENDOR "All")
+    endif()
+  endif()
+
+  # LAPACK in the Intel MKL 10+ library?
+  if(BLA_VENDOR MATCHES "Intel" OR BLA_VENDOR STREQUAL "All")
+    if(NOT LAPACK_LIBRARIES)
+      if(CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED)
+        # System-specific settings
+        if(NOT WIN32)
+          set(LAPACK_mkl_LM "-lm")
+          set(LAPACK_mkl_LDL "-ldl")
+        endif()
+
+        if(LAPACK_FIND_QUIETLY OR NOT LAPACK_FIND_REQUIRED)
+          find_package(Threads)
+        else()
+          find_package(Threads REQUIRED)
+        endif()
+
+        if(BLA_VENDOR MATCHES "_64ilp")
+          set(LAPACK_mkl_ILP_MODE "ilp64")
+        else()
+          set(LAPACK_mkl_ILP_MODE "lp64")
+        endif()
+
+        set(LAPACK_SEARCH_LIBS "")
+
+        if(BLA_F95)
+          set(LAPACK_mkl_SEARCH_SYMBOL "cheev_f95")
+          set(_LIBRARIES LAPACK95_LIBRARIES)
+          set(_BLAS_LIBRARIES ${BLAS95_LIBRARIES})
+
+          # old
+          list(APPEND LAPACK_SEARCH_LIBS
+            "mkl_lapack95")
+          # new >= 10.3
+          list(APPEND LAPACK_SEARCH_LIBS
+            "mkl_intel_c")
+          list(APPEND LAPACK_SEARCH_LIBS
+            "mkl_lapack95_${LAPACK_mkl_ILP_MODE}")
+        else()
+          set(LAPACK_mkl_SEARCH_SYMBOL "cheev")
+          set(_LIBRARIES LAPACK_LIBRARIES)
+          set(_BLAS_LIBRARIES ${BLAS_LIBRARIES})
+
+          # old and new >= 10.3
+          list(APPEND LAPACK_SEARCH_LIBS
+            "mkl_lapack")
+        endif()
+
+        # MKL uses a multitude of partially platform-specific subdirectories:
+        if(BLA_VENDOR STREQUAL "Intel10_32")
+          set(LAPACK_mkl_ARCH_NAME "ia32")
+        else()
+          set(LAPACK_mkl_ARCH_NAME "intel64")
+        endif()
+        if(WIN32)
+          set(LAPACK_mkl_OS_NAME "win")
+        elseif(APPLE)
+          set(LAPACK_mkl_OS_NAME "mac")
+        else()
+          set(LAPACK_mkl_OS_NAME "lin")
+        endif()
+        if(DEFINED ENV{MKLROOT})
+          file(TO_CMAKE_PATH "$ENV{MKLROOT}" LAPACK_mkl_MKLROOT)
+          # If MKLROOT points to the subdirectory 'mkl', use the parent directory instead
+          # so we can better detect other relevant libraries in 'compiler' or 'tbb':
+          get_filename_component(LAPACK_mkl_MKLROOT_LAST_DIR "${LAPACK_mkl_MKLROOT}" NAME)
+          if(LAPACK_mkl_MKLROOT_LAST_DIR STREQUAL "mkl")
+              get_filename_component(LAPACK_mkl_MKLROOT "${LAPACK_mkl_MKLROOT}" DIRECTORY)
+          endif()
+        endif()
+        set(LAPACK_mkl_LIB_PATH_SUFFIXES
+            "compiler/lib" "compiler/lib/${LAPACK_mkl_ARCH_NAME}_${LAPACK_mkl_OS_NAME}"
+            "mkl/lib" "mkl/lib/${LAPACK_mkl_ARCH_NAME}_${LAPACK_mkl_OS_NAME}"
+            "lib/${LAPACK_mkl_ARCH_NAME}_${LAPACK_mkl_OS_NAME}")
+
+        # First try empty lapack libs
+        if(NOT ${_LIBRARIES})
+          check_lapack_libraries(
+            ${_LIBRARIES}
+            LAPACK
+            ${LAPACK_mkl_SEARCH_SYMBOL}
+            ""
+            ""
+            "${CMAKE_THREAD_LIBS_INIT};${LAPACK_mkl_LM};${LAPACK_mkl_LDL}"
+            "${LAPACK_mkl_MKLROOT}"
+            "${LAPACK_mkl_LIB_PATH_SUFFIXES}"
+            "${_BLAS_LIBRARIES}"
+          )
+        endif()
+
+        # Then try the search libs
+        foreach(IT ${LAPACK_SEARCH_LIBS})
+          string(REPLACE " " ";" SEARCH_LIBS ${IT})
+          if(NOT ${_LIBRARIES})
+            check_lapack_libraries(
+              ${_LIBRARIES}
+              LAPACK
+              ${LAPACK_mkl_SEARCH_SYMBOL}
+              ""
+              "${SEARCH_LIBS}"
+              "${CMAKE_THREAD_LIBS_INIT};${LAPACK_mkl_LM};${LAPACK_mkl_LDL}"
+              "${LAPACK_mkl_MKLROOT}"
+              "${LAPACK_mkl_LIB_PATH_SUFFIXES}"
+              "${_BLAS_LIBRARIES}"
+            )
+          endif()
+        endforeach()
+
+        unset(LAPACK_mkl_ILP_MODE)
+        unset(LAPACK_mkl_SEARCH_SYMBOL)
+        unset(LAPACK_mkl_LM)
+        unset(LAPACK_mkl_LDL)
+        unset(LAPACK_mkl_MKLROOT)
+        unset(LAPACK_mkl_ARCH_NAME)
+        unset(LAPACK_mkl_OS_NAME)
+        unset(LAPACK_mkl_LIB_PATH_SUFFIXES)
+      endif()
+    endif()
+  endif()
+
+  # gotoblas? (http://www.tacc.utexas.edu/tacc-projects/gotoblas2)
+  if(BLA_VENDOR STREQUAL "Goto" OR BLA_VENDOR STREQUAL "All")
+    if(NOT LAPACK_LIBRARIES)
+      check_lapack_libraries(
+        LAPACK_LIBRARIES
+        LAPACK
+        cheev
+        ""
+        "goto2"
+        ""
+        ""
+        ""
+        "${BLAS_LIBRARIES}"
+      )
+    endif()
+  endif()
+
+  # OpenBLAS? (http://www.openblas.net)
+  if(BLA_VENDOR STREQUAL "OpenBLAS" OR BLA_VENDOR STREQUAL "All")
+    if(NOT LAPACK_LIBRARIES)
+      check_lapack_libraries(
+        LAPACK_LIBRARIES
+        LAPACK
+        cheev
+        ""
+        "openblas"
+        ""
+        ""
+        ""
+        "${BLAS_LIBRARIES}"
+      )
+    endif()
+  endif()
+
+  # ArmPL? (https://developer.arm.com/tools-and-software/server-and-hpc/compile/arm-compiler-for-linux/arm-performance-libraries)
+  if(BLA_VENDOR MATCHES "Arm" OR BLA_VENDOR STREQUAL "All")
+
+    # Check for 64bit Integer support
+    if(BLA_VENDOR MATCHES "_ilp64")
+      set(LAPACK_armpl_LIB "armpl_ilp64")
+    else()
+      set(LAPACK_armpl_LIB "armpl_lp64")
+    endif()
+
+    # Check for OpenMP support, VIA BLA_VENDOR of Arm_mp or Arm_ipl64_mp
+    if(BLA_VENDOR MATCHES "_mp")
+     set(LAPACK_armpl_LIB "${LAPACK_armpl_LIB}_mp")
+    endif()
+
+    if(NOT LAPACK_LIBRARIES)
+      check_lapack_libraries(
+        LAPACK_LIBRARIES
+        LAPACK
+        cheev
+        ""
+        "${LAPACK_armpl_LIB}"
+        ""
+        ""
+        ""
+        "${BLAS_LIBRARIES}"
+      )
+    endif()
+  endif()
+
+  # FLAME's blis library? (https://github.com/flame/blis)
+  if(BLA_VENDOR STREQUAL "FLAME" OR BLA_VENDOR STREQUAL "All")
+    if(NOT LAPACK_LIBRARIES)
+      check_lapack_libraries(
+        LAPACK_LIBRARIES
+        LAPACK
+        cheev
+        ""
+        "flame"
+        ""
+        ""
+        ""
+        "${BLAS_LIBRARIES}"
+      )
+    endif()
+  endif()
+
+  # BLAS in acml library?
+  if(BLA_VENDOR MATCHES "ACML" OR BLA_VENDOR STREQUAL "All")
+    if(BLAS_LIBRARIES MATCHES ".+acml.+")
+      set(LAPACK_LIBRARIES ${BLAS_LIBRARIES})
+    endif()
+  endif()
+
+  # Apple LAPACK library?
+  if(BLA_VENDOR STREQUAL "Apple" OR BLA_VENDOR STREQUAL "All")
+    if(NOT LAPACK_LIBRARIES)
+      check_lapack_libraries(
+        LAPACK_LIBRARIES
+        LAPACK
+        cheev
+        ""
+        "Accelerate"
+        ""
+        ""
+        ""
+        "${BLAS_LIBRARIES}"
+      )
+    endif()
+  endif()
+
+  # Apple NAS (vecLib) library?
+  if(BLA_VENDOR STREQUAL "NAS" OR BLA_VENDOR STREQUAL "All")
+    if(NOT LAPACK_LIBRARIES)
+      check_lapack_libraries(
+        LAPACK_LIBRARIES
+        LAPACK
+        cheev
+        ""
+        "vecLib"
+        ""
+        ""
+        ""
+        "${BLAS_LIBRARIES}"
+      )
+    endif()
+  endif()
+
+  # Generic LAPACK library?
+  if(BLA_VENDOR STREQUAL "Generic" OR
+      BLA_VENDOR STREQUAL "ATLAS" OR
+      BLA_VENDOR STREQUAL "All")
+    if(NOT LAPACK_LIBRARIES)
+      check_lapack_libraries(
+        LAPACK_LIBRARIES
+        LAPACK
+        cheev
+        ""
+        "lapack"
+        ""
+        ""
+        ""
+        "${BLAS_LIBRARIES}"
+      )
+    endif()
+    if(NOT LAPACK_LIBRARIES AND NOT WIN32)
+      check_lapack_libraries(
+        LAPACK_LIBRARIES
+        LAPACK
+        cheev
+        ""
+        "lapack;m;gfortran"
+        ""
+        ""
+        ""
+        "${BLAS_LIBRARIES}"
+      )
+    endif()
+  endif()
+else()
+  message(STATUS "LAPACK requires BLAS")
+endif()
+
+if(BLA_F95)
+  if(LAPACK95_LIBRARIES)
+    set(LAPACK95_FOUND TRUE)
+  else()
+    set(LAPACK95_FOUND FALSE)
+  endif()
+  if(NOT LAPACK_FIND_QUIETLY)
+    if(LAPACK95_FOUND)
+      message(STATUS "A library with LAPACK95 API found.")
+    else()
+      if(LAPACK_FIND_REQUIRED)
+        message(FATAL_ERROR
+          "A required library with LAPACK95 API not found. Please specify library location."
+        )
+      else()
+        message(STATUS
+          "A library with LAPACK95 API not found. Please specify library location."
+        )
+      endif()
+    endif()
+  endif()
+  set(LAPACK_FOUND "${LAPACK95_FOUND}")
+  set(LAPACK_LIBRARIES "${LAPACK95_LIBRARIES}")
+else()
+  if(LAPACK_LIBRARIES)
+    set(LAPACK_FOUND TRUE)
+  else()
+    set(LAPACK_FOUND FALSE)
+  endif()
+
+  if(NOT LAPACK_FIND_QUIETLY)
+    if(LAPACK_FOUND)
+      message(STATUS "A library with LAPACK API found.")
+    else()
+      if(LAPACK_FIND_REQUIRED)
+        message(FATAL_ERROR
+          "A required library with LAPACK API not found. Please specify library location."
+        )
+      else()
+        message(STATUS
+          "A library with LAPACK API not found. Please specify library location."
+        )
+      endif()
+    endif()
+  endif()
+endif()
+
+# On compilers that implicitly link LAPACK (such as ftn, cc, and CC on Cray HPC machines)
+# we used a placeholder for empty LAPACK_LIBRARIES to get through our logic above.
+if(LAPACK_LIBRARIES STREQUAL "LAPACK_LIBRARIES-PLACEHOLDER-FOR-EMPTY-LIBRARIES")
+  set(LAPACK_LIBRARIES "")
+endif()
+
+if(NOT TARGET LAPACK::LAPACK)
+  add_library(LAPACK::LAPACK INTERFACE IMPORTED)
+  set(_lapack_libs "${LAPACK_LIBRARIES}")
+  if(_lapack_libs AND TARGET BLAS::BLAS)
+    # remove the ${BLAS_LIBRARIES} from the interface and replace it
+    # with the BLAS::BLAS target
+    list(REMOVE_ITEM _lapack_libs "${BLAS_LIBRARIES}")
+  endif()
+
+  if(_lapack_libs)
+    set_target_properties(LAPACK::LAPACK PROPERTIES
+      INTERFACE_LINK_LIBRARIES "${_lapack_libs}"
+    )
+  endif()
+  unset(_lapack_libs)
+endif()
+
+cmake_pop_check_state()
+# restore original values for CMAKE_FIND_LIBRARY_SUFFIXES
+set(CMAKE_FIND_LIBRARY_SUFFIXES ${_lapack_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
diff --git a/CMakeModules/vcpkg/ports/lapack-reference/lapacke.patch b/CMakeModules/vcpkg/ports/lapack-reference/lapacke.patch
new file mode 100644
index 0000000000..964f0e3192
--- /dev/null
+++ b/CMakeModules/vcpkg/ports/lapack-reference/lapacke.patch
@@ -0,0 +1,16 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 1ee66f1..7cec7ca 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -392,8 +392,9 @@ endif()
+ set(LAPACK_INSTALL_EXPORT_NAME ${LAPACK_INSTALL_EXPORT_NAME_CACHE})
+ unset(LAPACK_INSTALL_EXPORT_NAME_CACHE)
+ 
+-add_subdirectory(LAPACKE)
+-
++if(LAPACKE)
++    add_subdirectory(LAPACKE)
++endif()
+ 
+ #-------------------------------------
+ # BLAS++ / LAPACK++
diff --git a/CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake b/CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake
new file mode 100644
index 0000000000..ba8999d36e
--- /dev/null
+++ b/CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake
@@ -0,0 +1,164 @@
+#TODO: Features to add:
+# USE_XBLAS??? extended precision blas. needs xblas
+# LAPACKE should be its own PORT
+# USE_OPTIMIZED_LAPACK (Probably not what we want. Does a find_package(LAPACK): probably for LAPACKE only builds _> own port?)
+# LAPACKE Builds LAPACKE
+# LAPACKE_WITH_TMG Build LAPACKE with tmglib routines
+if(EXISTS "${CURRENT_INSTALLED_DIR}/share/clapack/copyright")
+    message(FATAL_ERROR "Can't build ${PORT} if clapack is installed. Please remove clapack:${TARGET_TRIPLET}, and try to install ${PORT}:${TARGET_TRIPLET} again.")
+endif()
+
+include(vcpkg_find_fortran)
+SET(VCPKG_POLICY_EMPTY_INCLUDE_FOLDER enabled)
+
+set(lapack_ver 3.10.1)
+
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO  "Reference-LAPACK/lapack"
+    REF "v${lapack_ver}"
+    SHA512 0500bbbb48483208c0a35b74972ff0059c389da6032824a2079637266a99fa980882eedf7f1fc490219ee4ff27812ac8c6afe118e25f40a9c2387e7b997762fb
+    HEAD_REF master
+    PATCHES
+        lapacke.patch
+)
+
+if(NOT VCPKG_TARGET_IS_WINDOWS)
+    set(ENV{FFLAGS} "$ENV{FFLAGS} -fPIC")
+endif()
+
+set(CBLAS OFF)
+if("cblas" IN_LIST FEATURES)
+    set(CBLAS ON)
+    if("noblas" IN_LIST FEATURES)
+        message(FATAL_ERROR "Cannot built feature 'cblas' together with feature 'noblas'. cblas requires blas!")
+    endif()
+endif()
+
+set(USE_OPTIMIZED_BLAS OFF) 
+if("noblas" IN_LIST FEATURES)
+    set(USE_OPTIMIZED_BLAS ON)
+    set(pcfile "${CURRENT_INSTALLED_DIR}/lib/pkgconfig/openblas.pc")
+    if(EXISTS "${pcfile}")
+        file(CREATE_LINK "${pcfile}" "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/blas.pc" COPY_ON_ERROR)
+    endif()
+    set(pcfile "${CURRENT_INSTALLED_DIR}/debug/lib/pkgconfig/openblas.pc")
+    if(EXISTS "${pcfile}")
+        file(CREATE_LINK "${pcfile}" "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/blas.pc" COPY_ON_ERROR)
+    endif()
+endif()
+
+set(VCPKG_CRT_LINKAGE_BACKUP ${VCPKG_CRT_LINKAGE})
+vcpkg_find_fortran(FORTRAN_CMAKE)
+if(VCPKG_USE_INTERNAL_Fortran)
+    if(VCPKG_CRT_LINKAGE_BACKUP STREQUAL static) 
+    # If openblas has been built with static crt linkage we cannot use it with gfortran!
+        set(USE_OPTIMIZED_BLAS OFF) 
+        #Cannot use openblas from vcpkg if we are building with gfortran here. 
+        if("noblas" IN_LIST FEATURES)
+            message(FATAL_ERROR "Feature 'noblas' cannot be used without supplying an external fortran compiler")
+        endif()
+    endif()
+else()
+    set(USE_OPTIMIZED_BLAS ON)
+endif()
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        "-DUSE_OPTIMIZED_BLAS=${USE_OPTIMIZED_BLAS}"
+        "-DCBLAS=${CBLAS}"
+	"-DLAPACKE=ON"
+        ${FORTRAN_CMAKE}
+)
+
+vcpkg_cmake_install()
+
+vcpkg_cmake_config_fixup(PACKAGE_NAME lapack-${lapack_ver} CONFIG_PATH lib/cmake/lapack-${lapack_ver}) #Should the target path be lapack and not lapack-reference?
+
+message("CURRENT_PACKAGES_DIR: ${CURRENT_PACKAGES_DIR}")
+set(pcfile "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/lapack.pc")
+if(EXISTS "${pcfile}")
+    file(READ "${pcfile}" _contents)
+    set(_contents "prefix=${CURRENT_INSTALLED_DIR}\n${_contents}")
+    file(WRITE "${pcfile}" "${_contents}")
+endif()
+set(pcfile "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/lapack.pc")
+if(EXISTS "${pcfile}")
+    file(READ "${pcfile}" _contents)
+    set(_contents "prefix=${CURRENT_INSTALLED_DIR}/debug\n${_contents}")
+    file(WRITE "${pcfile}" "${_contents}")
+endif()
+set(pcfile "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/lapacke.pc")
+if(EXISTS "${pcfile}")
+    file(READ "${pcfile}" _contents)
+    set(_contents "prefix=${CURRENT_INSTALLED_DIR}\n${_contents}")
+    file(WRITE "${pcfile}" "${_contents}")
+endif()
+set(pcfile "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/lapacke.pc")
+if(EXISTS "${pcfile}")
+    file(READ "${pcfile}" _contents)
+    set(_contents "prefix=${CURRENT_INSTALLED_DIR}/debug\n${_contents}")
+    file(WRITE "${pcfile}" "${_contents}")
+endif()
+if(NOT USE_OPTIMIZED_BLAS AND NOT (VCPKG_TARGET_IS_WINDOWS AND VCPKG_LIBRARY_LINKAGE STREQUAL "static"))
+    set(pcfile "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/blas.pc")
+    if(EXISTS "${pcfile}")
+        file(READ "${pcfile}" _contents)
+        set(_contents "prefix=${CURRENT_INSTALLED_DIR}\n${_contents}")
+        file(WRITE "${pcfile}" "${_contents}")
+    endif()
+    set(pcfile "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/blas.pc")
+    if(EXISTS "${pcfile}")
+        file(READ "${pcfile}" _contents)
+        set(_contents "prefix=${CURRENT_INSTALLED_DIR}/debug\n${_contents}")
+        file(WRITE "${pcfile}" "${_contents}")
+    endif()
+endif()
+if("cblas" IN_LIST FEATURES)
+    set(pcfile "${CURRENT_PACKAGES_DIR}/lib/pkgconfig/cblas.pc")
+    if(EXISTS "${pcfile}")
+        file(READ "${pcfile}" _contents)
+        set(_contents "prefix=${CURRENT_INSTALLED_DIR}\n${_contents}")
+        file(WRITE "${pcfile}" "${_contents}")
+    endif()
+    set(pcfile "${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/cblas.pc")
+    if(EXISTS "${pcfile}")
+        file(READ "${pcfile}" _contents)
+        set(_contents "prefix=${CURRENT_INSTALLED_DIR}/debug\n${_contents}")
+        file(WRITE "${pcfile}" "${_contents}")
+    endif()
+endif()
+#vcpkg_fixup_pkgconfig()
+
+# Handle copyright
+file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)
+
+# remove debug includes
+file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include)
+
+if(VCPKG_TARGET_IS_WINDOWS)
+    if(EXISTS "${CURRENT_PACKAGES_DIR}/lib/liblapack.lib")
+        file(RENAME "${CURRENT_PACKAGES_DIR}/lib/liblapack.lib" "${CURRENT_PACKAGES_DIR}/lib/lapack.lib")
+    endif()
+    if(EXISTS "${CURRENT_PACKAGES_DIR}/debug/lib/liblapack.lib")
+        file(RENAME "${CURRENT_PACKAGES_DIR}/debug/lib/liblapack.lib" "${CURRENT_PACKAGES_DIR}/debug/lib/lapack.lib")
+    endif()
+    if(EXISTS "${CURRENT_PACKAGES_DIR}/lib/liblapacke.lib")
+        file(RENAME "${CURRENT_PACKAGES_DIR}/lib/liblapacke.lib" "${CURRENT_PACKAGES_DIR}/lib/lapacke.lib")
+    endif()
+    if(EXISTS "${CURRENT_PACKAGES_DIR}/debug/lib/liblapacke.lib")
+        file(RENAME "${CURRENT_PACKAGES_DIR}/debug/lib/liblapacke.lib" "${CURRENT_PACKAGES_DIR}/debug/lib/lapacke.lib")
+    endif()
+    if(NOT USE_OPTIMIZED_BLAS)
+        if(EXISTS "${CURRENT_PACKAGES_DIR}/lib/libblas.lib")
+            file(RENAME "${CURRENT_PACKAGES_DIR}/lib/libblas.lib" "${CURRENT_PACKAGES_DIR}/lib/blas.lib")
+        endif()
+        if(EXISTS "${CURRENT_PACKAGES_DIR}/debug/lib/libblas.lib")
+            file(RENAME "${CURRENT_PACKAGES_DIR}/debug/lib/libblas.lib" "${CURRENT_PACKAGES_DIR}/debug/lib/blas.lib")
+        endif()
+    endif()
+endif()
+
+file(COPY ${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/lapack)
+file(COPY ${CMAKE_CURRENT_LIST_DIR}/FindLAPACK.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/lapack)
diff --git a/CMakeModules/vcpkg/ports/lapack-reference/vcpkg-cmake-wrapper.cmake b/CMakeModules/vcpkg/ports/lapack-reference/vcpkg-cmake-wrapper.cmake
new file mode 100644
index 0000000000..b3a7128fff
--- /dev/null
+++ b/CMakeModules/vcpkg/ports/lapack-reference/vcpkg-cmake-wrapper.cmake
@@ -0,0 +1,11 @@
+message(STATUS "Using VCPKG FindLAPACK from package 'lapack-reference'")
+set(LAPACK_PREV_MODULE_PATH ${CMAKE_MODULE_PATH})
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
+
+list(REMOVE_ITEM ARGS "NO_MODULE")
+list(REMOVE_ITEM ARGS "CONFIG")
+list(REMOVE_ITEM ARGS "MODULE")
+
+_find_package(${ARGS})
+
+set(CMAKE_MODULE_PATH ${LAPACK_PREV_MODULE_PATH})
diff --git a/CMakeModules/vcpkg/ports/lapack-reference/vcpkg.json b/CMakeModules/vcpkg/ports/lapack-reference/vcpkg.json
new file mode 100644
index 0000000000..b2fe5d6998
--- /dev/null
+++ b/CMakeModules/vcpkg/ports/lapack-reference/vcpkg.json
@@ -0,0 +1,48 @@
+{
+  "name": "lapack-reference",
+  "version": "3.10.1",
+  "description": "LAPACK - Linear Algebra PACKage",
+  "homepage": "http://www.netlib.org/lapack/",
+  "license": "BSD-3-Clause-Open-MPI",
+  "dependencies": [
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    },
+    {
+      "name": "vcpkg-gfortran",
+      "platform": "windows"
+    }
+  ],
+  "default-features": [
+    "blas-select"
+  ],
+  "features": {
+    "blas-select": {
+      "description": "Use external optimized BLAS",
+      "dependencies": [
+        {
+          "name": "lapack-reference",
+          "default-features": false,
+          "features": [
+            "noblas"
+          ],
+          "platform": "!windows | !static"
+        }
+      ]
+    },
+    "cblas": {
+      "description": "Builds CBLAS"
+    },
+    "noblas": {
+      "description": "Use external optimized BLAS",
+      "dependencies": [
+        "blas"
+      ]
+    }
+  }
+}
diff --git a/CMakeModules/vcpkg-triplets/x64-windows.cmake b/CMakeModules/vcpkg/vcpkg-triplets/x64-windows.cmake
similarity index 100%
rename from CMakeModules/vcpkg-triplets/x64-windows.cmake
rename to CMakeModules/vcpkg/vcpkg-triplets/x64-windows.cmake

From 60ee8506e717b7c5123a16167877454716b2befb Mon Sep 17 00:00:00 2001
From: Yurkevitch <stephen.yurkevitch@intel.com>
Date: Thu, 1 Sep 2022 18:16:25 -0400
Subject: [PATCH 092/473] Initial oneAPI backend code

* afoneapi compiles and links

* Files that compile:
                      Array.cpp
                      Event.cpp
                      wrap.cpp
                      where.cpp
                      unwrap.cpp
                      triangle.cpp
                      transpose.cpp
                      transform.cpp
                      topk.cpp
                      tile.cpp
                      svd.cpp
                      susan.cpp
                      surface.cpp
                      sum.cpp
                      sparse_blas.cpp
                      sparse_arith.cpp
                      sparse.cpp
                      sort_index.cpp
                      sort.cpp
                      solve.cpp
                      sobel.cpp
                      sift.cpp
                      set.cpp
                      select.cpp
                      scan_by_key.cpp
                      scan.cpp
                      rotate.cpp
                      resize.cpp
                      reshape.cpp
                      reorder.cpp
                      regions.cpp
                      range.cpp
                      random_engine.cpp
                      qr.cpp
                      product.cpp
                      plot.cpp
                      orb.cpp
                      nearest_neighbor.cpp
                      morph.cpp
                      moments.cpp
                      min.cpp
                      memory.cpp
                      medfilt.cpp
                      meanshift.cpp
                      mean.cpp
                      max.cpp
                      platform.cpp
---
 CMakeLists.txt                                |   6 +
 src/api/c/det.cpp                             |   2 +-
 src/api/c/morph.cpp                           |   2 +
 src/backend/common/EventBase.hpp              |   3 +-
 src/backend/common/Logger.hpp                 |  32 +
 src/backend/common/forge_loader.hpp           |  31 +-
 src/backend/common/jit/BinaryNode.cpp         |   1 +
 src/backend/common/jit/BufferNodeBase.hpp     |   3 +-
 src/backend/common/jit/Node.hpp               |  48 +-
 src/backend/common/kernel_cache.cpp           |   2 +-
 src/backend/oneapi/Array.cpp                  | 563 ++++++++++++++++++
 src/backend/oneapi/Array.hpp                  | 327 ++++++++++
 src/backend/oneapi/CMakeLists.txt             | 240 ++++++++
 src/backend/oneapi/Event.cpp                  |  78 +++
 src/backend/oneapi/Event.hpp                  |  64 ++
 .../oneapi/GraphicsResourceManager.cpp        |  20 +
 .../oneapi/GraphicsResourceManager.hpp        |  33 +
 src/backend/oneapi/Kernel.hpp                 |  90 +++
 src/backend/oneapi/Module.hpp                 |  40 ++
 src/backend/oneapi/Param.hpp                  |  36 ++
 src/backend/oneapi/all.cpp                    |  30 +
 src/backend/oneapi/anisotropic_diffusion.cpp  |  31 +
 src/backend/oneapi/anisotropic_diffusion.hpp  |  17 +
 src/backend/oneapi/any.cpp                    |  30 +
 src/backend/oneapi/approx.cpp                 |  89 +++
 src/backend/oneapi/approx.hpp                 |  24 +
 src/backend/oneapi/arith.hpp                  |  30 +
 src/backend/oneapi/assign.cpp                 |  48 ++
 src/backend/oneapi/assign.hpp                 |  18 +
 src/backend/oneapi/backend.hpp                |  22 +
 src/backend/oneapi/bilateral.cpp              |  41 ++
 src/backend/oneapi/bilateral.hpp              |  16 +
 src/backend/oneapi/binary.hpp                 | 127 ++++
 src/backend/oneapi/blas.cpp                   |  84 +++
 src/backend/oneapi/blas.hpp                   |  41 ++
 src/backend/oneapi/canny.cpp                  |  28 +
 src/backend/oneapi/canny.hpp                  |  19 +
 src/backend/oneapi/cast.hpp                   |  73 +++
 src/backend/oneapi/cholesky.cpp               |  70 +++
 src/backend/oneapi/cholesky.hpp               |  18 +
 src/backend/oneapi/compile_module.cpp         | 131 ++++
 src/backend/oneapi/complex.hpp                |  90 +++
 src/backend/oneapi/convolve.cpp               | 125 ++++
 src/backend/oneapi/convolve.hpp               |  39 ++
 src/backend/oneapi/convolve_separable.cpp     |  45 ++
 src/backend/oneapi/copy.cpp                   | 150 +++++
 src/backend/oneapi/copy.hpp                   |  67 +++
 src/backend/oneapi/count.cpp                  |  30 +
 src/backend/oneapi/device_manager.cpp         |  95 +++
 src/backend/oneapi/device_manager.hpp         | 163 +++++
 src/backend/oneapi/diagonal.cpp               |  58 ++
 src/backend/oneapi/diagonal.hpp               |  18 +
 src/backend/oneapi/diff.cpp                   |  61 ++
 src/backend/oneapi/diff.hpp                   |  18 +
 src/backend/oneapi/err_oneapi.hpp             |  18 +
 src/backend/oneapi/errorcodes.cpp             |  18 +
 src/backend/oneapi/errorcodes.hpp             |  14 +
 src/backend/oneapi/exampleFunction.cpp        |  65 ++
 src/backend/oneapi/exampleFunction.hpp        |  16 +
 src/backend/oneapi/fast.cpp                   |  44 ++
 src/backend/oneapi/fast.hpp                   |  23 +
 src/backend/oneapi/fft.cpp                    | 106 ++++
 src/backend/oneapi/fft.hpp                    |  25 +
 src/backend/oneapi/fftconvolve.cpp            |  82 +++
 src/backend/oneapi/fftconvolve.hpp            |  16 +
 src/backend/oneapi/flood_fill.cpp             |  36 ++
 src/backend/oneapi/flood_fill.hpp             |  21 +
 src/backend/oneapi/gradient.cpp               |  31 +
 src/backend/oneapi/gradient.hpp               |  15 +
 src/backend/oneapi/harris.cpp                 |  40 ++
 src/backend/oneapi/harris.hpp                 |  24 +
 src/backend/oneapi/hist_graphics.cpp          |  32 +
 src/backend/oneapi/hist_graphics.hpp          |  18 +
 src/backend/oneapi/histogram.cpp              |  49 ++
 src/backend/oneapi/histogram.hpp              |  17 +
 src/backend/oneapi/homography.cpp             |  44 ++
 src/backend/oneapi/homography.hpp             |  21 +
 src/backend/oneapi/hsv_rgb.cpp                |  37 ++
 src/backend/oneapi/hsv_rgb.hpp                |  20 +
 src/backend/oneapi/identity.cpp               |  43 ++
 src/backend/oneapi/identity.hpp               |  15 +
 src/backend/oneapi/iir.cpp                    |  37 ++
 src/backend/oneapi/iir.hpp                    |  16 +
 src/backend/oneapi/image.cpp                  |  36 ++
 src/backend/oneapi/image.hpp                  |  18 +
 src/backend/oneapi/index.cpp                  |  46 ++
 src/backend/oneapi/index.hpp                  |  18 +
 src/backend/oneapi/inverse.cpp                |  54 ++
 src/backend/oneapi/inverse.hpp                |  15 +
 src/backend/oneapi/iota.cpp                   |  43 ++
 src/backend/oneapi/iota.hpp                   |  16 +
 src/backend/oneapi/ireduce.cpp                |  78 +++
 src/backend/oneapi/ireduce.hpp                |  24 +
 src/backend/oneapi/jit.cpp                    |  71 +++
 src/backend/oneapi/jit/BufferNode.hpp         |  34 ++
 src/backend/oneapi/jit/kernel_generators.hpp  | 112 ++++
 src/backend/oneapi/join.cpp                   |  91 +++
 src/backend/oneapi/join.hpp                   |  18 +
 src/backend/oneapi/kernel/KParam.hpp          |  26 +
 src/backend/oneapi/logic.hpp                  |  30 +
 src/backend/oneapi/lookup.cpp                 |  63 ++
 src/backend/oneapi/lookup.hpp                 |  16 +
 src/backend/oneapi/lu.cpp                     |  86 +++
 src/backend/oneapi/lu.hpp                     |  21 +
 src/backend/oneapi/match_template.cpp         |  38 ++
 src/backend/oneapi/match_template.hpp         |  18 +
 src/backend/oneapi/math.cpp                   |  53 ++
 src/backend/oneapi/math.hpp                   | 155 +++++
 src/backend/oneapi/max.cpp                    |  30 +
 src/backend/oneapi/mean.cpp                   |  94 +++
 src/backend/oneapi/mean.hpp                   |  26 +
 src/backend/oneapi/meanshift.cpp              |  48 ++
 src/backend/oneapi/meanshift.hpp              |  17 +
 src/backend/oneapi/medfilt.cpp                |  67 +++
 src/backend/oneapi/medfilt.hpp                |  22 +
 src/backend/oneapi/memory.cpp                 | 351 +++++++++++
 src/backend/oneapi/memory.hpp                 |  94 +++
 src/backend/oneapi/min.cpp                    |  30 +
 src/backend/oneapi/moments.cpp                |  57 ++
 src/backend/oneapi/moments.hpp                |  15 +
 src/backend/oneapi/morph.cpp                  |  70 +++
 src/backend/oneapi/morph.hpp                  |  18 +
 src/backend/oneapi/nearest_neighbour.cpp      |  89 +++
 src/backend/oneapi/nearest_neighbour.hpp      |  23 +
 src/backend/oneapi/orb.cpp                    |  69 +++
 src/backend/oneapi/orb.hpp                    |  24 +
 src/backend/oneapi/platform.cpp               | 462 ++++++++++++++
 src/backend/oneapi/platform.hpp               | 121 ++++
 src/backend/oneapi/plot.cpp                   |  79 +++
 src/backend/oneapi/plot.hpp                   |  18 +
 src/backend/oneapi/print.hpp                  |  24 +
 src/backend/oneapi/product.cpp                |  30 +
 src/backend/oneapi/qr.cpp                     | 142 +++++
 src/backend/oneapi/qr.hpp                     |  18 +
 src/backend/oneapi/random_engine.cpp          | 160 +++++
 src/backend/oneapi/random_engine.hpp          |  41 ++
 src/backend/oneapi/range.cpp                  |  57 ++
 src/backend/oneapi/range.hpp                  |  16 +
 src/backend/oneapi/reduce.hpp                 |  27 +
 src/backend/oneapi/reduce_impl.hpp            |  56 ++
 src/backend/oneapi/regions.cpp                |  42 ++
 src/backend/oneapi/regions.hpp                |  17 +
 src/backend/oneapi/reorder.cpp                |  52 ++
 src/backend/oneapi/reorder.hpp                |  15 +
 src/backend/oneapi/reshape.cpp                |  82 +++
 src/backend/oneapi/resize.cpp                 |  48 ++
 src/backend/oneapi/resize.hpp                 |  16 +
 src/backend/oneapi/rotate.cpp                 |  59 ++
 src/backend/oneapi/rotate.hpp                 |  16 +
 src/backend/oneapi/scalar.hpp                 |  23 +
 src/backend/oneapi/scan.cpp                   |  58 ++
 src/backend/oneapi/scan.hpp                   |  16 +
 src/backend/oneapi/scan_by_key.cpp            |  65 ++
 src/backend/oneapi/scan_by_key.hpp            |  17 +
 src/backend/oneapi/select.cpp                 | 145 +++++
 src/backend/oneapi/select.hpp                 |  29 +
 src/backend/oneapi/set.cpp                    | 157 +++++
 src/backend/oneapi/set.hpp                    |  23 +
 src/backend/oneapi/shift.cpp                  |  73 +++
 src/backend/oneapi/shift.hpp                  |  15 +
 src/backend/oneapi/sift.cpp                   |  76 +++
 src/backend/oneapi/sift.hpp                   |  26 +
 src/backend/oneapi/sobel.cpp                  |  49 ++
 src/backend/oneapi/sobel.hpp                  |  19 +
 src/backend/oneapi/solve.cpp                  | 368 ++++++++++++
 src/backend/oneapi/solve.hpp                  |  20 +
 src/backend/oneapi/sort.cpp                   |  67 +++
 src/backend/oneapi/sort.hpp                   |  15 +
 src/backend/oneapi/sort_by_key.cpp            |  55 ++
 src/backend/oneapi/sort_by_key.hpp            |  16 +
 src/backend/oneapi/sort_index.cpp             |  82 +++
 src/backend/oneapi/sort_index.hpp             |  16 +
 src/backend/oneapi/sparse.cpp                 | 225 +++++++
 src/backend/oneapi/sparse.hpp                 |  27 +
 src/backend/oneapi/sparse_arith.cpp           | 180 ++++++
 src/backend/oneapi/sparse_arith.hpp           |  30 +
 src/backend/oneapi/sparse_blas.cpp            |  99 +++
 src/backend/oneapi/sparse_blas.hpp            |  20 +
 src/backend/oneapi/sum.cpp                    |  39 ++
 src/backend/oneapi/surface.cpp                |  81 +++
 src/backend/oneapi/surface.hpp                |  18 +
 src/backend/oneapi/susan.cpp                  |  75 +++
 src/backend/oneapi/susan.hpp                  |  24 +
 src/backend/oneapi/svd.cpp                    | 268 +++++++++
 src/backend/oneapi/svd.hpp                    |  18 +
 src/backend/oneapi/tile.cpp                   |  51 ++
 src/backend/oneapi/tile.hpp                   |  15 +
 src/backend/oneapi/topk.cpp                   | 182 ++++++
 src/backend/oneapi/topk.hpp                   |  14 +
 src/backend/oneapi/traits.hpp                 |  56 ++
 src/backend/oneapi/transform.cpp              |  58 ++
 src/backend/oneapi/transform.hpp              |  17 +
 src/backend/oneapi/transpose.cpp              |  54 ++
 src/backend/oneapi/transpose.hpp              |  20 +
 src/backend/oneapi/transpose_inplace.cpp      |  44 ++
 src/backend/oneapi/triangle.cpp               |  56 ++
 src/backend/oneapi/triangle.hpp               |  20 +
 src/backend/oneapi/types.hpp                  | 163 +++++
 src/backend/oneapi/unary.hpp                  | 111 ++++
 src/backend/oneapi/unwrap.cpp                 |  63 ++
 src/backend/oneapi/unwrap.hpp                 |  17 +
 src/backend/oneapi/vector_field.cpp           |  36 ++
 src/backend/oneapi/vector_field.hpp           |  18 +
 src/backend/oneapi/where.cpp                  |  44 ++
 src/backend/oneapi/where.hpp                  |  15 +
 src/backend/oneapi/wrap.cpp                   |  76 +++
 src/backend/oneapi/wrap.hpp                   |  24 +
 test/CMakeLists.txt                           |   4 +
 208 files changed, 12095 insertions(+), 29 deletions(-)
 create mode 100644 src/backend/oneapi/Array.cpp
 create mode 100644 src/backend/oneapi/Array.hpp
 create mode 100644 src/backend/oneapi/CMakeLists.txt
 create mode 100644 src/backend/oneapi/Event.cpp
 create mode 100644 src/backend/oneapi/Event.hpp
 create mode 100644 src/backend/oneapi/GraphicsResourceManager.cpp
 create mode 100644 src/backend/oneapi/GraphicsResourceManager.hpp
 create mode 100644 src/backend/oneapi/Kernel.hpp
 create mode 100644 src/backend/oneapi/Module.hpp
 create mode 100644 src/backend/oneapi/Param.hpp
 create mode 100644 src/backend/oneapi/all.cpp
 create mode 100644 src/backend/oneapi/anisotropic_diffusion.cpp
 create mode 100644 src/backend/oneapi/anisotropic_diffusion.hpp
 create mode 100644 src/backend/oneapi/any.cpp
 create mode 100644 src/backend/oneapi/approx.cpp
 create mode 100644 src/backend/oneapi/approx.hpp
 create mode 100644 src/backend/oneapi/arith.hpp
 create mode 100644 src/backend/oneapi/assign.cpp
 create mode 100644 src/backend/oneapi/assign.hpp
 create mode 100644 src/backend/oneapi/backend.hpp
 create mode 100644 src/backend/oneapi/bilateral.cpp
 create mode 100644 src/backend/oneapi/bilateral.hpp
 create mode 100644 src/backend/oneapi/binary.hpp
 create mode 100644 src/backend/oneapi/blas.cpp
 create mode 100644 src/backend/oneapi/blas.hpp
 create mode 100644 src/backend/oneapi/canny.cpp
 create mode 100644 src/backend/oneapi/canny.hpp
 create mode 100644 src/backend/oneapi/cast.hpp
 create mode 100644 src/backend/oneapi/cholesky.cpp
 create mode 100644 src/backend/oneapi/cholesky.hpp
 create mode 100644 src/backend/oneapi/compile_module.cpp
 create mode 100644 src/backend/oneapi/complex.hpp
 create mode 100644 src/backend/oneapi/convolve.cpp
 create mode 100644 src/backend/oneapi/convolve.hpp
 create mode 100644 src/backend/oneapi/convolve_separable.cpp
 create mode 100644 src/backend/oneapi/copy.cpp
 create mode 100644 src/backend/oneapi/copy.hpp
 create mode 100644 src/backend/oneapi/count.cpp
 create mode 100644 src/backend/oneapi/device_manager.cpp
 create mode 100644 src/backend/oneapi/device_manager.hpp
 create mode 100644 src/backend/oneapi/diagonal.cpp
 create mode 100644 src/backend/oneapi/diagonal.hpp
 create mode 100644 src/backend/oneapi/diff.cpp
 create mode 100644 src/backend/oneapi/diff.hpp
 create mode 100644 src/backend/oneapi/err_oneapi.hpp
 create mode 100644 src/backend/oneapi/errorcodes.cpp
 create mode 100644 src/backend/oneapi/errorcodes.hpp
 create mode 100644 src/backend/oneapi/exampleFunction.cpp
 create mode 100644 src/backend/oneapi/exampleFunction.hpp
 create mode 100644 src/backend/oneapi/fast.cpp
 create mode 100644 src/backend/oneapi/fast.hpp
 create mode 100644 src/backend/oneapi/fft.cpp
 create mode 100644 src/backend/oneapi/fft.hpp
 create mode 100644 src/backend/oneapi/fftconvolve.cpp
 create mode 100644 src/backend/oneapi/fftconvolve.hpp
 create mode 100644 src/backend/oneapi/flood_fill.cpp
 create mode 100644 src/backend/oneapi/flood_fill.hpp
 create mode 100644 src/backend/oneapi/gradient.cpp
 create mode 100644 src/backend/oneapi/gradient.hpp
 create mode 100644 src/backend/oneapi/harris.cpp
 create mode 100644 src/backend/oneapi/harris.hpp
 create mode 100644 src/backend/oneapi/hist_graphics.cpp
 create mode 100644 src/backend/oneapi/hist_graphics.hpp
 create mode 100644 src/backend/oneapi/histogram.cpp
 create mode 100644 src/backend/oneapi/histogram.hpp
 create mode 100644 src/backend/oneapi/homography.cpp
 create mode 100644 src/backend/oneapi/homography.hpp
 create mode 100644 src/backend/oneapi/hsv_rgb.cpp
 create mode 100644 src/backend/oneapi/hsv_rgb.hpp
 create mode 100644 src/backend/oneapi/identity.cpp
 create mode 100644 src/backend/oneapi/identity.hpp
 create mode 100644 src/backend/oneapi/iir.cpp
 create mode 100644 src/backend/oneapi/iir.hpp
 create mode 100644 src/backend/oneapi/image.cpp
 create mode 100644 src/backend/oneapi/image.hpp
 create mode 100644 src/backend/oneapi/index.cpp
 create mode 100644 src/backend/oneapi/index.hpp
 create mode 100644 src/backend/oneapi/inverse.cpp
 create mode 100644 src/backend/oneapi/inverse.hpp
 create mode 100644 src/backend/oneapi/iota.cpp
 create mode 100644 src/backend/oneapi/iota.hpp
 create mode 100644 src/backend/oneapi/ireduce.cpp
 create mode 100644 src/backend/oneapi/ireduce.hpp
 create mode 100644 src/backend/oneapi/jit.cpp
 create mode 100644 src/backend/oneapi/jit/BufferNode.hpp
 create mode 100644 src/backend/oneapi/jit/kernel_generators.hpp
 create mode 100644 src/backend/oneapi/join.cpp
 create mode 100644 src/backend/oneapi/join.hpp
 create mode 100644 src/backend/oneapi/kernel/KParam.hpp
 create mode 100644 src/backend/oneapi/logic.hpp
 create mode 100644 src/backend/oneapi/lookup.cpp
 create mode 100644 src/backend/oneapi/lookup.hpp
 create mode 100644 src/backend/oneapi/lu.cpp
 create mode 100644 src/backend/oneapi/lu.hpp
 create mode 100644 src/backend/oneapi/match_template.cpp
 create mode 100644 src/backend/oneapi/match_template.hpp
 create mode 100644 src/backend/oneapi/math.cpp
 create mode 100644 src/backend/oneapi/math.hpp
 create mode 100644 src/backend/oneapi/max.cpp
 create mode 100644 src/backend/oneapi/mean.cpp
 create mode 100644 src/backend/oneapi/mean.hpp
 create mode 100644 src/backend/oneapi/meanshift.cpp
 create mode 100644 src/backend/oneapi/meanshift.hpp
 create mode 100644 src/backend/oneapi/medfilt.cpp
 create mode 100644 src/backend/oneapi/medfilt.hpp
 create mode 100644 src/backend/oneapi/memory.cpp
 create mode 100644 src/backend/oneapi/memory.hpp
 create mode 100644 src/backend/oneapi/min.cpp
 create mode 100644 src/backend/oneapi/moments.cpp
 create mode 100644 src/backend/oneapi/moments.hpp
 create mode 100644 src/backend/oneapi/morph.cpp
 create mode 100644 src/backend/oneapi/morph.hpp
 create mode 100644 src/backend/oneapi/nearest_neighbour.cpp
 create mode 100644 src/backend/oneapi/nearest_neighbour.hpp
 create mode 100644 src/backend/oneapi/orb.cpp
 create mode 100644 src/backend/oneapi/orb.hpp
 create mode 100644 src/backend/oneapi/platform.cpp
 create mode 100644 src/backend/oneapi/platform.hpp
 create mode 100644 src/backend/oneapi/plot.cpp
 create mode 100644 src/backend/oneapi/plot.hpp
 create mode 100644 src/backend/oneapi/print.hpp
 create mode 100644 src/backend/oneapi/product.cpp
 create mode 100644 src/backend/oneapi/qr.cpp
 create mode 100644 src/backend/oneapi/qr.hpp
 create mode 100644 src/backend/oneapi/random_engine.cpp
 create mode 100644 src/backend/oneapi/random_engine.hpp
 create mode 100644 src/backend/oneapi/range.cpp
 create mode 100644 src/backend/oneapi/range.hpp
 create mode 100644 src/backend/oneapi/reduce.hpp
 create mode 100644 src/backend/oneapi/reduce_impl.hpp
 create mode 100644 src/backend/oneapi/regions.cpp
 create mode 100644 src/backend/oneapi/regions.hpp
 create mode 100644 src/backend/oneapi/reorder.cpp
 create mode 100644 src/backend/oneapi/reorder.hpp
 create mode 100644 src/backend/oneapi/reshape.cpp
 create mode 100644 src/backend/oneapi/resize.cpp
 create mode 100644 src/backend/oneapi/resize.hpp
 create mode 100644 src/backend/oneapi/rotate.cpp
 create mode 100644 src/backend/oneapi/rotate.hpp
 create mode 100644 src/backend/oneapi/scalar.hpp
 create mode 100644 src/backend/oneapi/scan.cpp
 create mode 100644 src/backend/oneapi/scan.hpp
 create mode 100644 src/backend/oneapi/scan_by_key.cpp
 create mode 100644 src/backend/oneapi/scan_by_key.hpp
 create mode 100644 src/backend/oneapi/select.cpp
 create mode 100644 src/backend/oneapi/select.hpp
 create mode 100644 src/backend/oneapi/set.cpp
 create mode 100644 src/backend/oneapi/set.hpp
 create mode 100644 src/backend/oneapi/shift.cpp
 create mode 100644 src/backend/oneapi/shift.hpp
 create mode 100644 src/backend/oneapi/sift.cpp
 create mode 100644 src/backend/oneapi/sift.hpp
 create mode 100644 src/backend/oneapi/sobel.cpp
 create mode 100644 src/backend/oneapi/sobel.hpp
 create mode 100644 src/backend/oneapi/solve.cpp
 create mode 100644 src/backend/oneapi/solve.hpp
 create mode 100644 src/backend/oneapi/sort.cpp
 create mode 100644 src/backend/oneapi/sort.hpp
 create mode 100644 src/backend/oneapi/sort_by_key.cpp
 create mode 100644 src/backend/oneapi/sort_by_key.hpp
 create mode 100644 src/backend/oneapi/sort_index.cpp
 create mode 100644 src/backend/oneapi/sort_index.hpp
 create mode 100644 src/backend/oneapi/sparse.cpp
 create mode 100644 src/backend/oneapi/sparse.hpp
 create mode 100644 src/backend/oneapi/sparse_arith.cpp
 create mode 100644 src/backend/oneapi/sparse_arith.hpp
 create mode 100644 src/backend/oneapi/sparse_blas.cpp
 create mode 100644 src/backend/oneapi/sparse_blas.hpp
 create mode 100644 src/backend/oneapi/sum.cpp
 create mode 100644 src/backend/oneapi/surface.cpp
 create mode 100644 src/backend/oneapi/surface.hpp
 create mode 100644 src/backend/oneapi/susan.cpp
 create mode 100644 src/backend/oneapi/susan.hpp
 create mode 100644 src/backend/oneapi/svd.cpp
 create mode 100644 src/backend/oneapi/svd.hpp
 create mode 100644 src/backend/oneapi/tile.cpp
 create mode 100644 src/backend/oneapi/tile.hpp
 create mode 100644 src/backend/oneapi/topk.cpp
 create mode 100644 src/backend/oneapi/topk.hpp
 create mode 100644 src/backend/oneapi/traits.hpp
 create mode 100644 src/backend/oneapi/transform.cpp
 create mode 100644 src/backend/oneapi/transform.hpp
 create mode 100644 src/backend/oneapi/transpose.cpp
 create mode 100644 src/backend/oneapi/transpose.hpp
 create mode 100644 src/backend/oneapi/transpose_inplace.cpp
 create mode 100644 src/backend/oneapi/triangle.cpp
 create mode 100644 src/backend/oneapi/triangle.hpp
 create mode 100644 src/backend/oneapi/types.hpp
 create mode 100644 src/backend/oneapi/unary.hpp
 create mode 100644 src/backend/oneapi/unwrap.cpp
 create mode 100644 src/backend/oneapi/unwrap.hpp
 create mode 100644 src/backend/oneapi/vector_field.cpp
 create mode 100644 src/backend/oneapi/vector_field.hpp
 create mode 100644 src/backend/oneapi/where.cpp
 create mode 100644 src/backend/oneapi/where.hpp
 create mode 100644 src/backend/oneapi/wrap.cpp
 create mode 100644 src/backend/oneapi/wrap.hpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 08445a986f..60df46c5a3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -76,6 +76,7 @@ include(config_ccache)
 option(AF_BUILD_CPU      "Build ArrayFire with a CPU backend"        ON)
 option(AF_BUILD_CUDA     "Build ArrayFire with a CUDA backend"       ${CUDA_FOUND})
 option(AF_BUILD_OPENCL   "Build ArrayFire with a OpenCL backend"     ${OpenCL_FOUND})
+option(AF_BUILD_ONEAPI   "Build ArrayFire with a oneAPI backend"     ${IntelDPCPP_FOUND})
 option(AF_BUILD_UNIFIED  "Build Backend-Independent ArrayFire API"   ON)
 option(AF_BUILD_DOCS     "Create ArrayFire Documentation"            ${DOXYGEN_FOUND})
 option(AF_BUILD_EXAMPLES "Build Examples"                            ON)
@@ -355,6 +356,7 @@ add_subdirectory(src/api/cpp)
 
 conditional_directory(AF_BUILD_CPU     src/backend/cpu)
 conditional_directory(AF_BUILD_CUDA    src/backend/cuda)
+conditional_directory(AF_BUILD_ONEAPI  src/backend/oneapi)
 conditional_directory(AF_BUILD_OPENCL  src/backend/opencl)
 conditional_directory(AF_BUILD_UNIFIED src/api/unified)
 
@@ -370,6 +372,10 @@ if(TARGET afcuda)
   list(APPEND built_backends afcuda)
 endif()
 
+if(TARGET afoneapi)
+  list(APPEND built_backends afoneapi)
+endif()
+
 if(TARGET afopencl)
   list(APPEND built_backends afopencl)
 endif()
diff --git a/src/api/c/det.cpp b/src/api/c/det.cpp
index 8507675b85..0d0e5cc1d7 100644
--- a/src/api/c/det.cpp
+++ b/src/api/c/det.cpp
@@ -24,9 +24,9 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::createEmptyArray;
+using detail::scalar;
 using detail::imag;
 using detail::real;
-using detail::scalar;
 
 template<typename T>
 T det(const af_array a) {
diff --git a/src/api/c/morph.cpp b/src/api/c/morph.cpp
index e95ee06b25..948effd652 100644
--- a/src/api/c/morph.cpp
+++ b/src/api/c/morph.cpp
@@ -62,6 +62,8 @@ af_array morph<char>(const af_array &input, const af_array &mask,
     constexpr unsigned fftMethodThreshold = 17;
 #elif defined(AF_OPENCL)
     constexpr unsigned fftMethodThreshold = 19;
+#elif defined(AF_ONEAPI)
+    constexpr unsigned fftMethodThreshold = 19;
 #endif  // defined(AF_CPU)
 
     const Array<float> se = castArray<float>(mask);
diff --git a/src/backend/common/EventBase.hpp b/src/backend/common/EventBase.hpp
index 46c35e9389..874ec5b6c6 100644
--- a/src/backend/common/EventBase.hpp
+++ b/src/backend/common/EventBase.hpp
@@ -36,7 +36,8 @@ class EventBase {
 
     /// \brief Event destructor. Calls the destroy event call on the native API
     ~EventBase() noexcept {
-        if (e_) NativeEventPolicy::destroyEvent(&e_);
+      //if (e_)
+          NativeEventPolicy::destroyEvent(&e_);
     }
 
     /// \brief Creates the event object by calling the native create API
diff --git a/src/backend/common/Logger.hpp b/src/backend/common/Logger.hpp
index aa56fc4ed0..50e74ae03b 100644
--- a/src/backend/common/Logger.hpp
+++ b/src/backend/common/Logger.hpp
@@ -13,8 +13,40 @@
 #include <string>
 #include <type_traits>
 
+#if defined(__clang__)
+/* Clang/LLVM */
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wignored-attributes"
+#elif defined(__ICC) || defined(__INTEL_COMPILER)
+/* Intel ICC/ICPC */
+// Fix the warning code here, if any
+#elif defined(__GNUC__) || defined(__GNUG__)
+/* GNU GCC/G++ */
+#elif defined(_MSC_VER)
+/* Microsoft Visual Studio */
+#else
+/* Other */
+#endif
+
 #include <spdlog/spdlog.h>
 
+#if defined(__clang__)
+/* Clang/LLVM */
+#pragma clang diagnostic pop
+#elif defined(__ICC) || defined(__INTEL_COMPILER)
+/* Intel ICC/ICPC */
+// Fix the warning code here, if any
+#elif defined(__GNUC__) || defined(__GNUG__)
+/* GNU GCC/G++ */
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+/* Microsoft Visual Studio */
+#pragma warning(pop)
+#else
+/* Other */
+#endif
+
+
 namespace common {
 std::shared_ptr<spdlog::logger> loggerFactory(const std::string& name);
 std::string bytesToString(size_t bytes);
diff --git a/src/backend/common/forge_loader.hpp b/src/backend/common/forge_loader.hpp
index bf1cce8c5d..1e3edc7125 100644
--- a/src/backend/common/forge_loader.hpp
+++ b/src/backend/common/forge_loader.hpp
@@ -10,10 +10,39 @@
 #pragma once
 
 #include <common/DependencyModule.hpp>
+#include <forge.h>
+
+#if defined(__clang__)
+/* Clang/LLVM */
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wignored-attributes"
+#elif defined(__ICC) || defined(__INTEL_COMPILER)
+/* Intel ICC/ICPC */
+// Fix the warning code here, if any
+#elif defined(_MSC_VER)
+/* Microsoft Visual Studio */
+#else
+/* Other */
+#endif
 
 #include <glad/glad.h>
 
-#include <forge.h>
+#if defined(__clang__)
+/* Clang/LLVM */
+#pragma clang diagnostic pop
+#elif defined(__ICC) || defined(__INTEL_COMPILER)
+/* Intel ICC/ICPC */
+// Fix the warning code here, if any
+#elif defined(__GNUC__) || defined(__GNUG__)
+/* GNU GCC/G++ */
+#pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+/* Microsoft Visual Studio */
+#pragma warning(pop)
+#else
+/* Other */
+#endif
+
 
 class ForgeModule : public common::DependencyModule {
    public:
diff --git a/src/backend/common/jit/BinaryNode.cpp b/src/backend/common/jit/BinaryNode.cpp
index f67015b9fa..1277aa10be 100644
--- a/src/backend/common/jit/BinaryNode.cpp
+++ b/src/backend/common/jit/BinaryNode.cpp
@@ -4,6 +4,7 @@
 #include <common/jit/BinaryNode.hpp>
 #include <complex.hpp>
 #include <types.hpp>
+#include <af/traits.hpp>
 
 #include <memory>
 
diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index 8bb8185378..6b3d56162b 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -13,6 +13,7 @@
 #include <jit/kernel_generators.hpp>
 
 #include <sstream>
+#include <cstring>
 
 namespace common {
 
@@ -95,7 +96,7 @@ class BufferNodeBase : public common::Node {
     size_t getHash() const noexcept {
         size_t out = 0;
         auto ptr   = m_data.get();
-        memcpy(&out, &ptr, std::max(sizeof(Node *), sizeof(size_t)));
+        std::memcpy(&out, &ptr, std::max(sizeof(Node *), sizeof(size_t)));
         return out;
     }
 
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index bbe3fcb859..3062935909 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -71,18 +71,18 @@ using Node_ptr = std::shared_ptr<Node>;
 
 static const char *getFullName(af::dtype type) {
     switch (type) {
-        case f32: return detail::getFullName<float>();
-        case f64: return detail::getFullName<double>();
-        case c32: return detail::getFullName<detail::cfloat>();
-        case c64: return detail::getFullName<detail::cdouble>();
-        case u32: return detail::getFullName<unsigned>();
-        case s32: return detail::getFullName<int>();
-        case u64: return detail::getFullName<unsigned long long>();
-        case s64: return detail::getFullName<long long>();
-        case u16: return detail::getFullName<unsigned short>();
-        case s16: return detail::getFullName<short>();
-        case b8: return detail::getFullName<char>();
-        case u8: return detail::getFullName<unsigned char>();
+    case f32: return detail::getFullName<float>();
+    case f64: return detail::getFullName<double>();
+    case c32: return detail::getFullName<detail::cfloat>();
+    case c64: return detail::getFullName<detail::cdouble>();
+    case u32: return detail::getFullName<unsigned>();
+    case s32: return detail::getFullName<int>();
+    case u64: return detail::getFullName<unsigned long long>();
+    case s64: return detail::getFullName<long long>();
+    case u16: return detail::getFullName<unsigned short>();
+    case s16: return detail::getFullName<short>();
+    case b8: return  detail::getFullName<char>();
+    case u8: return  detail::getFullName<unsigned char>();
         case f16: return "half";
     }
     return "";
@@ -90,18 +90,18 @@ static const char *getFullName(af::dtype type) {
 
 static const char *getShortName(af::dtype type) {
     switch (type) {
-        case f32: return detail::shortname<float>();
-        case f64: return detail::shortname<double>();
-        case c32: return detail::shortname<detail::cfloat>();
-        case c64: return detail::shortname<detail::cdouble>();
-        case u32: return detail::shortname<unsigned>();
-        case s32: return detail::shortname<int>();
-        case u64: return detail::shortname<unsigned long long>();
-        case s64: return detail::shortname<long long>();
-        case u16: return detail::shortname<unsigned short>();
-        case s16: return detail::shortname<short>();
-        case b8: return detail::shortname<char>();
-        case u8: return detail::shortname<unsigned char>();
+    case f32: return detail::shortname<float>();
+    case f64: return detail::shortname<double>();
+    case c32: return detail::shortname<detail::cfloat>();
+    case c64: return detail::shortname<detail::cdouble>();
+    case u32: return detail::shortname<unsigned>();
+    case s32: return detail::shortname<int>();
+    case u64: return detail::shortname<unsigned long long>();
+    case s64: return detail::shortname<long long>();
+    case u16: return detail::shortname<unsigned short>();
+    case s16: return detail::shortname<short>();
+    case b8: return  detail::shortname<char>();
+    case u8: return  detail::shortname<unsigned char>();
         case f16: return "h";
     }
     return "";
diff --git a/src/backend/common/kernel_cache.cpp b/src/backend/common/kernel_cache.cpp
index 981d544511..869ea8d5e9 100644
--- a/src/backend/common/kernel_cache.cpp
+++ b/src/backend/common/kernel_cache.cpp
@@ -7,7 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#if !defined(AF_CPU)
+#if !defined(AF_CPU) && !defined(AF_ONEAPI)
 
 #include <common/compile_module.hpp>
 #include <common/kernel_cache.hpp>
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
new file mode 100644
index 0000000000..5f53e37052
--- /dev/null
+++ b/src/backend/oneapi/Array.cpp
@@ -0,0 +1,563 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+#include <common/half.hpp>
+#include <common/jit/NodeIterator.hpp>
+#include <common/jit/ScalarNode.hpp>
+#include <jit/BufferNode.hpp>
+#include <common/util.hpp>
+#include <err_oneapi.hpp>
+#include <copy.hpp>
+#include <memory.hpp>
+#include <platform.hpp>
+#include <scalar.hpp>
+#include <traits.hpp>
+#include <af/dim4.hpp>
+
+#include <cstddef>
+#include <cstdlib>
+#include <memory>
+#include <numeric>
+
+#include <cstdio>
+#include <cstdlib>
+#include <iostream>
+
+#include <vector>
+
+using af::dim4;
+using af::dtype_traits;
+
+using oneapi::jit::BufferNode;
+using common::half;
+using common::Node;
+using common::Node_ptr;
+using common::NodeIterator;
+
+using nonstd::span;
+using std::accumulate;
+using std::is_standard_layout;
+using std::make_shared;
+using std::shared_ptr;
+using std::vector;
+
+using sycl::buffer;
+
+namespace oneapi {
+namespace {
+template<typename T>
+shared_ptr<BufferNode<T>> bufferNodePtr() {
+    return make_shared<BufferNode<T>>(
+        static_cast<af::dtype>(dtype_traits<T>::af_type));
+}
+
+template<typename T>
+void verifyTypeSupport() {}
+
+template<>
+void verifyTypeSupport<double>() {
+    if (!isDoubleSupported(getActiveDeviceId())) {
+        AF_ERROR("Double precision not supported", AF_ERR_NO_DBL);
+    }
+}
+
+template<>
+void verifyTypeSupport<cdouble>() {
+    if (!isDoubleSupported(getActiveDeviceId())) {
+        AF_ERROR("Double precision not supported", AF_ERR_NO_DBL);
+    }
+}
+
+template<>
+void verifyTypeSupport<common::half>() {
+    if (!isHalfSupported(getActiveDeviceId())) {
+        AF_ERROR("Half precision not supported", AF_ERR_NO_HALF);
+    }
+}
+}  // namespace
+
+template<typename T>
+Array<T>::Array(const dim4 &dims)
+    : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
+           static_cast<af_dtype>(dtype_traits<T>::af_type))
+    , data(memAlloc<T>(info.elements()).release(), bufferFree<T>)
+    , data_dims(dims)
+    , node()
+    , owner(true) {}
+
+template<typename T>
+Array<T>::Array(const dim4 &dims, Node_ptr n)
+    : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
+           static_cast<af_dtype>(dtype_traits<T>::af_type))
+    , data_dims(dims)
+    , node(std::move(n))
+    , owner(true) {
+    if (node->isBuffer()) {
+        data = std::static_pointer_cast<BufferNode<T>>(node)->getDataPointer();
+    }
+}
+
+template<typename T>
+Array<T>::Array(const dim4 &dims, const T *const in_data)
+    : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
+           static_cast<af_dtype>(dtype_traits<T>::af_type))
+    , data(memAlloc<T>(info.elements()).release(), bufferFree<T>)
+    , data_dims(dims)
+    , node()
+    , owner(true) {
+    static_assert(is_standard_layout<Array<T>>::value,
+                  "Array<T> must be a standard layout type");
+    static_assert(std::is_nothrow_move_assignable<Array<T>>::value,
+                  "Array<T> is not move assignable");
+    static_assert(std::is_nothrow_move_constructible<Array<T>>::value,
+                  "Array<T> is not move constructible");
+    static_assert(
+        offsetof(Array<T>, info) == 0,
+        "Array<T>::info must be the first member variable of Array<T>");
+    // TODO(oneapi): Copy to buffer
+    //getQueue().enqueueWriteBuffer(*data.get(), CL_TRUE, 0,
+    //sizeof(T) * info.elements(), in_data);
+}
+
+template<typename T>
+Array<T>::Array(const af::dim4 &dims, buffer<T> *const mem, size_t offset,
+                bool copy)
+    : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
+           static_cast<af_dtype>(dtype_traits<T>::af_type))
+    , data(
+           copy ? memAlloc<T>(info.elements()).release() : new buffer<T>(*mem),
+          bufferFree<T>)
+    , data_dims(dims)
+    , node()
+    , owner(true) {
+    if (copy) {
+      //clRetainMemObject(mem);
+      //buffer src_buf = buffer(mem);
+        // TODO(oneapi): copy buffer
+        ONEAPI_NOT_SUPPORTED("Buffer constructor not implamented");
+        //getQueue().enqueueCopyBuffer(src_buf, *data.get(), src_offset, 0,
+        //sizeof(T) * info.elements());
+    }
+}
+
+template<typename T>
+Array<T>::Array(const Array<T> &parent, const dim4 &dims, const dim_t &offset_,
+                const dim4 &stride)
+    : info(parent.getDevId(), dims, offset_, stride,
+           static_cast<af_dtype>(dtype_traits<T>::af_type))
+    , data(parent.getData())
+    , data_dims(parent.getDataDims())
+    , node()
+    , owner(false) {}
+
+template<typename T>
+Array<T>::Array(Param<T> &tmp, bool owner_)
+    : info(getActiveDeviceId(),
+           dim4(tmp.info.dims[0], tmp.info.dims[1], tmp.info.dims[2],
+                tmp.info.dims[3]),
+           0,
+           dim4(tmp.info.strides[0], tmp.info.strides[1], tmp.info.strides[2],
+                tmp.info.strides[3]),
+           static_cast<af_dtype>(dtype_traits<T>::af_type))
+    , data(
+           tmp.data, owner_ ? bufferFree<T> : [](buffer<T> * /*unused*/) {})
+    , data_dims(dim4(tmp.info.dims[0], tmp.info.dims[1], tmp.info.dims[2],
+                     tmp.info.dims[3]))
+    , node()
+    , owner(owner_) {}
+
+template<typename T>
+Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
+                const T *const in_data, bool is_device)
+    : info(getActiveDeviceId(), dims, offset_, strides,
+           static_cast<af_dtype>(dtype_traits<T>::af_type))
+    , data(is_device ? (new buffer<T>(*reinterpret_cast<buffer<T>*>(
+                                                                    const_cast<T *>(in_data))))
+                     : (memAlloc<T>(info.elements()).release()),
+           bufferFree<T>)
+    , data_dims(dims)
+    , node()
+    , owner(true) {
+    if (!is_device) {
+      ONEAPI_NOT_SUPPORTED("Write to buffer from Host");
+          //getQueue().enqueueWriteBuffer(*data.get(), CL_TRUE, 0,
+          //sizeof(T) * info.total(), in_data);
+    }
+}
+
+template<typename T>
+void Array<T>::eval() {
+    if (isReady()) { return; }
+
+    this->setId(getActiveDeviceId());
+    data = std::shared_ptr<buffer<T>>(memAlloc<T>(info.elements()).release(),
+                                       bufferFree<T>);
+
+    ONEAPI_NOT_SUPPORTED("JIT Not supported");
+    // Do not replace this with cast operator
+    Param<T> info; //= {{dims()[0], dims()[1], dims()[2], dims()[3]},
+                  // {strides()[0], strides()[1], strides()[2], strides()[3]},
+                  // 0};
+
+    Param<T> res;// = {data.get(), info};
+
+    evalNodes(res, getNode().get());
+    node.reset();
+}
+
+template<typename T>
+void Array<T>::eval() const {
+    const_cast<Array<T> *>(this)->eval();
+}
+
+template<typename T>
+buffer<T> *Array<T>::device() {
+    if (!isOwner() || getOffset() || data.use_count() > 1) {
+        *this = copyArray<T>(*this);
+    }
+    return this->get();
+}
+
+template<typename T>
+void evalMultiple(vector<Array<T> *> arrays) {
+    vector<Param<T>> outputs;
+    vector<Array<T> *> output_arrays;
+    vector<Node *> nodes;
+
+    ONEAPI_NOT_SUPPORTED("JIT Not supported");
+    // // Check if all the arrays have the same dimension
+    // auto it = std::adjacent_find(begin(arrays), end(arrays),
+    //                              [](const Array<T> *l, const Array<T> *r) {
+    //                                  return l->dims() != r->dims();
+    //                              });
+
+    // // If they are not the same. eval individually
+    // if (it != end(arrays)) {
+    //     for (auto ptr : arrays) { ptr->eval(); }
+    //     return;
+    // }
+
+    // for (Array<T> *array : arrays) {
+    //     if (array->isReady()) { continue; }
+
+    //     const ArrayInfo info = array->info;
+
+    //     array->setId(getActiveDeviceId());
+    //     array->data = std::shared_ptr<buffer<T>>(
+    //         memAlloc<T>(info.elements()).release(), bufferFree<T>);
+
+    //     // Do not replace this with cast operator
+    //     Param<T> kInfo = {
+    //         {info.dims()[0], info.dims()[1], info.dims()[2], info.dims()[3]},
+    //         {info.strides()[0], info.strides()[1], info.strides()[2],
+    //          info.strides()[3]},
+    //         0};
+
+    //     outputs.emplace_back(array->data.get(), kInfo);
+    //     output_arrays.push_back(array);
+    //     nodes.push_back(array->getNode().get());
+    // }
+
+    // evalNodes(outputs, nodes);
+
+    // for (Array<T> *array : output_arrays) { array->node.reset(); }
+}
+
+template<typename T>
+Node_ptr Array<T>::getNode() {
+    if (node) { return node; }
+
+    KParam kinfo   = *this;
+    unsigned bytes = this->dims().elements() * sizeof(T);
+    auto nn        = bufferNodePtr<T>();
+    nn->setData(kinfo, data, bytes, isLinear());
+
+    return nn;
+}
+
+template<typename T>
+Node_ptr Array<T>::getNode() const {
+    return const_cast<Array<T> *>(this)->getNode();
+}
+
+/// This function should be called after a new JIT node is created. It will
+/// return true if the newly created node will generate a valid kernel. If
+/// false the node will fail to compile or the node and its referenced buffers
+/// are consuming too many resources. If false, the node's child nodes should
+/// be evaluated before continuing.
+///
+/// We eval in the following cases:
+///
+/// 1. Too many bytes are locked up by JIT causing memory
+///    pressure. Too many bytes is assumed to be half of all bytes
+///    allocated so far.
+///
+/// 2. The number of parameters we are passing into the kernel exceeds the
+///    limitation on the platform. For NVIDIA this is 4096 bytes. The
+template<typename T>
+kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
+    if (!evalFlag()) { return kJITHeuristics::Pass; }
+    for (const Node *n : root_nodes) {
+        if (n->getHeight() > static_cast<int>(getMaxJitSize())) {
+            return kJITHeuristics::TreeHeight;
+        }
+    }
+
+    bool isBufferLimit = getMemoryPressure() >= getMemoryPressureThreshold();
+    auto platform      = getActivePlatform();
+
+    // The Apple platform can have the nvidia card or the AMD card
+    ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
+    // bool isIntel = platform == AFCL_PLATFORM_INTEL;
+
+    // /// Intels param_size limit is much smaller than the other platforms
+    // /// so we need to start checking earlier with smaller trees
+    // int heightCheckLimit =
+    //     isIntel && getDeviceType() == CL_DEVICE_TYPE_GPU ? 3 : 6;
+
+    // // A lightweight check based on the height of the node. This is
+    // // an inexpensive operation and does not traverse the JIT tree.
+    // bool atHeightLimit =
+    //     std::any_of(std::begin(root_nodes), std::end(root_nodes),
+    //                 [heightCheckLimit](Node *n) {
+    //                     return (n->getHeight() + 1 >= heightCheckLimit);
+    //                 });
+
+    // if (atHeightLimit || isBufferLimit) {
+    //     // This is the base parameter size if the kernel had no
+    //     // arguments
+    //     size_t base_param_size =
+    //         (sizeof(T *) + sizeof(Param<T>)) * root_nodes.size() +
+    //         (3 * sizeof(uint));
+
+    //     const cl::Device &device = getDevice();
+    //     size_t max_param_size = device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>();
+    //     // typical values:
+    //     //   NVIDIA     = 4096
+    //     //   AMD        = 3520  (AMD A10 iGPU = 1024)
+    //     //   Intel iGPU = 1024
+    //     max_param_size -= base_param_size;
+
+    //     struct tree_info {
+    //         size_t total_buffer_size;
+    //         size_t num_buffers;
+    //         size_t param_scalar_size;
+    //     };
+
+    //     tree_info info{0, 0, 0};
+    //     for (Node *n : root_nodes) {
+    //         NodeIterator<> it(n);
+    //         info = accumulate(
+    //             it, NodeIterator<>(), info, [](tree_info &prev, Node &n) {
+    //                 if (n.isBuffer()) {
+    //                     auto &buf_node = static_cast<BufferNode &>(n);
+    //                     // getBytes returns the size of the data Array.
+    //                     // Sub arrays will be represented by their parent
+    //                     // size.
+    //                     prev.total_buffer_size += buf_node.getBytes();
+    //                     prev.num_buffers++;
+    //                 } else {
+    //                     prev.param_scalar_size += n.getParamBytes();
+    //                 }
+    //                 return prev;
+    //             });
+    //     }
+    //     isBufferLimit = jitTreeExceedsMemoryPressure(info.total_buffer_size);
+
+    //     size_t param_size = (info.num_buffers * (sizeof(Param<T>) + sizeof(T *)) +
+    //                          info.param_scalar_size);
+
+    //     bool isParamLimit = param_size >= max_param_size;
+
+    //     if (isParamLimit) { return kJITHeuristics::KernelParameterSize; }
+    //     if (isBufferLimit) { return kJITHeuristics::MemoryPressure; }
+    // }
+    return kJITHeuristics::Pass;
+}
+
+template<typename T>
+void *getDevicePtr(const Array<T> &arr) {
+    const buffer<T> *buf = arr.device();
+    //if (!buf) { return NULL; }
+    //memLock(buf);
+    //cl_mem mem = (*buf)();
+    return (void *)buf;
+}
+
+template<typename T>
+Array<T> createNodeArray(const dim4 &dims, Node_ptr node) {
+    verifyTypeSupport<T>();
+    Array<T> out = Array<T>(dims, node);
+    return out;
+}
+
+template<typename T>
+Array<T> createSubArray(const Array<T> &parent, const vector<af_seq> &index,
+                        bool copy) {
+    parent.eval();
+
+    dim4 dDims          = parent.getDataDims();
+    dim4 parent_strides = parent.strides();
+
+    if (parent.isLinear() == false) {
+        const Array<T> parentCopy = copyArray(parent);
+        return createSubArray(parentCopy, index, copy);
+    }
+
+    const dim4 &pDims = parent.dims();
+
+    dim4 dims    = toDims(index, pDims);
+    dim4 strides = toStride(index, dDims);
+
+    // Find total offsets after indexing
+    dim4 offsets = toOffset(index, pDims);
+    dim_t offset = parent.getOffset();
+    for (int i = 0; i < 4; i++) { offset += offsets[i] * parent_strides[i]; }
+
+    Array<T> out = Array<T>(parent, dims, offset, strides);
+
+    if (!copy) { return out; }
+
+    if (strides[0] != 1 || strides[1] < 0 || strides[2] < 0 || strides[3] < 0) {
+        out = copyArray(out);
+    }
+
+    return out;
+}
+
+template<typename T>
+Array<T> createHostDataArray(const dim4 &dims, const T *const data) {
+    verifyTypeSupport<T>();
+    return Array<T>(dims, data);
+}
+
+template<typename T>
+Array<T> createDeviceDataArray(const dim4 &dims, void *data) {
+    verifyTypeSupport<T>();
+
+    bool copy_device = false;
+    return Array<T>(dims, static_cast<buffer<T>*>(data), 0, copy_device);
+}
+
+template<typename T>
+Array<T> createValueArray(const dim4 &dims, const T &value) {
+    verifyTypeSupport<T>();
+    return createScalarNode<T>(dims, value);
+}
+
+template<typename T>
+Array<T> createEmptyArray(const dim4 &dims) {
+    verifyTypeSupport<T>();
+    return Array<T>(dims);
+}
+
+template<typename T>
+Array<T> createParamArray(Param<T> &tmp, bool owner) {
+    verifyTypeSupport<T>();
+    return Array<T>(tmp, owner);
+}
+
+template<typename T>
+void destroyArray(Array<T> *A) {
+    delete A;
+}
+
+template<typename T>
+void writeHostDataArray(Array<T> &arr, const T *const data,
+                        const size_t bytes) {
+    if (!arr.isOwner()) { arr = copyArray<T>(arr); }
+
+    ONEAPI_NOT_SUPPORTED("writeHostDataArray Not supported");
+    //getQueue().enqueueWriteBuffer(*arr.get(), CL_TRUE, arr.getOffset(), bytes,
+    //data);
+}
+
+template<typename T>
+void writeDeviceDataArray(Array<T> &arr, const void *const data,
+                          const size_t bytes) {
+    if (!arr.isOwner()) { arr = copyArray<T>(arr); }
+
+    buffer<T> &buf = *arr.get();
+
+    //clRetainMemObject(
+    //    reinterpret_cast<buffer<T> *>(const_cast<void *>(data)));
+    //buffer<T> data_buf =
+    //  buffer<T>(reinterpret_cast<buffer<T>*>(const_cast<void *>(data)));
+
+    ONEAPI_NOT_SUPPORTED("writeDeviceDataArray not supported");
+    //getQueue().enqueueCopyBuffer(data_buf, buf, 0,
+    //static_cast<size_t>(arr.getOffset()), bytes);
+}
+
+template<typename T>
+void Array<T>::setDataDims(const dim4 &new_dims) {
+    data_dims = new_dims;
+    modDims(new_dims);
+}
+
+template<typename T>
+size_t Array<T>::getAllocatedBytes() const {
+    return 0;
+    /*
+    if (!isReady()) { return 0; }
+    size_t bytes = memoryManager().allocated(data.get());
+    // External device pointer
+    if (bytes == 0 && data.get()) { return data_dims.elements() * sizeof(T); }
+    return bytes;
+    */
+}
+
+#define INSTANTIATE(T)                                                        \
+    template Array<T> createHostDataArray<T>(const dim4 &dims,                \
+                                             const T *const data);            \
+    template Array<T> createDeviceDataArray<T>(const dim4 &dims, void *data); \
+    template Array<T> createValueArray<T>(const dim4 &dims, const T &value);  \
+    template Array<T> createEmptyArray<T>(const dim4 &dims);                  \
+    template Array<T> createParamArray<T>(Param<T> & tmp, bool owner);           \
+    template Array<T> createSubArray<T>(                                      \
+        const Array<T> &parent, const vector<af_seq> &index, bool copy);      \
+    template void destroyArray<T>(Array<T> * A);                              \
+    template Array<T> createNodeArray<T>(const dim4 &dims, Node_ptr node);    \
+    template Array<T>::Array(const dim4 &dims, const dim4 &strides,           \
+                             dim_t offset, const T *const in_data,            \
+                             bool is_device);                                 \
+    template Array<T>::Array(const dim4 &dims, buffer<T>* mem, size_t src_offset, \
+                             bool copy);                                      \
+    template Node_ptr Array<T>::getNode();                                    \
+    template Node_ptr Array<T>::getNode() const;                              \
+    template void Array<T>::eval();                                           \
+    template void Array<T>::eval() const;                                     \
+    template buffer<T> *Array<T>::device();                             \
+    template void writeHostDataArray<T>(Array<T> & arr, const T *const data,  \
+                                        const size_t bytes);                  \
+    template void writeDeviceDataArray<T>(                                    \
+        Array<T> & arr, const void *const data, const size_t bytes);          \
+    template void evalMultiple<T>(vector<Array<T> *> arrays);                 \
+    template kJITHeuristics passesJitHeuristics<T>(span<Node *> node);        \
+    template void *getDevicePtr<T>(const Array<T> &arr);                      \
+    template void Array<T>::setDataDims(const dim4 &new_dims);                \
+    template size_t Array<T>::getAllocatedBytes() const;
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
new file mode 100644
index 0000000000..eb010385d4
--- /dev/null
+++ b/src/backend/oneapi/Array.hpp
@@ -0,0 +1,327 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <CL/sycl.hpp>
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/ArrayInfo.hpp>
+#include <common/MemoryManagerBase.hpp>
+#include <common/jit/Node.hpp>
+//#include <err_opencl.hpp>
+//#include <jit/BufferNode.hpp>
+//#include <memory.hpp>
+//#include <platform.hpp>
+//#include <traits.hpp>
+//#include <types.hpp>
+
+//#include <af/dim4.hpp>
+
+#include <nonstd/span.hpp>
+#include <algorithm>
+#include <cstdlib>
+#include <memory>
+#include <vector>
+
+namespace common {
+template<typename T>
+class SparseArray;
+}
+
+namespace oneapi {
+
+template<typename T>
+using Buffer_ptr = std::shared_ptr<sycl::buffer<T>>;
+using af::dim4;
+template<typename T>
+class Array;
+
+template<typename T>
+void evalMultiple(std::vector<Array<T> *> arrays);
+
+  template<typename T>
+void evalNodes(Param<T> &out, common::Node *node);
+
+  template<typename T>
+  void evalNodes(std::vector<Param<T>> &outputs,
+                 const std::vector<common::Node *> &nodes);
+
+  /// Creates a new Array object on the heap and returns a reference to it.
+  template<typename T>
+  Array<T> createNodeArray(const af::dim4 &dims, common::Node_ptr node);
+
+  /// Creates a new Array object on the heap and returns a reference to it.
+  template<typename T>
+  Array<T> createValueArray(const af::dim4 &dims, const T &value);
+
+  /// Creates a new Array object on the heap and returns a reference to it.
+  template<typename T>
+  Array<T> createHostDataArray(const af::dim4 &dims, const T *const data);
+
+  template<typename T>
+  Array<T> createDeviceDataArray(const af::dim4 &dims, void *data);
+
+  template<typename T>
+  Array<T> createStridedArray(const af::dim4 &dims, const af::dim4 &strides,
+                              dim_t offset, const T *const in_data,
+                              bool is_device) {
+      return Array<T>(dims, strides, offset, in_data, is_device);
+}
+
+/// Copies data to an existing Array object from a host pointer
+template<typename T>
+void writeHostDataArray(Array<T> &arr, const T *const data, const size_t bytes);
+
+/// Copies data to an existing Array object from a device pointer
+template<typename T>
+void writeDeviceDataArray(Array<T> &arr, const void *const data,
+                          const size_t bytes);
+
+/// Creates an empty array of a given size. No data is initialized
+///
+/// \param[in] size The dimension of the output array
+template<typename T>
+Array<T> createEmptyArray(const af::dim4 &dims);
+
+/// Create an Array object from Param object.
+///
+/// \param[in] in    The Param array that is created.
+/// \param[in] owner If true, the new Array<T> object is the owner of the data.
+/// If false
+///                  the Array<T> will not delete the object on destruction
+template<typename T>
+Array<T> createParamArray(Param<T> &tmp, bool owner);
+
+template<typename T>
+Array<T> createSubArray(const Array<T> &parent,
+                        const std::vector<af_seq> &index, bool copy = true);
+
+/// Creates a new Array object on the heap and returns a reference to it.
+template<typename T>
+void destroyArray(Array<T> *A);
+
+/// \brief Checks if the Node can be compiled successfully and the buffers
+///        references are not consuming most of the allocated memory
+///
+/// \param [in] node The root node which needs to be checked
+///
+/// \returns false if the kernel generated by this node will fail to compile
+///          or its nodes are consuming too much memory.
+template<typename T>
+kJITHeuristics passesJitHeuristics(nonstd::span<common::Node *> node);
+
+template<typename T>
+void *getDevicePtr(const Array<T> &arr);
+
+template<typename T>
+void *getRawPtr(const Array<T> &arr) {
+  //const sycl::buffer<T> *buf = arr.get();
+  //if (!buf) return NULL;
+  //cl_mem mem = (*buf)();
+  //return (void *)mem;
+
+  // TODO:
+  return nullptr;
+}
+
+template<typename T>
+using mapped_ptr = std::unique_ptr<T, std::function<void(void *)>>;
+
+template<typename T>
+class Array {
+    ArrayInfo info;  // This must be the first element of Array<T>
+
+    /// Pointer to the data
+    std::shared_ptr<sycl::buffer<T>> data;
+
+    /// The shape of the underlying parent data.
+    af::dim4 data_dims;
+
+    /// Null if this a buffer node. Otherwise this points to a JIT node
+    common::Node_ptr node;
+
+    /// If true, the Array object is the parent. If false the data object points
+    /// to another array's data
+    bool owner;
+
+    Array(const af::dim4 &dims);
+
+    Array(const Array<T> &parent, const dim4 &dims, const dim_t &offset,
+          const dim4 &stride);
+    Array(Param<T> &tmp, bool owner);
+    explicit Array(const af::dim4 &dims, common::Node_ptr n);
+    explicit Array(const af::dim4 &dims, const T *const in_data);
+
+  explicit Array(const af::dim4 &dims, sycl::buffer<T>* const mem, size_t offset, bool copy);
+
+   public:
+    Array(const Array<T> &other) = default;
+
+    Array(Array<T> &&other) noexcept = default;
+
+    Array<T> &operator=(Array<T> other) noexcept {
+        swap(other);
+        return *this;
+    }
+
+    void swap(Array<T> &other) noexcept {
+        using std::swap;
+        swap(info, other.info);
+        swap(data, other.data);
+        swap(data_dims, other.data_dims);
+        swap(node, other.node);
+        swap(owner, other.owner);
+    }
+
+    Array(const af::dim4 &dims, const af::dim4 &strides, dim_t offset,
+          const T *const in_data, bool is_device = false);
+    void resetInfo(const af::dim4 &dims) { info.resetInfo(dims); }
+    void resetDims(const af::dim4 &dims) { info.resetDims(dims); }
+    void modDims(const af::dim4 &newDims) { info.modDims(newDims); }
+    void modStrides(const af::dim4 &newStrides) { info.modStrides(newStrides); }
+    void setId(int id) { info.setId(id); }
+
+#define INFO_FUNC(RET_TYPE, NAME) \
+    RET_TYPE NAME() const { return info.NAME(); }
+
+    INFO_FUNC(const af_dtype &, getType)
+    INFO_FUNC(const af::dim4 &, strides)
+    INFO_FUNC(dim_t, elements)
+    INFO_FUNC(dim_t, ndims)
+    INFO_FUNC(const af::dim4 &, dims)
+    INFO_FUNC(int, getDevId)
+
+#undef INFO_FUNC
+
+#define INFO_IS_FUNC(NAME) \
+    bool NAME() const { return info.NAME(); }
+
+    INFO_IS_FUNC(isEmpty);
+    INFO_IS_FUNC(isScalar);
+    INFO_IS_FUNC(isRow);
+    INFO_IS_FUNC(isColumn);
+    INFO_IS_FUNC(isVector);
+    INFO_IS_FUNC(isComplex);
+    INFO_IS_FUNC(isReal);
+    INFO_IS_FUNC(isDouble);
+    INFO_IS_FUNC(isSingle);
+    INFO_IS_FUNC(isHalf);
+    INFO_IS_FUNC(isRealFloating);
+    INFO_IS_FUNC(isFloating);
+    INFO_IS_FUNC(isInteger);
+    INFO_IS_FUNC(isBool);
+    INFO_IS_FUNC(isLinear);
+    INFO_IS_FUNC(isSparse);
+
+#undef INFO_IS_FUNC
+    ~Array() = default;
+
+    bool isReady() const { return static_cast<bool>(node) == false; }
+    bool isOwner() const { return owner; }
+
+    void eval();
+    void eval() const;
+
+    sycl::buffer<T> *device();
+    sycl::buffer<T> *device() const {
+        return const_cast<Array<T> *>(this)->device();
+    }
+
+    // FIXME: This should do a copy if it is not owner. You do not want to
+    // overwrite parents data
+    sycl::buffer<T> *get() {
+        if (!isReady()) eval();
+        return data.get();
+    }
+
+    const sycl::buffer<T> *get() const {
+        if (!isReady()) eval();
+        return data.get();
+    }
+
+    int useCount() const { return data.use_count(); }
+
+    dim_t getOffset() const { return info.getOffset(); }
+
+    std::shared_ptr<sycl::buffer<T>> getData() const { return data; }
+
+    dim4 getDataDims() const { return data_dims; }
+
+    void setDataDims(const dim4 &new_dims);
+
+    size_t getAllocatedBytes() const;
+
+    operator Param<T>() const {
+        KParam info = {{dims()[0], dims()[1], dims()[2], dims()[3]},
+                       {strides()[0], strides()[1], strides()[2], strides()[3]},
+                       getOffset()};
+
+        Param<T> out{(sycl::buffer<T> *)this->get(), info};
+        return out;
+    }
+
+  operator KParam() const {
+      KParam kinfo = {
+          {dims()[0], dims()[1], dims()[2], dims()[3]},
+          {strides()[0], strides()[1], strides()[2], strides()[3]},
+          getOffset()};
+
+      return kinfo;
+  }
+
+    common::Node_ptr getNode() const;
+    common::Node_ptr getNode();
+
+   public:
+    mapped_ptr<T> getMappedPtr(cl_map_flags map_flags = CL_MAP_READ |
+                                                        CL_MAP_WRITE) const {
+        if (!isReady()) eval();
+        auto func = [data = data](void *ptr) {
+            if (ptr != nullptr) {
+              //cl_int err = getQueue().enqueueUnmapMemObject(*data, ptr);
+              //UNUSED(err);
+                ptr = nullptr;
+            }
+        };
+
+        //T *ptr = (T *)getQueue().enqueueMapBuffer(
+            //*static_cast<const sycl::buffer<T> *>(get()), CL_TRUE, map_flags,
+            //getOffset() * sizeof(T), elements() * sizeof(T), nullptr, nullptr,
+            //nullptr);
+
+        return mapped_ptr<T>(nullptr, func);
+    }
+
+    friend void evalMultiple<T>(std::vector<Array<T> *> arrays);
+
+    friend Array<T> createValueArray<T>(const af::dim4 &dims, const T &value);
+    friend Array<T> createHostDataArray<T>(const af::dim4 &dims,
+                                           const T *const data);
+    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data);
+    friend Array<T> createStridedArray<T>(const af::dim4 &dims,
+                                          const af::dim4 &strides, dim_t offset,
+                                          const T *const in_data,
+                                          bool is_device);
+
+    friend Array<T> createEmptyArray<T>(const af::dim4 &dims);
+    friend Array<T> createParamArray<T>(Param<T> &tmp, bool owner);
+    friend Array<T> createNodeArray<T>(const af::dim4 &dims,
+                                       common::Node_ptr node);
+
+    friend Array<T> createSubArray<T>(const Array<T> &parent,
+                                      const std::vector<af_seq> &index,
+                                      bool copy);
+
+    friend void destroyArray<T>(Array<T> *arr);
+    friend void *getDevicePtr<T>(const Array<T> &arr);
+    friend void *getRawPtr<T>(const Array<T> &arr);
+};
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
new file mode 100644
index 0000000000..61ce0f1eae
--- /dev/null
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -0,0 +1,240 @@
+# Copyright (c) 2022, ArrayFire
+# All rights reserved.
+#
+# This file is distributed under 3-clause BSD license.
+# The complete license agreement can be obtained at:
+# http://arrayfire.com/licenses/BSD-3-Clause
+
+include(InternalUtils)
+include(build_cl2hpp)
+
+add_library(afoneapi
+  Array.cpp
+  Array.hpp
+  Event.cpp
+  Event.hpp
+  GraphicsResourceManager.cpp
+  GraphicsResourceManager.hpp
+  Module.hpp
+  Param.hpp
+  all.cpp
+  anisotropic_diffusion.cpp
+  anisotropic_diffusion.hpp
+  any.cpp
+  approx.cpp
+  approx.hpp
+  arith.hpp
+  assign.cpp
+  assign.hpp
+  backend.hpp
+  bilateral.cpp
+  bilateral.hpp
+  binary.hpp
+  blas.cpp
+  blas.hpp
+  canny.cpp
+  canny.hpp
+  cast.hpp
+  cholesky.cpp
+  cholesky.hpp
+  compile_module.cpp
+  complex.hpp
+  convolve.cpp
+  convolve.hpp
+  convolve_separable.cpp
+  copy.cpp
+  copy.hpp
+  count.cpp
+  device_manager.cpp
+  device_manager.hpp
+  diagonal.cpp
+  diagonal.hpp
+  diff.cpp
+  diff.hpp
+  err_oneapi.hpp
+  errorcodes.cpp
+  errorcodes.hpp
+  exampleFunction.cpp
+  exampleFunction.hpp
+  fast.cpp
+  fast.hpp
+  fft.cpp
+  fft.hpp
+  fftconvolve.cpp
+  fftconvolve.hpp
+  flood_fill.cpp
+  flood_fill.hpp
+  gradient.cpp
+  gradient.hpp
+  harris.cpp
+  harris.hpp
+  hist_graphics.cpp
+  hist_graphics.hpp
+  histogram.cpp
+  histogram.hpp
+  homography.cpp
+  homography.hpp
+  hsv_rgb.cpp
+  hsv_rgb.hpp
+  identity.cpp
+  identity.hpp
+  iir.cpp
+  iir.hpp
+  image.cpp
+  image.hpp
+  index.cpp
+  index.hpp
+  inverse.cpp
+  inverse.hpp
+  iota.cpp
+  iota.hpp
+  ireduce.cpp
+  ireduce.hpp
+  jit.cpp
+  join.cpp
+  join.hpp
+  logic.hpp
+  lookup.cpp
+  lookup.hpp
+  lu.cpp
+  lu.hpp
+  match_template.cpp
+  match_template.hpp
+  math.cpp
+  math.hpp
+  max.cpp
+  mean.cpp
+  mean.hpp
+  meanshift.cpp
+  meanshift.hpp
+  medfilt.cpp
+  medfilt.hpp
+  memory.cpp
+  memory.hpp
+  min.cpp
+  moments.cpp
+  moments.hpp
+  morph.cpp
+  morph.hpp
+  nearest_neighbour.cpp
+  nearest_neighbour.hpp
+  orb.cpp
+  orb.hpp
+  platform.cpp
+  platform.hpp
+  plot.cpp
+  plot.hpp
+  print.hpp
+  product.cpp
+  qr.cpp
+  qr.hpp
+  random_engine.cpp
+  random_engine.hpp
+  range.cpp
+  range.hpp
+  reduce.hpp
+  reduce_impl.hpp
+  regions.cpp
+  regions.hpp
+  reorder.cpp
+  reorder.hpp
+  reshape.cpp
+  resize.cpp
+  resize.hpp
+  rotate.cpp
+  rotate.hpp
+  scalar.hpp
+  scan.cpp
+  scan.hpp
+  scan_by_key.cpp
+  scan_by_key.hpp
+  select.cpp
+  select.hpp
+  set.cpp
+  set.hpp
+  shift.cpp
+  shift.hpp
+  sift.cpp
+  sift.hpp
+  sobel.cpp
+  sobel.hpp
+  solve.cpp
+  solve.hpp
+  sort.cpp
+  sort.hpp
+  sort_by_key.cpp
+  sort_by_key.hpp
+  sort_index.cpp
+  sort_index.hpp
+  sparse.cpp
+  sparse.hpp
+  sparse_arith.cpp
+  sparse_arith.hpp
+  sparse_blas.cpp
+  sparse_blas.hpp
+  sum.cpp
+  surface.cpp
+  surface.hpp
+  susan.cpp
+  susan.hpp
+  svd.cpp
+  svd.hpp
+  tile.cpp
+  tile.hpp
+  topk.cpp
+  topk.hpp
+  transform.cpp
+  transform.hpp
+  transpose.cpp
+  transpose_inplace.cpp
+  transpose.hpp
+  triangle.cpp
+  triangle.hpp
+  types.hpp
+  unwrap.cpp
+  unwrap.hpp
+  vector_field.cpp
+  vector_field.hpp
+  where.cpp
+  where.hpp
+  wrap.cpp
+  wrap.hpp
+  )
+
+add_library(ArrayFire::afoneapi ALIAS afoneapi)
+
+arrayfire_set_default_cxx_flags(afoneapi)
+
+target_include_directories(afoneapi
+  PUBLIC
+    $<BUILD_INTERFACE:${ArrayFire_SOURCE_DIR}/include>
+    $<BUILD_INTERFACE:${ArrayFire_BINARY_DIR}/include>
+    $<INSTALL_INTERFACE:${AF_INSTALL_INC_DIR}>
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${SYCL_INCLUDE_DIR}
+  )
+
+target_compile_options(afoneapi
+  PRIVATE -fsycl)
+
+target_compile_definitions(afoneapi
+  PRIVATE
+    AF_ONEAPI
+  )
+
+target_link_libraries(afoneapi
+  PRIVATE
+    c_api_interface
+    cpp_api_interface
+    afcommon_interface
+    -fsycl
+  )
+
+source_group(include REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/include/*)
+source_group(api\\cpp REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/cpp/*)
+source_group(api\\c   REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/c/*)
+source_group(backend  REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/backend/common/*|${CMAKE_CURRENT_SOURCE_DIR}/*)
+source_group(backend\\kernel  REGULAR_EXPRESSION ${CMAKE_CURRENT_SOURCE_DIR}/kernel/*)
+source_group("generated files" FILES ${ArrayFire_BINARY_DIR}/version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h)
+source_group("" FILES CMakeLists.txt)
diff --git a/src/backend/oneapi/Event.cpp b/src/backend/oneapi/Event.cpp
new file mode 100644
index 0000000000..7e08c2fd44
--- /dev/null
+++ b/src/backend/oneapi/Event.cpp
@@ -0,0 +1,78 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Event.hpp>
+
+#include <events.hpp>
+#include <platform.hpp>
+#include <af/event.h>
+#include <memory>
+#include <err_oneapi.hpp>
+
+#include <memory>
+
+using std::make_unique;
+using std::unique_ptr;
+
+namespace oneapi {
+/// \brief Creates a new event and marks it in the queue
+Event makeEvent(sycl::queue& queue) {
+    ONEAPI_NOT_SUPPORTED("makeEvent");
+    return Event();
+}
+
+af_event createEvent() {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+    // auto e = make_unique<Event>();
+    // // Ensure the default CL command queue is initialized
+    // getQueue();
+    // if (e->create() != CL_SUCCESS) {
+    //     AF_ERROR("Could not create event", AF_ERR_RUNTIME);
+    // }
+    // Event& ref = *e.release();
+    // return getHandle(ref);
+}
+
+void markEventOnActiveQueue(af_event eventHandle) {
+    ONEAPI_NOT_SUPPORTED("");
+    //Event& event = getEvent(eventHandle);
+    //// Use the currently-active stream
+    //if (event.mark(getQueue()()) != CL_SUCCESS) {
+    //    AF_ERROR("Could not mark event on active queue", AF_ERR_RUNTIME);
+    //}
+}
+
+void enqueueWaitOnActiveQueue(af_event eventHandle) {
+    ONEAPI_NOT_SUPPORTED("");
+    //Event& event = getEvent(eventHandle);
+    //// Use the currently-active stream
+    //if (event.enqueueWait(getQueue()()) != CL_SUCCESS) {
+    //    AF_ERROR("Could not enqueue wait on active queue for event",
+    //             AF_ERR_RUNTIME);
+    //}
+}
+
+void block(af_event eventHandle) {
+    ONEAPI_NOT_SUPPORTED("");
+    //Event& event = getEvent(eventHandle);
+    //if (event.block() != CL_SUCCESS) {
+    //    AF_ERROR("Could not block on active queue for event", AF_ERR_RUNTIME);
+    //}
+}
+
+af_event createAndMarkEvent() {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+    //af_event handle = createEvent();
+    //markEventOnActiveQueue(handle);
+    //return handle;
+}
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/Event.hpp b/src/backend/oneapi/Event.hpp
new file mode 100644
index 0000000000..bc143283d0
--- /dev/null
+++ b/src/backend/oneapi/Event.hpp
@@ -0,0 +1,64 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <CL/sycl.hpp>
+#include <common/EventBase.hpp>
+#include <af/event.h>
+
+namespace oneapi {
+class OneAPIEventPolicy {
+   public:
+    using EventType = sycl::event;
+    using QueueType = sycl::queue;
+    //using ErrorType = sycl::exception; //does this make sense
+    using ErrorType = int;
+
+    static ErrorType createAndMarkEvent(EventType *e) noexcept {
+        // Events are created when you mark them
+        return 0;
+    }
+
+    static ErrorType markEvent(EventType *e, QueueType stream) noexcept {
+        //return clEnqueueMarkerWithWaitList(stream, 0, nullptr, e);
+      return 0;
+    }
+
+    static ErrorType waitForEvent(EventType *e, QueueType stream) noexcept {
+        //return clEnqueueMarkerWithWaitList(stream, 1, e, nullptr);
+      return 0;
+    }
+
+    static ErrorType syncForEvent(EventType *e) noexcept {
+        //return clWaitForEvents(1, e);
+      return 0;
+    }
+
+    static ErrorType destroyEvent(EventType *e) noexcept {
+        //return clReleaseEvent(*e);
+      return 0;
+    }
+};
+
+using Event = common::EventBase<OneAPIEventPolicy>;
+
+/// \brief Creates a new event and marks it in the queue
+Event makeEvent(sycl::queue &queue);
+
+af_event createEvent();
+
+void markEventOnActiveQueue(af_event eventHandle);
+
+void enqueueWaitOnActiveQueue(af_event eventHandle);
+
+void block(af_event eventHandle);
+
+af_event createAndMarkEvent();
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/GraphicsResourceManager.cpp b/src/backend/oneapi/GraphicsResourceManager.cpp
new file mode 100644
index 0000000000..8cf078e8be
--- /dev/null
+++ b/src/backend/oneapi/GraphicsResourceManager.cpp
@@ -0,0 +1,20 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <GraphicsResourceManager.hpp>
+#include <platform.hpp>
+
+namespace oneapi {
+GraphicsResourceManager::ShrdResVector
+GraphicsResourceManager::registerResources(
+    const std::vector<uint32_t>& resources) {
+    ShrdResVector output;
+    return output;
+}
+}  // namespace oneapi
diff --git a/src/backend/oneapi/GraphicsResourceManager.hpp b/src/backend/oneapi/GraphicsResourceManager.hpp
new file mode 100644
index 0000000000..bdc889708a
--- /dev/null
+++ b/src/backend/oneapi/GraphicsResourceManager.hpp
@@ -0,0 +1,33 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/InteropManager.hpp>
+
+#include <map>
+#include <memory>
+#include <vector>
+
+
+namespace oneapi {
+class GraphicsResourceManager
+    : public common::InteropManager<GraphicsResourceManager, std::byte> {
+   public:
+    using ShrdResVector = std::vector<std::shared_ptr<std::byte>>;
+
+    GraphicsResourceManager() {}
+    static ShrdResVector registerResources(
+        const std::vector<uint32_t>& resources);
+
+   protected:
+    GraphicsResourceManager(GraphicsResourceManager const&);
+    void operator=(GraphicsResourceManager const&);
+};
+}  // namespace oneapi
diff --git a/src/backend/oneapi/Kernel.hpp b/src/backend/oneapi/Kernel.hpp
new file mode 100644
index 0000000000..823fc511ef
--- /dev/null
+++ b/src/backend/oneapi/Kernel.hpp
@@ -0,0 +1,90 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/KernelInterface.hpp>
+#include <common/Logger.hpp>
+
+#include <backend.hpp>
+#include <CL/sycl.hpp>
+#include <string>
+
+namespace oneapi {
+namespace kernel_logger {
+inline auto getLogger() -> spdlog::logger* {
+    static auto logger = common::loggerFactory("kernel");
+    return logger.get();
+}
+}  // namespace kernel_logger
+
+/*
+struct Enqueuer {
+    template<typename... Args>
+    void operator()(std::string name, sycl::kernel ker,
+                    const cl::EnqueueArgs& qArgs, Args&&... args) {
+        auto launchOp = cl::KernelFunctor<Args...>(ker);
+        using namespace kernel_logger;
+        AF_TRACE("Launching {}", name);
+        launchOp(qArgs, std::forward<Args>(args)...);
+    }
+};
+
+class Kernel
+    : public common::KernelInterface<const cl::Program*, cl::Kernel, Enqueuer,
+                                     cl::Buffer*> {
+   public:
+    using ModuleType = const sycl::program*;
+    using KernelType = sycl::kernel;
+    using DevPtrType<T> = sycl::buffer<T>*;
+    using BaseClass =
+        common::KernelInterface<ModuleType, KernelType, Enqueuer, DevPtrType<T>>;
+
+    Kernel() : BaseClass("", nullptr, cl::Kernel{nullptr, false}) {}
+    Kernel(std::string name, ModuleType mod, KernelType ker)
+        : BaseClass(name, mod, ker) {}
+
+    // clang-format off
+    [[deprecated("OpenCL backend doesn't need Kernel::getDevPtr method")]]
+    DevPtrType<T> getDevPtr(const char* name) final;
+    // clang-format on
+
+    void copyToReadOnly(DevPtrType<T> dst, DevPtrType<T> src, size_t bytes) final;
+
+    void setFlag(DevPtrType<T> dst, int* scalarValPtr,
+                 const bool syncCopy = false) final;
+
+    int getFlag(DevPtrType<T> src) final;
+};
+*/
+
+class Kernel {
+   public:
+    using ModuleType = const sycl::kernel_bundle<sycl::bundle_state::executable> *;
+    using KernelType = sycl::kernel;
+  template<typename T>
+    using DevPtrType = sycl::buffer<T>*;
+    //using BaseClass =
+        //common::KernelInterface<ModuleType, KernelType, Enqueuer, DevPtrType<T>>;
+
+    Kernel() {}
+    Kernel(std::string name, ModuleType mod, KernelType ker) {}
+
+    template<typename T>
+    void copyToReadOnly(DevPtrType<T> dst, DevPtrType<T> src, size_t bytes);
+
+    template<typename T>
+    void setFlag(DevPtrType<T> dst, int* scalarValPtr,
+                 const bool syncCopy = false);
+
+    template<typename T>
+    int getFlag(DevPtrType<T> src);
+};
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/Module.hpp b/src/backend/oneapi/Module.hpp
new file mode 100644
index 0000000000..1c34306d68
--- /dev/null
+++ b/src/backend/oneapi/Module.hpp
@@ -0,0 +1,40 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/ModuleInterface.hpp>
+#include <CL/sycl.hpp>
+
+
+namespace oneapi {
+
+/// oneapi backend wrapper for cl::Program object
+  class Module : public common::ModuleInterface<sycl::kernel_bundle<sycl::bundle_state::executable>> {
+   public:
+    using ModuleType = sycl::kernel_bundle<sycl::bundle_state::executable>;
+    using BaseClass  = common::ModuleInterface<ModuleType>;
+
+    /// \brief Create an uninitialized Module
+    Module() = default;
+
+    /// \brief Create a module given a sycl::program type
+    Module(ModuleType mod) : BaseClass(mod) {}
+
+    /// \brief Unload module
+    operator bool() const final { return get().empty(); }
+
+    /// Unload the module
+    void unload() final {
+      // TODO(oneapi): Unload kernel/program
+      ;
+    }
+};
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
new file mode 100644
index 0000000000..0536d3dc0c
--- /dev/null
+++ b/src/backend/oneapi/Param.hpp
@@ -0,0 +1,36 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <CL/sycl.hpp>
+#include <kernel/KParam.hpp>
+
+namespace oneapi {
+
+template<typename T>
+struct Param {
+    sycl::buffer<T>* data;
+    KParam info;
+    Param& operator=(const Param& other) = default;
+    Param(const Param& other)            = default;
+    Param(Param&& other)                 = default;
+
+    // AF_DEPRECATED("Use Array<T>")
+    Param();
+    // AF_DEPRECATED("Use Array<T>")
+    Param(sycl::buffer<T>* data_, KParam info_);
+    ~Param() = default;
+};
+
+// AF_DEPRECATED("Use Array<T>")
+template<typename T>
+Param<T> makeParam(sycl::buffer<T>& mem, int off, const int dims[4],
+                   const int strides[4]);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/all.cpp b/src/backend/oneapi/all.cpp
new file mode 100644
index 0000000000..e74df9806c
--- /dev/null
+++ b/src/backend/oneapi/all.cpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/half.hpp>
+#include "reduce_impl.hpp"
+
+using common::half;
+
+namespace oneapi {
+// alltrue
+INSTANTIATE(af_and_t, float, char)
+INSTANTIATE(af_and_t, double, char)
+INSTANTIATE(af_and_t, cfloat, char)
+INSTANTIATE(af_and_t, cdouble, char)
+INSTANTIATE(af_and_t, int, char)
+INSTANTIATE(af_and_t, uint, char)
+INSTANTIATE(af_and_t, intl, char)
+INSTANTIATE(af_and_t, uintl, char)
+INSTANTIATE(af_and_t, char, char)
+INSTANTIATE(af_and_t, uchar, char)
+INSTANTIATE(af_and_t, short, char)
+INSTANTIATE(af_and_t, ushort, char)
+INSTANTIATE(af_and_t, half, char)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/anisotropic_diffusion.cpp b/src/backend/oneapi/anisotropic_diffusion.cpp
new file mode 100644
index 0000000000..c063736c21
--- /dev/null
+++ b/src/backend/oneapi/anisotropic_diffusion.cpp
@@ -0,0 +1,31 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <anisotropic_diffusion.hpp>
+#include <err_oneapi.hpp>
+#include <copy.hpp>
+#include <af/dim4.hpp>
+
+namespace oneapi {
+template<typename T>
+void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
+                          const af::fluxFunction fftype,
+                          const af::diffusionEq eq) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE(T)                                     \
+    template void anisotropicDiffusion<T>(                 \
+        Array<T> & inout, const float dt, const float mct, \
+        const af::fluxFunction fftype, const af::diffusionEq eq);
+
+INSTANTIATE(double)
+INSTANTIATE(float)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/anisotropic_diffusion.hpp b/src/backend/oneapi/anisotropic_diffusion.hpp
new file mode 100644
index 0000000000..e71d8928ef
--- /dev/null
+++ b/src/backend/oneapi/anisotropic_diffusion.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
+                          const af::fluxFunction fftype,
+                          const af::diffusionEq eq);
+}
diff --git a/src/backend/oneapi/any.cpp b/src/backend/oneapi/any.cpp
new file mode 100644
index 0000000000..3a3e62431f
--- /dev/null
+++ b/src/backend/oneapi/any.cpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/half.hpp>
+#include "reduce_impl.hpp"
+
+using common::half;
+
+namespace oneapi {
+// anytrue
+INSTANTIATE(af_or_t, float, char)
+INSTANTIATE(af_or_t, double, char)
+INSTANTIATE(af_or_t, cfloat, char)
+INSTANTIATE(af_or_t, cdouble, char)
+INSTANTIATE(af_or_t, int, char)
+INSTANTIATE(af_or_t, uint, char)
+INSTANTIATE(af_or_t, intl, char)
+INSTANTIATE(af_or_t, uintl, char)
+INSTANTIATE(af_or_t, char, char)
+INSTANTIATE(af_or_t, uchar, char)
+INSTANTIATE(af_or_t, short, char)
+INSTANTIATE(af_or_t, ushort, char)
+INSTANTIATE(af_or_t, half, char)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/approx.cpp b/src/backend/oneapi/approx.cpp
new file mode 100644
index 0000000000..df22448704
--- /dev/null
+++ b/src/backend/oneapi/approx.cpp
@@ -0,0 +1,89 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <approx.hpp>
+#include <err_oneapi.hpp>
+
+namespace oneapi {
+template<typename Ty, typename Tp>
+void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
+             const int xdim, const Tp &xi_beg, const Tp &xi_step,
+             const af_interp_type method, const float offGrid) {
+
+    ONEAPI_NOT_SUPPORTED("");
+    return;
+    switch (method) {
+        case AF_INTERP_NEAREST:
+        case AF_INTERP_LOWER:
+            //kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    //method, 1);
+            break;
+        case AF_INTERP_LINEAR:
+        case AF_INTERP_LINEAR_COSINE:
+            //kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    //method, 2);
+            break;
+        case AF_INTERP_CUBIC:
+        case AF_INTERP_CUBIC_SPLINE:
+            //kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    //method, 3);
+            break;
+        default: break;
+    }
+}
+
+template<typename Ty, typename Tp>
+void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
+             const int xdim, const Tp &xi_beg, const Tp &xi_step,
+             const Array<Tp> &yo, const int ydim, const Tp &yi_beg,
+             const Tp &yi_step, const af_interp_type method,
+             const float offGrid) {
+    ONEAPI_NOT_SUPPORTED("");
+    return;
+    switch (method) {
+        case AF_INTERP_NEAREST:
+        case AF_INTERP_LOWER:
+            //kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    //yi_beg, yi_step, offGrid, method, 1);
+            break;
+        case AF_INTERP_LINEAR:
+        case AF_INTERP_BILINEAR:
+        case AF_INTERP_LINEAR_COSINE:
+        case AF_INTERP_BILINEAR_COSINE:
+            //kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    //yi_beg, yi_step, offGrid, method, 2);
+            break;
+        case AF_INTERP_CUBIC:
+        case AF_INTERP_BICUBIC:
+        case AF_INTERP_CUBIC_SPLINE:
+        case AF_INTERP_BICUBIC_SPLINE:
+            //kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    //yi_beg, yi_step, offGrid, method, 3);
+            break;
+        default: break;
+    }
+}
+
+#define INSTANTIATE(Ty, Tp)                                       \
+    template void approx1<Ty, Tp>(                                \
+        Array<Ty> & yo, const Array<Ty> &yi, const Array<Tp> &xo, \
+        const int xdim, const Tp &xi_beg, const Tp &xi_step,      \
+        const af_interp_type method, const float offGrid);        \
+    template void approx2<Ty, Tp>(                                \
+        Array<Ty> & zo, const Array<Ty> &zi, const Array<Tp> &xo, \
+        const int xdim, const Tp &xi_beg, const Tp &xi_step,      \
+        const Array<Tp> &yo, const int ydim, const Tp &yi_beg,    \
+        const Tp &yi_step, const af_interp_type method, const float offGrid);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+INSTANTIATE(cfloat, float)
+INSTANTIATE(cdouble, double)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/approx.hpp b/src/backend/oneapi/approx.hpp
new file mode 100644
index 0000000000..68d06967eb
--- /dev/null
+++ b/src/backend/oneapi/approx.hpp
@@ -0,0 +1,24 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename Ty, typename Tp>
+void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
+             const int xdim, const Tp &xi_beg, const Tp &xi_step,
+             const af_interp_type method, const float offGrid);
+
+template<typename Ty, typename Tp>
+void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
+             const int xdim, const Tp &xi_beg, const Tp &xi_step,
+             const Array<Tp> &yo, const int ydim, const Tp &yi_beg,
+             const Tp &yi_step, const af_interp_type method,
+             const float offGrid);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/arith.hpp b/src/backend/oneapi/arith.hpp
new file mode 100644
index 0000000000..2a004b5766
--- /dev/null
+++ b/src/backend/oneapi/arith.hpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Array.hpp>
+#include <common/jit/BinaryNode.hpp>
+#include <optypes.hpp>
+#include <af/dim4.hpp>
+
+namespace oneapi {
+
+template<typename T, af_op_t op>
+Array<T> arithOp(const Array<T> &&lhs, const Array<T> &&rhs,
+                 const af::dim4 &odims) {
+    return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
+}
+
+template<typename T, af_op_t op>
+Array<T> arithOp(const Array<T> &lhs, const Array<T> &rhs,
+                 const af::dim4 &odims) {
+    return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
+}
+}  // namespace oneapi
diff --git a/src/backend/oneapi/assign.cpp b/src/backend/oneapi/assign.cpp
new file mode 100644
index 0000000000..06e0f63abf
--- /dev/null
+++ b/src/backend/oneapi/assign.cpp
@@ -0,0 +1,48 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <assign.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <handle.hpp>
+#include <memory.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+using common::half;
+
+namespace oneapi {
+
+template<typename T>
+void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs) {
+    ONEAPI_NOT_SUPPORTED("");
+    return;
+}
+
+#define INSTANTIATE(T)                                                \
+    template void assign<T>(Array<T> & out, const af_index_t idxrs[], \
+                            const Array<T>& rhs);
+
+INSTANTIATE(cdouble)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(float)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/assign.hpp b/src/backend/oneapi/assign.hpp
new file mode 100644
index 0000000000..7cb69fb9f4
--- /dev/null
+++ b/src/backend/oneapi/assign.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/index.h>
+
+namespace oneapi {
+
+template<typename T>
+void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs);
+
+}
diff --git a/src/backend/oneapi/backend.hpp b/src/backend/oneapi/backend.hpp
new file mode 100644
index 0000000000..5c805903c5
--- /dev/null
+++ b/src/backend/oneapi/backend.hpp
@@ -0,0 +1,22 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include "types.hpp"
+#ifdef __DH__
+#undef __DH__
+#endif
+
+#ifdef __CUDACC__
+#include <opencl_runtime.h>
+#define __DH__ __device__ __host__
+#else
+#define __DH__
+#endif
+
+namespace detail = oneapi;
diff --git a/src/backend/oneapi/bilateral.cpp b/src/backend/oneapi/bilateral.cpp
new file mode 100644
index 0000000000..4fef2afd5e
--- /dev/null
+++ b/src/backend/oneapi/bilateral.cpp
@@ -0,0 +1,41 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <bilateral.hpp>
+#include <af/dim4.hpp>
+#include <err_oneapi.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename inType, typename outType>
+Array<outType> bilateral(const Array<inType> &in, const float &sSigma,
+                         const float &cSigma) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<outType> out = createEmptyArray<outType>(in.dims());
+    return out;
+
+}
+
+#define INSTANTIATE(inT, outT)                                    \
+    template Array<outT> bilateral<inT, outT>(const Array<inT> &, \
+                                              const float &, const float &);
+
+INSTANTIATE(double, double)
+INSTANTIATE(float, float)
+INSTANTIATE(char, float)
+INSTANTIATE(int, float)
+INSTANTIATE(uint, float)
+INSTANTIATE(uchar, float)
+INSTANTIATE(short, float)
+INSTANTIATE(ushort, float)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/bilateral.hpp b/src/backend/oneapi/bilateral.hpp
new file mode 100644
index 0000000000..14a221f48f
--- /dev/null
+++ b/src/backend/oneapi/bilateral.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename inType, typename outType>
+Array<outType> bilateral(const Array<inType> &in, const float &spatialSigma,
+                         const float &chromaticSigma);
+}
diff --git a/src/backend/oneapi/binary.hpp b/src/backend/oneapi/binary.hpp
new file mode 100644
index 0000000000..b0d02195b6
--- /dev/null
+++ b/src/backend/oneapi/binary.hpp
@@ -0,0 +1,127 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <optypes.hpp>
+
+namespace oneapi {
+
+template<typename To, typename Ti, af_op_t op>
+struct BinOp;
+
+#define BINARY_TYPE_1(fn)                            \
+    template<typename To, typename Ti>               \
+    struct BinOp<To, Ti, af_##fn##_t> {              \
+        const char *name() { return "__" #fn; }      \
+    };                                               \
+                                                     \
+    template<typename To>                            \
+    struct BinOp<To, cfloat, af_##fn##_t> {          \
+        const char *name() { return "__c" #fn "f"; } \
+    };                                               \
+                                                     \
+    template<typename To>                            \
+    struct BinOp<To, cdouble, af_##fn##_t> {         \
+        const char *name() { return "__c" #fn; }     \
+    };
+
+BINARY_TYPE_1(eq)
+BINARY_TYPE_1(neq)
+BINARY_TYPE_1(lt)
+BINARY_TYPE_1(le)
+BINARY_TYPE_1(gt)
+BINARY_TYPE_1(ge)
+BINARY_TYPE_1(add)
+BINARY_TYPE_1(sub)
+BINARY_TYPE_1(mul)
+BINARY_TYPE_1(div)
+BINARY_TYPE_1(and)
+BINARY_TYPE_1(or)
+BINARY_TYPE_1(bitand)
+BINARY_TYPE_1(bitor)
+BINARY_TYPE_1(bitxor)
+BINARY_TYPE_1(bitshiftl)
+BINARY_TYPE_1(bitshiftr)
+
+#undef BINARY_TYPE_1
+
+#define BINARY_TYPE_2(fn)                            \
+    template<typename To, typename Ti>               \
+    struct BinOp<To, Ti, af_##fn##_t> {              \
+        const char *name() { return "__" #fn; }      \
+    };                                               \
+    template<typename To>                            \
+    struct BinOp<To, float, af_##fn##_t> {           \
+        const char *name() { return "f" #fn; }       \
+    };                                               \
+    template<typename To>                            \
+    struct BinOp<To, double, af_##fn##_t> {          \
+        const char *name() { return "f" #fn; }       \
+    };                                               \
+    template<typename To>                            \
+    struct BinOp<To, cfloat, af_##fn##_t> {          \
+        const char *name() { return "__c" #fn "f"; } \
+    };                                               \
+                                                     \
+    template<typename To>                            \
+    struct BinOp<To, cdouble, af_##fn##_t> {         \
+        const char *name() { return "__c" #fn; }     \
+    };
+
+BINARY_TYPE_2(min)
+BINARY_TYPE_2(max)
+BINARY_TYPE_2(rem)
+BINARY_TYPE_2(mod)
+
+template<typename To, typename Ti>
+struct BinOp<To, Ti, af_pow_t> {
+    const char *name() { return "__pow"; }
+};
+
+#define POW_BINARY_OP(INTYPE, OPNAME)         \
+    template<typename To>                     \
+    struct BinOp<To, INTYPE, af_pow_t> {      \
+        const char *name() { return OPNAME; } \
+    };
+
+POW_BINARY_OP(double, "pow")
+POW_BINARY_OP(float, "pow")
+POW_BINARY_OP(intl, "__powll")
+POW_BINARY_OP(uintl, "__powul")
+POW_BINARY_OP(uint, "__powui")
+POW_BINARY_OP(int, "__powsi")
+
+#undef POW_BINARY_OP
+
+template<typename Ti>
+struct BinOp<cfloat, Ti, af_cplx2_t> {
+    const char *name() { return "__cplx2f"; }
+};
+
+template<typename Ti>
+struct BinOp<cdouble, Ti, af_cplx2_t> {
+    const char *name() { return "__cplx2"; }
+};
+
+template<typename To, typename Ti>
+struct BinOp<To, Ti, af_cplx2_t> {
+    const char *name() { return "noop"; }
+};
+
+template<typename To, typename Ti>
+struct BinOp<To, Ti, af_atan2_t> {
+    const char *name() { return "atan2"; }
+};
+
+template<typename To, typename Ti>
+struct BinOp<To, Ti, af_hypot_t> {
+    const char *name() { return "hypot"; }
+};
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
new file mode 100644
index 0000000000..852b277870
--- /dev/null
+++ b/src/backend/oneapi/blas.cpp
@@ -0,0 +1,84 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <blas.hpp>
+
+#include <Array.hpp>
+#include <arith.hpp>
+#include <common/half.hpp>
+#include <common/traits.hpp>
+#include <complex.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+#include <reduce.hpp>
+#include <transpose.hpp>
+
+#include <complex>
+#include <vector>
+
+using common::half;
+
+namespace oneapi {
+
+void initBlas() { /*gpu_blas_init();*/ }
+
+void deInitBlas() { /*gpu_blas_deinit();*/ }
+
+template<typename T>
+void gemm_fallback(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+                   const T *alpha, const Array<T> &lhs, const Array<T> &rhs,
+                   const T *beta) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<>
+void gemm_fallback<half>(Array<half> & /*out*/, af_mat_prop /*optLhs*/,
+                         af_mat_prop /*optRhs*/, const half * /*alpha*/,
+                         const Array<half> & /*lhs*/,
+                         const Array<half> & /*rhs*/, const half * /*beta*/) {
+    ONEAPI_NOT_SUPPORTED("");
+    assert(false && "CPU fallback not implemented for f16");
+}
+
+template<typename T>
+void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
+          const Array<T> &lhs, const Array<T> &rhs, const T *beta) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<typename T>
+Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
+             af_mat_prop optRhs) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE_GEMM(TYPE)                                               \
+    template void gemm<TYPE>(Array<TYPE> & out, af_mat_prop optLhs,          \
+                             af_mat_prop optRhs, const TYPE *alpha,          \
+                             const Array<TYPE> &lhs, const Array<TYPE> &rhs, \
+                             const TYPE *beta);
+
+INSTANTIATE_GEMM(float)
+INSTANTIATE_GEMM(cfloat)
+INSTANTIATE_GEMM(double)
+INSTANTIATE_GEMM(cdouble)
+INSTANTIATE_GEMM(half)
+
+#define INSTANTIATE_DOT(TYPE)                                                  \
+    template Array<TYPE> dot<TYPE>(const Array<TYPE> &lhs,                     \
+                                   const Array<TYPE> &rhs, af_mat_prop optLhs, \
+                                   af_mat_prop optRhs);
+
+INSTANTIATE_DOT(float)
+INSTANTIATE_DOT(double)
+INSTANTIATE_DOT(cfloat)
+INSTANTIATE_DOT(cdouble)
+INSTANTIATE_DOT(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/blas.hpp b/src/backend/oneapi/blas.hpp
new file mode 100644
index 0000000000..7371d4884f
--- /dev/null
+++ b/src/backend/oneapi/blas.hpp
@@ -0,0 +1,41 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Array.hpp>
+
+// This file contains the common interface for OneAPI BLAS
+// functions
+
+namespace oneapi {
+
+void initBlas();
+void deInitBlas();
+
+template<typename T>
+void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
+          const Array<T> &lhs, const Array<T> &rhs, const T *beta);
+
+template<typename T>
+Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
+                af_mat_prop optRhs) {
+    int Mdim     = optLhs == AF_MAT_NONE ? 0 : 1;
+    int Ndim     = optRhs == AF_MAT_NONE ? 1 : 0;
+    Array<T> res = createEmptyArray<T>(
+        dim4(lhs.dims()[Mdim], rhs.dims()[Ndim], lhs.dims()[2], lhs.dims()[3]));
+    static const T alpha = T(1.0);
+    static const T beta  = T(0.0);
+    gemm(res, optLhs, optRhs, &alpha, lhs, rhs, &beta);
+    return res;
+}
+
+template<typename T>
+Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
+             af_mat_prop optRhs);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/canny.cpp b/src/backend/oneapi/canny.cpp
new file mode 100644
index 0000000000..ac85af2e1b
--- /dev/null
+++ b/src/backend/oneapi/canny.cpp
@@ -0,0 +1,28 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <canny.hpp>
+#include <err_oneapi.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+Array<float> nonMaximumSuppression(const Array<float>& mag,
+                                   const Array<float>& gx,
+                                   const Array<float>& gy) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
+                                     const Array<char>& weak) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/canny.hpp b/src/backend/oneapi/canny.hpp
new file mode 100644
index 0000000000..25f7f5458b
--- /dev/null
+++ b/src/backend/oneapi/canny.hpp
@@ -0,0 +1,19 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+Array<float> nonMaximumSuppression(const Array<float>& mag,
+                                   const Array<float>& gx,
+                                   const Array<float>& gy);
+
+Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
+                                     const Array<char>& weak);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/cast.hpp b/src/backend/oneapi/cast.hpp
new file mode 100644
index 0000000000..aef3711589
--- /dev/null
+++ b/src/backend/oneapi/cast.hpp
@@ -0,0 +1,73 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Array.hpp>
+#include <common/jit/UnaryNode.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+#include <optypes.hpp>
+#include <traits.hpp>
+#include <types.hpp>
+#include <af/dim4.hpp>
+#include <complex>
+
+namespace oneapi {
+
+template<typename To, typename Ti>
+struct CastOp {
+    const char *name() { return ""; }
+};
+
+#define CAST_FN(TYPE)                                   \
+    template<typename Ti>                               \
+    struct CastOp<TYPE, Ti> {                           \
+        const char *name() { return "convert_" #TYPE; } \
+    };
+
+CAST_FN(int)
+CAST_FN(uint)
+CAST_FN(uchar)
+CAST_FN(float)
+CAST_FN(double)
+
+#define CAST_CFN(TYPE)                                    \
+    template<typename Ti>                                 \
+    struct CastOp<TYPE, Ti> {                             \
+        const char *name() { return "__convert_" #TYPE; } \
+    };
+
+CAST_CFN(cfloat)
+CAST_CFN(cdouble)
+CAST_CFN(char)
+
+template<>
+struct CastOp<cfloat, cdouble> {
+    const char *name() { return "__convert_z2c"; }
+};
+
+template<>
+struct CastOp<cdouble, cfloat> {
+    const char *name() { return "__convert_c2z"; }
+};
+
+template<>
+struct CastOp<cfloat, cfloat> {
+    const char *name() { return "__convert_c2c"; }
+};
+
+template<>
+struct CastOp<cdouble, cdouble> {
+    const char *name() { return "__convert_z2z"; }
+};
+
+#undef CAST_FN
+#undef CAST_CFN
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/cholesky.cpp b/src/backend/oneapi/cholesky.cpp
new file mode 100644
index 0000000000..bd6b286654
--- /dev/null
+++ b/src/backend/oneapi/cholesky.cpp
@@ -0,0 +1,70 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <blas.hpp>
+#include <cholesky.hpp>
+#include <copy.hpp>
+#include <err_oneapi.hpp>
+
+#if defined(WITH_LINEAR_ALGEBRA)
+//#include <triangle.hpp>
+
+namespace oneapi {
+
+template<typename T>
+int cholesky_inplace(Array<T> &in, const bool is_upper) {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+template<typename T>
+Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper) {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+#define INSTANTIATE_CH(T)                                                 \
+    template int cholesky_inplace<T>(Array<T> & in, const bool is_upper); \
+    template Array<T> cholesky<T>(int *info, const Array<T> &in,          \
+                                  const bool is_upper);
+
+INSTANTIATE_CH(float)
+INSTANTIATE_CH(cfloat)
+INSTANTIATE_CH(double)
+INSTANTIATE_CH(cdouble)
+
+}  // namespace oneapi
+
+#else  // WITH_LINEAR_ALGEBRA
+
+namespace oneapi {
+
+template<typename T>
+Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper) {
+    AF_ERROR("Linear Algebra is disabled on OpenCL", AF_ERR_NOT_CONFIGURED);
+}
+
+template<typename T>
+int cholesky_inplace(Array<T> &in, const bool is_upper) {
+    AF_ERROR("Linear Algebra is disabled on OpenCL", AF_ERR_NOT_CONFIGURED);
+}
+
+#define INSTANTIATE_CH(T)                                                 \
+    template int cholesky_inplace<T>(Array<T> & in, const bool is_upper); \
+    template Array<T> cholesky<T>(int *info, const Array<T> &in,          \
+                                  const bool is_upper);
+
+INSTANTIATE_CH(float)
+INSTANTIATE_CH(cfloat)
+INSTANTIATE_CH(double)
+INSTANTIATE_CH(cdouble)
+
+}  // namespace oneapi
+
+#endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/cholesky.hpp b/src/backend/oneapi/cholesky.hpp
new file mode 100644
index 0000000000..d934beb566
--- /dev/null
+++ b/src/backend/oneapi/cholesky.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
+
+template<typename T>
+int cholesky_inplace(Array<T> &in, const bool is_upper);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/compile_module.cpp b/src/backend/oneapi/compile_module.cpp
new file mode 100644
index 0000000000..a682ac7bfd
--- /dev/null
+++ b/src/backend/oneapi/compile_module.cpp
@@ -0,0 +1,131 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/compile_module.hpp>  //compileModule & loadModuleFromDisk
+#include <common/kernel_cache.hpp>    //getKernel(Module&, ...)
+
+#include <CL/sycl.hpp>
+#include <common/Logger.hpp>
+#include <common/defines.hpp>
+#include <common/util.hpp>
+//#include <debug_opencl.hpp> TODO: remove?
+#include <err_oneapi.hpp>
+//#include <kernel_headers/KParam.hpp>
+#include <platform.hpp>
+#include <traits.hpp>
+
+#include <algorithm>
+#include <cctype>
+#include <cstdio>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+using common::loggerFactory;
+using fmt::format;
+//using oneapi::getActiveDeviceId;
+//using oneapi::getDevice;
+using sycl::kernel_bundle;
+using sycl::bundle_state;
+using oneapi::Kernel;
+using oneapi::Module;
+using spdlog::logger;
+
+using std::begin;
+using std::end;
+using std::ofstream;
+using std::ostringstream;
+using std::shared_ptr;
+using std::string;
+using std::to_string;
+using std::transform;
+using std::vector;
+using std::chrono::duration_cast;
+using std::chrono::high_resolution_clock;
+using std::chrono::milliseconds;
+
+logger *getLogger() {
+    static shared_ptr<logger> logger(loggerFactory("jit"));
+    return logger.get();
+}
+
+string getProgramBuildLog(const kernel_bundle<bundle_state::executable> &prog) {
+    ONEAPI_NOT_SUPPORTED("");
+    return "";
+}
+
+//#define THROW_BUILD_LOG_EXCEPTION(PROG)                              \
+//    do {                                                             \
+//        string build_error = getProgramBuildLog(PROG);               \
+//        string info        = getEnvVar("AF_OPENCL_SHOW_BUILD_INFO"); \
+//        if (!info.empty() && info != "0") puts(build_error.c_str()); \
+//        AF_ERROR(build_error, AF_ERR_INTERNAL);                      \
+//    } while (0)
+
+namespace oneapi {
+
+//const static string DEFAULT_MACROS_STR(
+    //"\n\
+                                           //#ifdef USE_DOUBLE\n\
+                                           //#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n\
+                                           //#endif\n                     \
+                                           //#ifdef USE_HALF\n\
+                                           //#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\
+                                           //#else\n                     \
+                                           //#define half short\n          \
+                                           //#endif\n                      \
+                                           //#ifndef M_PI\n               \
+                                           //#define M_PI 3.1415926535897932384626433832795028841971693993751058209749445923078164\n \
+                                           //#endif\n                     \
+                                           //");
+
+/*
+get_kernel_bundle<>() needs sycl::context
+kernel_bundle<bundle_state::executable> buildProgram(const vector<string> &kernelSources,
+                           const vector<string> &compileOpts) {
+    ONEAPI_NOT_SUPPORTED("");
+    kernel_bundle<bundle_state::executable> bb;
+    return bb;
+}
+*/
+
+}  // namespace oneapi
+
+string getKernelCacheFilename(const int device, const string &key) {
+    ONEAPI_NOT_SUPPORTED("");
+    return "";
+}
+
+namespace common {
+
+/*
+Module compileModule(const string &moduleKey, const vector<string> &sources,
+                     const vector<string> &options,
+                     const vector<string> &kInstances, const bool isJIT) {
+    ONEAPI_NOT_SUPPORTED("");
+    Module m{}
+    return m;
+}
+
+Module loadModuleFromDisk(const int device, const string &moduleKey,
+                          const bool isJIT) {
+    ONEAPI_NOT_SUPPORTED("");
+    Module m{}
+    return m;
+}
+
+Kernel getKernel(const Module &mod, const string &nameExpr,
+                 const bool sourceWasJIT) {
+    ONEAPI_NOT_SUPPORTED("");
+    return {nameExpr, &mod.get(), sycl::Kernel()};
+}
+*/
+
+}  // namespace common
diff --git a/src/backend/oneapi/complex.hpp b/src/backend/oneapi/complex.hpp
new file mode 100644
index 0000000000..c087959b42
--- /dev/null
+++ b/src/backend/oneapi/complex.hpp
@@ -0,0 +1,90 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <binary.hpp>
+#include <common/jit/BinaryNode.hpp>
+#include <common/jit/UnaryNode.hpp>
+#include <optypes.hpp>
+#include <traits.hpp>
+#include <af/dim4.hpp>
+
+namespace oneapi {
+template<typename To, typename Ti>
+Array<To> cplx(const Array<Ti> &lhs, const Array<Ti> &rhs,
+               const af::dim4 &odims) {
+    return common::createBinaryNode<To, Ti, af_cplx2_t>(lhs, rhs, odims);
+}
+
+template<typename To, typename Ti>
+Array<To> real(const Array<Ti> &in) {
+    common::Node_ptr in_node = in.getNode();
+    common::UnaryNode *node =
+        new common::UnaryNode(static_cast<af::dtype>(dtype_traits<To>::af_type),
+                              "__creal", in_node, af_real_t);
+
+    return createNodeArray<To>(in.dims(), common::Node_ptr(node));
+}
+
+template<typename To, typename Ti>
+Array<To> imag(const Array<Ti> &in) {
+    common::Node_ptr in_node = in.getNode();
+    common::UnaryNode *node =
+        new common::UnaryNode(static_cast<af::dtype>(dtype_traits<To>::af_type),
+                              "__cimag", in_node, af_imag_t);
+
+    return createNodeArray<To>(in.dims(), common::Node_ptr(node));
+}
+
+template<typename T>
+static const char *abs_name() {
+    return "fabs";
+}
+template<>
+inline const char *abs_name<cfloat>() {
+    return "__cabsf";
+}
+template<>
+inline const char *abs_name<cdouble>() {
+    return "__cabs";
+}
+
+template<typename To, typename Ti>
+Array<To> abs(const Array<Ti> &in) {
+    common::Node_ptr in_node = in.getNode();
+    common::UnaryNode *node =
+        new common::UnaryNode(static_cast<af::dtype>(dtype_traits<To>::af_type),
+                              abs_name<Ti>(), in_node, af_abs_t);
+
+    return createNodeArray<To>(in.dims(), common::Node_ptr(node));
+}
+
+template<typename T>
+static const char *conj_name() {
+    return "__noop";
+}
+template<>
+inline const char *conj_name<cfloat>() {
+    return "__cconjf";
+}
+template<>
+inline const char *conj_name<cdouble>() {
+    return "__cconj";
+}
+
+template<typename T>
+Array<T> conj(const Array<T> &in) {
+    common::Node_ptr in_node = in.getNode();
+    common::UnaryNode *node =
+        new common::UnaryNode(static_cast<af::dtype>(dtype_traits<T>::af_type),
+                              conj_name<T>(), in_node, af_conj_t);
+
+    return createNodeArray<T>(in.dims(), common::Node_ptr(node));
+}
+}  // namespace oneapi
diff --git a/src/backend/oneapi/convolve.cpp b/src/backend/oneapi/convolve.cpp
new file mode 100644
index 0000000000..94e6d48d09
--- /dev/null
+++ b/src/backend/oneapi/convolve.cpp
@@ -0,0 +1,125 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <blas.hpp>
+#include <common/half.hpp>
+#include <common/indexing_helpers.hpp>
+#include <common/moddims.hpp>
+#include <convolve.hpp>
+#include <err_oneapi.hpp>
+#include <reorder.hpp>
+#include <transpose.hpp>
+#include <unwrap.hpp>
+#include <wrap.hpp>
+#include <af/defines.h>
+#include <af/dim4.hpp>
+#include <vector>
+
+using af::dim4;
+using common::flip;
+using common::half;
+using common::modDims;
+using std::vector;
+
+namespace oneapi {
+
+template<typename T, typename accT>
+Array<T> convolve(Array<T> const &signal, Array<accT> const &filter,
+                  AF_BATCH_KIND kind, const int rank, const bool expand) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(dim4(1));
+    return out;
+}
+
+#define INSTANTIATE(T, accT)                                                   \
+    template Array<T> convolve<T, accT>(Array<T> const &, Array<accT> const &, \
+                                        AF_BATCH_KIND, const int, const bool);
+
+INSTANTIATE(cdouble, cdouble)
+INSTANTIATE(cfloat, cfloat)
+INSTANTIATE(double, double)
+INSTANTIATE(float, float)
+INSTANTIATE(uint, float)
+INSTANTIATE(int, float)
+INSTANTIATE(uchar, float)
+INSTANTIATE(char, float)
+INSTANTIATE(ushort, float)
+INSTANTIATE(short, float)
+INSTANTIATE(uintl, float)
+INSTANTIATE(intl, float)
+#undef INSTANTIATE
+
+template<typename T>
+Array<T> convolve2_unwrap(const Array<T> &signal, const Array<T> &filter,
+                          const dim4 &stride, const dim4 &padding,
+                          const dim4 &dilation) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(dim4(1));
+    return out;
+}
+
+template<typename T>
+Array<T> convolve2(Array<T> const &signal, Array<T> const &filter,
+                   const dim4 stride, const dim4 padding, const dim4 dilation) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(dim4(1));
+    return out;
+}
+
+#define INSTANTIATE(T)                                                        \
+    template Array<T> convolve2<T>(Array<T> const &signal,                    \
+                                   Array<T> const &filter, const dim4 stride, \
+                                   const dim4 padding, const dim4 dilation);
+
+INSTANTIATE(double)
+INSTANTIATE(float)
+INSTANTIATE(half)
+#undef INSTANTIATE
+
+template<typename T>
+Array<T> conv2DataGradient(const Array<T> &incoming_gradient,
+                           const Array<T> &original_signal,
+                           const Array<T> &original_filter,
+                           const Array<T> & /*convolved_output*/,
+                           af::dim4 stride, af::dim4 padding,
+                           af::dim4 dilation) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(dim4(1));
+    return out;
+}
+
+template<typename T>
+Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
+                             const Array<T> &original_signal,
+                             const Array<T> &original_filter,
+                             const Array<T> & /*convolved_output*/,
+                             af::dim4 stride, af::dim4 padding,
+                             af::dim4 dilation) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(dim4(1));
+    return out;
+}
+
+#define INSTANTIATE(T)                                                      \
+    template Array<T> conv2DataGradient<T>(                                 \
+        Array<T> const &incoming_gradient, Array<T> const &original_signal, \
+        Array<T> const &original_filter, Array<T> const &convolved_output,  \
+        const dim4 stride, const dim4 padding, const dim4 dilation);        \
+    template Array<T> conv2FilterGradient<T>(                               \
+        Array<T> const &incoming_gradient, Array<T> const &original_signal, \
+        Array<T> const &original_filter, Array<T> const &convolved_output,  \
+        const dim4 stride, const dim4 padding, const dim4 dilation);
+
+INSTANTIATE(double)
+INSTANTIATE(float)
+INSTANTIATE(half)
+#undef INSTANTIATE
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/convolve.hpp b/src/backend/oneapi/convolve.hpp
new file mode 100644
index 0000000000..7fbf2e86a1
--- /dev/null
+++ b/src/backend/oneapi/convolve.hpp
@@ -0,0 +1,39 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+template<typename T, typename accT>
+Array<T> convolve(Array<T> const &signal, Array<accT> const &filter,
+                  AF_BATCH_KIND kind, const int rank, const bool expand);
+
+template<typename T, typename accT>
+Array<T> convolve2(Array<T> const &signal, Array<accT> const &c_filter,
+                   Array<accT> const &r_filter, const bool expand);
+
+template<typename T>
+Array<T> convolve2(Array<T> const &signal, Array<T> const &filter,
+                   const dim4 stride, const dim4 padding, const dim4 dilation);
+
+template<typename T>
+Array<T> conv2DataGradient(const Array<T> &incoming_gradient,
+                           const Array<T> &original_signal,
+                           const Array<T> &original_filter,
+                           const Array<T> &convolved_output, af::dim4 stride,
+                           af::dim4 padding, af::dim4 dilation);
+
+template<typename T>
+Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
+                             const Array<T> &original_signal,
+                             const Array<T> &original_filter,
+                             const Array<T> &convolved_output, af::dim4 stride,
+                             af::dim4 padding, af::dim4 dilation);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/convolve_separable.cpp b/src/backend/oneapi/convolve_separable.cpp
new file mode 100644
index 0000000000..d9b1e1f64a
--- /dev/null
+++ b/src/backend/oneapi/convolve_separable.cpp
@@ -0,0 +1,45 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <convolve.hpp>
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename T, typename accT>
+Array<T> convolve2(Array<T> const& signal, Array<accT> const& c_filter,
+                   Array<accT> const& r_filter, const bool expand) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(dim4(1));
+    return out;
+}
+
+#define INSTANTIATE(T, accT)                                                  \
+    template Array<T> convolve2<T, accT>(Array<T> const&, Array<accT> const&, \
+                                         Array<accT> const&, const bool);
+
+INSTANTIATE(cdouble, cdouble)
+INSTANTIATE(cfloat, cfloat)
+INSTANTIATE(double, double)
+INSTANTIATE(float, float)
+INSTANTIATE(uint, float)
+INSTANTIATE(int, float)
+INSTANTIATE(uchar, float)
+INSTANTIATE(char, float)
+INSTANTIATE(short, float)
+INSTANTIATE(ushort, float)
+INSTANTIATE(intl, float)
+INSTANTIATE(uintl, float)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
new file mode 100644
index 0000000000..5e708bb593
--- /dev/null
+++ b/src/backend/oneapi/copy.cpp
@@ -0,0 +1,150 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#include <copy.hpp>
+
+#include <Array.hpp>
+#include <common/complex.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+
+using common::half;
+using common::is_complex;
+
+namespace oneapi {
+
+template<typename T>
+void copyData(T *data, const Array<T> &A) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<typename T>
+Array<T> copyArray(const Array<T> &A) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(dim4(1));
+    return out;
+}
+
+template<typename T>
+void multiply_inplace(Array<T> &in, double val) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<typename inType, typename outType>
+struct copyWrapper {
+    void operator()(Array<outType> &out, Array<inType> const &in) {
+        ONEAPI_NOT_SUPPORTED("");
+    }
+};
+
+template<typename T>
+struct copyWrapper<T, T> {
+    void operator()(Array<T> &out, Array<T> const &in) {
+        ONEAPI_NOT_SUPPORTED("");
+    }
+};
+
+template<typename inType, typename outType>
+void copyArray(Array<outType> &out, Array<inType> const &in) {
+    static_assert(!(is_complex<inType>::value && !is_complex<outType>::value),
+                  "Cannot copy from complex value to a non complex value");
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE(T)                                         \
+    template void copyData<T>(T * data, const Array<T> &from); \
+    template Array<T> copyArray<T>(const Array<T> &A);         \
+    template void multiply_inplace<T>(Array<T> & in, double norm);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+
+#define INSTANTIATE_COPY_ARRAY(SRC_T)                                 \
+    template void copyArray<SRC_T, float>(Array<float> & dst,         \
+                                          Array<SRC_T> const &src);   \
+    template void copyArray<SRC_T, double>(Array<double> & dst,       \
+                                           Array<SRC_T> const &src);  \
+    template void copyArray<SRC_T, cfloat>(Array<cfloat> & dst,       \
+                                           Array<SRC_T> const &src);  \
+    template void copyArray<SRC_T, cdouble>(Array<cdouble> & dst,     \
+                                            Array<SRC_T> const &src); \
+    template void copyArray<SRC_T, int>(Array<int> & dst,             \
+                                        Array<SRC_T> const &src);     \
+    template void copyArray<SRC_T, uint>(Array<uint> & dst,           \
+                                         Array<SRC_T> const &src);    \
+    template void copyArray<SRC_T, intl>(Array<intl> & dst,           \
+                                         Array<SRC_T> const &src);    \
+    template void copyArray<SRC_T, uintl>(Array<uintl> & dst,         \
+                                          Array<SRC_T> const &src);   \
+    template void copyArray<SRC_T, short>(Array<short> & dst,         \
+                                          Array<SRC_T> const &src);   \
+    template void copyArray<SRC_T, ushort>(Array<ushort> & dst,       \
+                                           Array<SRC_T> const &src);  \
+    template void copyArray<SRC_T, uchar>(Array<uchar> & dst,         \
+                                          Array<SRC_T> const &src);   \
+    template void copyArray<SRC_T, char>(Array<char> & dst,           \
+                                         Array<SRC_T> const &src);    \
+    template void copyArray<SRC_T, half>(Array<half> & dst,           \
+                                         Array<SRC_T> const &src);
+
+INSTANTIATE_COPY_ARRAY(float)
+INSTANTIATE_COPY_ARRAY(double)
+INSTANTIATE_COPY_ARRAY(int)
+INSTANTIATE_COPY_ARRAY(uint)
+INSTANTIATE_COPY_ARRAY(intl)
+INSTANTIATE_COPY_ARRAY(uintl)
+INSTANTIATE_COPY_ARRAY(uchar)
+INSTANTIATE_COPY_ARRAY(char)
+INSTANTIATE_COPY_ARRAY(short)
+INSTANTIATE_COPY_ARRAY(ushort)
+INSTANTIATE_COPY_ARRAY(half)
+
+#define INSTANTIATE_COPY_ARRAY_COMPLEX(SRC_T)                        \
+    template void copyArray<SRC_T, cfloat>(Array<cfloat> & dst,      \
+                                           Array<SRC_T> const &src); \
+    template void copyArray<SRC_T, cdouble>(Array<cdouble> & dst,    \
+                                            Array<SRC_T> const &src);
+
+INSTANTIATE_COPY_ARRAY_COMPLEX(cfloat)
+INSTANTIATE_COPY_ARRAY_COMPLEX(cdouble)
+
+template<typename T>
+T getScalar(const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+    return (T)0;
+}
+
+#define INSTANTIATE_GETSCALAR(T) template T getScalar(const Array<T> &in);
+
+INSTANTIATE_GETSCALAR(float)
+INSTANTIATE_GETSCALAR(double)
+INSTANTIATE_GETSCALAR(cfloat)
+INSTANTIATE_GETSCALAR(cdouble)
+INSTANTIATE_GETSCALAR(int)
+INSTANTIATE_GETSCALAR(uint)
+INSTANTIATE_GETSCALAR(uchar)
+INSTANTIATE_GETSCALAR(char)
+INSTANTIATE_GETSCALAR(intl)
+INSTANTIATE_GETSCALAR(uintl)
+INSTANTIATE_GETSCALAR(short)
+INSTANTIATE_GETSCALAR(ushort)
+INSTANTIATE_GETSCALAR(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/copy.hpp b/src/backend/oneapi/copy.hpp
new file mode 100644
index 0000000000..00f01a8ac4
--- /dev/null
+++ b/src/backend/oneapi/copy.hpp
@@ -0,0 +1,67 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <Array.hpp>
+//#include <kernel/pad_array_borders.hpp>
+
+namespace oneapi {
+template<typename T>
+void copyData(T *data, const Array<T> &A);
+
+template<typename T>
+Array<T> copyArray(const Array<T> &A);
+
+template<typename inType, typename outType>
+void copyArray(Array<outType> &out, const Array<inType> &in);
+
+// Resize Array to target dimensions and convert type
+//
+// Depending on the \p outDims, the output Array can be either truncated
+// or padded (towards end of respective dimensions).
+//
+// While resizing copying, if output dimensions are larger than input, then
+// elements beyond the input dimensions are set to the \p defaultValue.
+//
+// \param[in] in is input Array
+// \param[in] outDims is the target output dimensions
+// \param[in] defaultValue is the value to which padded locations are set.
+// \param[in] scale is the value by which all output elements are scaled.
+//
+// \returns Array<outType>
+template<typename inType, typename outType>
+Array<outType> reshape(const Array<inType> &in, const dim4 &outDims,
+                       outType defaultValue = outType(0), double scale = 1.0);
+
+template<typename T>
+Array<T> padArrayBorders(Array<T> const &in, dim4 const &lowerBoundPadding,
+                         dim4 const &upperBoundPadding,
+                         const af::borderType btype) {
+    auto iDims = in.dims();
+
+    dim4 oDims(lowerBoundPadding[0] + iDims[0] + upperBoundPadding[0],
+               lowerBoundPadding[1] + iDims[1] + upperBoundPadding[1],
+               lowerBoundPadding[2] + iDims[2] + upperBoundPadding[2],
+               lowerBoundPadding[3] + iDims[3] + upperBoundPadding[3]);
+
+    if (oDims == iDims) { return in; }
+
+    auto ret = createEmptyArray<T>(oDims);
+
+    //kernel::padBorders<T>(ret, in, lowerBoundPadding, btype);
+
+    return ret;
+}
+
+template<typename T>
+void multiply_inplace(Array<T> &in, double val);
+
+template<typename T>
+T getScalar(const Array<T> &in);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/count.cpp b/src/backend/oneapi/count.cpp
new file mode 100644
index 0000000000..d50f35b694
--- /dev/null
+++ b/src/backend/oneapi/count.cpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/half.hpp>
+#include "reduce_impl.hpp"
+
+using common::half;
+
+namespace oneapi {
+// count
+INSTANTIATE(af_notzero_t, float, uint)
+INSTANTIATE(af_notzero_t, double, uint)
+INSTANTIATE(af_notzero_t, cfloat, uint)
+INSTANTIATE(af_notzero_t, cdouble, uint)
+INSTANTIATE(af_notzero_t, int, uint)
+INSTANTIATE(af_notzero_t, uint, uint)
+INSTANTIATE(af_notzero_t, intl, uint)
+INSTANTIATE(af_notzero_t, uintl, uint)
+INSTANTIATE(af_notzero_t, char, uint)
+INSTANTIATE(af_notzero_t, uchar, uint)
+INSTANTIATE(af_notzero_t, short, uint)
+INSTANTIATE(af_notzero_t, ushort, uint)
+INSTANTIATE(af_notzero_t, half, uint)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
new file mode 100644
index 0000000000..5ef59d2682
--- /dev/null
+++ b/src/backend/oneapi/device_manager.cpp
@@ -0,0 +1,95 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/graphics_common.hpp>
+
+#include <GraphicsResourceManager.hpp>
+#include <common/DefaultMemoryManager.hpp>
+#include <common/MemoryManagerBase.hpp>
+#include <common/Logger.hpp>
+#include <common/defines.hpp>
+#include <common/host_memory.hpp>
+#include <common/util.hpp>
+#include <device_manager.hpp>
+#include <err_oneapi.hpp>
+//#include <errorcodes.hpp>
+#include <version.hpp>
+//#include <af/oneapi.h>
+#include <af/version.h>
+#include <memory>
+
+#ifdef OS_MAC
+#include <OpenGL/CGLCurrent.h>
+#endif
+
+#include <algorithm>
+#include <iterator>
+#include <sstream>
+#include <string>
+#include <vector>
+
+using std::begin;
+using std::end;
+using std::find;
+using std::make_unique;
+using std::string;
+using std::stringstream;
+using std::unique_ptr;
+using std::vector;
+using sycl::device;
+
+namespace oneapi {
+
+bool checkExtnAvailability(const device& pDevice, const string& pName) {
+    ONEAPI_NOT_SUPPORTED("");
+    return false;
+}
+
+DeviceManager::DeviceManager()
+    : logger(common::loggerFactory("platform"))
+    , mUserDeviceOffset(0)
+    , fgMngr(nullptr) {
+}
+
+spdlog::logger* DeviceManager::getLogger() { return logger.get(); }
+
+DeviceManager& DeviceManager::getInstance() {
+    ONEAPI_NOT_SUPPORTED("");
+    static auto* my_instance = new DeviceManager();
+    return *my_instance;
+}
+
+void DeviceManager::setMemoryManager(
+    std::unique_ptr<MemoryManagerBase> newMgr) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void DeviceManager::resetMemoryManager() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void DeviceManager::setMemoryManagerPinned(
+    std::unique_ptr<MemoryManagerBase> newMgr) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void DeviceManager::resetMemoryManagerPinned() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+DeviceManager::~DeviceManager() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void DeviceManager::markDeviceForInterop(const int device,
+                                         const void* wHandle) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
new file mode 100644
index 0000000000..b4f291afc2
--- /dev/null
+++ b/src/backend/oneapi/device_manager.hpp
@@ -0,0 +1,163 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <CL/sycl.hpp>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#ifndef AF_OPENCL_MEM_DEBUG
+#define AF_OPENCL_MEM_DEBUG 0
+#endif
+
+namespace boost {
+template<typename T>
+class shared_ptr;
+}  // namespace boost
+
+namespace spdlog {
+class logger;
+}
+
+namespace graphics {
+class ForgeManager;
+}
+
+namespace common {
+namespace memory {
+class MemoryManagerBase;
+}
+}  // namespace common
+
+using common::memory::MemoryManagerBase;
+
+namespace oneapi {
+
+// opencl namespace forward declarations
+class GraphicsResourceManager;
+struct kc_entry_t;  // kernel cache entry
+
+class DeviceManager {
+    friend MemoryManagerBase& memoryManager();
+
+    friend void setMemoryManager(std::unique_ptr<MemoryManagerBase> mgr);
+
+    void setMemoryManager(std::unique_ptr<MemoryManagerBase> mgr);
+
+    friend void resetMemoryManager();
+
+    void resetMemoryManager();
+
+    friend MemoryManagerBase& pinnedMemoryManager();
+
+    friend void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
+
+    void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
+
+    friend void resetMemoryManagerPinned();
+
+    void resetMemoryManagerPinned();
+
+    friend graphics::ForgeManager& forgeManager();
+
+    friend GraphicsResourceManager& interopManager();
+
+    //friend PlanCache& fftManager();
+
+    friend void addKernelToCache(int device, const std::string& key,
+                                 const kc_entry_t entry);
+
+    friend void removeKernelFromCache(int device, const std::string& key);
+
+    friend kc_entry_t kernelCache(int device, const std::string& key);
+
+    friend std::string getDeviceInfo() noexcept;
+
+    friend int getDeviceCount() noexcept;
+
+    //friend int getDeviceIdFromNativeId(cl_device_id id);
+
+    friend const sycl::context& getContext();
+
+    friend sycl::queue& getQueue();
+
+    friend const sycl::device& getDevice(int id);
+
+    friend size_t getDeviceMemorySize(int device);
+
+    friend bool isGLSharingSupported();
+
+    friend bool isDoubleSupported(unsigned device);
+
+    friend bool isHalfSupported(unsigned device);
+
+    friend void devprop(char* d_name, char* d_platform, char* d_toolkit,
+                        char* d_compute);
+
+    friend int setDevice(int device);
+
+/*
+    friend void addDeviceContext(cl_device_id dev, cl_context ctx,
+                                 cl_command_queue que);
+
+    friend void setDeviceContext(cl_device_id dev, cl_context ctx);
+
+    friend void removeDeviceContext(cl_device_id dev, cl_context ctx);
+*/
+
+    friend int getActiveDeviceType();
+
+    friend int getActivePlatform();
+
+   public:
+    static const int MAX_DEVICES = 32;
+
+    static DeviceManager& getInstance();
+
+    ~DeviceManager();
+
+    spdlog::logger* getLogger();
+
+   protected:
+    DeviceManager();
+
+    // Following two declarations are required to
+    // avoid copying accidental copy/assignment
+    // of instance returned by getInstance to other
+    // variables
+    DeviceManager(DeviceManager const&);
+    void operator=(DeviceManager const&);
+    void markDeviceForInterop(const int device, const void* wHandle);
+
+   private:
+    // Attributes
+    std::shared_ptr<spdlog::logger> logger;
+    std::mutex deviceMutex;
+    std::vector<std::unique_ptr<sycl::device>> mDevices;
+    std::vector<std::unique_ptr<sycl::context>> mContexts;
+    std::vector<std::unique_ptr<sycl::queue>> mQueues;
+    std::vector<bool> mIsGLSharingOn;
+    std::vector<int> mDeviceTypes;
+    std::vector<int> mPlatforms;
+    unsigned mUserDeviceOffset;
+
+    std::unique_ptr<graphics::ForgeManager> fgMngr;
+    std::unique_ptr<MemoryManagerBase> memManager;
+    std::unique_ptr<MemoryManagerBase> pinnedMemManager;
+    std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
+    std::mutex mutex;
+
+  //using BoostProgCache = boost::shared_ptr<boost::compute::program_cache>;
+  //std::vector<BoostProgCache*> mBoostProgCacheVector;
+};
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/diagonal.cpp b/src/backend/oneapi/diagonal.cpp
new file mode 100644
index 0000000000..f22b2440c2
--- /dev/null
+++ b/src/backend/oneapi/diagonal.cpp
@@ -0,0 +1,58 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <diagonal.hpp>
+#include <err_oneapi.hpp>
+//#include <kernel/diagonal.hpp>
+#include <math.hpp>
+#include <af/dim4.hpp>
+
+using common::half;
+
+namespace oneapi {
+template<typename T>
+Array<T> diagCreate(const Array<T> &in, const int num) {
+    ONEAPI_NOT_SUPPORTED("");
+    int size     = in.dims()[0] + std::abs(num);
+    int batch    = in.dims()[1];
+    Array<T> out = createEmptyArray<T>(dim4(size, size, batch));
+    return out;
+}
+
+template<typename T>
+Array<T> diagExtract(const Array<T> &in, const int num) {
+    ONEAPI_NOT_SUPPORTED("");
+    const dim_t *idims = in.dims().get();
+    dim_t size         = std::min(idims[0], idims[1]) - std::abs(num);
+    Array<T> out       = createEmptyArray<T>(dim4(size, 1, idims[2], idims[3]));
+
+    return out;
+}
+
+#define INSTANTIATE_DIAGONAL(T)                                          \
+    template Array<T> diagExtract<T>(const Array<T> &in, const int num); \
+    template Array<T> diagCreate<T>(const Array<T> &in, const int num);
+
+INSTANTIATE_DIAGONAL(float)
+INSTANTIATE_DIAGONAL(double)
+INSTANTIATE_DIAGONAL(cfloat)
+INSTANTIATE_DIAGONAL(cdouble)
+INSTANTIATE_DIAGONAL(int)
+INSTANTIATE_DIAGONAL(uint)
+INSTANTIATE_DIAGONAL(intl)
+INSTANTIATE_DIAGONAL(uintl)
+INSTANTIATE_DIAGONAL(char)
+INSTANTIATE_DIAGONAL(uchar)
+INSTANTIATE_DIAGONAL(short)
+INSTANTIATE_DIAGONAL(ushort)
+INSTANTIATE_DIAGONAL(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/diagonal.hpp b/src/backend/oneapi/diagonal.hpp
new file mode 100644
index 0000000000..28b4f46df6
--- /dev/null
+++ b/src/backend/oneapi/diagonal.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> diagCreate(const Array<T> &in, const int num);
+
+template<typename T>
+Array<T> diagExtract(const Array<T> &in, const int num);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/diff.cpp b/src/backend/oneapi/diff.cpp
new file mode 100644
index 0000000000..7dfffc1881
--- /dev/null
+++ b/src/backend/oneapi/diff.cpp
@@ -0,0 +1,61 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <diff.hpp>
+//#include <kernel/diff.hpp>
+#include <af/dim4.hpp>
+#include <stdexcept>
+#include <err_oneapi.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> diff(const Array<T> &in, const int dim, const bool isDiff2) {
+    ONEAPI_NOT_SUPPORTED("");
+    const af::dim4 &iDims = in.dims();
+    af::dim4 oDims        = iDims;
+    oDims[dim] -= (isDiff2 + 1);
+
+    if (iDims.elements() == 0 || oDims.elements() == 0) {
+        throw std::runtime_error("Elements are 0");
+    }
+    Array<T> out = createEmptyArray<T>(oDims);
+    return out;
+}
+
+template<typename T>
+Array<T> diff1(const Array<T> &in, const int dim) {
+    ONEAPI_NOT_SUPPORTED("");
+    return diff<T>(in, dim, false);
+}
+
+template<typename T>
+Array<T> diff2(const Array<T> &in, const int dim) {
+    ONEAPI_NOT_SUPPORTED("");
+    return diff<T>(in, dim, true);
+}
+
+#define INSTANTIATE(T)                                             \
+    template Array<T> diff1<T>(const Array<T> &in, const int dim); \
+    template Array<T> diff2<T>(const Array<T> &in, const int dim);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(char)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/diff.hpp b/src/backend/oneapi/diff.hpp
new file mode 100644
index 0000000000..d7f5aaf477
--- /dev/null
+++ b/src/backend/oneapi/diff.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> diff1(const Array<T> &in, const int dim);
+
+template<typename T>
+Array<T> diff2(const Array<T> &in, const int dim);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/err_oneapi.hpp b/src/backend/oneapi/err_oneapi.hpp
new file mode 100644
index 0000000000..ff6c83d6ca
--- /dev/null
+++ b/src/backend/oneapi/err_oneapi.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/err_common.hpp>
+
+#define ONEAPI_NOT_SUPPORTED(message)                                       \
+    do {                                                                    \
+        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \
+                           boost::stacktrace::stacktrace());                \
+    } while (0)
diff --git a/src/backend/oneapi/errorcodes.cpp b/src/backend/oneapi/errorcodes.cpp
new file mode 100644
index 0000000000..615bbb94e7
--- /dev/null
+++ b/src/backend/oneapi/errorcodes.cpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <errorcodes.hpp>
+#include <err_oneapi.hpp>
+
+
+std::string getErrorMessage(int error_code) {
+    ONEAPI_NOT_SUPPORTED("");
+    //return boost::compute::opencl_error::to_string(error_code);
+    return "";
+}
diff --git a/src/backend/oneapi/errorcodes.hpp b/src/backend/oneapi/errorcodes.hpp
new file mode 100644
index 0000000000..ff30326ae9
--- /dev/null
+++ b/src/backend/oneapi/errorcodes.hpp
@@ -0,0 +1,14 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <string>
+
+std::string getErrorMessage(int error_code);
diff --git a/src/backend/oneapi/exampleFunction.cpp b/src/backend/oneapi/exampleFunction.cpp
new file mode 100644
index 0000000000..dc5c6a8680
--- /dev/null
+++ b/src/backend/oneapi/exampleFunction.cpp
@@ -0,0 +1,65 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>  // header with oneapi backend specific
+                      // Array class implementation that inherits
+                      // ArrayInfo base class
+
+#include <exampleFunction.hpp>  // oneapi backend function header
+
+#include <err_oneapi.hpp>  // error check functions and Macros
+                           // specific to oneapi backend
+
+//#include <kernel/exampleFunction.hpp>  // this header under the folder src/oneapi/kernel
+                                       // defines the OneAPI kernel wrapper
+// function to which the main computation of your
+// algorithm should be relayed to
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> exampleFunction(const Array<T> &a, const Array<T> &b,
+                         const af_someenum_t method) {
+    ONEAPI_NOT_SUPPORTED("");
+    dim4 outputDims;  // this should be '= in.dims();' in most cases
+                      // but would definitely depend on the type of
+                      // algorithm you are implementing.
+
+    Array<T> out = createEmptyArray<T>(outputDims);
+    // Please use the create***Array<T> helper
+    // functions defined in Array.hpp to create
+    // different types of Arrays. Please check the
+    // file to know what are the different types you
+    // can create.
+
+    // Relay the actual computation to OneAPI kernel wrapper
+    //kernel::exampleFunc<T>(out, a, b, method);
+
+    return out;  // return the result
+}
+
+#define INSTANTIATE(T)                                                         \
+    template Array<T> exampleFunction<T>(const Array<T> &a, const Array<T> &b, \
+                                         const af_someenum_t method);
+
+// INSTANTIATIONS for all the types which
+// are present in the switch case statement
+// in src/api/c/exampleFunction.cpp should be available
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/exampleFunction.hpp b/src/backend/oneapi/exampleFunction.hpp
new file mode 100644
index 0000000000..7f51018f83
--- /dev/null
+++ b/src/backend/oneapi/exampleFunction.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> exampleFunction(const Array<T> &a, const Array<T> &b,
+                         const af_someenum_t method);
+}
diff --git a/src/backend/oneapi/fast.cpp b/src/backend/oneapi/fast.cpp
new file mode 100644
index 0000000000..25f8c47e6a
--- /dev/null
+++ b/src/backend/oneapi/fast.cpp
@@ -0,0 +1,44 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+#include <af/dim4.hpp>
+#include <af/features.h>
+
+using af::dim4;
+using af::features;
+
+namespace oneapi {
+
+template<typename T>
+unsigned fast(Array<float> &x_out, Array<float> &y_out, Array<float> &score_out,
+              const Array<T> &in, const float thr, const unsigned arc_length,
+              const bool non_max, const float feature_ratio,
+              const unsigned edge) {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+#define INSTANTIATE(T)                                                        \
+    template unsigned fast<T>(                                                \
+        Array<float> & x_out, Array<float> & y_out, Array<float> & score_out, \
+        const Array<T> &in, const float thr, const unsigned arc_length,       \
+        const bool nonmax, const float feature_ratio, const unsigned edge);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/fast.hpp b/src/backend/oneapi/fast.hpp
new file mode 100644
index 0000000000..19667cf49e
--- /dev/null
+++ b/src/backend/oneapi/fast.hpp
@@ -0,0 +1,23 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/features.h>
+
+using af::features;
+
+namespace oneapi {
+
+template<typename T>
+unsigned fast(Array<float> &x_out, Array<float> &y_out, Array<float> &score_out,
+              const Array<T> &in, const float thr, const unsigned arc_length,
+              const bool non_max, const float feature_ratio,
+              const unsigned edge);
+
+}
diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
new file mode 100644
index 0000000000..684cc860b7
--- /dev/null
+++ b/src/backend/oneapi/fft.cpp
@@ -0,0 +1,106 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <fft.hpp>
+
+#include <copy.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+void setFFTPlanCacheSize(size_t numPlans) {
+}
+
+/*
+template<typename T>
+struct Precision;
+template<>
+struct Precision<cfloat> {
+    enum { type = CLFFT_SINGLE };
+};
+template<>
+struct Precision<cdouble> {
+    enum { type = CLFFT_DOUBLE };
+};
+*/
+
+void computeDims(size_t rdims[AF_MAX_DIMS], const dim4 &idims) {
+    for (int i = 0; i < AF_MAX_DIMS; i++) {
+        rdims[i] = static_cast<size_t>(idims[i]);
+    }
+}
+
+//(currently) true is in clFFT if length is a power of 2,3,5
+inline bool isSupLen(dim_t length) {
+    while (length > 1) {
+        if (length % 2 == 0) {
+            length /= 2;
+        } else if (length % 3 == 0) {
+            length /= 3;
+        } else if (length % 5 == 0) {
+            length /= 5;
+        } else if (length % 7 == 0) {
+            length /= 7;
+        } else if (length % 11 == 0) {
+            length /= 11;
+        } else if (length % 13 == 0) {
+            length /= 13;
+        } else {
+            return false;
+        }
+    }
+    return true;
+}
+
+void verifySupported(const int rank, const dim4 &dims) {
+    for (int i = 0; i < rank; i++) { ARG_ASSERT(1, isSupLen(dims[i])); }
+}
+
+template<typename T>
+void fft_inplace(Array<T> &in, const int rank, const bool direction) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<typename Tc, typename Tr>
+Array<Tc> fft_r2c(const Array<Tr> &in, const int rank) {
+    ONEAPI_NOT_SUPPORTED("");
+    dim4 odims = in.dims();
+
+    odims[0] = odims[0] / 2 + 1;
+
+    Array<Tc> out = createEmptyArray<Tc>(odims);
+    return out;
+}
+
+template<typename Tr, typename Tc>
+Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<Tr> out = createEmptyArray<Tr>(odims);
+    return out;
+}
+
+#define INSTANTIATE(T) \
+    template void fft_inplace<T>(Array<T> &, const int, const bool);
+
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+
+#define INSTANTIATE_REAL(Tr, Tc)                                        \
+    template Array<Tc> fft_r2c<Tc, Tr>(const Array<Tr> &, const int);   \
+    template Array<Tr> fft_c2r<Tr, Tc>(const Array<Tc> &, const dim4 &, \
+                                       const int);
+
+INSTANTIATE_REAL(float, cfloat)
+INSTANTIATE_REAL(double, cdouble)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/fft.hpp b/src/backend/oneapi/fft.hpp
new file mode 100644
index 0000000000..57de589db2
--- /dev/null
+++ b/src/backend/oneapi/fft.hpp
@@ -0,0 +1,25 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+void setFFTPlanCacheSize(size_t numPlans);
+
+template<typename T>
+void fft_inplace(Array<T> &in, const int rank, const bool direction);
+
+template<typename Tc, typename Tr>
+Array<Tc> fft_r2c(const Array<Tr> &in, const int rank);
+
+template<typename Tr, typename Tc>
+Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank);
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/fftconvolve.cpp b/src/backend/oneapi/fftconvolve.cpp
new file mode 100644
index 0000000000..5a2a64d869
--- /dev/null
+++ b/src/backend/oneapi/fftconvolve.cpp
@@ -0,0 +1,82 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <fftconvolve.hpp>
+
+#include <Array.hpp>
+#include <common/dispatch.hpp>
+#include <fft.hpp>
+#include <af/dim4.hpp>
+#include <err_oneapi.hpp>
+
+#include <cmath>
+#include <type_traits>
+#include <vector>
+
+using af::dim4;
+using std::ceil;
+using std::conditional;
+using std::is_integral;
+using std::is_same;
+using std::vector;
+
+namespace oneapi {
+
+template<typename T>
+dim4 calcPackedSize(Array<T> const& i1, Array<T> const& i2, const dim_t rank) {
+    const dim4& i1d = i1.dims();
+    const dim4& i2d = i2.dims();
+
+    dim_t pd[4] = {1, 1, 1, 1};
+
+    // Pack both signal and filter on same memory array, this will ensure
+    // better use of batched cuFFT capabilities
+    pd[0] = nextpow2(static_cast<unsigned>(
+        static_cast<int>(ceil(i1d[0] / 2.f)) + i2d[0] - 1));
+
+    for (dim_t k = 1; k < rank; k++) {
+        pd[k] = nextpow2(static_cast<unsigned>(i1d[k] + i2d[k] - 1));
+    }
+
+    dim_t i1batch = 1;
+    dim_t i2batch = 1;
+    for (int k = rank; k < 4; k++) {
+        i1batch *= i1d[k];
+        i2batch *= i2d[k];
+    }
+    pd[rank] = (i1batch + i2batch);
+
+    return dim4(pd[0], pd[1], pd[2], pd[3]);
+}
+
+template<typename T>
+Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
+                     const bool expand, AF_BATCH_KIND kind, const int rank) {
+    ONEAPI_NOT_SUPPORTED("");
+    dim4 oDims(1);
+    Array<T> out = createEmptyArray<T>(oDims);
+    return out;
+}
+
+#define INSTANTIATE(T)                                                 \
+    template Array<T> fftconvolve<T>(Array<T> const&, Array<T> const&, \
+                                     const bool, AF_BATCH_KIND, const int);
+
+INSTANTIATE(double)
+INSTANTIATE(float)
+INSTANTIATE(uint)
+INSTANTIATE(int)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(uintl)
+INSTANTIATE(intl)
+INSTANTIATE(ushort)
+INSTANTIATE(short)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/fftconvolve.hpp b/src/backend/oneapi/fftconvolve.hpp
new file mode 100644
index 0000000000..7eac7750aa
--- /dev/null
+++ b/src/backend/oneapi/fftconvolve.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
+                     const bool expand, AF_BATCH_KIND kind, const int rank);
+}
diff --git a/src/backend/oneapi/flood_fill.cpp b/src/backend/oneapi/flood_fill.cpp
new file mode 100644
index 0000000000..a336a441ec
--- /dev/null
+++ b/src/backend/oneapi/flood_fill.cpp
@@ -0,0 +1,36 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <flood_fill.hpp>
+
+#include <err_oneapi.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
+                   const Array<uint>& seedsY, const T newValue,
+                   const T lowValue, const T highValue,
+                   const af::connectivity nlookup) {
+    ONEAPI_NOT_SUPPORTED("");
+    auto out = createValueArray(image.dims(), T(0));
+    return out;
+}
+
+#define INSTANTIATE(T)                                                         \
+    template Array<T> floodFill(const Array<T>&, const Array<uint>&,           \
+                                const Array<uint>&, const T, const T, const T, \
+                                const af::connectivity);
+
+INSTANTIATE(float)
+INSTANTIATE(uint)
+INSTANTIATE(ushort)
+INSTANTIATE(uchar)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/flood_fill.hpp b/src/backend/oneapi/flood_fill.hpp
new file mode 100644
index 0000000000..6590f33e59
--- /dev/null
+++ b/src/backend/oneapi/flood_fill.hpp
@@ -0,0 +1,21 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Array.hpp>
+#include <af/defines.h>
+
+namespace oneapi {
+template<typename T>
+Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
+                   const Array<uint>& seedsY, const T newValue,
+                   const T lowValue, const T highValue,
+                   const af::connectivity nlookup = AF_CONNECTIVITY_8);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/gradient.cpp b/src/backend/oneapi/gradient.cpp
new file mode 100644
index 0000000000..0755b7a691
--- /dev/null
+++ b/src/backend/oneapi/gradient.cpp
@@ -0,0 +1,31 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <gradient.hpp>
+#include <err_oneapi.hpp>
+//#include <kernel/gradient.hpp>
+#include <math.hpp>
+#include <stdexcept>
+
+namespace oneapi {
+template<typename T>
+void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE(T)                                            \
+    template void gradient<T>(Array<T> & grad0, Array<T> & grad1, \
+                              const Array<T> &in);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/gradient.hpp b/src/backend/oneapi/gradient.hpp
new file mode 100644
index 0000000000..e5ebff012c
--- /dev/null
+++ b/src/backend/oneapi/gradient.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in);
+}
diff --git a/src/backend/oneapi/harris.cpp b/src/backend/oneapi/harris.cpp
new file mode 100644
index 0000000000..ef6b844fd4
--- /dev/null
+++ b/src/backend/oneapi/harris.cpp
@@ -0,0 +1,40 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+#include <af/dim4.hpp>
+#include <af/features.h>
+
+using af::dim4;
+using af::features;
+
+namespace oneapi {
+
+template<typename T, typename convAccT>
+unsigned harris(Array<float> &x_out, Array<float> &y_out,
+                Array<float> &score_out, const Array<T> &in,
+                const unsigned max_corners, const float min_response,
+                const float sigma, const unsigned filter_len,
+                const float k_thr) {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+#define INSTANTIATE(T, convAccT)                                              \
+    template unsigned harris<T, convAccT>(                                    \
+        Array<float> & x_out, Array<float> & y_out, Array<float> & score_out, \
+        const Array<T> &in, const unsigned max_corners,                       \
+        const float min_response, const float sigma,                          \
+        const unsigned filter_len, const float k_thr);
+
+INSTANTIATE(double, double)
+INSTANTIATE(float, float)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/harris.hpp b/src/backend/oneapi/harris.hpp
new file mode 100644
index 0000000000..8eeef1dcc3
--- /dev/null
+++ b/src/backend/oneapi/harris.hpp
@@ -0,0 +1,24 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/features.h>
+
+using af::features;
+
+namespace oneapi {
+
+template<typename T, typename convAccT>
+unsigned harris(Array<float> &x_out, Array<float> &y_out,
+                Array<float> &score_out, const Array<T> &in,
+                const unsigned max_corners, const float min_response,
+                const float sigma, const unsigned filter_len,
+                const float k_thr);
+
+}
diff --git a/src/backend/oneapi/hist_graphics.cpp b/src/backend/oneapi/hist_graphics.cpp
new file mode 100644
index 0000000000..12d9bb2b33
--- /dev/null
+++ b/src/backend/oneapi/hist_graphics.cpp
@@ -0,0 +1,32 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <GraphicsResourceManager.hpp>
+#include <err_oneapi.hpp>
+#include <hist_graphics.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void copy_histogram(const Array<T> &data, fg_histogram hist) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE(T) \
+    template void copy_histogram<T>(const Array<T> &, fg_histogram);
+
+INSTANTIATE(float)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(uchar)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/hist_graphics.hpp b/src/backend/oneapi/hist_graphics.hpp
new file mode 100644
index 0000000000..4be3935750
--- /dev/null
+++ b/src/backend/oneapi/hist_graphics.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/graphics_common.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void copy_histogram(const Array<T> &data, fg_histogram hist);
+
+}
diff --git a/src/backend/oneapi/histogram.cpp b/src/backend/oneapi/histogram.cpp
new file mode 100644
index 0000000000..cf85c4e844
--- /dev/null
+++ b/src/backend/oneapi/histogram.cpp
@@ -0,0 +1,49 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <histogram.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+using common::half;
+
+namespace oneapi {
+
+template<typename T>
+Array<uint> histogram(const Array<T> &in, const unsigned &nbins,
+                      const double &minval, const double &maxval,
+                      const bool isLinear) {
+    ONEAPI_NOT_SUPPORTED("");
+    const dim4 &dims = in.dims();
+    dim4 outDims     = dim4(nbins, 1, dims[2], dims[3]);
+    Array<uint> out  = createValueArray<uint>(outDims, uint(0));
+    return out;
+}
+
+#define INSTANTIATE(T)                                                    \
+    template Array<uint> histogram<T>(const Array<T> &, const unsigned &, \
+                                      const double &, const double &,     \
+                                      const bool);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/histogram.hpp b/src/backend/oneapi/histogram.hpp
new file mode 100644
index 0000000000..f899faffbe
--- /dev/null
+++ b/src/backend/oneapi/histogram.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<uint> histogram(const Array<T> &in, const unsigned &nbins,
+                      const double &minval, const double &maxval,
+                      const bool isLinear);
+}
diff --git a/src/backend/oneapi/homography.cpp b/src/backend/oneapi/homography.cpp
new file mode 100644
index 0000000000..e9b08cc475
--- /dev/null
+++ b/src/backend/oneapi/homography.cpp
@@ -0,0 +1,44 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <homography.hpp>
+
+#include <arith.hpp>
+#include <af/dim4.hpp>
+#include <err_oneapi.hpp>
+
+#include <algorithm>
+#include <limits>
+
+using af::dim4;
+using std::numeric_limits;
+
+namespace oneapi {
+
+template<typename T>
+int homography(Array<T> &bestH, const Array<float> &x_src,
+               const Array<float> &y_src, const Array<float> &x_dst,
+               const Array<float> &y_dst, const Array<float> &initial,
+               const af_homography_type htype, const float inlier_thr,
+               const unsigned iterations) {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+#define INSTANTIATE(T)                                                     \
+    template int homography(                                               \
+        Array<T> &H, const Array<float> &x_src, const Array<float> &y_src, \
+        const Array<float> &x_dst, const Array<float> &y_dst,              \
+        const Array<float> &initial, const af_homography_type htype,       \
+        const float inlier_thr, const unsigned iterations);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/homography.hpp b/src/backend/oneapi/homography.hpp
new file mode 100644
index 0000000000..6c4e54be66
--- /dev/null
+++ b/src/backend/oneapi/homography.hpp
@@ -0,0 +1,21 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+template<typename T>
+int homography(Array<T> &H, const Array<float> &x_src,
+               const Array<float> &y_src, const Array<float> &x_dst,
+               const Array<float> &y_dst, const Array<float> &initial,
+               const af_homography_type htype, const float inlier_thr,
+               const unsigned iterations);
+
+}
diff --git a/src/backend/oneapi/hsv_rgb.cpp b/src/backend/oneapi/hsv_rgb.cpp
new file mode 100644
index 0000000000..6902f0f6c2
--- /dev/null
+++ b/src/backend/oneapi/hsv_rgb.cpp
@@ -0,0 +1,37 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <hsv_rgb.hpp>
+
+#include <err_oneapi.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> hsv2rgb(const Array<T>& in) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(in.dims());
+    return out;
+}
+
+template<typename T>
+Array<T> rgb2hsv(const Array<T>& in) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(in.dims());
+    return out;
+}
+
+#define INSTANTIATE(T)                                \
+    template Array<T> hsv2rgb<T>(const Array<T>& in); \
+    template Array<T> rgb2hsv<T>(const Array<T>& in);
+
+INSTANTIATE(double)
+INSTANTIATE(float)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/hsv_rgb.hpp b/src/backend/oneapi/hsv_rgb.hpp
new file mode 100644
index 0000000000..e46da55a80
--- /dev/null
+++ b/src/backend/oneapi/hsv_rgb.hpp
@@ -0,0 +1,20 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> hsv2rgb(const Array<T>& in);
+
+template<typename T>
+Array<T> rgb2hsv(const Array<T>& in);
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/identity.cpp b/src/backend/oneapi/identity.cpp
new file mode 100644
index 0000000000..ccb633aef2
--- /dev/null
+++ b/src/backend/oneapi/identity.cpp
@@ -0,0 +1,43 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#include <identity.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <af/dim4.hpp>
+
+using common::half;
+
+namespace oneapi {
+template<typename T>
+Array<T> identity(const dim4& dims) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(dims);
+    return out;
+}
+
+#define INSTANTIATE_IDENTITY(T) \
+    template Array<T> identity<T>(const af::dim4& dims);
+
+INSTANTIATE_IDENTITY(float)
+INSTANTIATE_IDENTITY(double)
+INSTANTIATE_IDENTITY(cfloat)
+INSTANTIATE_IDENTITY(cdouble)
+INSTANTIATE_IDENTITY(int)
+INSTANTIATE_IDENTITY(uint)
+INSTANTIATE_IDENTITY(intl)
+INSTANTIATE_IDENTITY(uintl)
+INSTANTIATE_IDENTITY(char)
+INSTANTIATE_IDENTITY(uchar)
+INSTANTIATE_IDENTITY(short)
+INSTANTIATE_IDENTITY(ushort)
+INSTANTIATE_IDENTITY(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/identity.hpp b/src/backend/oneapi/identity.hpp
new file mode 100644
index 0000000000..b9fed4aa03
--- /dev/null
+++ b/src/backend/oneapi/identity.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> identity(const dim4& dim);
+}
diff --git a/src/backend/oneapi/iir.cpp b/src/backend/oneapi/iir.cpp
new file mode 100644
index 0000000000..9051e34b5f
--- /dev/null
+++ b/src/backend/oneapi/iir.cpp
@@ -0,0 +1,37 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <arith.hpp>
+#include <convolve.hpp>
+#include <err_oneapi.hpp>
+#include <iir.hpp>
+//#include <kernel/iir.hpp>
+#include <math.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+template<typename T>
+Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> y = createEmptyArray<T>(dim4(1));
+    return y;
+}
+
+#define INSTANTIATE(T)                                          \
+    template Array<T> iir(const Array<T> &b, const Array<T> &a, \
+                          const Array<T> &x);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/iir.hpp b/src/backend/oneapi/iir.hpp
new file mode 100644
index 0000000000..6f7d052119
--- /dev/null
+++ b/src/backend/oneapi/iir.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x);
+}
diff --git a/src/backend/oneapi/image.cpp b/src/backend/oneapi/image.cpp
new file mode 100644
index 0000000000..8406294a44
--- /dev/null
+++ b/src/backend/oneapi/image.cpp
@@ -0,0 +1,36 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <GraphicsResourceManager.hpp>
+#include <err_oneapi.hpp>
+#include <image.hpp>
+
+#include <stdexcept>
+#include <vector>
+
+namespace oneapi {
+
+template<typename T>
+void copy_image(const Array<T> &in, fg_image image) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE(T) template void copy_image<T>(const Array<T> &, fg_image);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(ushort)
+INSTANTIATE(short)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/image.hpp b/src/backend/oneapi/image.hpp
new file mode 100644
index 0000000000..5647efea36
--- /dev/null
+++ b/src/backend/oneapi/image.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/graphics_common.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void copy_image(const Array<T> &in, fg_image image);
+
+}
diff --git a/src/backend/oneapi/index.cpp b/src/backend/oneapi/index.cpp
new file mode 100644
index 0000000000..481da0f9ec
--- /dev/null
+++ b/src/backend/oneapi/index.cpp
@@ -0,0 +1,46 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <index.hpp>
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+#include <handle.hpp>
+#include <memory.hpp>
+#include <af/dim4.hpp>
+
+using common::half;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> index(const Array<T>& in, const af_index_t idxrs[]) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(af::dim4(1));
+    return out;
+}
+
+#define INSTANTIATE(T) \
+    template Array<T> index<T>(const Array<T>& in, const af_index_t idxrs[]);
+
+INSTANTIATE(cdouble)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(float)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/index.hpp b/src/backend/oneapi/index.hpp
new file mode 100644
index 0000000000..d8fdb674b5
--- /dev/null
+++ b/src/backend/oneapi/index.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/index.h>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> index(const Array<T>& in, const af_index_t idxrs[]);
+
+}
diff --git a/src/backend/oneapi/inverse.cpp b/src/backend/oneapi/inverse.cpp
new file mode 100644
index 0000000000..60026719db
--- /dev/null
+++ b/src/backend/oneapi/inverse.cpp
@@ -0,0 +1,54 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <err_oneapi.hpp>
+#include <identity.hpp>
+#include <solve.hpp>
+
+#if defined(WITH_LINEAR_ALGEBRA)
+#include <platform.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> inverse(const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> I = identity<T>(in.dims());
+    return I;
+}
+
+#define INSTANTIATE(T) template Array<T> inverse<T>(const Array<T> &in);
+
+INSTANTIATE(float)
+INSTANTIATE(cfloat)
+INSTANTIATE(double)
+INSTANTIATE(cdouble)
+
+}  // namespace oneapi
+
+#else  // WITH_LINEAR_ALGEBRA
+
+namespace oneapi {
+
+template<typename T>
+Array<T> inverse(const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+    AF_ERROR("Linear Algebra is disabled on OneAPI backend", AF_ERR_NOT_CONFIGURED);
+}
+
+#define INSTANTIATE(T) template Array<T> inverse<T>(const Array<T> &in);
+
+INSTANTIATE(float)
+INSTANTIATE(cfloat)
+INSTANTIATE(double)
+INSTANTIATE(cdouble)
+
+}  // namespace oneapi
+
+#endif
diff --git a/src/backend/oneapi/inverse.hpp b/src/backend/oneapi/inverse.hpp
new file mode 100644
index 0000000000..2011950ed1
--- /dev/null
+++ b/src/backend/oneapi/inverse.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> inverse(const Array<T> &in);
+}
diff --git a/src/backend/oneapi/iota.cpp b/src/backend/oneapi/iota.cpp
new file mode 100644
index 0000000000..92fbbd2ede
--- /dev/null
+++ b/src/backend/oneapi/iota.cpp
@@ -0,0 +1,43 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#include <iota.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+
+#include <stdexcept>
+
+using common::half;
+
+namespace oneapi {
+template<typename T>
+Array<T> iota(const dim4 &dims, const dim4 &tile_dims) {
+    ONEAPI_NOT_SUPPORTED("");
+    dim4 outdims = dims * tile_dims;
+
+    Array<T> out = createEmptyArray<T>(outdims);
+    return out;
+}
+
+#define INSTANTIATE(T) \
+    template Array<T> iota<T>(const af::dim4 &dims, const af::dim4 &tile_dims);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/iota.hpp b/src/backend/oneapi/iota.hpp
new file mode 100644
index 0000000000..fe9b1cdf8c
--- /dev/null
+++ b/src/backend/oneapi/iota.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> iota(const dim4 &dim, const dim4 &tile_dims = dim4(1));
+}
diff --git a/src/backend/oneapi/ireduce.cpp b/src/backend/oneapi/ireduce.cpp
new file mode 100644
index 0000000000..cf97ad3a4a
--- /dev/null
+++ b/src/backend/oneapi/ireduce.cpp
@@ -0,0 +1,78 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#include <ireduce.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <optypes.hpp>
+#include <af/dim4.hpp>
+#include <complex>
+
+using af::dim4;
+using common::half;
+
+namespace oneapi {
+
+template<af_op_t op, typename T>
+void ireduce(Array<T> &out, Array<uint> &loc, const Array<T> &in,
+             const int dim) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<af_op_t op, typename T>
+void rreduce(Array<T> &out, Array<uint> &loc, const Array<T> &in, const int dim,
+             const Array<uint> &rlen) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<af_op_t op, typename T>
+T ireduce_all(unsigned *loc, const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+    return T(0);
+}
+
+#define INSTANTIATE(ROp, T)                                           \
+    template void ireduce<ROp, T>(Array<T> & out, Array<uint> & loc,  \
+                                  const Array<T> &in, const int dim); \
+    template void rreduce<ROp, T>(Array<T> & out, Array<uint> & loc,  \
+                                  const Array<T> &in, const int dim,  \
+                                  const Array<uint> &rlen);           \
+    template T ireduce_all<ROp, T>(unsigned *loc, const Array<T> &in);
+
+// min
+INSTANTIATE(af_min_t, float)
+INSTANTIATE(af_min_t, double)
+INSTANTIATE(af_min_t, cfloat)
+INSTANTIATE(af_min_t, cdouble)
+INSTANTIATE(af_min_t, int)
+INSTANTIATE(af_min_t, uint)
+INSTANTIATE(af_min_t, intl)
+INSTANTIATE(af_min_t, uintl)
+INSTANTIATE(af_min_t, char)
+INSTANTIATE(af_min_t, uchar)
+INSTANTIATE(af_min_t, short)
+INSTANTIATE(af_min_t, ushort)
+INSTANTIATE(af_min_t, half)
+
+// max
+INSTANTIATE(af_max_t, float)
+INSTANTIATE(af_max_t, double)
+INSTANTIATE(af_max_t, cfloat)
+INSTANTIATE(af_max_t, cdouble)
+INSTANTIATE(af_max_t, int)
+INSTANTIATE(af_max_t, uint)
+INSTANTIATE(af_max_t, intl)
+INSTANTIATE(af_max_t, uintl)
+INSTANTIATE(af_max_t, char)
+INSTANTIATE(af_max_t, uchar)
+INSTANTIATE(af_max_t, short)
+INSTANTIATE(af_max_t, ushort)
+INSTANTIATE(af_max_t, half)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/ireduce.hpp b/src/backend/oneapi/ireduce.hpp
new file mode 100644
index 0000000000..3ae1b6c476
--- /dev/null
+++ b/src/backend/oneapi/ireduce.hpp
@@ -0,0 +1,24 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <optypes.hpp>
+
+namespace oneapi {
+template<af_op_t op, typename T>
+void ireduce(Array<T> &out, Array<uint> &loc, const Array<T> &in,
+             const int dim);
+
+template<af_op_t op, typename T>
+void rreduce(Array<T> &out, Array<uint> &loc, const Array<T> &in, const int dim,
+             const Array<uint> &rlen);
+
+template<af_op_t op, typename T>
+T ireduce_all(unsigned *loc, const Array<T> &in);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
new file mode 100644
index 0000000000..c957c86c1d
--- /dev/null
+++ b/src/backend/oneapi/jit.cpp
@@ -0,0 +1,71 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/compile_module.hpp>
+#include <common/dispatch.hpp>
+#include <common/jit/ModdimNode.hpp>
+#include <common/jit/Node.hpp>
+#include <common/jit/NodeIterator.hpp>
+#include <common/kernel_cache.hpp>
+#include <common/util.hpp>
+#include <copy.hpp>
+#include <device_manager.hpp>
+#include <err_oneapi.hpp>
+#include <af/dim4.hpp>
+
+//#include <jit/BufferNode.hpp>
+
+#include <cstdio>
+#include <functional>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+using common::getFuncName;
+using common::Node;
+using common::Node_ids;
+using common::Node_map_t;
+
+using std::string;
+using std::stringstream;
+using std::to_string;
+using std::vector;
+
+namespace oneapi {
+
+string getKernelString(const string &funcName, const vector<Node *> &full_nodes,
+                       const vector<Node_ids> &full_ids,
+                       const vector<int> &output_ids, bool is_linear) {
+    ONEAPI_NOT_SUPPORTED("");
+    return "";
+}
+
+/*
+cl::Kernel getKernel(const vector<Node *> &output_nodes,
+                     const vector<int> &output_ids,
+                     const vector<Node *> &full_nodes,
+                     const vector<Node_ids> &full_ids, const bool is_linear) {
+    ONEAPI_NOT_SUPPORTED("");
+    return common::getKernel("", "", true).get();
+}
+*/
+
+/*
+void evalNodes(vector<Param> &outputs, const vector<Node *> &output_nodes) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void evalNodes(Param &out, Node *node) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+*/
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/jit/BufferNode.hpp b/src/backend/oneapi/jit/BufferNode.hpp
new file mode 100644
index 0000000000..2e6ef7fe34
--- /dev/null
+++ b/src/backend/oneapi/jit/BufferNode.hpp
@@ -0,0 +1,34 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <common/jit/BufferNodeBase.hpp>
+
+#include <memory>
+
+namespace oneapi {
+namespace jit {
+  template<typename T>
+  using BufferNode = common::BufferNodeBase<std::shared_ptr<sycl::buffer<T>>, KParam>;
+}
+}  // namespace opencl
+
+namespace common {
+
+template<typename DataType, typename ParamType>
+bool BufferNodeBase<DataType, ParamType>::operator==(
+    const BufferNodeBase<DataType, ParamType> &other) const noexcept {
+    // clang-format off
+    return m_data.get() == other.m_data.get() &&
+           m_bytes == other.m_bytes &&
+           m_param.offset == other.m_param.offset;
+    // clang-format on
+}
+
+}  // namespace common
diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
new file mode 100644
index 0000000000..607d85ce98
--- /dev/null
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -0,0 +1,112 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <sstream>
+#include <string>
+
+#include <err_oneapi.hpp>
+
+namespace oneapi {
+
+namespace {
+
+/// Creates a string that will be used to declare the parameter of kernel
+void generateParamDeclaration(std::stringstream& kerStream, int id,
+                              bool is_linear, const std::string& m_type_str) {
+    if (is_linear) {
+        kerStream << "__global " << m_type_str << " *in" << id
+                  << ", dim_t iInfo" << id << "_offset, \n";
+    } else {
+        kerStream << "__global " << m_type_str << " *in" << id
+                  << ", Param iInfo" << id << ", \n";
+    }
+}
+
+/// Calls the setArg function to set the arguments for a kernel call
+template<typename T>
+inline int setKernelArguments(
+    int start_id, bool is_linear,
+    std::function<void(int id, const void* ptr, size_t arg_size)>& setArg,
+    const std::shared_ptr<sycl::buffer<T>>& ptr, const KParam& info) {
+  // TODO(oneapi)
+  ONEAPI_NOT_SUPPORTED("ERROR");
+  //setArg(start_id + 0, static_cast<const void*>(&ptr.get()->operator()()),
+  //sizeof(cl_mem));
+    if (is_linear) {
+      //setArg(start_id + 1, static_cast<const void*>(&info.offset),
+      //sizeof(dim_t));
+    } else {
+      //setArg(start_id + 1, static_cast<const void*>(&info), sizeof(KParam));
+    }
+    return start_id + 2;
+}
+
+/// Generates the code to calculate the offsets for a buffer
+inline void generateBufferOffsets(std::stringstream& kerStream, int id,
+                                  bool is_linear, const std::string& type_str) {
+    UNUSED(type_str);
+    std::string idx_str  = std::string("int idx") + std::to_string(id);
+    std::string info_str = std::string("iInfo") + std::to_string(id);
+
+    if (is_linear) {
+        kerStream << idx_str << " = idx + " << info_str << "_offset;\n";
+    } else {
+        kerStream << idx_str << " = (id3 < " << info_str << ".dims[3]) * "
+                  << info_str << ".strides[3] * id3 + (id2 < " << info_str
+                  << ".dims[2]) * " << info_str << ".strides[2] * id2 + (id1 < "
+                  << info_str << ".dims[1]) * " << info_str
+                  << ".strides[1] * id1 + (id0 < " << info_str
+                  << ".dims[0]) * id0 + " << info_str << ".offset;\n";
+    }
+}
+
+/// Generates the code to read a buffer and store it in a local variable
+inline void generateBufferRead(std::stringstream& kerStream, int id,
+                               const std::string& type_str) {
+    kerStream << type_str << " val" << id << " = in" << id << "[idx" << id
+              << "];\n";
+}
+
+inline void generateShiftNodeOffsets(std::stringstream& kerStream, int id,
+                                     bool is_linear,
+                                     const std::string& type_str) {
+    UNUSED(is_linear);
+    UNUSED(type_str);
+    std::string idx_str   = std::string("idx") + std::to_string(id);
+    std::string info_str  = std::string("iInfo") + std::to_string(id);
+    std::string id_str    = std::string("sh_id_") + std::to_string(id) + "_";
+    std::string shift_str = std::string("shift") + std::to_string(id) + "_";
+
+    for (int i = 0; i < 4; i++) {
+        kerStream << "int " << id_str << i << " = __circular_mod(id" << i
+                  << " + " << shift_str << i << ", " << info_str << ".dims["
+                  << i << "]);\n";
+    }
+
+    kerStream << "int " << idx_str << " = (" << id_str << "3 < " << info_str
+              << ".dims[3]) * " << info_str << ".strides[3] * " << id_str
+              << "3;\n";
+    kerStream << idx_str << " += (" << id_str << "2 < " << info_str
+              << ".dims[2]) * " << info_str << ".strides[2] * " << id_str
+              << "2;\n";
+    kerStream << idx_str << " += (" << id_str << "1 < " << info_str
+              << ".dims[1]) * " << info_str << ".strides[1] * " << id_str
+              << "1;\n";
+    kerStream << idx_str << " += (" << id_str << "0 < " << info_str
+              << ".dims[0]) * " << id_str << "0 + " << info_str << ".offset;\n";
+}
+
+inline void generateShiftNodeRead(std::stringstream& kerStream, int id,
+                                  const std::string& type_str) {
+    kerStream << type_str << " val" << id << " = in" << id << "[idx" << id
+              << "];\n";
+}
+}  // namespace
+}  // namespace opencl
diff --git a/src/backend/oneapi/join.cpp b/src/backend/oneapi/join.cpp
new file mode 100644
index 0000000000..a645ea56f5
--- /dev/null
+++ b/src/backend/oneapi/join.cpp
@@ -0,0 +1,91 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <join.hpp>
+
+#include <algorithm>
+#include <stdexcept>
+#include <vector>
+
+using af::dim4;
+using common::half;
+using std::transform;
+using std::vector;
+
+namespace oneapi {
+dim4 calcOffset(const dim4 &dims, int dim) {
+    dim4 offset;
+    offset[0] = (dim == 0) ? dims[0] : 0;
+    offset[1] = (dim == 1) ? dims[1] : 0;
+    offset[2] = (dim == 2) ? dims[2] : 0;
+    offset[3] = (dim == 3) ? dims[3] : 0;
+    return offset;
+}
+
+template<typename T>
+Array<T> join(const int dim, const Array<T> &first, const Array<T> &second) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> out = createEmptyArray<T>(af::dim4(1));
+    return out;
+}
+
+template<typename T>
+void join_wrapper(const int dim, Array<T> &out,
+                  const vector<Array<T>> &inputs) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<typename T>
+void join(Array<T> &out, const int dim, const vector<Array<T>> &inputs) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE(T)                                              \
+    template Array<T> join<T>(const int dim, const Array<T> &first, \
+                              const Array<T> &second);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(half)
+
+#undef INSTANTIATE
+
+#define INSTANTIATE(T)                                   \
+    template void join<T>(Array<T> & out, const int dim, \
+                          const vector<Array<T>> &inputs);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(half)
+
+#undef INSTANTIATE
+}  // namespace oneapi
diff --git a/src/backend/oneapi/join.hpp b/src/backend/oneapi/join.hpp
new file mode 100644
index 0000000000..25763f063e
--- /dev/null
+++ b/src/backend/oneapi/join.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
+
+template<typename T>
+void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/KParam.hpp b/src/backend/oneapi/kernel/KParam.hpp
new file mode 100644
index 0000000000..b5bb98e850
--- /dev/null
+++ b/src/backend/oneapi/kernel/KParam.hpp
@@ -0,0 +1,26 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#ifndef __KPARAM_H
+#define __KPARAM_H
+
+//#ifndef __OPENCL_VERSION__
+// Only define dim_t in host code. dim_t is defined when setting the program
+// options in program.cpp
+#include <af/defines.h>
+//#endif
+
+// Defines the size and shape of the data in the OpenCL buffer
+typedef struct {
+    dim_t dims[4];
+    dim_t strides[4];
+    dim_t offset;
+} KParam;
+
+#endif
diff --git a/src/backend/oneapi/logic.hpp b/src/backend/oneapi/logic.hpp
new file mode 100644
index 0000000000..e1706583e2
--- /dev/null
+++ b/src/backend/oneapi/logic.hpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <binary.hpp>
+#include <common/jit/BinaryNode.hpp>
+#include <err_oneapi.hpp>
+#include <optypes.hpp>
+#include <af/defines.h>
+#include <af/dim4.hpp>
+
+namespace oneapi {
+template<typename T, af_op_t op>
+Array<char> logicOp(const Array<T> &lhs, const Array<T> &rhs,
+                    const af::dim4 &odims) {
+    return common::createBinaryNode<char, T, op>(lhs, rhs, odims);
+}
+
+template<typename T, af_op_t op>
+Array<T> bitOp(const Array<T> &lhs, const Array<T> &rhs,
+               const af::dim4 &odims) {
+    return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
+}
+}  // namespace oneapi
diff --git a/src/backend/oneapi/lookup.cpp b/src/backend/oneapi/lookup.cpp
new file mode 100644
index 0000000000..304ab9afa7
--- /dev/null
+++ b/src/backend/oneapi/lookup.cpp
@@ -0,0 +1,63 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <lookup.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <af/dim4.hpp>
+
+using common::half;
+
+namespace oneapi {
+template<typename in_t, typename idx_t>
+Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
+                   const unsigned dim) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<in_t> out = createEmptyArray<in_t>(af::dim4(1));
+    return out;
+}
+
+#define INSTANTIATE(T)                                                         \
+    template Array<T> lookup<T, float>(const Array<T> &, const Array<float> &, \
+                                       const unsigned);                        \
+    template Array<T> lookup<T, double>(                                       \
+        const Array<T> &, const Array<double> &, const unsigned);              \
+    template Array<T> lookup<T, int>(const Array<T> &, const Array<int> &,     \
+                                     const unsigned);                          \
+    template Array<T> lookup<T, unsigned>(                                     \
+        const Array<T> &, const Array<unsigned> &, const unsigned);            \
+    template Array<T> lookup<T, short>(const Array<T> &, const Array<short> &, \
+                                       const unsigned);                        \
+    template Array<T> lookup<T, ushort>(                                       \
+        const Array<T> &, const Array<ushort> &, const unsigned);              \
+    template Array<T> lookup<T, intl>(const Array<T> &, const Array<intl> &,   \
+                                      const unsigned);                         \
+    template Array<T> lookup<T, uintl>(const Array<T> &, const Array<uintl> &, \
+                                       const unsigned);                        \
+    template Array<T> lookup<T, uchar>(const Array<T> &, const Array<uchar> &, \
+                                       const unsigned);                        \
+    template Array<T> lookup<T, half>(const Array<T> &, const Array<half> &,   \
+                                      const unsigned)
+
+INSTANTIATE(float);
+INSTANTIATE(cfloat);
+INSTANTIATE(double);
+INSTANTIATE(cdouble);
+INSTANTIATE(int);
+INSTANTIATE(unsigned);
+INSTANTIATE(intl);
+INSTANTIATE(uintl);
+INSTANTIATE(uchar);
+INSTANTIATE(char);
+INSTANTIATE(ushort);
+INSTANTIATE(short);
+INSTANTIATE(half);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/lookup.hpp b/src/backend/oneapi/lookup.hpp
new file mode 100644
index 0000000000..2fe9b0240c
--- /dev/null
+++ b/src/backend/oneapi/lookup.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename in_t, typename idx_t>
+Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
+                   const unsigned dim);
+}
diff --git a/src/backend/oneapi/lu.cpp b/src/backend/oneapi/lu.cpp
new file mode 100644
index 0000000000..849fea1426
--- /dev/null
+++ b/src/backend/oneapi/lu.cpp
@@ -0,0 +1,86 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <err_oneapi.hpp>
+#include <lu.hpp>
+
+#if defined(WITH_LINEAR_ALGEBRA)
+#include <blas.hpp>
+#include <copy.hpp>
+#include <platform.hpp>
+
+namespace oneapi {
+
+Array<int> convertPivot(int *ipiv, int in_sz, int out_sz) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<int> out = createEmptyArray<in_t>(af::dim4(1));
+    return out;
+}
+
+template<typename T>
+void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
+        const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<typename T>
+Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<int> out = createEmptyArray<in_t>(af::dim4(1));
+    return out;
+}
+
+bool isLAPACKAvailable() { return true; }
+
+#define INSTANTIATE_LU(T)                                        \
+    template Array<int> lu_inplace<T>(Array<T> & in,             \
+                                      const bool convert_pivot); \
+    template void lu<T>(Array<T> & lower, Array<T> & upper,      \
+                        Array<int> & pivot, const Array<T> &in);
+
+INSTANTIATE_LU(float)
+INSTANTIATE_LU(cfloat)
+INSTANTIATE_LU(double)
+INSTANTIATE_LU(cdouble)
+
+}  // namespace oneapi
+
+#else  // WITH_LINEAR_ALGEBRA
+
+namespace oneapi {
+
+template<typename T>
+void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
+        const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+    AF_ERROR("Linear Algebra is disabled on OneAPI backend", AF_ERR_NOT_CONFIGURED);
+}
+
+template<typename T>
+Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
+    ONEAPI_NOT_SUPPORTED("");
+    AF_ERROR("Linear Algebra is disabled on OneAPI backend", AF_ERR_NOT_CONFIGURED);
+}
+
+bool isLAPACKAvailable() { return false; }
+
+#define INSTANTIATE_LU(T)                                        \
+    template Array<int> lu_inplace<T>(Array<T> & in,             \
+                                      const bool convert_pivot); \
+    template void lu<T>(Array<T> & lower, Array<T> & upper,      \
+                        Array<int> & pivot, const Array<T> &in);
+
+INSTANTIATE_LU(float)
+INSTANTIATE_LU(cfloat)
+INSTANTIATE_LU(double)
+INSTANTIATE_LU(cdouble)
+
+}  // namespace oneapi
+
+#endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/lu.hpp b/src/backend/oneapi/lu.hpp
new file mode 100644
index 0000000000..8ab1f25a7a
--- /dev/null
+++ b/src/backend/oneapi/lu.hpp
@@ -0,0 +1,21 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
+        const Array<T> &in);
+
+template<typename T>
+Array<int> lu_inplace(Array<T> &in, const bool convert_pivot = true);
+
+bool isLAPACKAvailable();
+}  // namespace oneapi
diff --git a/src/backend/oneapi/match_template.cpp b/src/backend/oneapi/match_template.cpp
new file mode 100644
index 0000000000..6a0182f7bd
--- /dev/null
+++ b/src/backend/oneapi/match_template.cpp
@@ -0,0 +1,38 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <match_template.hpp>
+
+#include <err_oneapi.hpp>
+
+namespace oneapi {
+
+template<typename inType, typename outType>
+Array<outType> match_template(const Array<inType> &sImg,
+                              const Array<inType> &tImg,
+                              const af::matchType mType) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<outType> out = createEmptyArray<outType>(sImg.dims());
+    return out;
+}
+
+#define INSTANTIATE(in_t, out_t)                       \
+    template Array<out_t> match_template<in_t, out_t>( \
+        const Array<in_t> &, const Array<in_t> &, const af::matchType);
+
+INSTANTIATE(double, double)
+INSTANTIATE(float, float)
+INSTANTIATE(char, float)
+INSTANTIATE(int, float)
+INSTANTIATE(uint, float)
+INSTANTIATE(uchar, float)
+INSTANTIATE(short, float)
+INSTANTIATE(ushort, float)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/match_template.hpp b/src/backend/oneapi/match_template.hpp
new file mode 100644
index 0000000000..9e79f3e19b
--- /dev/null
+++ b/src/backend/oneapi/match_template.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/defines.h>
+
+namespace oneapi {
+template<typename inType, typename outType>
+Array<outType> match_template(const Array<inType> &sImg,
+                              const Array<inType> &tImg,
+                              const af::matchType mType);
+}
diff --git a/src/backend/oneapi/math.cpp b/src/backend/oneapi/math.cpp
new file mode 100644
index 0000000000..a3b9d07e7a
--- /dev/null
+++ b/src/backend/oneapi/math.cpp
@@ -0,0 +1,53 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include "math.hpp"
+#include <common/half.hpp>
+
+namespace oneapi {
+cfloat operator+(cfloat lhs, cfloat rhs) {
+    //cfloat res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
+    cfloat res;
+    return res;
+}
+
+cdouble operator+(cdouble lhs, cdouble rhs) {
+    //cdouble res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
+    cdouble res;
+    return res;
+}
+
+cfloat operator*(cfloat lhs, cfloat rhs) {
+    cfloat out;
+    //out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1];
+    //out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0];
+    return out;
+}
+
+cdouble operator*(cdouble lhs, cdouble rhs) {
+    cdouble out;
+    //out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1];
+    //out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0];
+    return out;
+}
+
+cfloat division(cfloat lhs, double rhs) {
+    cfloat retVal;
+    //retVal.s[0] = real(lhs) / rhs;
+    //retVal.s[1] = imag(lhs) / rhs;
+    return retVal;
+}
+
+cdouble division(cdouble lhs, double rhs) {
+    cdouble retVal;
+    //retVal.s[0] = real(lhs) / rhs;
+    //retVal.s[1] = imag(lhs) / rhs;
+    return retVal;
+}
+}  // namespace oneapi
diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp
new file mode 100644
index 0000000000..2b4182d811
--- /dev/null
+++ b/src/backend/oneapi/math.hpp
@@ -0,0 +1,155 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/defines.hpp>
+#include <common/half.hpp>
+#include <af/defines.h>
+
+#include <backend.hpp>
+#include <types.hpp>
+
+#include <algorithm>
+#include <complex>
+#include <limits>
+
+#if defined(__GNUC__) || defined(__GNUG__)
+/* GCC/G++, Clang/LLVM, Intel ICC */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+#else
+/* Other */
+#endif
+
+namespace oneapi {
+
+template<typename T>
+static inline T abs(T val) {
+    return std::abs(val);
+}
+template<typename T>
+static inline T min(T lhs, T rhs) {
+    return std::min(lhs, rhs);
+}
+template<typename T>
+static inline T max(T lhs, T rhs) {
+    return std::max(lhs, rhs);
+}
+
+template<typename T>
+static inline T division(T lhs, double rhs) {
+    return lhs / rhs;
+}
+cfloat division(cfloat lhs, double rhs);
+cdouble division(cdouble lhs, double rhs);
+
+template<>
+inline cfloat max<cfloat>(cfloat lhs, cfloat rhs) {
+    return abs(lhs) > abs(rhs) ? lhs : rhs;
+}
+
+template<>
+inline cdouble max<cdouble>(cdouble lhs, cdouble rhs) {
+    return abs(lhs) > abs(rhs) ? lhs : rhs;
+}
+
+template<>
+inline cfloat min<cfloat>(cfloat lhs, cfloat rhs) {
+    return abs(lhs) < abs(rhs) ? lhs : rhs;
+}
+
+template<>
+inline cdouble min<cdouble>(cdouble lhs, cdouble rhs) {
+    return abs(lhs) < abs(rhs) ? lhs : rhs;
+}
+
+template<typename T>
+static T scalar(double val) {
+    return (T)(val);
+}
+
+template<>
+inline cfloat scalar<cfloat>(double val) {
+    cfloat cval(static_cast<float>(val));
+    // cval.real() = (float)val;
+    // cval.imag() = 0;
+    return cval;
+}
+
+template<>
+inline cdouble scalar<cdouble>(double val) {
+    cdouble cval(val);
+    return cval;
+}
+
+template<typename To, typename Ti>
+static To scalar(Ti real, Ti imag) {
+    To cval(real, imag);
+    return cval;
+}
+
+template<typename T>
+inline T maxval() {
+    return std::numeric_limits<T>::max();
+}
+template<typename T>
+inline T minval() {
+    return std::numeric_limits<T>::min();
+}
+template<>
+inline float maxval() {
+    return std::numeric_limits<float>::infinity();
+}
+template<>
+inline double maxval() {
+    return std::numeric_limits<double>::infinity();
+}
+
+template<>
+inline common::half maxval() {
+    return std::numeric_limits<common::half>::infinity();
+}
+
+template<>
+inline float minval() {
+    return -std::numeric_limits<float>::infinity();
+}
+
+template<>
+inline double minval() {
+    return -std::numeric_limits<double>::infinity();
+}
+template<>
+inline common::half minval() {
+    return -std::numeric_limits<common::half>::infinity();
+}
+
+template<typename T>
+static inline T real(T in) {
+    return std::real(in);
+}
+
+template<typename T>
+static inline T imag(T in) {
+    return std::imag(in);
+}
+
+inline common::half operator+(common::half lhs, common::half rhs) noexcept {
+    return common::half(static_cast<float>(lhs) + static_cast<float>(rhs));
+}
+}  // namespace oneapi
+
+
+#if defined(__GNUC__) || defined(__GNUG__)
+/* GCC/G++, Clang/LLVM, Intel ICC */
+#pragma GCC diagnostic pop
+#else
+/* Other */
+#endif
diff --git a/src/backend/oneapi/max.cpp b/src/backend/oneapi/max.cpp
new file mode 100644
index 0000000000..4ae8efeaee
--- /dev/null
+++ b/src/backend/oneapi/max.cpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/half.hpp>
+#include "reduce_impl.hpp"
+
+using common::half;
+
+namespace oneapi {
+// max
+INSTANTIATE(af_max_t, float, float)
+INSTANTIATE(af_max_t, double, double)
+INSTANTIATE(af_max_t, cfloat, cfloat)
+INSTANTIATE(af_max_t, cdouble, cdouble)
+INSTANTIATE(af_max_t, int, int)
+INSTANTIATE(af_max_t, uint, uint)
+INSTANTIATE(af_max_t, intl, intl)
+INSTANTIATE(af_max_t, uintl, uintl)
+INSTANTIATE(af_max_t, char, char)
+INSTANTIATE(af_max_t, uchar, uchar)
+INSTANTIATE(af_max_t, short, short)
+INSTANTIATE(af_max_t, ushort, ushort)
+INSTANTIATE(af_max_t, half, half)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/mean.cpp b/src/backend/oneapi/mean.cpp
new file mode 100644
index 0000000000..2fb632eb75
--- /dev/null
+++ b/src/backend/oneapi/mean.cpp
@@ -0,0 +1,94 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <mean.hpp>
+#include <err_oneapi.hpp>
+
+#include <common/half.hpp>
+// #include <kernel/mean.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+using common::half;
+using std::swap;
+
+namespace oneapi {
+template<typename Ti, typename Tw, typename To>
+To mean(const Array<Ti>& in) {
+
+    ONEAPI_NOT_SUPPORTED("mean Not supported");
+
+    return To(0);
+    // return kernel::meanAll<Ti, Tw, To>(in);
+}
+
+template<typename T, typename Tw>
+T mean(const Array<T>& in, const Array<Tw>& wts) {
+
+    ONEAPI_NOT_SUPPORTED("mean Not supported");
+
+    return T(0);
+    // return kernel::meanAllWeighted<T, Tw>(in, wts);
+}
+
+template<typename Ti, typename Tw, typename To>
+Array<To> mean(const Array<Ti>& in, const int dim) {
+
+    ONEAPI_NOT_SUPPORTED("mean Not supported");
+
+    dim4 odims    = in.dims();
+    odims[dim]    = 1;
+    Array<To> out = createEmptyArray<To>(odims);
+    // kernel::mean<Ti, Tw, To>(out, in, dim);
+    return out;
+}
+
+template<typename T, typename Tw>
+Array<T> mean(const Array<T>& in, const Array<Tw>& wts, const int dim) {
+
+    ONEAPI_NOT_SUPPORTED("mean Not supported");
+
+    dim4 odims   = in.dims();
+    odims[dim]   = 1;
+    Array<T> out = createEmptyArray<T>(odims);
+    // kernel::meanWeighted<T, Tw, T>(out, in, wts, dim);
+    return out;
+}
+
+#define INSTANTIATE(Ti, Tw, To)                        \
+    template To mean<Ti, Tw, To>(const Array<Ti>& in); \
+    template Array<To> mean<Ti, Tw, To>(const Array<Ti>& in, const int dim);
+
+INSTANTIATE(double, double, double);
+INSTANTIATE(float, float, float);
+INSTANTIATE(int, float, float);
+INSTANTIATE(unsigned, float, float);
+INSTANTIATE(intl, double, double);
+INSTANTIATE(uintl, double, double);
+INSTANTIATE(short, float, float);
+INSTANTIATE(ushort, float, float);
+INSTANTIATE(uchar, float, float);
+INSTANTIATE(char, float, float);
+INSTANTIATE(cfloat, float, cfloat);
+INSTANTIATE(cdouble, double, cdouble);
+INSTANTIATE(half, float, half);
+INSTANTIATE(half, float, float);
+
+#define INSTANTIATE_WGT(T, Tw)                                              \
+    template T mean<T, Tw>(const Array<T>& in, const Array<Tw>& wts);       \
+    template Array<T> mean<T, Tw>(const Array<T>& in, const Array<Tw>& wts, \
+                                  const int dim);
+
+INSTANTIATE_WGT(double, double);
+INSTANTIATE_WGT(float, float);
+INSTANTIATE_WGT(cfloat, float);
+INSTANTIATE_WGT(cdouble, double);
+INSTANTIATE_WGT(half, float);
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/mean.hpp b/src/backend/oneapi/mean.hpp
new file mode 100644
index 0000000000..c682fa8d5f
--- /dev/null
+++ b/src/backend/oneapi/mean.hpp
@@ -0,0 +1,26 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename Ti, typename Tw, typename To>
+To mean(const Array<Ti>& in);
+
+template<typename T, typename Tw>
+T mean(const Array<T>& in, const Array<Tw>& wts);
+
+template<typename Ti, typename Tw, typename To>
+Array<To> mean(const Array<Ti>& in, const int dim);
+
+template<typename T, typename Tw>
+Array<T> mean(const Array<T>& in, const Array<Tw>& wts, const int dim);
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/meanshift.cpp b/src/backend/oneapi/meanshift.cpp
new file mode 100644
index 0000000000..61823f1467
--- /dev/null
+++ b/src/backend/oneapi/meanshift.cpp
@@ -0,0 +1,48 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/meanshift.hpp>
+#include <meanshift.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+template<typename T>
+Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
+                   const float &chromaticSigma, const unsigned &numIterations,
+                   const bool &isColor) {
+
+    ONEAPI_NOT_SUPPORTED("meanshift Not supported");
+
+    const dim4 &dims = in.dims();
+    Array<T> out     = createEmptyArray<T>(dims);
+    // kernel::meanshift<T>(out, in, spatialSigma, chromaticSigma, numIterations,
+    //                      isColor);
+    return out;
+}
+
+#define INSTANTIATE(T)                                              \
+    template Array<T> meanshift<T>(const Array<T> &, const float &, \
+                                   const float &, const unsigned &, \
+                                   const bool &);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/meanshift.hpp b/src/backend/oneapi/meanshift.hpp
new file mode 100644
index 0000000000..014c0f2468
--- /dev/null
+++ b/src/backend/oneapi/meanshift.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
+                   const float &chromaticSigma, const unsigned &numIterations,
+                   const bool &isColor);
+}
diff --git a/src/backend/oneapi/medfilt.cpp b/src/backend/oneapi/medfilt.cpp
new file mode 100644
index 0000000000..526f505244
--- /dev/null
+++ b/src/backend/oneapi/medfilt.cpp
@@ -0,0 +1,67 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/medfilt.hpp>
+#include <medfilt.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> medfilt1(const Array<T> &in, const int w_wid,
+                  const af::borderType pad) {
+
+    ONEAPI_NOT_SUPPORTED("medfilt1 Not supported");
+
+    // ARG_ASSERT(2, (w_wid <= kernel::MAX_MEDFILTER1_LEN));
+    // ARG_ASSERT(2, (w_wid % 2 != 0));
+
+    const dim4 &dims = in.dims();
+
+    Array<T> out = createEmptyArray<T>(dims);
+
+    // kernel::medfilt1<T>(out, in, w_wid, pad);
+
+    return out;
+}
+
+template<typename T>
+Array<T> medfilt2(const Array<T> &in, const int w_len, const int w_wid,
+                  const af::borderType pad) {
+
+    ONEAPI_NOT_SUPPORTED("medfilt2 Not supported");
+
+    // ARG_ASSERT(2, (w_len % 2 != 0));
+    // ARG_ASSERT(2, (w_len <= kernel::MAX_MEDFILTER2_LEN));
+
+    Array<T> out = createEmptyArray<T>(in.dims());
+    // kernel::medfilt2<T>(out, in, pad, w_len, w_wid);
+    return out;
+}
+
+#define INSTANTIATE(T)                                                 \
+    template Array<T> medfilt1<T>(const Array<T> &in, const int w_wid, \
+                                  const af::borderType);               \
+    template Array<T> medfilt2<T>(const Array<T> &in, const int w_len, \
+                                  const int w_wid, const af::borderType);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/medfilt.hpp b/src/backend/oneapi/medfilt.hpp
new file mode 100644
index 0000000000..1e356a23bb
--- /dev/null
+++ b/src/backend/oneapi/medfilt.hpp
@@ -0,0 +1,22 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> medfilt1(const Array<T> &in, const int w_wid,
+                  const af::borderType edge_pad);
+
+template<typename T>
+Array<T> medfilt2(const Array<T> &in, const int w_len, const int w_wid,
+                  const af::borderType edge_pad);
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
new file mode 100644
index 0000000000..2f869d3147
--- /dev/null
+++ b/src/backend/oneapi/memory.cpp
@@ -0,0 +1,351 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/Logger.hpp>
+#include <common/MemoryManagerBase.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <errorcodes.hpp>
+#include <memory.hpp>
+#include <platform.hpp>
+#include <spdlog/spdlog.h>
+#include <types.hpp>
+#include <af/dim4.hpp>
+
+#include <utility>
+
+using common::bytesToString;
+
+using af::dim4;
+using std::function;
+using std::move;
+using std::unique_ptr;
+
+namespace oneapi {
+float getMemoryPressure() { return memoryManager().getMemoryPressure(); }
+float getMemoryPressureThreshold() {
+    return memoryManager().getMemoryPressureThreshold();
+}
+
+bool jitTreeExceedsMemoryPressure(size_t bytes) {
+    return memoryManager().jitTreeExceedsMemoryPressure(bytes);
+}
+
+void setMemStepSize(size_t step_bytes) {
+    memoryManager().setMemStepSize(step_bytes);
+}
+
+size_t getMemStepSize() { return memoryManager().getMemStepSize(); }
+
+void signalMemoryCleanup() { memoryManager().signalMemoryCleanup(); }
+
+void shutdownMemoryManager() { memoryManager().shutdown(); }
+
+void shutdownPinnedMemoryManager() { /*pinnedMemoryManager().shutdown();*/ }
+
+void printMemInfo(const char *msg, const int device) {
+    memoryManager().printInfo(msg, device);
+}
+
+template<typename T>
+// unique_ptr<cl::Buffer, function<void(cl::Buffer *)>> memAlloc(
+//unique_ptr<int, function<void(int *)>> memAlloc(
+std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>> memAlloc(
+    const size_t &elements) {
+    ONEAPI_NOT_SUPPORTED("memAlloc Not supported");
+    //return unique_ptr<int, function<void(int *)>>();
+    return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>();
+    // // TODO: make memAlloc aware of array shapes
+    // if (elements) {
+    //     dim4 dims(elements);
+    //     void *ptr = memoryManager().alloc(false, 1, dims.get(), sizeof(T));
+    //     auto buf  = static_cast<cl_mem>(ptr);
+    //     cl::Buffer *bptr = new cl::Buffer(buf, true);
+    //     return unique_ptr<cl::Buffer, function<void(cl::Buffer *)>>(bptr,
+    //                                                                 bufferFree);
+    // } else {
+    //     return unique_ptr<cl::Buffer, function<void(cl::Buffer *)>>(nullptr,
+    //                                                                 bufferFree);
+    // }
+}
+
+void *memAllocUser(const size_t &bytes) {
+
+    ONEAPI_NOT_SUPPORTED("memAllocUser Not supported");
+    return nullptr;
+
+    // dim4 dims(bytes);
+    // void *ptr = memoryManager().alloc(true, 1, dims.get(), 1);
+    // auto buf  = static_cast<cl_mem>(ptr);
+    // return new cl::Buffer(buf, true);
+}
+
+template<typename T>
+void memFree(T *ptr) {
+
+    ONEAPI_NOT_SUPPORTED("memFree Not supported");
+
+    // cl::Buffer *buf = reinterpret_cast<cl::Buffer *>(ptr);
+    // cl_mem mem      = static_cast<cl_mem>((*buf)());
+    // delete buf;
+    // return memoryManager().unlock(static_cast<void *>(mem), false);
+}
+
+void memFreeUser(void *ptr) {
+
+    ONEAPI_NOT_SUPPORTED("memFreeUser Not supported");
+
+    // cl::Buffer *buf = static_cast<cl::Buffer *>(ptr);
+    // cl_mem mem      = (*buf)();
+    // delete buf;
+    // memoryManager().unlock(mem, true);
+}
+
+template<typename T>
+sycl::buffer<T> *bufferAlloc(const size_t &bytes) {
+
+    ONEAPI_NOT_SUPPORTED("bufferAlloc Not supported");
+    return nullptr;
+
+    // dim4 dims(bytes);
+    // if (bytes) {
+    //     void *ptr       = memoryManager().alloc(false, 1, dims.get(), 1);
+    //     cl_mem mem      = static_cast<cl_mem>(ptr);
+    //     cl::Buffer *buf = new cl::Buffer(mem, true);
+    //     return buf;
+    // } else {
+    //     return nullptr;
+    // }
+}
+
+template<typename T>
+void bufferFree(sycl::buffer<T> *buf) {
+
+    ONEAPI_NOT_SUPPORTED("bufferFree Not supported");
+
+    // if (buf) {
+    //     cl_mem mem = (*buf)();
+    //     delete buf;
+    //     memoryManager().unlock(static_cast<void *>(mem), false);
+    // }
+}
+
+template<typename T>
+void memLock(const sycl::buffer<T> *ptr) {
+
+    ONEAPI_NOT_SUPPORTED("memLock Not supported");
+
+    // cl_mem mem = static_cast<cl_mem>((*ptr)());
+    // memoryManager().userLock(static_cast<void *>(mem));
+}
+
+template<typename T>
+void memUnlock(const sycl::buffer<T> *ptr) {
+
+    ONEAPI_NOT_SUPPORTED("memUnlock Not supported");
+
+    // cl_mem mem = static_cast<cl_mem>((*ptr)());
+    // memoryManager().userUnlock(static_cast<void *>(mem));
+}
+
+bool isLocked(const void *ptr) {
+    return memoryManager().isUserLocked(const_cast<void *>(ptr));
+}
+
+void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
+                      size_t *lock_bytes, size_t *lock_buffers) {
+    memoryManager().usageInfo(alloc_bytes, alloc_buffers, lock_bytes,
+                              lock_buffers);
+}
+
+template<typename T>
+T *pinnedAlloc(const size_t &elements) {
+
+    ONEAPI_NOT_SUPPORTED("pinnedAlloc Not supported");
+
+    // // TODO: make pinnedAlloc aware of array shapes
+    // dim4 dims(elements);
+    // void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), sizeof(T));
+    return static_cast<T *>(nullptr);
+}
+
+template<typename T>
+void pinnedFree(T *ptr) {
+    //pinnedMemoryManager().unlock(static_cast<void *>(ptr), false);
+}
+
+//template unique_ptr<int, function<void(int *)>> memAlloc<T>(
+#define INSTANTIATE(T)                                                                          \
+    template std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>> memAlloc( \
+        const size_t &elements);                                                                \
+    template void memFree(T *ptr);                                                              \
+    template T *pinnedAlloc(const size_t &elements);                                            \
+    template void pinnedFree(T *ptr);                                                           \
+    template void bufferFree(sycl::buffer<T> *buf);                                             \
+    template void memLock(const sycl::buffer<T> *buf);                                          \
+    template void memUnlock(const sycl::buffer<T> *buf);
+
+INSTANTIATE(float)
+INSTANTIATE(cfloat)
+INSTANTIATE(double)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(char)
+INSTANTIATE(uchar)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(common::half)
+
+Allocator::Allocator() { logger = common::loggerFactory("mem"); }
+
+void Allocator::shutdown() {
+
+    ONEAPI_NOT_SUPPORTED("Allocator::shutdown Not supported");
+
+    // for (int n = 0; n < opencl::getDeviceCount(); n++) {
+    //     try {
+    //         opencl::setDevice(n);
+    //         shutdownMemoryManager();
+    //     } catch (const AfError &err) {
+    //         continue;  // Do not throw any errors while shutting down
+    //     }
+    // }
+}
+
+int Allocator::getActiveDeviceId() {
+
+    ONEAPI_NOT_SUPPORTED("Allocator::getActiveDeviceId Not supported");
+
+    return 0;
+    // return opencl::getActiveDeviceId();
+}
+
+size_t Allocator::getMaxMemorySize(int id) {
+
+    ONEAPI_NOT_SUPPORTED("Allocator::getMaxMemorySize Not supported");
+
+    return 0;
+    // return opencl::getDeviceMemorySize(id);
+}
+
+void *Allocator::nativeAlloc(const size_t bytes) {
+
+    ONEAPI_NOT_SUPPORTED("Allocator::nativeAlloc Not supported");
+    return nullptr;
+
+    // cl_int err = CL_SUCCESS;
+    // auto ptr   = static_cast<void *>(clCreateBuffer(
+    //     getContext()(), CL_MEM_READ_WRITE,  // NOLINT(hicpp-signed-bitwise)
+    //     bytes, nullptr, &err));
+
+    // if (err != CL_SUCCESS) {
+    //     auto str = fmt::format("Failed to allocate device memory of size {}",
+    //                            bytesToString(bytes));
+    //     AF_ERROR(str, AF_ERR_NO_MEM);
+    // }
+
+    // AF_TRACE("nativeAlloc: {} {}", bytesToString(bytes), ptr);
+    // return ptr;
+}
+
+void Allocator::nativeFree(void *ptr) {
+
+    ONEAPI_NOT_SUPPORTED("Allocator::nativeFree Not supported");
+
+    // cl_mem buffer = static_cast<cl_mem>(ptr);
+    // AF_TRACE("nativeFree:          {}", ptr);
+    // cl_int err = clReleaseMemObject(buffer);
+    // if (err != CL_SUCCESS) {
+    //     AF_ERROR("Failed to release device memory.", AF_ERR_RUNTIME);
+    // }
+}
+
+AllocatorPinned::AllocatorPinned() : pinnedMaps(oneapi::getDeviceCount()) {
+    logger = common::loggerFactory("mem");
+}
+
+void AllocatorPinned::shutdown() {
+
+    ONEAPI_NOT_SUPPORTED("AllocatorPinned::shutdown Not supported");
+
+//     for (int n = 0; n < opencl::getDeviceCount(); n++) {
+//         opencl::setDevice(n);
+//         shutdownPinnedMemoryManager();
+//         auto currIterator = pinnedMaps[n].begin();
+//         auto endIterator  = pinnedMaps[n].end();
+//         while (currIterator != endIterator) {
+//             pinnedMaps[n].erase(currIterator++);
+//         }
+//     }
+}
+
+int AllocatorPinned::getActiveDeviceId() {
+
+    ONEAPI_NOT_SUPPORTED("AllocatorPinned::getActiveDeviceId Not supported");
+    return 0;
+
+    // opencl::getActiveDeviceId();
+}
+
+size_t AllocatorPinned::getMaxMemorySize(int id) {
+
+    ONEAPI_NOT_SUPPORTED("AllocatorPinned::getMaxMemorySize Not supported");
+    return 0;
+    // return opencl::getDeviceMemorySize(id);
+}
+
+void *AllocatorPinned::nativeAlloc(const size_t bytes) {
+
+    ONEAPI_NOT_SUPPORTED("AllocatorPinned::nativeAlloc Not supported");
+    return nullptr;
+//     void *ptr = NULL;
+
+//     cl_int err = CL_SUCCESS;
+//     auto buf   = clCreateBuffer(getContext()(), CL_MEM_ALLOC_HOST_PTR, bytes,
+//                               nullptr, &err);
+//     if (err != CL_SUCCESS) {
+//         AF_ERROR("Failed to allocate pinned memory.", AF_ERR_NO_MEM);
+//     }
+
+//     ptr = clEnqueueMapBuffer(getQueue()(), buf, CL_TRUE,
+//                              CL_MAP_READ | CL_MAP_WRITE, 0, bytes, 0, nullptr,
+//                              nullptr, &err);
+//     if (err != CL_SUCCESS) {
+//         AF_ERROR("Failed to map pinned memory", AF_ERR_RUNTIME);
+//     }
+//     AF_TRACE("Pinned::nativeAlloc: {:>7} {}", bytesToString(bytes), ptr);
+//     pinnedMaps[opencl::getActiveDeviceId()].emplace(ptr, new cl::Buffer(buf));
+//     return ptr;
+}
+
+void AllocatorPinned::nativeFree(void *ptr) {
+
+    ONEAPI_NOT_SUPPORTED("AllocatorPinned::nativeFree Not supported");
+
+    // AF_TRACE("Pinned::nativeFree:          {}", ptr);
+    // int n     = opencl::getActiveDeviceId();
+    // auto &map = pinnedMaps[n];
+    // auto iter = map.find(ptr);
+
+    // if (iter != map.end()) {
+    //     cl::Buffer *buf = map[ptr];
+    //     if (cl_int err = getQueue().enqueueUnmapMemObject(*buf, ptr)) {
+    //         getLogger()->warn(
+    //             "Pinned::nativeFree: Error unmapping pinned memory({}:{}). "
+    //             "Ignoring",
+    //             err, getErrorMessage(err));
+    //     }
+    //     delete buf;
+    //     map.erase(iter);
+    // }
+}
+}  // namespace oneapi
diff --git a/src/backend/oneapi/memory.hpp b/src/backend/oneapi/memory.hpp
new file mode 100644
index 0000000000..2e18a13ae4
--- /dev/null
+++ b/src/backend/oneapi/memory.hpp
@@ -0,0 +1,94 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <common/AllocatorInterface.hpp>
+
+#include <cstdlib>
+#include <functional>
+#include <map>
+#include <memory>
+#include <vector>
+
+
+namespace oneapi {
+template<typename T>
+sycl::buffer<T> *bufferAlloc(const size_t &bytes);
+
+template<typename T>
+void bufferFree(sycl::buffer<T> *buf);
+
+template<typename T>
+using bufptr =
+  std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>>;
+
+template<typename T>
+bufptr<T> memAlloc(const size_t &elements);
+void *memAllocUser(const size_t &bytes);
+
+// Need these as 2 separate function and not a default argument
+// This is because it is used as the deleter in shared pointer
+// which cannot support default arguments
+template<typename T>
+void memFree(T *ptr);
+void memFreeUser(void *ptr);
+
+template<typename T>
+void memLock(const sycl::buffer<T> *ptr);
+
+template<typename T>
+void memUnlock(const sycl::buffer<T> *ptr);
+
+  bool isLocked(const void *ptr);
+
+  template<typename T>
+  T *pinnedAlloc(const size_t &elements);
+  template<typename T>
+  void pinnedFree(T *ptr);
+
+  void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
+                        size_t *lock_bytes, size_t *lock_buffers);
+  void signalMemoryCleanup();
+  void shutdownMemoryManager();
+  void pinnedGarbageCollect();
+
+  void printMemInfo(const char *msg, const int device);
+
+  float getMemoryPressure();
+  float getMemoryPressureThreshold();
+  bool jitTreeExceedsMemoryPressure(size_t bytes);
+  void setMemStepSize(size_t step_bytes);
+  size_t getMemStepSize(void);
+
+  class Allocator final : public common::memory::AllocatorInterface {
+     public:
+      Allocator();
+      ~Allocator() = default;
+      void shutdown() override;
+      int getActiveDeviceId() override;
+      size_t getMaxMemorySize(int id) override;
+      void *nativeAlloc(const size_t bytes) override;
+      void nativeFree(void *ptr) override;
+};
+
+class AllocatorPinned final : public common::memory::AllocatorInterface {
+   public:
+    AllocatorPinned();
+    ~AllocatorPinned() = default;
+    void shutdown() override;
+    int getActiveDeviceId() override;
+    size_t getMaxMemorySize(int id) override;
+    void *nativeAlloc(const size_t bytes) override;
+    void nativeFree(void *ptr) override;
+
+   private:
+    std::vector<std::map<void *, void *>> pinnedMaps;
+};
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/min.cpp b/src/backend/oneapi/min.cpp
new file mode 100644
index 0000000000..3afa0d9787
--- /dev/null
+++ b/src/backend/oneapi/min.cpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/half.hpp>
+#include "reduce_impl.hpp"
+
+using common::half;
+
+namespace oneapi {
+// min
+INSTANTIATE(af_min_t, float, float)
+INSTANTIATE(af_min_t, double, double)
+INSTANTIATE(af_min_t, cfloat, cfloat)
+INSTANTIATE(af_min_t, cdouble, cdouble)
+INSTANTIATE(af_min_t, int, int)
+INSTANTIATE(af_min_t, uint, uint)
+INSTANTIATE(af_min_t, intl, intl)
+INSTANTIATE(af_min_t, uintl, uintl)
+INSTANTIATE(af_min_t, char, char)
+INSTANTIATE(af_min_t, uchar, uchar)
+INSTANTIATE(af_min_t, short, short)
+INSTANTIATE(af_min_t, ushort, ushort)
+INSTANTIATE(af_min_t, half, half)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/moments.cpp b/src/backend/oneapi/moments.cpp
new file mode 100644
index 0000000000..aa595c9269
--- /dev/null
+++ b/src/backend/oneapi/moments.cpp
@@ -0,0 +1,57 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+// #include <debug_opencl.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/moments.hpp>
+
+namespace oneapi {
+
+static inline unsigned bitCount(unsigned v) {
+    v = v - ((v >> 1U) & 0x55555555U);
+    v = (v & 0x33333333U) + ((v >> 2U) & 0x33333333U);
+    return (((v + (v >> 4U)) & 0xF0F0F0FU) * 0x1010101U) >> 24U;
+}
+
+template<typename T>
+Array<float> moments(const Array<T> &in, const af_moment_type moment) {
+
+    ONEAPI_NOT_SUPPORTED("moments Not supported");
+
+    in.eval();
+    dim4 odims, idims = in.dims();
+    dim_t moments_dim = bitCount(moment);
+
+    odims[0] = moments_dim;
+    odims[1] = 1;
+    odims[2] = idims[2];
+    odims[3] = idims[3];
+
+    Array<float> out = createValueArray<float>(odims, 0.f);
+    out.eval();
+
+    // kernel::moments<T>(out, in, moment);
+    return out;
+}
+
+#define INSTANTIATE(T)                                   \
+    template Array<float> moments<T>(const Array<T> &in, \
+                                     const af_moment_type moment);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(ushort)
+INSTANTIATE(short)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/moments.hpp b/src/backend/oneapi/moments.hpp
new file mode 100644
index 0000000000..6201ccb897
--- /dev/null
+++ b/src/backend/oneapi/moments.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<float> moments(const Array<T> &in, const af_moment_type moment);
+}
diff --git a/src/backend/oneapi/morph.cpp b/src/backend/oneapi/morph.cpp
new file mode 100644
index 0000000000..de38b446ac
--- /dev/null
+++ b/src/backend/oneapi/morph.cpp
@@ -0,0 +1,70 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/morph.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+#include <morph.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation) {
+
+    ONEAPI_NOT_SUPPORTED("morph Not supported");
+
+    // const dim4 mdims = mask.dims();
+    // if (mdims[0] != mdims[1]) {
+    //     OPENCL_NOT_SUPPORTED("Rectangular masks are not suported");
+    // }
+    // if (mdims[0] > 19) {
+    //     OPENCL_NOT_SUPPORTED("Kernels > 19x19 are not supported");
+    // }
+    const dim4 dims = in.dims();
+    Array<T> out    = createEmptyArray<T>(dims);
+    // kernel::morph<T>(out, in, mask, isDilation);
+    return out;
+}
+
+template<typename T>
+Array<T> morph3d(const Array<T> &in, const Array<T> &mask, bool isDilation) {
+
+    ONEAPI_NOT_SUPPORTED("morph3d Not supported");
+
+    // const dim4 mdims = mask.dims();
+    // if (mdims[0] != mdims[1] || mdims[0] != mdims[2]) {
+    //     OPENCL_NOT_SUPPORTED("Only cubic masks are supported");
+    // }
+    // if (mdims[0] > 7) {
+    //     OPENCL_NOT_SUPPORTED("Kernels > 7x7x7 masks are not supported");
+    // }
+    Array<T> out = createEmptyArray<T>(in.dims());
+    // kernel::morph3d<T>(out, in, mask, isDilation);
+    return out;
+}
+
+#define INSTANTIATE(T)                                                    \
+    template Array<T> morph<T>(const Array<T> &, const Array<T> &, bool); \
+    template Array<T> morph3d<T>(const Array<T> &, const Array<T> &, bool);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/morph.hpp b/src/backend/oneapi/morph.hpp
new file mode 100644
index 0000000000..086baf2a90
--- /dev/null
+++ b/src/backend/oneapi/morph.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
+
+template<typename T>
+Array<T> morph3d(const Array<T> &in, const Array<T> &mask, bool isDilation);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/nearest_neighbour.cpp b/src/backend/oneapi/nearest_neighbour.cpp
new file mode 100644
index 0000000000..e4705f1126
--- /dev/null
+++ b/src/backend/oneapi/nearest_neighbour.cpp
@@ -0,0 +1,89 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/nearest_neighbour.hpp>
+#include <math.hpp>
+#include <topk.hpp>
+#include <transpose.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+// nsing cl::Device;
+
+namespace oneapi {
+
+template<typename T, typename To, af_match_type dist_type>
+void nearest_neighbour_(Array<uint>& idx, Array<To>& dist,
+                        const Array<T>& query, const Array<T>& train,
+                        const uint dist_dim, const uint n_dist) {
+
+    ONEAPI_NOT_SUPPORTED("nearest_neighbour_ Not supported");
+
+    uint sample_dim   = (dist_dim == 0) ? 1 : 0;
+    const dim4& qDims = query.dims();
+    const dim4& tDims = train.dims();
+
+    const dim4 outDims(n_dist, qDims[sample_dim]);
+    const dim4 distDims(tDims[sample_dim], qDims[sample_dim]);
+
+    Array<To> tmp_dists = createEmptyArray<To>(distDims);
+
+    idx  = createEmptyArray<uint>(outDims);
+    dist = createEmptyArray<To>(outDims);
+
+    Array<T> queryT = dist_dim == 0 ? transpose(query, false) : query;
+    Array<T> trainT = dist_dim == 0 ? transpose(train, false) : train;
+
+    // kernel::allDistances<T, To>(tmp_dists, queryT, trainT, 1, dist_type);
+
+    topk(dist, idx, tmp_dists, n_dist, 0, AF_TOPK_MIN);
+}
+
+template<typename T, typename To>
+void nearest_neighbour(Array<uint>& idx, Array<To>& dist, const Array<T>& query,
+                       const Array<T>& train, const uint dist_dim,
+                       const uint n_dist, const af_match_type dist_type) {
+    switch (dist_type) {
+        case AF_SAD:
+            nearest_neighbour_<T, To, AF_SAD>(idx, dist, query, train, dist_dim,
+                                              n_dist);
+            break;
+        case AF_SSD:
+            nearest_neighbour_<T, To, AF_SSD>(idx, dist, query, train, dist_dim,
+                                              n_dist);
+            break;
+        case AF_SHD:
+            nearest_neighbour_<T, To, AF_SHD>(idx, dist, query, train, dist_dim,
+                                              n_dist);
+            break;
+        default: AF_ERROR("Unsupported dist_type", AF_ERR_NOT_CONFIGURED);
+    }
+}
+
+#define INSTANTIATE(T, To)                                             \
+    template void nearest_neighbour<T, To>(                            \
+        Array<uint> & idx, Array<To> & dist, const Array<T>& query,    \
+        const Array<T>& train, const uint dist_dim, const uint n_dist, \
+        const af_match_type dist_type);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+INSTANTIATE(int, int)
+INSTANTIATE(uint, uint)
+INSTANTIATE(intl, intl)
+INSTANTIATE(uintl, uintl)
+INSTANTIATE(short, int)
+INSTANTIATE(ushort, uint)
+INSTANTIATE(uchar, uint)
+
+INSTANTIATE(uintl, uint)  // For Hamming
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/nearest_neighbour.hpp b/src/backend/oneapi/nearest_neighbour.hpp
new file mode 100644
index 0000000000..f16b709d8e
--- /dev/null
+++ b/src/backend/oneapi/nearest_neighbour.hpp
@@ -0,0 +1,23 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/features.h>
+
+using af::features;
+
+namespace oneapi {
+
+template<typename T, typename To>
+void nearest_neighbour(Array<uint>& idx, Array<To>& dist, const Array<T>& query,
+                       const Array<T>& train, const uint dist_dim,
+                       const uint n_dist,
+                       const af_match_type dist_type = AF_SSD);
+
+}
diff --git a/src/backend/oneapi/orb.cpp b/src/backend/oneapi/orb.cpp
new file mode 100644
index 0000000000..db7bd31207
--- /dev/null
+++ b/src/backend/oneapi/orb.cpp
@@ -0,0 +1,69 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/orb.hpp>
+#include <math.hpp>
+#include <af/dim4.hpp>
+#include <af/features.h>
+
+using af::dim4;
+using af::features;
+
+namespace oneapi {
+
+template<typename T, typename convAccT>
+unsigned orb(Array<float> &x_out, Array<float> &y_out, Array<float> &score_out,
+             Array<float> &ori_out, Array<float> &size_out,
+             Array<uint> &desc_out, const Array<T> &image, const float fast_thr,
+             const unsigned max_feat, const float scl_fctr,
+             const unsigned levels, const bool blur_img) {
+
+    ONEAPI_NOT_SUPPORTED("orb Not supported");
+    return 0;
+
+    // unsigned nfeat;
+
+    // Param x;
+    // Param y;
+    // Param score;
+    // Param ori;
+    // Param size;
+    // Param desc;
+
+    // kernel::orb<T, convAccT>(&nfeat, x, y, score, ori, size, desc, image,
+    //                          fast_thr, max_feat, scl_fctr, levels, blur_img);
+
+    // if (nfeat > 0) {
+    //     const dim4 out_dims(nfeat);
+    //     const dim4 desc_dims(8, nfeat);
+
+    //     x_out     = createParamArray<float>(x, true);
+    //     y_out     = createParamArray<float>(y, true);
+    //     score_out = createParamArray<float>(score, true);
+    //     ori_out   = createParamArray<float>(ori, true);
+    //     size_out  = createParamArray<float>(size, true);
+    //     desc_out  = createParamArray<unsigned>(desc, true);
+    // }
+
+    // return nfeat;
+}
+
+#define INSTANTIATE(T, convAccT)                                              \
+    template unsigned orb<T, convAccT>(                                       \
+        Array<float> & x, Array<float> & y, Array<float> & score,             \
+        Array<float> & ori, Array<float> & size, Array<uint> & desc,          \
+        const Array<T> &image, const float fast_thr, const unsigned max_feat, \
+        const float scl_fctr, const unsigned levels, const bool blur_img);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/orb.hpp b/src/backend/oneapi/orb.hpp
new file mode 100644
index 0000000000..aa1fe324bb
--- /dev/null
+++ b/src/backend/oneapi/orb.hpp
@@ -0,0 +1,24 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/features.h>
+
+using af::features;
+
+namespace oneapi {
+
+template<typename T, typename convAccT>
+unsigned orb(Array<float> &x, Array<float> &y, Array<float> &score,
+             Array<float> &orientation, Array<float> &size,
+             Array<unsigned> &desc, const Array<T> &image, const float fast_thr,
+             const unsigned max_feat, const float scl_fctr,
+             const unsigned levels, const bool blur_img);
+
+}
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
new file mode 100644
index 0000000000..ef28dadbdb
--- /dev/null
+++ b/src/backend/oneapi/platform.cpp
@@ -0,0 +1,462 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/graphics_common.hpp>
+#include <GraphicsResourceManager.hpp>
+#include <blas.hpp>
+#include <common/DefaultMemoryManager.hpp>
+#include <common/Logger.hpp>
+#include <common/host_memory.hpp>
+#include <common/util.hpp>
+#include <device_manager.hpp>
+#include <err_oneapi.hpp>
+#include <errorcodes.hpp>
+#include <version.hpp>
+#include <af/version.h>
+#include <memory.hpp>
+
+#ifdef OS_MAC
+#include <OpenGL/CGLCurrent.h>
+#endif
+
+#include <cctype>
+#include <cstdlib>
+#include <functional>
+#include <map>
+#include <mutex>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+using sycl::queue;
+using sycl::context;
+using sycl::device;
+using sycl::platform;
+using std::begin;
+using std::call_once;
+using std::end;
+using std::endl;
+using std::find_if;
+using std::get;
+using std::make_pair;
+using std::make_unique;
+using std::map;
+using std::move;
+using std::once_flag;
+using std::ostringstream;
+using std::pair;
+using std::string;
+using std::to_string;
+using std::unique_ptr;
+using std::vector;
+
+using common::memory::MemoryManagerBase;
+using oneapi::Allocator;
+using oneapi::AllocatorPinned;
+
+namespace oneapi {
+
+static string get_system() {
+    string arch = (sizeof(void*) == 4) ? "32-bit " : "64-bit ";
+
+    return arch +
+#if defined(OS_LNX)
+           "Linux";
+#elif defined(OS_WIN)
+           "Windows";
+#elif defined(OS_MAC)
+           "Mac OSX";
+#endif
+}
+
+int getBackend() { return AF_BACKEND_OPENCL; }
+
+bool verify_present(const string& pname, const string ref) {
+    auto iter =
+        search(begin(pname), end(pname), begin(ref), end(ref),
+               [](const string::value_type& l, const string::value_type& r) {
+                   return tolower(l) == tolower(r);
+               });
+
+    return iter != end(pname);
+}
+
+static string platformMap(string& platStr) {
+    using strmap_t                = map<string, string>;
+    static const strmap_t platMap = {
+        make_pair("NVIDIA CUDA", "NVIDIA"),
+        make_pair("Intel(R) OpenCL", "INTEL"),
+        make_pair("AMD Accelerated Parallel Processing", "AMD"),
+        make_pair("Intel Gen OCL Driver", "BEIGNET"),
+        make_pair("Intel(R) OpenCL HD Graphics", "INTEL"),
+        make_pair("Apple", "APPLE"),
+        make_pair("Portable Computing Language", "POCL"),
+    };
+
+    auto idx = platMap.find(platStr);
+
+    if (idx == platMap.end()) {
+        return platStr;
+    } else {
+        return idx->second;
+    }
+}
+
+/*
+afcl::platform getPlatformEnum(cl::Device dev) {
+    string pname = getPlatformName(dev);
+    if (verify_present(pname, "AMD"))
+        return AFCL_PLATFORM_AMD;
+    else if (verify_present(pname, "NVIDIA"))
+        return AFCL_PLATFORM_NVIDIA;
+    else if (verify_present(pname, "INTEL"))
+        return AFCL_PLATFORM_INTEL;
+    else if (verify_present(pname, "APPLE"))
+        return AFCL_PLATFORM_APPLE;
+    else if (verify_present(pname, "BEIGNET"))
+        return AFCL_PLATFORM_BEIGNET;
+    else if (verify_present(pname, "POCL"))
+        return AFCL_PLATFORM_POCL;
+    return AFCL_PLATFORM_UNKNOWN;
+}
+*/
+
+string getDeviceInfo() noexcept {
+    ONEAPI_NOT_SUPPORTED("");
+    return "";
+}
+
+string getPlatformName(const sycl::device& device) {
+    ONEAPI_NOT_SUPPORTED("");
+    return "";
+}
+
+typedef pair<unsigned, unsigned> device_id_t;
+
+pair<unsigned, unsigned>& tlocalActiveDeviceId() {
+    // First element is active context id
+    // Second element is active queue id
+    thread_local device_id_t activeDeviceId(0, 0);
+
+    return activeDeviceId;
+}
+
+void setActiveContext(int device) {
+    tlocalActiveDeviceId() = make_pair(device, device);
+}
+
+int getDeviceCount() noexcept {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+void init() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+unsigned getActiveDeviceId() {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+/*
+int getDeviceIdFromNativeId(cl_device_id id) {
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    int nDevices = static_cast<int>(devMngr.mDevices.size());
+    int devId    = 0;
+    for (devId = 0; devId < nDevices; ++devId) {
+        if (id == devMngr.mDevices[devId]->operator()()) { break; }
+    }
+
+    return devId;
+}
+*/
+
+int getActiveDeviceType() {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+int getActivePlatform() {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+const context& getContext() {
+    ONEAPI_NOT_SUPPORTED("");
+    sycl::context c;
+    return c;
+    /*
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return *(devMngr.mContexts[get<0>(devId)]);
+    */
+}
+
+sycl::queue& getQueue() {
+    sycl::queue q;
+    return q; 
+    /*
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return *(devMngr.mQueues[get<1>(devId)]);
+    */
+}
+
+const sycl::device& getDevice(int id) {
+    sycl::device d;
+    return d;
+    /*
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    if (id == -1) { id = get<1>(devId); }
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+    return *(devMngr.mDevices[id]);
+    */
+}
+
+size_t getDeviceMemorySize(int device) {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+size_t getHostMemorySize() { return common::getHostMemorySize(); }
+
+/*
+cl_device_type getDeviceType() {
+    const sycl::device& device = getDevice();
+    cl_device_type type        = device.getInfo<CL_DEVICE_TYPE>();
+    return type;
+}
+*/
+
+bool isHostUnifiedMemory(const sycl::device& device) {
+    ONEAPI_NOT_SUPPORTED("");
+    return false;
+}
+
+bool OpenCLCPUOffload(bool forceOffloadOSX) {
+    ONEAPI_NOT_SUPPORTED("");
+    return false;
+}
+
+bool isGLSharingSupported() {
+    ONEAPI_NOT_SUPPORTED("");
+    return false;
+}
+
+bool isDoubleSupported(unsigned device) {
+    ONEAPI_NOT_SUPPORTED("");
+    return false;
+}
+
+bool isHalfSupported(unsigned device) {
+    ONEAPI_NOT_SUPPORTED("");
+    return false;
+}
+
+void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+int setDevice(int device) {
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+}
+
+void sync(int device) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void addDeviceContext(sycl::device dev, sycl::context ctx, sycl::queue que) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void setDeviceContext(sycl::device dev, sycl::context ctx) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void removeDeviceContext(sycl::device dev, sycl::context ctx) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+bool synchronize_calls() {
+    return false;
+}
+
+int& getMaxJitSize() {
+#if defined(OS_MAC)
+    constexpr int MAX_JIT_LEN = 50;
+#else
+    constexpr int MAX_JIT_LEN = 100;
+#endif
+    thread_local int length = 0;
+    if (length <= 0) {
+        string env_var = getEnvVar("AF_OPENCL_MAX_JIT_LEN");
+        if (!env_var.empty()) {
+            int input_len = stoi(env_var);
+            length        = input_len > 0 ? input_len : MAX_JIT_LEN;
+        } else {
+            length = MAX_JIT_LEN;
+        }
+    }
+    return length;
+}
+
+bool& evalFlag() {
+    ONEAPI_NOT_SUPPORTED("");
+    thread_local bool flag = true;
+    return flag;
+}
+
+MemoryManagerBase& memoryManager() {
+    static once_flag flag;
+
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    call_once(flag, [&]() {
+        // By default, create an instance of the default memory manager
+        inst.memManager = make_unique<common::DefaultMemoryManager>(
+            getDeviceCount(), common::MAX_BUFFERS,
+            AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG);
+        // Set the memory manager's device memory manager
+        unique_ptr<Allocator> deviceMemoryManager;
+        deviceMemoryManager = make_unique<Allocator>();
+        inst.memManager->setAllocator(move(deviceMemoryManager));
+        inst.memManager->initialize();
+    });
+
+    return *(inst.memManager.get());
+}
+
+/*
+MemoryManagerBase& pinnedMemoryManager() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+*/
+
+void setMemoryManager(unique_ptr<MemoryManagerBase> mgr) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void resetMemoryManager() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void setMemoryManagerPinned(unique_ptr<MemoryManagerBase> mgr) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+void resetMemoryManagerPinned() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+graphics::ForgeManager& forgeManager() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+GraphicsResourceManager& interopManager() {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+}  // namespace oneapi
+
+/*
+using namespace oneapi;
+
+af_err afcl_get_device_type(afcl_device_type* res) {
+    try {
+        *res = static_cast<afcl_device_type>(getActiveDeviceType());
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+
+af_err afcl_get_platform(afcl_platform* res) {
+    try {
+        *res = static_cast<afcl_platform>(getActivePlatform());
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+
+af_err afcl_get_context(cl_context* ctx, const bool retain) {
+    try {
+        *ctx = getContext()();
+        if (retain) { clRetainContext(*ctx); }
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+
+af_err afcl_get_queue(cl_command_queue* queue, const bool retain) {
+    try {
+        *queue = getQueue()();
+        if (retain) { clRetainCommandQueue(*queue); }
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+
+af_err afcl_get_device_id(cl_device_id* id) {
+    try {
+        *id = getDevice()();
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+
+af_err afcl_set_device_id(cl_device_id id) {
+    try {
+        setDevice(getDeviceIdFromNativeId(id));
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+
+af_err afcl_add_device_context(cl_device_id dev, cl_context ctx,
+                               cl_command_queue que) {
+    try {
+        addDeviceContext(dev, ctx, que);
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+
+af_err afcl_set_device_context(cl_device_id dev, cl_context ctx) {
+    try {
+        setDeviceContext(dev, ctx);
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+
+af_err afcl_delete_device_context(cl_device_id dev, cl_context ctx) {
+    try {
+        removeDeviceContext(dev, ctx);
+    }
+    CATCHALL;
+    return AF_SUCCESS;
+}
+*/
\ No newline at end of file
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
new file mode 100644
index 0000000000..da33f35690
--- /dev/null
+++ b/src/backend/oneapi/platform.hpp
@@ -0,0 +1,121 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <CL/sycl.hpp>
+//#include <af/oneapi.h>
+
+#include <memory>
+#include <string>
+
+// Forward declarations
+namespace spdlog {
+class logger;
+}
+
+namespace graphics {
+class ForgeManager;
+}
+
+namespace common {
+namespace memory {
+class MemoryManagerBase;
+}
+}  // namespace common
+
+using common::memory::MemoryManagerBase;
+
+namespace oneapi {
+
+// Forward declarations
+class GraphicsResourceManager;
+class PlanCache;  // clfft
+
+bool verify_present(const std::string& pname, const std::string ref);
+
+int getBackend();
+
+std::string getDeviceInfo() noexcept;
+
+int getDeviceCount() noexcept;
+
+void init();
+
+unsigned getActiveDeviceId();
+
+int& getMaxJitSize();
+
+const sycl::context& getContext();
+
+sycl::queue& getQueue();
+
+const sycl::device& getDevice(int id = -1);
+
+size_t getDeviceMemorySize(int device);
+
+size_t getHostMemorySize();
+
+//sycl::device::is_cpu,is_gpu,is_accelerator
+//cl_device_type getDeviceType();
+
+bool isHostUnifiedMemory(const sycl::device& device);
+
+bool OneAPICPUOffload(bool forceOffloadOSX = true);
+
+bool isGLSharingSupported();
+
+bool isDoubleSupported(unsigned device);
+
+// Returns true if 16-bit precision floats are supported by the device
+bool isHalfSupported(unsigned device);
+
+void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute);
+
+std::string getPlatformName(const sycl::device& device);
+
+int setDevice(int device);
+
+void addDeviceContext(sycl::device dev, sycl::context ctx, sycl::queue que);
+
+void setDeviceContext(sycl::device dev, sycl::context ctx);
+
+void removeDeviceContext(sycl::device dev, sycl::context ctx);
+
+void sync(int device);
+
+bool synchronize_calls();
+
+int getActiveDeviceType();
+
+int getActivePlatform();
+
+bool& evalFlag();
+
+MemoryManagerBase& memoryManager();
+
+void setMemoryManager(std::unique_ptr<MemoryManagerBase> mgr);
+
+void resetMemoryManager();
+
+MemoryManagerBase& pinnedMemoryManager();
+
+void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
+
+void resetMemoryManagerPinned();
+
+graphics::ForgeManager& forgeManager();
+
+GraphicsResourceManager& interopManager();
+
+//afcl::platform getPlatformEnum(cl::Device dev);
+
+void setActiveContext(int device);
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/plot.cpp b/src/backend/oneapi/plot.cpp
new file mode 100644
index 0000000000..544cc61568
--- /dev/null
+++ b/src/backend/oneapi/plot.cpp
@@ -0,0 +1,79 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+// #include <GraphicsResourceManager.hpp>
+// #include <debug_opencl.hpp>
+#include <err_oneapi.hpp>
+#include <plot.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename T>
+void copy_plot(const Array<T> &P, fg_plot plot) {
+    ONEAPI_NOT_SUPPORTED("copy_plot Not supported");
+
+    // ForgeModule &_ = graphics::forgePlugin();
+    // if (isGLSharingSupported()) {
+    //     CheckGL("Begin OpenCL resource copy");
+    //     const cl::Buffer *d_P = P.get();
+    //     unsigned bytes        = 0;
+    //     FG_CHECK(_.fg_get_plot_vertex_buffer_size(&bytes, plot));
+
+    //     auto res = interopManager().getPlotResources(plot);
+
+    //     std::vector<cl::Memory> shared_objects;
+    //     shared_objects.push_back(*(res[0].get()));
+
+    //     glFinish();
+
+    //     // Use of events:
+    //     // https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReleaseGLObjects.html
+    //     cl::Event event;
+
+    //     getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
+    //     event.wait();
+    //     getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes, NULL,
+    //                                  &event);
+    //     getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
+    //     event.wait();
+
+    //     CL_DEBUG_FINISH(getQueue());
+    //     CheckGL("End OpenCL resource copy");
+    // } else {
+    //     unsigned bytes = 0, buffer = 0;
+    //     FG_CHECK(_.fg_get_plot_vertex_buffer(&buffer, plot));
+    //     FG_CHECK(_.fg_get_plot_vertex_buffer_size(&bytes, plot));
+
+    //     CheckGL("Begin OpenCL fallback-resource copy");
+    //     glBindBuffer(GL_ARRAY_BUFFER, buffer);
+    //     auto *ptr =
+    //         static_cast<GLubyte *>(glMapBuffer(GL_ARRAY_BUFFER, GL_WRITE_ONLY));
+    //     if (ptr) {
+    //         getQueue().enqueueReadBuffer(*P.get(), CL_TRUE, 0, bytes, ptr);
+    //         glUnmapBuffer(GL_ARRAY_BUFFER);
+    //     }
+    //     glBindBuffer(GL_ARRAY_BUFFER, 0);
+    //     CheckGL("End OpenCL fallback-resource copy");
+    // }
+}
+
+#define INSTANTIATE(T) template void copy_plot<T>(const Array<T> &, fg_plot);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(uchar)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/plot.hpp b/src/backend/oneapi/plot.hpp
new file mode 100644
index 0000000000..c7c922e270
--- /dev/null
+++ b/src/backend/oneapi/plot.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/graphics_common.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void copy_plot(const Array<T> &P, fg_plot plot);
+
+}
diff --git a/src/backend/oneapi/print.hpp b/src/backend/oneapi/print.hpp
new file mode 100644
index 0000000000..787df41df2
--- /dev/null
+++ b/src/backend/oneapi/print.hpp
@@ -0,0 +1,24 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <backend.hpp>
+#include <ostream>
+
+namespace oneapi {
+static std::ostream& operator<<(std::ostream& out, const cfloat& var) {
+    out << "(" << std::real(var) << "," << std::imag(var) << ")";
+    return out;
+}
+
+static std::ostream& operator<<(std::ostream& out, const cdouble& var) {
+    out << "(" << std::real(var) << "," << std::imag(var) << ")";
+    return out;
+}
+}  // namespace oneapi
diff --git a/src/backend/oneapi/product.cpp b/src/backend/oneapi/product.cpp
new file mode 100644
index 0000000000..6d449e1fa7
--- /dev/null
+++ b/src/backend/oneapi/product.cpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/half.hpp>
+#include "reduce_impl.hpp"
+
+using common::half;
+
+namespace oneapi {
+// sum
+INSTANTIATE(af_mul_t, float, float)
+INSTANTIATE(af_mul_t, double, double)
+INSTANTIATE(af_mul_t, cfloat, cfloat)
+INSTANTIATE(af_mul_t, cdouble, cdouble)
+INSTANTIATE(af_mul_t, int, int)
+INSTANTIATE(af_mul_t, uint, uint)
+INSTANTIATE(af_mul_t, intl, intl)
+INSTANTIATE(af_mul_t, uintl, uintl)
+INSTANTIATE(af_mul_t, char, int)
+INSTANTIATE(af_mul_t, uchar, uint)
+INSTANTIATE(af_mul_t, short, int)
+INSTANTIATE(af_mul_t, ushort, uint)
+INSTANTIATE(af_mul_t, half, float)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/qr.cpp b/src/backend/oneapi/qr.cpp
new file mode 100644
index 0000000000..80fa226994
--- /dev/null
+++ b/src/backend/oneapi/qr.cpp
@@ -0,0 +1,142 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <qr.hpp>
+
+#include <err_oneapi.hpp>
+
+#if defined(WITH_LINEAR_ALGEBRA) && !defined(AF_ONEAPI)
+
+#include <blas.hpp>
+#include <copy.hpp>
+#include <cpu/cpu_qr.hpp>
+#include <identity.hpp>
+// #include <kernel/triangle.hpp>
+#include <magma/magma.h>
+#include <magma/magma_data.h>
+#include <magma/magma_helper.h>
+#include <platform.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &orig) {
+    if (OpenCLCPUOffload()) { return cpu::qr(q, r, t, orig); }
+
+    const dim4 NullShape(0, 0, 0, 0);
+
+    dim4 iDims = orig.dims();
+    int M      = iDims[0];
+    int N      = iDims[1];
+
+    dim4 endPadding(M - iDims[0], max(M, N) - iDims[1], 0, 0);
+    Array<T> in =
+        (endPadding == NullShape
+             ? copyArray(orig)
+             : padArrayBorders(orig, NullShape, endPadding, AF_PAD_ZERO));
+    in.resetDims(iDims);
+
+    int MN = std::min(M, N);
+    int NB = magma_get_geqrf_nb<T>(M);
+
+    int NUM      = (2 * MN + ((N + 31) / 32) * 32) * NB;
+    Array<T> tmp = createEmptyArray<T>(dim4(NUM));
+
+    std::vector<T> h_tau(MN);
+
+    int info           = 0;
+    cl::Buffer *in_buf = in.get();
+    cl::Buffer *dT     = tmp.get();
+
+    magma_geqrf3_gpu<T>(M, N, (*in_buf)(), in.getOffset(), in.strides()[1],
+                        &h_tau[0], (*dT)(), tmp.getOffset(), getQueue()(),
+                        &info);
+
+    r = createEmptyArray<T>(in.dims());
+    kernel::triangle<T>(r, in, true, false);
+
+    cl::Buffer *r_buf = r.get();
+    magmablas_swapdblk<T>(MN - 1, NB, (*r_buf)(), r.getOffset(), r.strides()[1],
+                          1, (*dT)(), tmp.getOffset() + MN * NB, NB, 0,
+                          getQueue()());
+
+    q = in;  // No need to copy
+    q.resetDims(dim4(M, M));
+    cl::Buffer *q_buf = q.get();
+
+    magma_ungqr_gpu<T>(q.dims()[0], q.dims()[1], std::min(M, N), (*q_buf)(),
+                       q.getOffset(), q.strides()[1], &h_tau[0], (*dT)(),
+                       tmp.getOffset(), NB, getQueue()(), &info);
+
+    t = createHostDataArray(dim4(MN), &h_tau[0]);
+}
+
+template<typename T>
+Array<T> qr_inplace(Array<T> &in) {
+    if (OpenCLCPUOffload()) { return cpu::qr_inplace(in); }
+
+    dim4 iDims = in.dims();
+    int M      = iDims[0];
+    int N      = iDims[1];
+    int MN     = std::min(M, N);
+
+    getQueue().finish();  // FIXME: Does this need to be here?
+    cl::CommandQueue Queue2(getContext(), getDevice());
+    cl_command_queue queues[] = {getQueue()(), Queue2()};
+
+    std::vector<T> h_tau(MN);
+    cl::Buffer *in_buf = in.get();
+
+    int info = 0;
+    magma_geqrf2_gpu<T>(M, N, (*in_buf)(), in.getOffset(), in.strides()[1],
+                        &h_tau[0], queues, &info);
+
+    Array<T> t = createHostDataArray(dim4(MN), &h_tau[0]);
+    return t;
+}
+
+#define INSTANTIATE_QR(T)                                         \
+    template Array<T> qr_inplace<T>(Array<T> & in);               \
+    template void qr<T>(Array<T> & q, Array<T> & r, Array<T> & t, \
+                        const Array<T> &in);
+
+INSTANTIATE_QR(float)
+INSTANTIATE_QR(cfloat)
+INSTANTIATE_QR(double)
+INSTANTIATE_QR(cdouble)
+
+}  // namespace oneapi
+
+#else  // WITH_LINEAR_ALGEBRA
+
+namespace oneapi {
+
+template<typename T>
+void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in) {
+    AF_ERROR("Linear Algebra is disabled on OneAPI", AF_ERR_NOT_CONFIGURED);
+}
+
+template<typename T>
+Array<T> qr_inplace(Array<T> &in) {
+    AF_ERROR("Linear Algebra is disabled on OneAPI", AF_ERR_NOT_CONFIGURED);
+}
+
+#define INSTANTIATE_QR(T)                                         \
+    template Array<T> qr_inplace<T>(Array<T> & in);               \
+    template void qr<T>(Array<T> & q, Array<T> & r, Array<T> & t, \
+                        const Array<T> &in);
+
+INSTANTIATE_QR(float)
+INSTANTIATE_QR(cfloat)
+INSTANTIATE_QR(double)
+INSTANTIATE_QR(cdouble)
+
+}  // namespace oneapi
+
+#endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/qr.hpp b/src/backend/oneapi/qr.hpp
new file mode 100644
index 0000000000..3ae750cf70
--- /dev/null
+++ b/src/backend/oneapi/qr.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &orig);
+
+template<typename T>
+Array<T> qr_inplace(Array<T> &in);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/random_engine.cpp b/src/backend/oneapi/random_engine.cpp
new file mode 100644
index 0000000000..9e9e7ba305
--- /dev/null
+++ b/src/backend/oneapi/random_engine.cpp
@@ -0,0 +1,160 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreemengt can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+#include <common/half.hpp>
+// #include <kernel/random_engine.hpp>
+#include <af/dim4.hpp>
+
+using common::half;
+
+namespace oneapi {
+void initMersenneState(Array<uint> &state, const uintl seed,
+                       const Array<uint> &tbl) {
+
+    ONEAPI_NOT_SUPPORTED("initMersenneState Not supported");
+
+    // kernel::initMersenneState(*state.get(), *tbl.get(), seed);
+}
+
+template<typename T>
+Array<T> uniformDistribution(const af::dim4 &dims,
+                             const af_random_engine_type type,
+                             const uintl &seed, uintl &counter) {
+
+    ONEAPI_NOT_SUPPORTED("uniformDistribution Not supported");
+
+    Array<T> out = createEmptyArray<T>(dims);
+    // kernel::uniformDistributionCBRNG<T>(*out.get(), out.elements(), type, seed,
+    //                                     counter);
+    return out;
+}
+
+template<typename T>
+Array<T> normalDistribution(const af::dim4 &dims,
+                            const af_random_engine_type type, const uintl &seed,
+                            uintl &counter) {
+
+    ONEAPI_NOT_SUPPORTED("normalDistribution Not supported");
+
+    Array<T> out = createEmptyArray<T>(dims);
+    // kernel::normalDistributionCBRNG<T>(*out.get(), out.elements(), type, seed,
+    //                                    counter);
+    return out;
+}
+
+template<typename T>
+Array<T> uniformDistribution(const af::dim4 &dims, Array<uint> pos,
+                             Array<uint> sh1, Array<uint> sh2, uint mask,
+                             Array<uint> recursion_table,
+                             Array<uint> temper_table, Array<uint> state) {
+
+    ONEAPI_NOT_SUPPORTED("uniformDistribution Not supported");
+
+    Array<T> out = createEmptyArray<T>(dims);
+    // kernel::uniformDistributionMT<T>(
+    //     *out.get(), out.elements(), *state.get(), *pos.get(), *sh1.get(),
+    //     *sh2.get(), mask, *recursion_table.get(), *temper_table.get());
+    return out;
+}
+
+template<typename T>
+Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
+                            Array<uint> sh1, Array<uint> sh2, uint mask,
+                            Array<uint> recursion_table,
+                            Array<uint> temper_table, Array<uint> state) {
+
+    ONEAPI_NOT_SUPPORTED("normalDistribution Not supported");
+
+    Array<T> out = createEmptyArray<T>(dims);
+    // kernel::normalDistributionMT<T>(
+    //     *out.get(), out.elements(), *state.get(), *pos.get(), *sh1.get(),
+    //     *sh2.get(), mask, *recursion_table.get(), *temper_table.get());
+    return out;
+}
+
+#define INSTANTIATE_UNIFORM(T)                                   \
+    template Array<T> uniformDistribution<T>(                    \
+        const af::dim4 &dims, const af_random_engine_type type,  \
+        const uintl &seed, uintl &counter);                      \
+    template Array<T> uniformDistribution<T>(                    \
+        const af::dim4 &dims, Array<uint> pos, Array<uint> sh1,  \
+        Array<uint> sh2, uint mask, Array<uint> recursion_table, \
+        Array<uint> temper_table, Array<uint> state);
+
+#define INSTANTIATE_NORMAL(T)                                    \
+    template Array<T> normalDistribution<T>(                     \
+        const af::dim4 &dims, const af_random_engine_type type,  \
+        const uintl &seed, uintl &counter);                      \
+    template Array<T> normalDistribution<T>(                     \
+        const af::dim4 &dims, Array<uint> pos, Array<uint> sh1,  \
+        Array<uint> sh2, uint mask, Array<uint> recursion_table, \
+        Array<uint> temper_table, Array<uint> state);
+
+#define COMPLEX_UNIFORM_DISTRIBUTION(T, TR)                                    \
+    template<>                                                                 \
+    Array<T> uniformDistribution<T>(const af::dim4 &dims,                      \
+                                    const af_random_engine_type type,          \
+                                    const uintl &seed, uintl &counter) {       \
+        ONEAPI_NOT_SUPPORTED("uniformDistribution Not supported");             \
+        Array<T> out    = createEmptyArray<T>(dims);                           \
+        return out;                                                            \
+    }                                                                          \
+    template<>                                                                 \
+    Array<T> uniformDistribution<T>(                                           \
+        const af::dim4 &dims, Array<uint> pos, Array<uint> sh1,                \
+        Array<uint> sh2, uint mask, Array<uint> recursion_table,               \
+        Array<uint> temper_table, Array<uint> state) {                         \
+        Array<T> out    = createEmptyArray<T>(dims);                           \
+        return out;                                                            \
+    }
+
+#define COMPLEX_NORMAL_DISTRIBUTION(T, TR)                                    \
+    template<>                                                                \
+    Array<T> normalDistribution<T>(const af::dim4 &dims,                      \
+                                   const af_random_engine_type type,          \
+                                   const uintl &seed, uintl &counter) {       \
+        ONEAPI_NOT_SUPPORTED("normalDistribution Not supported");             \
+        Array<T> out    = createEmptyArray<T>(dims);                          \
+        return out;                                                           \
+    }                                                                         \
+    template<>                                                                \
+    Array<T> normalDistribution<T>(                                           \
+        const af::dim4 &dims, Array<uint> pos, Array<uint> sh1,               \
+        Array<uint> sh2, uint mask, Array<uint> recursion_table,              \
+        Array<uint> temper_table, Array<uint> state) {                        \
+        ONEAPI_NOT_SUPPORTED("normalDistribution Not supported");             \
+        Array<T> out    = createEmptyArray<T>(dims);                          \
+        return out;                                                           \
+    }
+
+INSTANTIATE_UNIFORM(float)
+INSTANTIATE_UNIFORM(double)
+INSTANTIATE_UNIFORM(int)
+INSTANTIATE_UNIFORM(uint)
+INSTANTIATE_UNIFORM(intl)
+INSTANTIATE_UNIFORM(uintl)
+INSTANTIATE_UNIFORM(char)
+INSTANTIATE_UNIFORM(uchar)
+INSTANTIATE_UNIFORM(short)
+INSTANTIATE_UNIFORM(ushort)
+INSTANTIATE_UNIFORM(half)
+
+INSTANTIATE_NORMAL(float)
+INSTANTIATE_NORMAL(double)
+INSTANTIATE_NORMAL(half)
+
+COMPLEX_UNIFORM_DISTRIBUTION(cdouble, double)
+COMPLEX_UNIFORM_DISTRIBUTION(cfloat, float)
+
+COMPLEX_NORMAL_DISTRIBUTION(cdouble, double)
+COMPLEX_NORMAL_DISTRIBUTION(cfloat, float)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/random_engine.hpp b/src/backend/oneapi/random_engine.hpp
new file mode 100644
index 0000000000..0839d387b8
--- /dev/null
+++ b/src/backend/oneapi/random_engine.hpp
@@ -0,0 +1,41 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Array.hpp>
+#include <backend.hpp>
+#include <af/defines.h>
+
+namespace oneapi {
+void initMersenneState(Array<uint> &state, const uintl seed,
+                       const Array<uint> &tbl);
+
+template<typename T>
+Array<T> uniformDistribution(const af::dim4 &dims,
+                             const af_random_engine_type type,
+                             const uintl &seed, uintl &counter);
+
+template<typename T>
+Array<T> normalDistribution(const af::dim4 &dims,
+                            const af_random_engine_type type, const uintl &seed,
+                            uintl &counter);
+
+template<typename T>
+Array<T> uniformDistribution(const af::dim4 &dims, Array<uint> pos,
+                             Array<uint> sh1, Array<uint> sh2, uint mask,
+                             Array<uint> recursion_table,
+                             Array<uint> temper_table, Array<uint> state);
+
+template<typename T>
+Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
+                            Array<uint> sh1, Array<uint> sh2, uint mask,
+                            Array<uint> recursion_table,
+                            Array<uint> temper_table, Array<uint> state);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/range.cpp b/src/backend/oneapi/range.cpp
new file mode 100644
index 0000000000..e47a9cc664
--- /dev/null
+++ b/src/backend/oneapi/range.cpp
@@ -0,0 +1,57 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+// #include <kernel/range.hpp>
+#include <range.hpp>
+#include <err_oneapi.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+#include <stdexcept>
+
+using common::half;
+
+namespace oneapi {
+template<typename T>
+Array<T> range(const dim4& dim, const int seq_dim) {
+
+    ONEAPI_NOT_SUPPORTED("range Not supported");
+
+    // Set dimension along which the sequence should be
+    // Other dimensions are simply tiled
+    int _seq_dim = seq_dim;
+    if (seq_dim < 0) {
+        _seq_dim = 0;  // column wise sequence
+    }
+
+    if (_seq_dim < 0 || _seq_dim > 3) {
+        AF_ERROR("Invalid rep selection", AF_ERR_ARG);
+    }
+
+    Array<T> out = createEmptyArray<T>(dim);
+    // kernel::range<T>(out, _seq_dim);
+
+    return out;
+}
+
+#define INSTANTIATE(T) \
+    template Array<T> range<T>(const af::dim4& dims, const int seq_dims);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/range.hpp b/src/backend/oneapi/range.hpp
new file mode 100644
index 0000000000..7191152fb1
--- /dev/null
+++ b/src/backend/oneapi/range.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> range(const dim4& dim, const int seq_dim = -1);
+}
diff --git a/src/backend/oneapi/reduce.hpp b/src/backend/oneapi/reduce.hpp
new file mode 100644
index 0000000000..668fa1ac72
--- /dev/null
+++ b/src/backend/oneapi/reduce.hpp
@@ -0,0 +1,27 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Array.hpp>
+#include <optypes.hpp>
+
+namespace oneapi {
+template<af_op_t op, typename Ti, typename To>
+Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan = false,
+                 double nanval = 0);
+
+template<af_op_t op, typename Ti, typename Tk, typename To>
+void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
+                   const Array<Tk> &keys, const Array<Ti> &vals, const int dim,
+                   bool change_nan = false, double nanval = 0);
+
+template<af_op_t op, typename Ti, typename To>
+Array<To> reduce_all(const Array<Ti> &in, bool change_nan = false,
+                     double nanval = 0);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp
new file mode 100644
index 0000000000..d2763c92ac
--- /dev/null
+++ b/src/backend/oneapi/reduce_impl.hpp
@@ -0,0 +1,56 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+//#include <kernel/reduce.hpp>
+//#include <kernel/reduce_by_key.hpp>
+#include <reduce.hpp>
+#include <af/dim4.hpp>
+#include <complex>
+
+using af::dim4;
+using std::swap;
+namespace oneapi {
+template<af_op_t op, typename Ti, typename To>
+Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan,
+                 double nanval) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<To> out = createEmptyArray<To>(1);
+    return out;
+
+}
+
+template<af_op_t op, typename Ti, typename Tk, typename To>
+void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
+                   const Array<Tk> &keys, const Array<Ti> &vals, const int dim,
+                   bool change_nan, double nanval) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+template<af_op_t op, typename Ti, typename To>
+Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<To> out = createEmptyArray<To>(1);
+    return out;
+}
+
+}  // namespace oneapi
+
+#define INSTANTIATE(Op, Ti, To)                                                \
+    template Array<To> reduce<Op, Ti, To>(const Array<Ti> &in, const int dim,  \
+                                          bool change_nan, double nanval);     \
+    template void reduce_by_key<Op, Ti, int, To>(                              \
+        Array<int> & keys_out, Array<To> & vals_out, const Array<int> &keys,   \
+        const Array<Ti> &vals, const int dim, bool change_nan, double nanval); \
+    template void reduce_by_key<Op, Ti, uint, To>(                             \
+        Array<uint> & keys_out, Array<To> & vals_out, const Array<uint> &keys, \
+        const Array<Ti> &vals, const int dim, bool change_nan, double nanval); \
+    template Array<To> reduce_all<Op, Ti, To>(const Array<Ti> &in,             \
+                                              bool change_nan, double nanval);
diff --git a/src/backend/oneapi/regions.cpp b/src/backend/oneapi/regions.cpp
new file mode 100644
index 0000000000..cc74fb9543
--- /dev/null
+++ b/src/backend/oneapi/regions.cpp
@@ -0,0 +1,42 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/regions.hpp>
+#include <regions.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> regions(const Array<char> &in, af_connectivity connectivity) {
+
+    ONEAPI_NOT_SUPPORTED("regions Not supported");
+
+    const af::dim4 &dims = in.dims();
+    Array<T> out         = createEmptyArray<T>(dims);
+    // kernel::regions<T>(out, in, connectivity == AF_CONNECTIVITY_8, 2);
+    return out;
+}
+
+#define INSTANTIATE(T)                                  \
+    template Array<T> regions<T>(const Array<char> &in, \
+                                 af_connectivity connectivity);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/regions.hpp b/src/backend/oneapi/regions.hpp
new file mode 100644
index 0000000000..585f7e6e14
--- /dev/null
+++ b/src/backend/oneapi/regions.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> regions(const Array<char> &in, af_connectivity connectivity);
+
+}
diff --git a/src/backend/oneapi/reorder.cpp b/src/backend/oneapi/reorder.cpp
new file mode 100644
index 0000000000..7cced14197
--- /dev/null
+++ b/src/backend/oneapi/reorder.cpp
@@ -0,0 +1,52 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/reorder.hpp>
+#include <reorder.hpp>
+#include <stdexcept>
+
+using common::half;
+
+namespace oneapi {
+template<typename T>
+Array<T> reorder(const Array<T> &in, const af::dim4 &rdims) {
+
+    ONEAPI_NOT_SUPPORTED("reorder Not supported");
+
+    const af::dim4 &iDims = in.dims();
+    af::dim4 oDims(0);
+    for (int i = 0; i < 4; i++) { oDims[i] = iDims[rdims[i]]; }
+
+    Array<T> out = createEmptyArray<T>(oDims);
+
+    // kernel::reorder<T>(out, in, rdims.get());
+
+    return out;
+}
+
+#define INSTANTIATE(T) \
+    template Array<T> reorder<T>(const Array<T> &in, const af::dim4 &rdims);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/reorder.hpp b/src/backend/oneapi/reorder.hpp
new file mode 100644
index 0000000000..eb2cc8ef9c
--- /dev/null
+++ b/src/backend/oneapi/reorder.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> reorder(const Array<T> &in, const af::dim4 &rdims);
+}
diff --git a/src/backend/oneapi/reshape.cpp b/src/backend/oneapi/reshape.cpp
new file mode 100644
index 0000000000..9331038986
--- /dev/null
+++ b/src/backend/oneapi/reshape.cpp
@@ -0,0 +1,82 @@
+
+/*******************************************************
+ * Copyright (c) 2020, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <copy.hpp>
+#include <err_oneapi.hpp>
+
+#include <common/half.hpp>
+// #include <kernel/memcopy.hpp>
+
+using common::half;
+
+namespace oneapi {
+
+template<typename inType, typename outType>
+Array<outType> reshape(const Array<inType> &in, const dim4 &outDims,
+                       outType defaultValue, double scale) {
+
+    ONEAPI_NOT_SUPPORTED("reshape Not supported");
+
+    Array<outType> out = createEmptyArray<outType>(outDims);
+    // kernel::copy<inType, outType>(out, in, in.ndims(), defaultValue, scale,
+    //                               in.dims() == outDims);
+    return out;
+}
+
+#define INSTANTIATE(SRC_T)                                                    \
+    template Array<float> reshape<SRC_T, float>(Array<SRC_T> const &,         \
+                                                dim4 const &, float, double); \
+    template Array<double> reshape<SRC_T, double>(                            \
+        Array<SRC_T> const &, dim4 const &, double, double);                  \
+    template Array<cfloat> reshape<SRC_T, cfloat>(                            \
+        Array<SRC_T> const &, dim4 const &, cfloat, double);                  \
+    template Array<cdouble> reshape<SRC_T, cdouble>(                          \
+        Array<SRC_T> const &, dim4 const &, cdouble, double);                 \
+    template Array<int> reshape<SRC_T, int>(Array<SRC_T> const &,             \
+                                            dim4 const &, int, double);       \
+    template Array<uint> reshape<SRC_T, uint>(Array<SRC_T> const &,           \
+                                              dim4 const &, uint, double);    \
+    template Array<intl> reshape<SRC_T, intl>(Array<SRC_T> const &,           \
+                                              dim4 const &, intl, double);    \
+    template Array<uintl> reshape<SRC_T, uintl>(Array<SRC_T> const &,         \
+                                                dim4 const &, uintl, double); \
+    template Array<short> reshape<SRC_T, short>(Array<SRC_T> const &,         \
+                                                dim4 const &, short, double); \
+    template Array<ushort> reshape<SRC_T, ushort>(                            \
+        Array<SRC_T> const &, dim4 const &, ushort, double);                  \
+    template Array<uchar> reshape<SRC_T, uchar>(Array<SRC_T> const &,         \
+                                                dim4 const &, uchar, double); \
+    template Array<char> reshape<SRC_T, char>(Array<SRC_T> const &,           \
+                                              dim4 const &, char, double);    \
+    template Array<half> reshape<SRC_T, half>(Array<SRC_T> const &,           \
+                                              dim4 const &, half, double);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(half)
+
+#define INSTANTIATE_COMPLEX(SRC_T)                           \
+    template Array<cfloat> reshape<SRC_T, cfloat>(           \
+        Array<SRC_T> const &, dim4 const &, cfloat, double); \
+    template Array<cdouble> reshape<SRC_T, cdouble>(         \
+        Array<SRC_T> const &, dim4 const &, cdouble, double);
+
+INSTANTIATE_COMPLEX(cfloat)
+INSTANTIATE_COMPLEX(cdouble)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/resize.cpp b/src/backend/oneapi/resize.cpp
new file mode 100644
index 0000000000..89bdea49b1
--- /dev/null
+++ b/src/backend/oneapi/resize.cpp
@@ -0,0 +1,48 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/resize.hpp>
+#include <resize.hpp>
+#include <af/dim4.hpp>
+#include <stdexcept>
+
+namespace oneapi {
+template<typename T>
+Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
+                const af_interp_type method) {
+    const af::dim4 &iDims = in.dims();
+    af::dim4 oDims(odim0, odim1, iDims[2], iDims[3]);
+    Array<T> out = createEmptyArray<T>(oDims);
+
+    ONEAPI_NOT_SUPPORTED("resize Not supported");
+
+    // kernel::resize<T>(out, in, method);
+    return out;
+}
+
+#define INSTANTIATE(T)                                                 \
+    template Array<T> resize<T>(const Array<T> &in, const dim_t odim0, \
+                                const dim_t odim1,                     \
+                                const af_interp_type method);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/resize.hpp b/src/backend/oneapi/resize.hpp
new file mode 100644
index 0000000000..77b5972588
--- /dev/null
+++ b/src/backend/oneapi/resize.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
+                const af_interp_type method);
+}
diff --git a/src/backend/oneapi/rotate.cpp b/src/backend/oneapi/rotate.cpp
new file mode 100644
index 0000000000..fc49dd6baa
--- /dev/null
+++ b/src/backend/oneapi/rotate.cpp
@@ -0,0 +1,59 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <rotate.hpp>
+#include <err_oneapi.hpp>
+
+// #include <kernel/rotate.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
+                const af_interp_type method) {
+
+    ONEAPI_NOT_SUPPORTED("rotate Not supported");
+
+    Array<T> out = createEmptyArray<T>(odims);
+
+    // switch (method) {
+    //     case AF_INTERP_NEAREST:
+    //     case AF_INTERP_LOWER:
+    //         kernel::rotate<T>(out, in, theta, method, 1);
+    //         break;
+    //     case AF_INTERP_BILINEAR:
+    //     case AF_INTERP_BILINEAR_COSINE:
+    //         kernel::rotate<T>(out, in, theta, method, 2);
+    //         break;
+    //     case AF_INTERP_BICUBIC:
+    //     case AF_INTERP_BICUBIC_SPLINE:
+    //         kernel::rotate<T>(out, in, theta, method, 3);
+    //         break;
+    //     default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG);
+    // }
+    return out;
+}
+
+#define INSTANTIATE(T)                                              \
+    template Array<T> rotate(const Array<T> &in, const float theta, \
+                             const af::dim4 &odims,                 \
+                             const af_interp_type method);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/rotate.hpp b/src/backend/oneapi/rotate.hpp
new file mode 100644
index 0000000000..369bbd2521
--- /dev/null
+++ b/src/backend/oneapi/rotate.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
+                const af_interp_type method);
+}
diff --git a/src/backend/oneapi/scalar.hpp b/src/backend/oneapi/scalar.hpp
new file mode 100644
index 0000000000..fee814f9f2
--- /dev/null
+++ b/src/backend/oneapi/scalar.hpp
@@ -0,0 +1,23 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/jit/ScalarNode.hpp>
+#include <math.hpp>
+#include <optypes.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> createScalarNode(const dim4 &size, const T val) {
+    return createNodeArray<T>(size,
+                              std::make_shared<common::ScalarNode<T>>(val));
+}
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/scan.cpp b/src/backend/oneapi/scan.cpp
new file mode 100644
index 0000000000..572746035c
--- /dev/null
+++ b/src/backend/oneapi/scan.cpp
@@ -0,0 +1,58 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <scan.hpp>
+#include <err_oneapi.hpp>
+
+// #include <kernel/scan_dim.hpp>
+// #include <kernel/scan_first.hpp>
+
+namespace oneapi {
+template<af_op_t op, typename Ti, typename To>
+Array<To> scan(const Array<Ti>& in, const int dim, bool inclusiveScan) {
+
+    ONEAPI_NOT_SUPPORTED("scan Not supported");
+
+    Array<To> out = createEmptyArray<To>(in.dims());
+
+    // Param Out = out;
+    // Param In  = in;
+
+    // if (dim == 0) {
+    //     kernel::scanFirst<Ti, To, op>(Out, In, inclusiveScan);
+    // } else {
+    //     kernel::scanDim<Ti, To, op>(Out, In, dim, inclusiveScan);
+    // }
+
+    return out;
+}
+
+#define INSTANTIATE_SCAN(ROp, Ti, To) \
+    template Array<To> scan<ROp, Ti, To>(const Array<Ti>&, const int, bool);
+
+#define INSTANTIATE_SCAN_ALL(ROp)           \
+    INSTANTIATE_SCAN(ROp, float, float)     \
+    INSTANTIATE_SCAN(ROp, double, double)   \
+    INSTANTIATE_SCAN(ROp, cfloat, cfloat)   \
+    INSTANTIATE_SCAN(ROp, cdouble, cdouble) \
+    INSTANTIATE_SCAN(ROp, int, int)         \
+    INSTANTIATE_SCAN(ROp, uint, uint)       \
+    INSTANTIATE_SCAN(ROp, intl, intl)       \
+    INSTANTIATE_SCAN(ROp, uintl, uintl)     \
+    INSTANTIATE_SCAN(ROp, char, uint)       \
+    INSTANTIATE_SCAN(ROp, uchar, uint)      \
+    INSTANTIATE_SCAN(ROp, short, int)       \
+    INSTANTIATE_SCAN(ROp, ushort, uint)
+
+INSTANTIATE_SCAN(af_notzero_t, char, uint)
+INSTANTIATE_SCAN_ALL(af_add_t)
+INSTANTIATE_SCAN_ALL(af_mul_t)
+INSTANTIATE_SCAN_ALL(af_min_t)
+INSTANTIATE_SCAN_ALL(af_max_t)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/scan.hpp b/src/backend/oneapi/scan.hpp
new file mode 100644
index 0000000000..5e8508a8da
--- /dev/null
+++ b/src/backend/oneapi/scan.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <optypes.hpp>
+
+namespace oneapi {
+template<af_op_t op, typename Ti, typename To>
+Array<To> scan(const Array<Ti>& in, const int dim, bool inclusive_scan = true);
+}
diff --git a/src/backend/oneapi/scan_by_key.cpp b/src/backend/oneapi/scan_by_key.cpp
new file mode 100644
index 0000000000..08a4969905
--- /dev/null
+++ b/src/backend/oneapi/scan_by_key.cpp
@@ -0,0 +1,65 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+#include <scan.hpp>
+#include <af/dim4.hpp>
+#include <complex>
+
+// #include <kernel/scan_dim_by_key.hpp>
+// #include <kernel/scan_first_by_key.hpp>
+
+namespace oneapi {
+template<af_op_t op, typename Ti, typename Tk, typename To>
+Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
+               bool inclusive_scan) {
+
+    ONEAPI_NOT_SUPPORTED("scan Not supported");
+
+    Array<To> out = createEmptyArray<To>(in.dims());
+
+    // Param Out = out;
+    // Param Key = key;
+    // Param In  = in;
+
+    // if (dim == 0) {
+    //     // kernel::scanFirstByKey<Ti, Tk, To, op>(Out, In, Key, inclusive_scan);
+    // } else {
+    //     // kernel::scanDimByKey<Ti, Tk, To, op>(Out, In, Key, dim, inclusive_scan);
+    // }
+    return out;
+}
+
+#define INSTANTIATE_SCAN_BY_KEY(ROp, Ti, Tk, To)                  \
+    template Array<To> scan<ROp, Ti, Tk, To>(                     \
+        const Array<Tk>& key, const Array<Ti>& in, const int dim, \
+        bool inclusive_scan);
+
+#define INSTANTIATE_SCAN_BY_KEY_ALL(ROp, Tk)           \
+    INSTANTIATE_SCAN_BY_KEY(ROp, float, Tk, float)     \
+    INSTANTIATE_SCAN_BY_KEY(ROp, double, Tk, double)   \
+    INSTANTIATE_SCAN_BY_KEY(ROp, cfloat, Tk, cfloat)   \
+    INSTANTIATE_SCAN_BY_KEY(ROp, cdouble, Tk, cdouble) \
+    INSTANTIATE_SCAN_BY_KEY(ROp, int, Tk, int)         \
+    INSTANTIATE_SCAN_BY_KEY(ROp, uint, Tk, uint)       \
+    INSTANTIATE_SCAN_BY_KEY(ROp, intl, Tk, intl)       \
+    INSTANTIATE_SCAN_BY_KEY(ROp, uintl, Tk, uintl)
+
+#define INSTANTIATE_SCAN_BY_KEY_OP(ROp)    \
+    INSTANTIATE_SCAN_BY_KEY_ALL(ROp, int)  \
+    INSTANTIATE_SCAN_BY_KEY_ALL(ROp, uint) \
+    INSTANTIATE_SCAN_BY_KEY_ALL(ROp, intl) \
+    INSTANTIATE_SCAN_BY_KEY_ALL(ROp, uintl)
+
+INSTANTIATE_SCAN_BY_KEY_OP(af_add_t)
+INSTANTIATE_SCAN_BY_KEY_OP(af_mul_t)
+INSTANTIATE_SCAN_BY_KEY_OP(af_min_t)
+INSTANTIATE_SCAN_BY_KEY_OP(af_max_t)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/scan_by_key.hpp b/src/backend/oneapi/scan_by_key.hpp
new file mode 100644
index 0000000000..556d59f922
--- /dev/null
+++ b/src/backend/oneapi/scan_by_key.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <optypes.hpp>
+
+namespace oneapi {
+template<af_op_t op, typename Ti, typename Tk, typename To>
+Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
+               bool inclusive_scan = true);
+}
diff --git a/src/backend/oneapi/select.cpp b/src/backend/oneapi/select.cpp
new file mode 100644
index 0000000000..f15e2ab61c
--- /dev/null
+++ b/src/backend/oneapi/select.cpp
@@ -0,0 +1,145 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+// #include <kernel/select.hpp>
+#include <select.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <common/jit/NaryNode.hpp>
+#include <err_oneapi.hpp>
+#include <scalar.hpp>
+
+#include <nonstd/span.hpp>
+#include <memory>
+
+using af::dim4;
+
+using common::half;
+using common::NaryNode;
+
+using std::make_shared;
+using std::max;
+
+namespace oneapi {
+template<typename T>
+Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
+                          const Array<T> &b, const dim4 &odims) {
+
+    ONEAPI_NOT_SUPPORTED("createSelectNode Not supported");
+
+    auto cond_node   = cond.getNode();
+    auto a_node      = a.getNode();
+    auto b_node      = b.getNode();
+    auto a_height    = a_node->getHeight();
+    auto b_height    = b_node->getHeight();
+    auto cond_height = cond_node->getHeight();
+    const int height = max(max(a_height, b_height), cond_height) + 1;
+
+    auto node = make_shared<NaryNode>(
+        NaryNode(static_cast<af::dtype>(af::dtype_traits<T>::af_type), "__select",
+                 3, {{cond_node, a_node, b_node}}, af_select_t, height));
+    std::array<common::Node *, 1> nodes{node.get()};
+    if (detail::passesJitHeuristics<T>(nodes) != kJITHeuristics::Pass) {
+        if (a_height > max(b_height, cond_height)) {
+            a.eval();
+        } else if (b_height > cond_height) {
+            b.eval();
+        } else {
+            cond.eval();
+        }
+        return createSelectNode<T>(cond, a, b, odims);
+    }
+    return createNodeArray<T>(odims, node);
+}
+
+template<typename T, bool flip>
+Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
+                          const T &b_val, const dim4 &odims) {
+
+    ONEAPI_NOT_SUPPORTED("createSelectNode Not supported");
+
+    auto cond_node   = cond.getNode();
+    auto a_node      = a.getNode();
+    Array<T> b       = createScalarNode<T>(odims, b_val);
+    auto b_node      = b.getNode();
+    auto a_height    = a_node->getHeight();
+    auto b_height    = b_node->getHeight();
+    auto cond_height = cond_node->getHeight();
+    const int height = max(max(a_height, b_height), cond_height) + 1;
+
+    auto node = make_shared<NaryNode>(NaryNode(
+        static_cast<af::dtype>(af::dtype_traits<T>::af_type),
+        (flip ? "__not_select" : "__select"), 3, {{cond_node, a_node, b_node}},
+        (flip ? af_not_select_t : af_select_t), height));
+
+    std::array<common::Node *, 1> nodes{node.get()};
+    if (detail::passesJitHeuristics<T>(nodes) != kJITHeuristics::Pass) {
+        if (a_height > max(b_height, cond_height)) {
+            a.eval();
+        } else if (b_height > cond_height) {
+            b.eval();
+        } else {
+            cond.eval();
+        }
+        return createSelectNode<T, flip>(cond, a, b_val, odims);
+    }
+    return createNodeArray<T>(odims, node);
+}
+
+template<typename T>
+void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
+            const Array<T> &b) {
+    ONEAPI_NOT_SUPPORTED("select Not supported");
+
+    // kernel::select<T>(out, cond, a, b, out.ndims());
+}
+
+template<typename T, bool flip>
+void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
+                   const T &b) {
+    ONEAPI_NOT_SUPPORTED("select_scalar Not supported");
+
+    // kernel::select_scalar<T>(out, cond, a, b, out.ndims(), flip);
+}
+
+#define INSTANTIATE(T)                                                   \
+    template Array<T> createSelectNode<T>(                               \
+        const Array<char> &cond, const Array<T> &a, const Array<T> &b,   \
+        const af::dim4 &odims);                                          \
+    template Array<T> createSelectNode<T, true>(                         \
+        const Array<char> &cond, const Array<T> &a, const T &b_val,      \
+        const af::dim4 &odims);                                          \
+    template Array<T> createSelectNode<T, false>(                        \
+        const Array<char> &cond, const Array<T> &a, const T &b_val,      \
+        const af::dim4 &odims);                                          \
+    template void select<T>(Array<T> & out, const Array<char> &cond,     \
+                            const Array<T> &a, const Array<T> &b);       \
+    template void select_scalar<T, true>(Array<T> & out,                 \
+                                         const Array<char> &cond,        \
+                                         const Array<T> &a, const T &b); \
+    template void select_scalar<T, false>(Array<T> & out,                \
+                                          const Array<char> &cond,       \
+                                          const Array<T> &a, const T &b)
+
+INSTANTIATE(float);
+INSTANTIATE(double);
+INSTANTIATE(cfloat);
+INSTANTIATE(cdouble);
+INSTANTIATE(int);
+INSTANTIATE(uint);
+INSTANTIATE(intl);
+INSTANTIATE(uintl);
+INSTANTIATE(char);
+INSTANTIATE(uchar);
+INSTANTIATE(short);
+INSTANTIATE(ushort);
+INSTANTIATE(half);
+
+#undef INSTANTIATE
+}  // namespace oneapi
diff --git a/src/backend/oneapi/select.hpp b/src/backend/oneapi/select.hpp
new file mode 100644
index 0000000000..00d0eb06c6
--- /dev/null
+++ b/src/backend/oneapi/select.hpp
@@ -0,0 +1,29 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+#include <Array.hpp>
+#include <af/dim4.hpp>
+
+namespace oneapi {
+template<typename T>
+void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
+            const Array<T> &b);
+
+template<typename T, bool flip>
+void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
+                   const T &b);
+
+template<typename T>
+Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
+                          const Array<T> &b, const af::dim4 &odims);
+
+template<typename T, bool flip>
+Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
+                          const T &b_val, const af::dim4 &odims);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/set.cpp b/src/backend/oneapi/set.cpp
new file mode 100644
index 0000000000..2001729eca
--- /dev/null
+++ b/src/backend/oneapi/set.cpp
@@ -0,0 +1,157 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <copy.hpp>
+#include <err_oneapi.hpp>
+#include <set.hpp>
+#include <sort.hpp>
+#include <af/dim4.hpp>
+#include <common/deprecated.hpp>
+
+namespace oneapi {
+using af::dim4;
+
+using std::conditional;
+using std::is_same;
+template<typename T>
+using ltype_t = typename conditional<is_same<T, intl>::value, cl_long, T>::type;
+
+template<typename T>
+using type_t =
+    typename conditional<is_same<T, uintl>::value, cl_ulong, ltype_t<T>>::type;
+
+template<typename T>
+Array<T> setUnique(const Array<T> &in, const bool is_sorted) {
+
+    ONEAPI_NOT_SUPPORTED("setUnique Not supported");
+    return createEmptyArray<T>(dim4(1, 1, 1, 1));
+
+    // try {
+    //     Array<T> out = copyArray<T>(in);
+
+    //     compute::command_queue queue(getQueue()());
+
+    //     compute::buffer out_data((*out.get())());
+
+    //     compute::buffer_iterator<type_t<T>> begin(out_data, 0);
+    //     compute::buffer_iterator<type_t<T>> end(out_data, out.elements());
+
+    //     if (!is_sorted) { compute::sort(begin, end, queue); }
+
+    //     end = compute::unique(begin, end, queue);
+
+    //     out.resetDims(dim4(std::distance(begin, end), 1, 1, 1));
+
+    //     return out;
+    // } catch (const std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+}
+
+template<typename T>
+Array<T> setUnion(const Array<T> &first, const Array<T> &second,
+                  const bool is_unique) {
+
+    ONEAPI_NOT_SUPPORTED("setUnion Not supported");
+    return createEmptyArray<T>(dim4(1, 1, 1, 1));
+
+    // try {
+    //     Array<T> unique_first  = first;
+    //     Array<T> unique_second = second;
+
+    //     if (!is_unique) {
+    //         unique_first  = setUnique(first, false);
+    //         unique_second = setUnique(second, false);
+    //     }
+
+    //     size_t out_size = unique_first.elements() + unique_second.elements();
+    //     Array<T> out    = createEmptyArray<T>(dim4(out_size, 1, 1, 1));
+
+    //     compute::command_queue queue(getQueue()());
+
+    //     compute::buffer first_data((*unique_first.get())());
+    //     compute::buffer second_data((*unique_second.get())());
+    //     compute::buffer out_data((*out.get())());
+
+    //     compute::buffer_iterator<type_t<T>> first_begin(first_data, 0);
+    //     compute::buffer_iterator<type_t<T>> first_end(first_data,
+    //                                                   unique_first.elements());
+    //     compute::buffer_iterator<type_t<T>> second_begin(second_data, 0);
+    //     compute::buffer_iterator<type_t<T>> second_end(
+    //         second_data, unique_second.elements());
+    //     compute::buffer_iterator<type_t<T>> out_begin(out_data, 0);
+
+    //     compute::buffer_iterator<type_t<T>> out_end = compute::set_union(
+    //         first_begin, first_end, second_begin, second_end, out_begin, queue);
+
+    //     out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
+    //     return out;
+
+    // } catch (const std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+}
+
+template<typename T>
+Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
+                      const bool is_unique) {
+
+    ONEAPI_NOT_SUPPORTED("setIntersect Not supported");
+    return createEmptyArray<T>(dim4(1, 1, 1, 1));
+
+    // try {
+    //     Array<T> unique_first  = first;
+    //     Array<T> unique_second = second;
+
+    //     if (!is_unique) {
+    //         unique_first  = setUnique(first, false);
+    //         unique_second = setUnique(second, false);
+    //     }
+
+    //     size_t out_size =
+    //         std::max(unique_first.elements(), unique_second.elements());
+    //     Array<T> out = createEmptyArray<T>(dim4(out_size, 1, 1, 1));
+
+    //     compute::command_queue queue(getQueue()());
+
+    //     compute::buffer first_data((*unique_first.get())());
+    //     compute::buffer second_data((*unique_second.get())());
+    //     compute::buffer out_data((*out.get())());
+
+    //     compute::buffer_iterator<type_t<T>> first_begin(first_data, 0);
+    //     compute::buffer_iterator<type_t<T>> first_end(first_data,
+    //                                                   unique_first.elements());
+    //     compute::buffer_iterator<type_t<T>> second_begin(second_data, 0);
+    //     compute::buffer_iterator<type_t<T>> second_end(
+    //         second_data, unique_second.elements());
+    //     compute::buffer_iterator<type_t<T>> out_begin(out_data, 0);
+
+    //     compute::buffer_iterator<type_t<T>> out_end = compute::set_intersection(
+    //         first_begin, first_end, second_begin, second_end, out_begin, queue);
+
+    //     out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
+    //     return out;
+    // } catch (const std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+}
+
+#define INSTANTIATE(T)                                                        \
+    template Array<T> setUnique<T>(const Array<T> &in, const bool is_sorted); \
+    template Array<T> setUnion<T>(                                            \
+        const Array<T> &first, const Array<T> &second, const bool is_unique); \
+    template Array<T> setIntersect<T>(                                        \
+        const Array<T> &first, const Array<T> &second, const bool is_unique);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(char)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/set.hpp b/src/backend/oneapi/set.hpp
new file mode 100644
index 0000000000..7836873639
--- /dev/null
+++ b/src/backend/oneapi/set.hpp
@@ -0,0 +1,23 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> setUnique(const Array<T> &in, const bool is_sorted);
+
+template<typename T>
+Array<T> setUnion(const Array<T> &first, const Array<T> &second,
+                  const bool is_unique);
+
+template<typename T>
+Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
+                      const bool is_unique);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/shift.cpp b/src/backend/oneapi/shift.cpp
new file mode 100644
index 0000000000..b3941f1960
--- /dev/null
+++ b/src/backend/oneapi/shift.cpp
@@ -0,0 +1,73 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <shift.hpp>
+
+#include <common/jit/ShiftNodeBase.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+
+using af::dim4;
+using common::Node_ptr;
+using common::ShiftNodeBase;
+using std::array;
+using std::make_shared;
+using std::static_pointer_cast;
+using std::string;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> shift(const Array<T> &in, const int sdims[4]) {
+    ONEAPI_NOT_SUPPORTED("");
+    Array<T> o = createEmptyArray<T>(dim4(1));
+    return o;
+    /*
+    // Shift should only be the first node in the JIT tree.
+    // Force input to be evaluated so that in is always a buffer.
+    in.eval();
+
+    string name_str("Sh");
+    name_str += shortname<T>(true);
+    const dim4 &iDims = in.dims();
+    dim4 oDims        = iDims;
+
+    array<int, 4> shifts{};
+    for (int i = 0; i < 4; i++) {
+        // sdims_[i] will always be positive and always [0, oDims[i]].
+        // Negative shifts are converted to position by going the other way
+        // round
+        shifts[i] = -(sdims[i] % static_cast<int>(oDims[i])) +
+                    oDims[i] * (sdims[i] > 0);
+        assert(shifts[i] >= 0 && shifts[i] <= oDims[i]);
+    }
+
+    auto node = make_shared<ShiftNode>(
+        static_cast<af::dtype>(dtype_traits<T>::af_type),
+        static_pointer_cast<BufferNode>(in.getNode()), shifts);
+    return createNodeArray<T>(oDims, common::Node_ptr(node));
+    */
+}
+
+#define INSTANTIATE(T) \
+    template Array<T> shift<T>(const Array<T> &in, const int sdims[4]);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+}  // namespace opencl
diff --git a/src/backend/oneapi/shift.hpp b/src/backend/oneapi/shift.hpp
new file mode 100644
index 0000000000..f236018321
--- /dev/null
+++ b/src/backend/oneapi/shift.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> shift(const Array<T> &in, const int sdims[4]);
+}
diff --git a/src/backend/oneapi/sift.cpp b/src/backend/oneapi/sift.cpp
new file mode 100644
index 0000000000..af2f7bf10d
--- /dev/null
+++ b/src/backend/oneapi/sift.cpp
@@ -0,0 +1,76 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <sift.hpp>
+
+// #include <kernel/sift.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+
+using af::dim4;
+using af::features;
+
+namespace oneapi {
+
+template<typename T, typename convAccT>
+unsigned sift(Array<float>& x_out, Array<float>& y_out, Array<float>& score_out,
+              Array<float>& ori_out, Array<float>& size_out,
+              Array<float>& desc_out, const Array<T>& in,
+              const unsigned n_layers, const float contrast_thr,
+              const float edge_thr, const float init_sigma,
+              const bool double_input, const float img_scale,
+              const float feature_ratio, const bool compute_GLOH) {
+
+    ONEAPI_NOT_SUPPORTED("sift Not supported");
+    return 0;
+
+    // unsigned nfeat_out;
+    // unsigned desc_len;
+
+    // Param x;
+    // Param y;
+    // Param score;
+    // Param ori;
+    // Param size;
+    // Param desc;
+
+    // kernel::sift<T, convAccT>(&nfeat_out, &desc_len, x, y, score, ori, size,
+    //                           desc, in, n_layers, contrast_thr, edge_thr,
+    //                           init_sigma, double_input, img_scale,
+    //                           feature_ratio, compute_GLOH);
+
+    // if (nfeat_out > 0) {
+    //     const dim4 out_dims(nfeat_out);
+    //     const dim4 desc_dims(desc_len, nfeat_out);
+
+    //     x_out     = createParamArray<float>(x, true);
+    //     y_out     = createParamArray<float>(y, true);
+    //     score_out = createParamArray<float>(score, true);
+    //     ori_out   = createParamArray<float>(ori, true);
+    //     size_out  = createParamArray<float>(size, true);
+    //     desc_out  = createParamArray<float>(desc, true);
+    // }
+
+    // return nfeat_out;
+}
+
+#define INSTANTIATE(T, convAccT)                                              \
+    template unsigned sift<T, convAccT>(                                      \
+        Array<float> & x_out, Array<float> & y_out, Array<float> & score_out, \
+        Array<float> & ori_out, Array<float> & size_out,                      \
+        Array<float> & desc_out, const Array<T>& in, const unsigned n_layers, \
+        const float contrast_thr, const float edge_thr,                       \
+        const float init_sigma, const bool double_input,                      \
+        const float img_scale, const float feature_ratio,                     \
+        const bool compute_GLOH);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sift.hpp b/src/backend/oneapi/sift.hpp
new file mode 100644
index 0000000000..5c2a33dca6
--- /dev/null
+++ b/src/backend/oneapi/sift.hpp
@@ -0,0 +1,26 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/features.h>
+
+using af::features;
+
+namespace oneapi {
+
+template<typename T, typename convAccT>
+unsigned sift(Array<float>& x, Array<float>& y, Array<float>& score,
+              Array<float>& ori, Array<float>& size, Array<float>& desc,
+              const Array<T>& in, const unsigned n_layers,
+              const float contrast_thr, const float edge_thr,
+              const float init_sigma, const bool double_input,
+              const float img_scale, const float feature_ratio,
+              const bool compute_GLOH);
+
+}
diff --git a/src/backend/oneapi/sobel.cpp b/src/backend/oneapi/sobel.cpp
new file mode 100644
index 0000000000..f76b8685db
--- /dev/null
+++ b/src/backend/oneapi/sobel.cpp
@@ -0,0 +1,49 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/sobel.hpp>
+#include <sobel.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename Ti, typename To>
+std::pair<Array<To>, Array<To>> sobelDerivatives(const Array<Ti> &img,
+                                                 const unsigned &ker_size) {
+
+    ONEAPI_NOT_SUPPORTED("sobelDerivatives Not supported");
+
+    Array<To> dx = createEmptyArray<To>(img.dims());
+    Array<To> dy = createEmptyArray<To>(img.dims());
+
+    // switch (ker_size) {
+    //     case 3: kernel::sobel<Ti, To, 3>(dx, dy, img); break;
+    // }
+
+    return std::make_pair(dx, dy);
+}
+
+#define INSTANTIATE(Ti, To)                                    \
+    template std::pair<Array<To>, Array<To>> sobelDerivatives( \
+        const Array<Ti> &img, const unsigned &ker_size);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+INSTANTIATE(int, int)
+INSTANTIATE(uint, int)
+INSTANTIATE(char, int)
+INSTANTIATE(uchar, int)
+INSTANTIATE(short, int)
+INSTANTIATE(ushort, int)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sobel.hpp b/src/backend/oneapi/sobel.hpp
new file mode 100644
index 0000000000..94d3e06879
--- /dev/null
+++ b/src/backend/oneapi/sobel.hpp
@@ -0,0 +1,19 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <utility>
+
+namespace oneapi {
+
+template<typename Ti, typename To>
+std::pair<Array<To>, Array<To>> sobelDerivatives(const Array<Ti> &img,
+                                                 const unsigned &ker_size);
+
+}
diff --git a/src/backend/oneapi/solve.cpp b/src/backend/oneapi/solve.cpp
new file mode 100644
index 0000000000..b38461d0f1
--- /dev/null
+++ b/src/backend/oneapi/solve.cpp
@@ -0,0 +1,368 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <solve.hpp>
+
+#include <err_oneapi.hpp>
+
+#if defined(WITH_LINEAR_ALGEBRA) && !defined(AF_ONEAPI)
+#include <blas.hpp>
+#include <copy.hpp>
+#include <cpu/cpu_solve.hpp>
+#include <lu.hpp>
+#include <magma/magma.h>
+#include <magma/magma_blas.h>
+#include <magma/magma_data.h>
+#include <magma/magma_helper.h>
+#include <math.hpp>
+#include <platform.hpp>
+#include <transpose.hpp>
+#include <af/opencl.h>
+
+#include <algorithm>
+#include <vector>
+
+using cl::Buffer;
+using std::min;
+using std::vector;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> solveLU(const Array<T> &A, const Array<int> &pivot, const Array<T> &b,
+                 const af_mat_prop options) {
+
+    ONEAPI_NOT_SUPPORTED("solveLU Not supported");
+
+    if (OpenCLCPUOffload()) { return cpu::solveLU(A, pivot, b, options); }
+
+    int N    = A.dims()[0];
+    int NRHS = b.dims()[1];
+
+    vector<int> ipiv(N);
+    copyData(&ipiv[0], pivot);
+
+    Array<T> B = copyArray<T>(b);
+
+    const Buffer *A_buf = A.get();
+    Buffer *B_buf       = B.get();
+
+    int info = 0;
+    magma_getrs_gpu<T>(MagmaNoTrans, N, NRHS, (*A_buf)(), A.getOffset(),
+                       A.strides()[1], &ipiv[0], (*B_buf)(), B.getOffset(),
+                       B.strides()[1], getQueue()(), &info);
+    return B;
+}
+
+template<typename T>
+Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
+
+    ONEAPI_NOT_SUPPORTED("generalSolve Not supported");
+
+    // dim4 aDims = a.dims();
+    // int batchz = aDims[2];
+    // int batchw = aDims[3];
+
+    // Array<T> A = copyArray<T>(a);
+    Array<T> B = copyArray<T>(b);
+
+    // for (int i = 0; i < batchw; i++) {
+    //     for (int j = 0; j < batchz; j++) {
+    //         int M  = aDims[0];
+    //         int N  = aDims[1];
+    //         int MN = min(M, N);
+    //         vector<int> ipiv(MN);
+
+    //         Buffer *A_buf      = A.get();
+    //         int info           = 0;
+    //         cl_command_queue q = getQueue()();
+    //         auto aoffset =
+    //             A.getOffset() + j * A.strides()[2] + i * A.strides()[3];
+    //         magma_getrf_gpu<T>(M, N, (*A_buf)(), aoffset, A.strides()[1],
+    //                            &ipiv[0], q, &info);
+
+    //         Buffer *B_buf = B.get();
+    //         int K         = B.dims()[1];
+
+    //         auto boffset =
+    //             B.getOffset() + j * B.strides()[2] + i * B.strides()[3];
+    //         magma_getrs_gpu<T>(MagmaNoTrans, M, K, (*A_buf)(), aoffset,
+    //                            A.strides()[1], &ipiv[0], (*B_buf)(), boffset,
+    //                            B.strides()[1], q, &info);
+    //     }
+    // }
+    return B;
+}
+
+template<typename T>
+Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
+
+    ONEAPI_NOT_SUPPORTED("leastSquares Not supported");
+
+    int M  = a.dims()[0];
+    int N  = a.dims()[1];
+    int K  = b.dims()[1];
+    int MN = min(M, N);
+
+    Array<T> B = createEmptyArray<T>(dim4());
+    gpu_blas_trsm_func<T> gpu_blas_trsm;
+
+    cl_event event;
+    cl_command_queue queue = getQueue()();
+
+    if (M < N) {
+#define UNMQR 0  // FIXME: UNMQR == 1 should be faster but does not work
+
+        // Least squres for this case is solved using the following
+        // solve(A, B) == matmul(Q, Xpad);
+        // Where:
+        // Xpad == pad(Xt, N - M, 1);
+        // Xt   == tri_solve(R1, B);
+        // R1   == R(seq(M), seq(M));
+        // transpose(A) == matmul(Q, R);
+
+        // QR is performed on the transpose of A
+        Array<T> A = transpose<T>(a, true);
+
+#if UNMQR
+        const dim4 NullShape(0, 0, 0, 0);
+        dim4 endPadding(N - b.dims()[0], K - b.dims()[1], 0, 0);
+        B = (endPadding == NullShape
+                 ? copyArray(b)
+                 : padArrayBorders(b, NullShape, endPadding, AF_PAD_ZERO));
+        B.resetDims(dim4(M, K));
+#else
+        B = copyArray<T>(b);
+#endif
+
+        int NB       = magma_get_geqrf_nb<T>(A.dims()[1]);
+        int NUM      = (2 * MN + ((M + 31) / 32) * 32) * NB;
+        Array<T> tmp = createEmptyArray<T>(dim4(NUM));
+
+        vector<T> h_tau(MN);
+
+        int info   = 0;
+        Buffer *dA = A.get();
+        Buffer *dT = tmp.get();
+        Buffer *dB = B.get();
+
+        magma_geqrf3_gpu<T>(A.dims()[0], A.dims()[1], (*dA)(), A.getOffset(),
+                            A.strides()[1], &h_tau[0], (*dT)(), tmp.getOffset(),
+                            getQueue()(), &info);
+
+        A.resetDims(dim4(M, M));
+
+        magmablas_swapdblk<T>(MN - 1, NB, (*dA)(), A.getOffset(),
+                              A.strides()[1], 1, (*dT)(),
+                              tmp.getOffset() + MN * NB, NB, 0, queue);
+
+        OPENCL_BLAS_CHECK(
+            gpu_blas_trsm(OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_UPPER,
+                          OPENCL_BLAS_CONJ_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL,
+                          B.dims()[0], B.dims()[1], scalar<T>(1), (*dA)(),
+                          A.getOffset(), A.strides()[1], (*dB)(), B.getOffset(),
+                          B.strides()[1], 1, &queue, 0, nullptr, &event));
+
+        magmablas_swapdblk<T>(MN - 1, NB, (*dT)(), tmp.getOffset() + MN * NB,
+                              NB, 0, (*dA)(), A.getOffset(), A.strides()[1], 1,
+                              queue);
+
+#if UNMQR
+        int lwork = (B.dims()[0] - A.dims()[0] + NB) * (B.dims()[1] + 2 * NB);
+        vector<T> h_work(lwork);
+        B.resetDims(dim4(N, K));
+        magma_unmqr_gpu<T>(MagmaLeft, MagmaNoTrans, B.dims()[0], B.dims()[1],
+                           A.dims()[0], (*dA)(), A.getOffset(), A.strides()[1],
+                           &h_tau[0], (*dB)(), B.getOffset(), B.strides()[1],
+                           &h_work[0], lwork, (*dT)(), tmp.getOffset(), NB,
+                           queue, &info);
+#else
+        A.resetDims(dim4(N, M));
+        magma_ungqr_gpu<T>(A.dims()[0], A.dims()[1], min(M, N), (*dA)(),
+                           A.getOffset(), A.strides()[1], &h_tau[0], (*dT)(),
+                           tmp.getOffset(), NB, queue, &info);
+
+        Array<T> B_new = createEmptyArray<T>(dim4(A.dims()[0], B.dims()[1]));
+        T alpha        = scalar<T>(1.0);
+        T beta         = scalar<T>(0.0);
+        gemm<T>(B_new, AF_MAT_NONE, AF_MAT_NONE, &alpha, A, B, &beta);
+        B = B_new;
+#endif
+    } else if (M > N) {
+        // Least squres for this case is solved using the following
+        // solve(A, B) == tri_solve(R1, Bt);
+        // Where:
+        // R1 == R(seq(N), seq(N));
+        // Bt == matmul(transpose(Q1), B);
+        // Q1 == Q(span, seq(N));
+        // A  == matmul(Q, R);
+
+        Array<T> A = copyArray<T>(a);
+        B          = copyArray(b);
+
+        int MN = min(M, N);
+        int NB = magma_get_geqrf_nb<T>(M);
+
+        int NUM      = (2 * MN + ((N + 31) / 32) * 32) * NB;
+        Array<T> tmp = createEmptyArray<T>(dim4(NUM));
+
+        vector<T> h_tau(NUM);
+
+        int info      = 0;
+        Buffer *A_buf = A.get();
+        Buffer *B_buf = B.get();
+        Buffer *dT    = tmp.get();
+
+        magma_geqrf3_gpu<T>(M, N, (*A_buf)(), A.getOffset(), A.strides()[1],
+                            &h_tau[0], (*dT)(), tmp.getOffset(), getQueue()(),
+                            &info);
+
+        int NRHS   = B.dims()[1];
+        int lhwork = (M - N + NB) * (NRHS + NB) + NRHS * NB;
+
+        vector<T> h_work(lhwork);
+        h_work[0] = scalar<T>(lhwork);
+
+        magma_unmqr_gpu<T>(MagmaLeft, MagmaConjTrans, M, NRHS, N, (*A_buf)(),
+                           A.getOffset(), A.strides()[1], &h_tau[0], (*B_buf)(),
+                           B.getOffset(), B.strides()[1], &h_work[0], lhwork,
+                           (*dT)(), tmp.getOffset(), NB, queue, &info);
+
+        magmablas_swapdblk<T>(MN - 1, NB, (*A_buf)(), A.getOffset(),
+                              A.strides()[1], 1, (*dT)(),
+                              tmp.getOffset() + NB * MN, NB, 0, queue);
+
+        if (getActivePlatform() == AFCL_PLATFORM_NVIDIA) {
+            Array<T> AT    = transpose<T>(A, true);
+            Buffer *AT_buf = AT.get();
+            OPENCL_BLAS_CHECK(gpu_blas_trsm(
+                OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_LOWER,
+                OPENCL_BLAS_CONJ_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL, N, NRHS,
+                scalar<T>(1), (*AT_buf)(), AT.getOffset(), AT.strides()[1],
+                (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0,
+                nullptr, &event));
+        } else {
+            OPENCL_BLAS_CHECK(gpu_blas_trsm(
+                OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_UPPER,
+                OPENCL_BLAS_NO_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL, N, NRHS,
+                scalar<T>(1), (*A_buf)(), A.getOffset(), A.strides()[1],
+                (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0,
+                nullptr, &event));
+        }
+        B.resetDims(dim4(N, K));
+    }
+
+    return B;
+}
+
+template<typename T>
+Array<T> triangleSolve(const Array<T> &A, const Array<T> &b,
+                       const af_mat_prop options) {
+    gpu_blas_trsm_func<T> gpu_blas_trsm;
+
+    Array<T> B = copyArray<T>(b);
+
+    int N    = B.dims()[0];
+    int NRHS = B.dims()[1];
+
+    const Buffer *A_buf = A.get();
+    Buffer *B_buf       = B.get();
+
+    cl_event event         = 0;
+    cl_command_queue queue = getQueue()();
+
+    if (getActivePlatform() == AFCL_PLATFORM_NVIDIA &&
+        (options & AF_MAT_UPPER)) {
+        Array<T> AT = transpose<T>(A, true);
+
+        cl::Buffer *AT_buf = AT.get();
+        OPENCL_BLAS_CHECK(gpu_blas_trsm(
+            OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_LOWER,
+            OPENCL_BLAS_CONJ_TRANS,
+            options & AF_MAT_DIAG_UNIT ? OPENCL_BLAS_UNIT_DIAGONAL
+                                       : OPENCL_BLAS_NON_UNIT_DIAGONAL,
+            N, NRHS, scalar<T>(1), (*AT_buf)(), AT.getOffset(), AT.strides()[1],
+            (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0, nullptr,
+            &event));
+    } else {
+        OPENCL_BLAS_CHECK(gpu_blas_trsm(
+            OPENCL_BLAS_SIDE_LEFT,
+            options & AF_MAT_LOWER ? OPENCL_BLAS_TRIANGLE_LOWER
+                                   : OPENCL_BLAS_TRIANGLE_UPPER,
+            OPENCL_BLAS_NO_TRANS,
+            options & AF_MAT_DIAG_UNIT ? OPENCL_BLAS_UNIT_DIAGONAL
+                                       : OPENCL_BLAS_NON_UNIT_DIAGONAL,
+            N, NRHS, scalar<T>(1), (*A_buf)(), A.getOffset(), A.strides()[1],
+            (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0, nullptr,
+            &event));
+    }
+
+    return B;
+}
+
+template<typename T>
+Array<T> solve(const Array<T> &a, const Array<T> &b,
+               const af_mat_prop options) {
+    if (OpenCLCPUOffload()) { return cpu::solve(a, b, options); }
+
+    if (options & AF_MAT_UPPER || options & AF_MAT_LOWER) {
+        return triangleSolve<T>(a, b, options);
+    }
+
+    if (a.dims()[0] == a.dims()[1]) {
+        return generalSolve<T>(a, b);
+    } else {
+        return leastSquares<T>(a, b);
+    }
+}
+
+#define INSTANTIATE_SOLVE(T)                                                 \
+    template Array<T> solve<T>(const Array<T> &a, const Array<T> &b,         \
+                               const af_mat_prop options);                   \
+    template Array<T> solveLU<T>(const Array<T> &A, const Array<int> &pivot, \
+                                 const Array<T> &b,                          \
+                                 const af_mat_prop options);
+
+INSTANTIATE_SOLVE(float)
+INSTANTIATE_SOLVE(cfloat)
+INSTANTIATE_SOLVE(double)
+INSTANTIATE_SOLVE(cdouble)
+}  // namespace oneapi
+
+#else  // WITH_LINEAR_ALGEBRA
+
+namespace oneapi {
+
+template<typename T>
+Array<T> solveLU(const Array<T> &A, const Array<int> &pivot, const Array<T> &b,
+                 const af_mat_prop options) {
+    AF_ERROR("Linear Algebra is disabled on OneAPI", AF_ERR_NOT_CONFIGURED);
+}
+
+template<typename T>
+Array<T> solve(const Array<T> &a, const Array<T> &b,
+               const af_mat_prop options) {
+    AF_ERROR("Linear Algebra is disabled on OneAPI", AF_ERR_NOT_CONFIGURED);
+}
+
+#define INSTANTIATE_SOLVE(T)                                                 \
+    template Array<T> solve<T>(const Array<T> &a, const Array<T> &b,         \
+                               const af_mat_prop options);                   \
+    template Array<T> solveLU<T>(const Array<T> &A, const Array<int> &pivot, \
+                                 const Array<T> &b,                          \
+                                 const af_mat_prop options);
+
+INSTANTIATE_SOLVE(float)
+INSTANTIATE_SOLVE(cfloat)
+INSTANTIATE_SOLVE(double)
+INSTANTIATE_SOLVE(cdouble)
+
+}  // namespace oneapi
+
+#endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/solve.hpp b/src/backend/oneapi/solve.hpp
new file mode 100644
index 0000000000..330605aa35
--- /dev/null
+++ b/src/backend/oneapi/solve.hpp
@@ -0,0 +1,20 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> solve(const Array<T> &a, const Array<T> &b,
+               const af_mat_prop options = AF_MAT_NONE);
+
+template<typename T>
+Array<T> solveLU(const Array<T> &a, const Array<int> &pivot, const Array<T> &b,
+                 const af_mat_prop options = AF_MAT_NONE);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sort.cpp b/src/backend/oneapi/sort.cpp
new file mode 100644
index 0000000000..f9c13b7429
--- /dev/null
+++ b/src/backend/oneapi/sort.cpp
@@ -0,0 +1,67 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <copy.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/sort.hpp>
+#include <math.hpp>
+#include <reorder.hpp>
+#include <sort.hpp>
+#include <stdexcept>
+
+namespace oneapi {
+template<typename T>
+Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending) {
+
+    ONEAPI_NOT_SUPPORTED("sort Not supported");
+
+    try {
+        Array<T> out = copyArray<T>(in);
+        // switch (dim) {
+        //     case 0: kernel::sort0<T>(out, isAscending); break;
+        //     case 1: kernel::sortBatched<T>(out, 1, isAscending); break;
+        //     case 2: kernel::sortBatched<T>(out, 2, isAscending); break;
+        //     case 3: kernel::sortBatched<T>(out, 3, isAscending); break;
+        //     default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
+        // }
+
+        if (dim != 0) {
+            af::dim4 preorderDims = out.dims();
+            af::dim4 reorderDims(0, 1, 2, 3);
+            reorderDims[dim] = 0;
+            preorderDims[0]  = out.dims()[dim];
+            for (int i = 1; i <= static_cast<int>(dim); i++) {
+                reorderDims[i - 1] = i;
+                preorderDims[i]    = out.dims()[i - 1];
+            }
+
+            out.setDataDims(preorderDims);
+            out = reorder<T>(out, reorderDims);
+        }
+        return out;
+    } catch (std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+}
+
+#define INSTANTIATE(T)                                                \
+    template Array<T> sort<T>(const Array<T> &in, const unsigned dim, \
+                              bool isAscending);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(char)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sort.hpp b/src/backend/oneapi/sort.hpp
new file mode 100644
index 0000000000..ae7fdc9e6a
--- /dev/null
+++ b/src/backend/oneapi/sort.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending);
+}
diff --git a/src/backend/oneapi/sort_by_key.cpp b/src/backend/oneapi/sort_by_key.cpp
new file mode 100644
index 0000000000..f2a140c338
--- /dev/null
+++ b/src/backend/oneapi/sort_by_key.cpp
@@ -0,0 +1,55 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <copy.hpp>
+#include <err_oneapi.hpp>
+//#include <kernel/sort_by_key.hpp>
+#include <math.hpp>
+#include <reorder.hpp>
+#include <sort_by_key.hpp>
+#include <stdexcept>
+
+namespace oneapi {
+template<typename Tk, typename Tv>
+void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
+                 const Array<Tv> &ival, const unsigned dim, bool isAscending) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE(Tk, Tv)                                        \
+    template void sort_by_key<Tk, Tv>(                             \
+        Array<Tk> & okey, Array<Tv> & oval, const Array<Tk> &ikey, \
+        const Array<Tv> &ival, const uint dim, bool isAscending);
+
+#define INSTANTIATE1(Tk)     \
+    INSTANTIATE(Tk, float)   \
+    INSTANTIATE(Tk, double)  \
+    INSTANTIATE(Tk, cfloat)  \
+    INSTANTIATE(Tk, cdouble) \
+    INSTANTIATE(Tk, int)     \
+    INSTANTIATE(Tk, uint)    \
+    INSTANTIATE(Tk, short)   \
+    INSTANTIATE(Tk, ushort)  \
+    INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, uchar)   \
+    INSTANTIATE(Tk, intl)    \
+    INSTANTIATE(Tk, uintl)
+
+INSTANTIATE1(float)
+INSTANTIATE1(double)
+INSTANTIATE1(int)
+INSTANTIATE1(uint)
+INSTANTIATE1(short)
+INSTANTIATE1(ushort)
+INSTANTIATE1(char)
+INSTANTIATE1(uchar)
+INSTANTIATE1(intl)
+INSTANTIATE1(uintl)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sort_by_key.hpp b/src/backend/oneapi/sort_by_key.hpp
new file mode 100644
index 0000000000..2ba2c67ba3
--- /dev/null
+++ b/src/backend/oneapi/sort_by_key.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename Tk, typename Tv>
+void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
+                 const Array<Tv> &ival, const unsigned dim, bool isAscending);
+}
diff --git a/src/backend/oneapi/sort_index.cpp b/src/backend/oneapi/sort_index.cpp
new file mode 100644
index 0000000000..ebf5ce65f7
--- /dev/null
+++ b/src/backend/oneapi/sort_index.cpp
@@ -0,0 +1,82 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <copy.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/sort_by_key.hpp>
+#include <math.hpp>
+#include <range.hpp>
+#include <reorder.hpp>
+#include <sort_index.hpp>
+#include <stdexcept>
+
+using common::half;
+
+namespace oneapi {
+template<typename T>
+void sort_index(Array<T> &okey, Array<uint> &oval, const Array<T> &in,
+                const uint dim, bool isAscending) {
+
+    ONEAPI_NOT_SUPPORTED("sort_index Not supported");
+
+    try {
+        // okey contains values, oval contains indices
+        okey = copyArray<T>(in);
+        oval = range<uint>(in.dims(), dim);
+        oval.eval();
+
+        // switch (dim) {
+        //     case 0: kernel::sort0ByKey<T, uint>(okey, oval, isAscending); break;
+        //     case 1:
+        //     case 2:
+        //     case 3:
+        //         kernel::sortByKeyBatched<T, uint>(okey, oval, dim, isAscending);
+        //         break;
+        //     default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
+        // }
+
+        if (dim != 0) {
+            af::dim4 preorderDims = okey.dims();
+            af::dim4 reorderDims(0, 1, 2, 3);
+            reorderDims[dim] = 0;
+            preorderDims[0]  = okey.dims()[dim];
+            for (uint i = 1; i <= dim; i++) {
+                reorderDims[i - 1] = i;
+                preorderDims[i]    = okey.dims()[i - 1];
+            }
+
+            okey.setDataDims(preorderDims);
+            oval.setDataDims(preorderDims);
+
+            okey = reorder<T>(okey, reorderDims);
+            oval = reorder<uint>(oval, reorderDims);
+        }
+    } catch (const std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+}
+
+#define INSTANTIATE(T)                                              \
+    template void sort_index<T>(Array<T> & val, Array<uint> & idx,  \
+                                const Array<T> &in, const uint dim, \
+                                bool isAscending);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(char)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sort_index.hpp b/src/backend/oneapi/sort_index.hpp
new file mode 100644
index 0000000000..2e7f262e62
--- /dev/null
+++ b/src/backend/oneapi/sort_index.hpp
@@ -0,0 +1,16 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+void sort_index(Array<T> &okey, Array<unsigned> &oval, const Array<T> &in,
+                const unsigned dim, bool isAscending);
+}
diff --git a/src/backend/oneapi/sparse.cpp b/src/backend/oneapi/sparse.cpp
new file mode 100644
index 0000000000..70de66f6ee
--- /dev/null
+++ b/src/backend/oneapi/sparse.cpp
@@ -0,0 +1,225 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+// #include <kernel/sparse.hpp>
+#include <sparse.hpp>
+
+#include <arith.hpp>
+#include <common/cast.hpp>
+#include <common/moddims.hpp>
+#include <complex.hpp>
+#include <copy.hpp>
+#include <err_oneapi.hpp>
+#include <lookup.hpp>
+#include <math.hpp>
+#include <platform.hpp>
+#include <range.hpp>
+#include <reduce.hpp>
+#include <where.hpp>
+
+#include <stdexcept>
+#include <string>
+
+namespace oneapi {
+
+using namespace common;
+
+// Partial template specialization of sparseConvertDenseToStorage for COO
+// However, template specialization is not allowed
+template<typename T>
+SparseArray<T> sparseConvertDenseToCOO(const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("sparseConvertDenseToCOO Not supported");
+    in.eval();
+
+    Array<uint> nonZeroIdx_ = where<T>(in);
+    Array<int> nonZeroIdx   = cast<int, uint>(nonZeroIdx_);
+
+    dim_t nNZ = nonZeroIdx.elements();
+
+    Array<int> constDim = createValueArray<int>(dim4(nNZ), in.dims()[0]);
+    constDim.eval();
+
+    Array<int> rowIdx =
+        arithOp<int, af_mod_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
+    Array<int> colIdx =
+        arithOp<int, af_div_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
+
+    Array<T> values = copyArray<T>(in);
+    values          = modDims(values, dim4(values.elements()));
+    values          = lookup<T, int>(values, nonZeroIdx, 0);
+
+    return createArrayDataSparseArray<T>(in.dims(), values, rowIdx, colIdx,
+                                         AF_STORAGE_COO);
+}
+
+template<typename T, af_storage stype>
+SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in_) {
+    ONEAPI_NOT_SUPPORTED("sparseConvertDenseToStorage Not supported");
+    in_.eval();
+
+    uint nNZ = getScalar<uint>(reduce_all<af_notzero_t, T, uint>(in_));
+
+    SparseArray<T> sparse_ = createEmptySparseArray<T>(in_.dims(), nNZ, stype);
+    sparse_.eval();
+
+    Array<T> &values   = sparse_.getValues();
+    Array<int> &rowIdx = sparse_.getRowIdx();
+    Array<int> &colIdx = sparse_.getColIdx();
+
+    // kernel::dense2csr<T>(values, rowIdx, colIdx, in_);
+
+    return sparse_;
+}
+
+// Partial template specialization of sparseConvertStorageToDense for COO
+// However, template specialization is not allowed
+template<typename T>
+Array<T> sparseConvertCOOToDense(const SparseArray<T> &in) {
+    ONEAPI_NOT_SUPPORTED("sparseConvertCOOToDense Not supported");
+    in.eval();
+
+    Array<T> dense = createValueArray<T>(in.dims(), scalar<T>(0));
+    dense.eval();
+
+    const Array<T> values   = in.getValues();
+    const Array<int> rowIdx = in.getRowIdx();
+    const Array<int> colIdx = in.getColIdx();
+
+    // kernel::coo2dense<T>(dense, values, rowIdx, colIdx);
+
+    return dense;
+}
+
+template<typename T, af_storage stype>
+Array<T> sparseConvertStorageToDense(const SparseArray<T> &in_) {
+    ONEAPI_NOT_SUPPORTED("sparseConvertStorageToDense Not supported");
+
+    if (stype != AF_STORAGE_CSR) {
+        AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
+                 AF_ERR_NOT_SUPPORTED);
+    }
+
+    in_.eval();
+
+    Array<T> dense_ = createValueArray<T>(in_.dims(), scalar<T>(0));
+    dense_.eval();
+
+    const Array<T> &values   = in_.getValues();
+    const Array<int> &rowIdx = in_.getRowIdx();
+    const Array<int> &colIdx = in_.getColIdx();
+
+    if (stype == AF_STORAGE_CSR) {
+      // kernel::csr2dense<T>(dense_, values, rowIdx, colIdx);
+    } else {
+        AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
+                 AF_ERR_NOT_SUPPORTED);
+    }
+
+    return dense_;
+}
+
+template<typename T, af_storage dest, af_storage src>
+SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
+    ONEAPI_NOT_SUPPORTED("sparseConvertStorageToStorage Not supported");
+    in.eval();
+
+    SparseArray<T> converted = createEmptySparseArray<T>(
+        in.dims(), static_cast<int>(in.getNNZ()), dest);
+    converted.eval();
+
+    if (src == AF_STORAGE_CSR && dest == AF_STORAGE_COO) {
+        Array<int> index = range<int>(in.getNNZ(), 0);
+        index.eval();
+
+        Array<T> &ovalues         = converted.getValues();
+        Array<int> &orowIdx       = converted.getRowIdx();
+        Array<int> &ocolIdx       = converted.getColIdx();
+        const Array<T> &ivalues   = in.getValues();
+        const Array<int> &irowIdx = in.getRowIdx();
+        const Array<int> &icolIdx = in.getColIdx();
+
+        // kernel::csr2coo<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, icolIdx,
+        //                    index);
+
+    } else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) {
+        Array<int> index = range<int>(in.getNNZ(), 0);
+        index.eval();
+
+        Array<T> &ovalues         = converted.getValues();
+        Array<int> &orowIdx       = converted.getRowIdx();
+        Array<int> &ocolIdx       = converted.getColIdx();
+        const Array<T> &ivalues   = in.getValues();
+        const Array<int> &irowIdx = in.getRowIdx();
+        const Array<int> &icolIdx = in.getColIdx();
+
+        Array<int> rowCopy = copyArray<int>(irowIdx);
+        rowCopy.eval();
+
+        // kernel::coo2csr<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, icolIdx,
+        //                    index, rowCopy, in.dims()[0]);
+
+    } else {
+        // Should never come here
+        AF_ERROR("OpenCL Backend invalid conversion combination",
+                 AF_ERR_NOT_SUPPORTED);
+    }
+
+    return converted;
+}
+
+#define INSTANTIATE_TO_STORAGE(T, S)                     \
+    template SparseArray<T>                              \
+    sparseConvertStorageToStorage<T, S, AF_STORAGE_CSR>( \
+        const SparseArray<T> &in);                       \
+    template SparseArray<T>                              \
+    sparseConvertStorageToStorage<T, S, AF_STORAGE_CSC>( \
+        const SparseArray<T> &in);                       \
+    template SparseArray<T>                              \
+    sparseConvertStorageToStorage<T, S, AF_STORAGE_COO>( \
+        const SparseArray<T> &in);
+
+#define INSTANTIATE_COO_SPECIAL(T)                                 \
+    template<>                                                     \
+    SparseArray<T> sparseConvertDenseToStorage<T, AF_STORAGE_COO>( \
+        const Array<T> &in) {                                      \
+        return sparseConvertDenseToCOO<T>(in);                     \
+    }                                                              \
+    template<>                                                     \
+    Array<T> sparseConvertStorageToDense<T, AF_STORAGE_COO>(       \
+        const SparseArray<T> &in) {                                \
+        return sparseConvertCOOToDense<T>(in);                     \
+    }
+
+#define INSTANTIATE_SPARSE(T)                                               \
+    template SparseArray<T> sparseConvertDenseToStorage<T, AF_STORAGE_CSR>( \
+        const Array<T> &in);                                                \
+    template SparseArray<T> sparseConvertDenseToStorage<T, AF_STORAGE_CSC>( \
+        const Array<T> &in);                                                \
+                                                                            \
+    template Array<T> sparseConvertStorageToDense<T, AF_STORAGE_CSR>(       \
+        const SparseArray<T> &in);                                          \
+    template Array<T> sparseConvertStorageToDense<T, AF_STORAGE_CSC>(       \
+        const SparseArray<T> &in);                                          \
+                                                                            \
+    INSTANTIATE_COO_SPECIAL(T)                                              \
+                                                                            \
+    INSTANTIATE_TO_STORAGE(T, AF_STORAGE_CSR)                               \
+    INSTANTIATE_TO_STORAGE(T, AF_STORAGE_CSC)                               \
+    INSTANTIATE_TO_STORAGE(T, AF_STORAGE_COO)
+
+INSTANTIATE_SPARSE(float)
+INSTANTIATE_SPARSE(double)
+INSTANTIATE_SPARSE(cfloat)
+INSTANTIATE_SPARSE(cdouble)
+
+#undef INSTANTIATE_TO_STORAGE
+#undef INSTANTIATE_COO_SPECIAL
+#undef INSTANTIATE_SPARSE
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sparse.hpp b/src/backend/oneapi/sparse.hpp
new file mode 100644
index 0000000000..3958dcea3b
--- /dev/null
+++ b/src/backend/oneapi/sparse.hpp
@@ -0,0 +1,27 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Array.hpp>
+#include <common/SparseArray.hpp>
+
+namespace oneapi {
+
+template<typename T, af_storage stype>
+common::SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in);
+
+template<typename T, af_storage stype>
+Array<T> sparseConvertStorageToDense(const common::SparseArray<T> &in);
+
+template<typename T, af_storage dest, af_storage src>
+common::SparseArray<T> sparseConvertStorageToStorage(
+    const common::SparseArray<T> &in);
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sparse_arith.cpp b/src/backend/oneapi/sparse_arith.cpp
new file mode 100644
index 0000000000..40e9e24ff4
--- /dev/null
+++ b/src/backend/oneapi/sparse_arith.cpp
@@ -0,0 +1,180 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+// #include <kernel/sparse_arith.hpp>
+#include <sparse.hpp>
+#include <err_oneapi.hpp>
+
+#include <stdexcept>
+#include <string>
+
+#include <arith.hpp>
+#include <common/cast.hpp>
+#include <common/err_common.hpp>
+#include <complex.hpp>
+#include <copy.hpp>
+#include <lookup.hpp>
+#include <math.hpp>
+#include <platform.hpp>
+#include <scan.hpp>
+#include <where.hpp>
+
+namespace oneapi {
+
+using namespace common;
+using std::numeric_limits;
+
+template<typename T>
+T getInf() {
+    return scalar<T>(numeric_limits<T>::infinity());
+}
+
+template<>
+cfloat getInf() {
+    return scalar<cfloat, float>(
+        NAN, NAN);  // Matches behavior of complex division by 0 in OpenCL
+}
+
+template<>
+cdouble getInf() {
+    return scalar<cdouble, double>(
+        NAN, NAN);  // Matches behavior of complex division by 0 in OpenCL
+}
+
+template<typename T, af_op_t op>
+Array<T> arithOpD(const SparseArray<T> &lhs, const Array<T> &rhs,
+                  const bool reverse) {
+    ONEAPI_NOT_SUPPORTED("arithOpD Not supported");
+    lhs.eval();
+    rhs.eval();
+
+    Array<T> out  = createEmptyArray<T>(dim4(0));
+    Array<T> zero = createValueArray<T>(rhs.dims(), scalar<T>(0));
+    switch (op) {
+        case af_add_t: out = copyArray<T>(rhs); break;
+        case af_sub_t:
+            out = reverse ? copyArray<T>(rhs)
+                          : arithOp<T, af_sub_t>(zero, rhs, rhs.dims());
+            break;
+        default: out = copyArray<T>(rhs);
+    }
+    out.eval();
+    switch (lhs.getStorage()) {
+        case AF_STORAGE_CSR:
+            // kernel::sparseArithOpCSR<T, op>(out, lhs.getValues(),
+            //                                 lhs.getRowIdx(), lhs.getColIdx(),
+            //                                 rhs, reverse);
+            break;
+        case AF_STORAGE_COO:
+            // kernel::sparseArithOpCOO<T, op>(out, lhs.getValues(),
+            //                                 lhs.getRowIdx(), lhs.getColIdx(),
+            //                                 rhs, reverse);
+            break;
+        default:
+            AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
+                     AF_ERR_NOT_SUPPORTED);
+    }
+
+    return out;
+}
+
+template<typename T, af_op_t op>
+SparseArray<T> arithOp(const SparseArray<T> &lhs, const Array<T> &rhs,
+                       const bool reverse) {
+    ONEAPI_NOT_SUPPORTED("arithOp Not supported");
+    lhs.eval();
+    rhs.eval();
+
+    SparseArray<T> out = createArrayDataSparseArray<T>(
+        lhs.dims(), lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(),
+        lhs.getStorage(), true);
+    out.eval();
+    switch (lhs.getStorage()) {
+        case AF_STORAGE_CSR:
+            // kernel::sparseArithOpCSR<T, op>(out.getValues(), out.getRowIdx(),
+            //                                 out.getColIdx(), rhs, reverse);
+            break;
+        case AF_STORAGE_COO:
+            // kernel::sparseArithOpCOO<T, op>(out.getValues(), out.getRowIdx(),
+            //                                 out.getColIdx(), rhs, reverse);
+            break;
+        default:
+            AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
+                     AF_ERR_NOT_SUPPORTED);
+    }
+
+    return out;
+}
+
+template<typename T, af_op_t op>
+SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
+    ONEAPI_NOT_SUPPORTED("arithOp Not supported");
+    lhs.eval();
+    rhs.eval();
+    af::storage sfmt = lhs.getStorage();
+
+    const dim4 &ldims = lhs.dims();
+
+    const uint M = ldims[0];
+    const uint N = ldims[1];
+
+    const dim_t nnzA = lhs.getNNZ();
+    const dim_t nnzB = rhs.getNNZ();
+
+    auto temp = createValueArray<int>(dim4(M + 1), scalar<int>(0));
+    temp.eval();
+
+    unsigned nnzC = 0;
+    // kernel::csrCalcOutNNZ(temp, nnzC, M, N, nnzA, lhs.getRowIdx(),
+    //                       lhs.getColIdx(), nnzB, rhs.getRowIdx(),
+    //                       rhs.getColIdx());
+
+    auto outRowIdx = scan<af_add_t, int, int>(temp, 0);
+
+    auto outColIdx = createEmptyArray<int>(dim4(nnzC));
+    auto outValues = createEmptyArray<T>(dim4(nnzC));
+
+    // kernel::ssArithCSR<T, op>(outValues, outColIdx, outRowIdx, M, N, nnzA,
+    //                           lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(),
+    //                           nnzB, rhs.getValues(), rhs.getRowIdx(),
+    //                           rhs.getColIdx());
+
+    SparseArray<T> retVal = createArrayDataSparseArray(
+        ldims, outValues, outRowIdx, outColIdx, sfmt);
+    return retVal;
+}
+
+#define INSTANTIATE(T)                                                         \
+    template Array<T> arithOpD<T, af_add_t>(                                   \
+        const SparseArray<T> &lhs, const Array<T> &rhs, const bool reverse);   \
+    template Array<T> arithOpD<T, af_sub_t>(                                   \
+        const SparseArray<T> &lhs, const Array<T> &rhs, const bool reverse);   \
+    template Array<T> arithOpD<T, af_mul_t>(                                   \
+        const SparseArray<T> &lhs, const Array<T> &rhs, const bool reverse);   \
+    template Array<T> arithOpD<T, af_div_t>(                                   \
+        const SparseArray<T> &lhs, const Array<T> &rhs, const bool reverse);   \
+    template SparseArray<T> arithOp<T, af_add_t>(                              \
+        const SparseArray<T> &lhs, const Array<T> &rhs, const bool reverse);   \
+    template SparseArray<T> arithOp<T, af_sub_t>(                              \
+        const SparseArray<T> &lhs, const Array<T> &rhs, const bool reverse);   \
+    template SparseArray<T> arithOp<T, af_mul_t>(                              \
+        const SparseArray<T> &lhs, const Array<T> &rhs, const bool reverse);   \
+    template SparseArray<T> arithOp<T, af_div_t>(                              \
+        const SparseArray<T> &lhs, const Array<T> &rhs, const bool reverse);   \
+    template SparseArray<T> arithOp<T, af_add_t>(                              \
+        const common::SparseArray<T> &lhs, const common::SparseArray<T> &rhs); \
+    template SparseArray<T> arithOp<T, af_sub_t>(                              \
+        const common::SparseArray<T> &lhs, const common::SparseArray<T> &rhs);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sparse_arith.hpp b/src/backend/oneapi/sparse_arith.hpp
new file mode 100644
index 0000000000..589620c314
--- /dev/null
+++ b/src/backend/oneapi/sparse_arith.hpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/SparseArray.hpp>
+#include <optypes.hpp>
+#include <sparse.hpp>
+
+namespace oneapi {
+
+// These two functions cannot be overloaded by return type.
+// So have to give them separate names.
+template<typename T, af_op_t op>
+Array<T> arithOpD(const common::SparseArray<T> &lhs, const Array<T> &rhs,
+                  const bool reverse = false);
+
+template<typename T, af_op_t op>
+common::SparseArray<T> arithOp(const common::SparseArray<T> &lhs,
+                               const Array<T> &rhs, const bool reverse = false);
+
+template<typename T, af_op_t op>
+common::SparseArray<T> arithOp(const common::SparseArray<T> &lhs,
+                               const common::SparseArray<T> &rhs);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sparse_blas.cpp b/src/backend/oneapi/sparse_blas.cpp
new file mode 100644
index 0000000000..bc06759dde
--- /dev/null
+++ b/src/backend/oneapi/sparse_blas.cpp
@@ -0,0 +1,99 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <sparse_blas.hpp>
+
+// #include <kernel/cscmm.hpp>
+// #include <kernel/cscmv.hpp>
+// #include <kernel/csrmm.hpp>
+// #include <kernel/csrmv.hpp>
+
+#include <cassert>
+#include <stdexcept>
+#include <string>
+
+#include <common/err_common.hpp>
+#include <complex.hpp>
+#include <err_oneapi.hpp>
+#include <math.hpp>
+#include <platform.hpp>
+#include <transpose.hpp>
+#include <af/dim4.hpp>
+
+#if defined(WITH_LINEAR_ALGEBRA)
+#include <cpu/cpu_sparse_blas.hpp>
+#endif  // WITH_LINEAR_ALGEBRA
+
+namespace oneapi {
+
+using namespace common;
+
+template<typename T>
+Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhsIn,
+                af_mat_prop optLhs, af_mat_prop optRhs) {
+  ONEAPI_NOT_SUPPORTED("sparse matmul Not supported");
+#if defined(WITH_LINEAR_ALGEBRA)
+    if (OpenCLCPUOffload(
+            false)) {  // Do not force offload gemm on OSX Intel devices
+        return cpu::matmul(lhs, rhsIn, optLhs, optRhs);
+    }
+#endif
+
+    int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1;
+    // int lColDim = (optLhs == AF_MAT_NONE) ? 1 : 0;
+    static const int rColDim =
+        1;  // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0;
+
+    dim4 lDims = lhs.dims();
+    dim4 rDims = rhsIn.dims();
+    int M      = lDims[lRowDim];
+    int N      = rDims[rColDim];
+    // int K = lDims[lColDim];
+
+    const Array<T> rhs =
+        (N != 1 && optLhs == AF_MAT_NONE) ? transpose(rhsIn, false) : rhsIn;
+    Array<T> out = createEmptyArray<T>(af::dim4(M, N, 1, 1));
+
+    static const T alpha = scalar<T>(1.0);
+    static const T beta  = scalar<T>(0.0);
+
+    const Array<T>& values   = lhs.getValues();
+    const Array<int>& rowIdx = lhs.getRowIdx();
+    const Array<int>& colIdx = lhs.getColIdx();
+
+    if (optLhs == AF_MAT_NONE) {
+        // if (N == 1) {
+        //     kernel::csrmv(out, values, rowIdx, colIdx, rhs, alpha, beta);
+        // } else {
+        //     kernel::csrmm_nt(out, values, rowIdx, colIdx, rhs, alpha, beta);
+        // }
+    } else {
+        // // CSR transpose is a CSC matrix
+        // if (N == 1) {
+        //     kernel::cscmv(out, values, rowIdx, colIdx, rhs, alpha, beta,
+        //                   optLhs == AF_MAT_CTRANS);
+        // } else {
+        //     kernel::cscmm_nn(out, values, rowIdx, colIdx, rhs, alpha, beta,
+        //                      optLhs == AF_MAT_CTRANS);
+        // }
+    }
+    return out;
+}
+
+#define INSTANTIATE_SPARSE(T)                                            \
+    template Array<T> matmul<T>(const common::SparseArray<T>& lhs,       \
+                                const Array<T>& rhs, af_mat_prop optLhs, \
+                                af_mat_prop optRhs);
+
+INSTANTIATE_SPARSE(float)
+INSTANTIATE_SPARSE(double)
+INSTANTIATE_SPARSE(cfloat)
+INSTANTIATE_SPARSE(cdouble)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sparse_blas.hpp b/src/backend/oneapi/sparse_blas.hpp
new file mode 100644
index 0000000000..d187a4422a
--- /dev/null
+++ b/src/backend/oneapi/sparse_blas.hpp
@@ -0,0 +1,20 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/SparseArray.hpp>
+#include <sparse.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhs,
+                af_mat_prop optLhs, af_mat_prop optRhs);
+
+}
diff --git a/src/backend/oneapi/sum.cpp b/src/backend/oneapi/sum.cpp
new file mode 100644
index 0000000000..30850564e8
--- /dev/null
+++ b/src/backend/oneapi/sum.cpp
@@ -0,0 +1,39 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/half.hpp>
+#include "reduce_impl.hpp"
+
+using common::half;
+
+namespace oneapi {
+// sum
+INSTANTIATE(af_add_t, float, float)
+INSTANTIATE(af_add_t, double, double)
+INSTANTIATE(af_add_t, cfloat, cfloat)
+INSTANTIATE(af_add_t, cdouble, cdouble)
+INSTANTIATE(af_add_t, int, int)
+INSTANTIATE(af_add_t, int, float)
+INSTANTIATE(af_add_t, uint, uint)
+INSTANTIATE(af_add_t, uint, float)
+INSTANTIATE(af_add_t, intl, intl)
+INSTANTIATE(af_add_t, intl, double)
+INSTANTIATE(af_add_t, uintl, uintl)
+INSTANTIATE(af_add_t, uintl, double)
+INSTANTIATE(af_add_t, char, int)
+INSTANTIATE(af_add_t, char, float)
+INSTANTIATE(af_add_t, uchar, uint)
+INSTANTIATE(af_add_t, uchar, float)
+INSTANTIATE(af_add_t, short, int)
+INSTANTIATE(af_add_t, short, float)
+INSTANTIATE(af_add_t, ushort, uint)
+INSTANTIATE(af_add_t, ushort, float)
+INSTANTIATE(af_add_t, half, half)
+INSTANTIATE(af_add_t, half, float)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/surface.cpp b/src/backend/oneapi/surface.cpp
new file mode 100644
index 0000000000..7efebfc43c
--- /dev/null
+++ b/src/backend/oneapi/surface.cpp
@@ -0,0 +1,81 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+// #include <GraphicsResourceManager.hpp>
+// #include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <surface.hpp>
+
+using af::dim4;
+// using cl::Memory;
+using std::vector;
+
+namespace oneapi {
+
+template<typename T>
+void copy_surface(const Array<T> &P, fg_surface surface) {
+    ONEAPI_NOT_SUPPORTED("copy_surface Not supported");
+    // ForgeModule &_ = graphics::forgePlugin();
+    // if (isGLSharingSupported()) {
+    //     CheckGL("Begin OpenCL resource copy");
+    //     const cl::Buffer *d_P = P.get();
+    //     unsigned bytes        = 0;
+    //     FG_CHECK(_.fg_get_surface_vertex_buffer_size(&bytes, surface));
+
+    //     auto res = interopManager().getSurfaceResources(surface);
+
+    //     vector<Memory> shared_objects;
+    //     shared_objects.push_back(*(res[0].get()));
+
+    //     glFinish();
+
+    //     // Use of events:
+    //     // https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReleaseGLObjects.html
+    //     cl::Event event;
+
+    //     getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
+    //     event.wait();
+    //     getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes, NULL,
+    //                                  &event);
+    //     getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
+    //     event.wait();
+
+    //     CL_DEBUG_FINISH(getQueue());
+    //     CheckGL("End OpenCL resource copy");
+    // } else {
+    //     unsigned bytes = 0, buffer = 0;
+    //     FG_CHECK(_.fg_get_surface_vertex_buffer(&buffer, surface));
+    //     FG_CHECK(_.fg_get_surface_vertex_buffer_size(&bytes, surface));
+
+    //     CheckGL("Begin OpenCL fallback-resource copy");
+    //     glBindBuffer(GL_ARRAY_BUFFER, buffer);
+    //     auto *ptr =
+    //         static_cast<GLubyte *>(glMapBuffer(GL_ARRAY_BUFFER, GL_WRITE_ONLY));
+    //     if (ptr) {
+    //         getQueue().enqueueReadBuffer(*P.get(), CL_TRUE, 0, bytes, ptr);
+    //         glUnmapBuffer(GL_ARRAY_BUFFER);
+    //     }
+    //     glBindBuffer(GL_ARRAY_BUFFER, 0);
+    //     CheckGL("End OpenCL fallback-resource copy");
+    // }
+}
+
+#define INSTANTIATE(T) \
+    template void copy_surface<T>(const Array<T> &, fg_surface);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(uchar)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/surface.hpp b/src/backend/oneapi/surface.hpp
new file mode 100644
index 0000000000..0c4110fd36
--- /dev/null
+++ b/src/backend/oneapi/surface.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/graphics_common.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void copy_surface(const Array<T> &P, fg_surface surface);
+
+}
diff --git a/src/backend/oneapi/susan.cpp b/src/backend/oneapi/susan.cpp
new file mode 100644
index 0000000000..e6fe536918
--- /dev/null
+++ b/src/backend/oneapi/susan.cpp
@@ -0,0 +1,75 @@
+/*******************************************************
+ * Copyright (c) 2022, Arrayfire
+ * all rights reserved.
+ *
+ * This file is distributed under 3-clause bsd license.
+ * the complete license agreement can be obtained at:
+ * http://Arrayfire.com/licenses/bsd-3-clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/susan.hpp>
+#include <af/features.h>
+#include <algorithm>
+#include <cmath>
+
+using af::features;
+using std::vector;
+
+namespace oneapi {
+
+template<typename T>
+unsigned susan(Array<float> &x_out, Array<float> &y_out, Array<float> &resp_out,
+               const Array<T> &in, const unsigned radius, const float diff_thr,
+               const float geom_thr, const float feature_ratio,
+               const unsigned edge) {
+    dim4 idims = in.dims();
+
+    const unsigned corner_lim = in.elements() * feature_ratio;
+    Array<float> x_corners    = createEmptyArray<float>({corner_lim});
+    Array<float> y_corners    = createEmptyArray<float>({corner_lim});
+    Array<float> resp_corners = createEmptyArray<float>({corner_lim});
+
+    // auto resp = memAlloc<float>(in.elements());
+
+    ONEAPI_NOT_SUPPORTED("");
+    return 0;
+
+    // kernel::susan<T>(resp.get(), in.get(), in.getOffset(), idims[0], idims[1],
+    //                  diff_thr, geom_thr, edge, radius);
+
+    // unsigned corners_found = kernel::nonMaximal<T>(
+    //     x_corners.get(), y_corners.get(), resp_corners.get(), idims[0],
+    //     idims[1], resp.get(), edge, corner_lim);
+
+    // const unsigned corners_out = std::min(corners_found, corner_lim);
+    // if (corners_out == 0) {
+    //     x_out    = createEmptyArray<float>(dim4());
+    //     y_out    = createEmptyArray<float>(dim4());
+    //     resp_out = createEmptyArray<float>(dim4());
+    // } else {
+    //     vector<af_seq> idx{{0., static_cast<double>(corners_out - 1.0), 1.}};
+    //     x_out    = createSubArray(x_corners, idx);
+    //     y_out    = createSubArray(y_corners, idx);
+    //     resp_out = createSubArray(resp_corners, idx);
+    // }
+    // return corners_out;
+}
+
+#define INSTANTIATE(T)                                                        \
+    template unsigned susan<T>(                                               \
+        Array<float> & x_out, Array<float> & y_out, Array<float> & score_out, \
+        const Array<T> &in, const unsigned radius, const float diff_thr,      \
+        const float geom_thr, const float feature_ratio, const unsigned edge);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+
+}  // namespace oneap
diff --git a/src/backend/oneapi/susan.hpp b/src/backend/oneapi/susan.hpp
new file mode 100644
index 0000000000..8510117dea
--- /dev/null
+++ b/src/backend/oneapi/susan.hpp
@@ -0,0 +1,24 @@
+/*******************************************************
+ * Copyright (c) 2022, Arrayfire
+ * all rights reserved.
+ *
+ * This file is distributed under 3-clause bsd license.
+ * the complete license agreement can be obtained at:
+ * http://Arrayfire.com/licenses/bsd-3-clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <af/features.h>
+
+using af::features;
+
+namespace oneapi {
+
+template<typename T>
+unsigned susan(Array<float> &x_out, Array<float> &y_out,
+               Array<float> &score_out, const Array<T> &in,
+               const unsigned radius, const float diff_thr,
+               const float geom_thr, const float feature_ratio,
+               const unsigned edge);
+
+}
diff --git a/src/backend/oneapi/svd.cpp b/src/backend/oneapi/svd.cpp
new file mode 100644
index 0000000000..8fef95ba6c
--- /dev/null
+++ b/src/backend/oneapi/svd.cpp
@@ -0,0 +1,268 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <blas.hpp>
+#include <copy.hpp>
+#include <err_oneapi.hpp>  // error check functions and Macros
+#include <math.hpp>
+#include <reduce.hpp>
+#include <svd.hpp>  // oneapi backend function header
+#include <transpose.hpp>
+
+#if defined(WITH_LINEAR_ALGEBRA)
+
+#include <cpu/cpu_svd.hpp>
+#include <magma/magma.h>
+#include <magma/magma_cpu_lapack.h>
+#include <magma/magma_helper.h>
+#include <platform.hpp>
+
+namespace oneapi {
+
+template<typename Tr>
+Tr calc_scale(Tr From, Tr To) {
+    // FIXME: I am not sure this is correct, removing this for now
+#if 0
+    //http://www.netlib.org/lapack/explore-3.1.1-html/dlascl.f.html
+    cpu_lapack_lamch_func<Tr> cpu_lapack_lamch;
+
+    Tr S = cpu_lapack_lamch('S');
+    Tr B = 1.0 / S;
+
+    Tr FromCopy = From, ToCopy = To;
+
+    Tr Mul = 1;
+
+    while (true) {
+        Tr From1 = FromCopy * S, To1 = ToCopy / B;
+        if (std::abs(From1) > std::abs(ToCopy) && ToCopy != 0) {
+            Mul *= S;
+            FromCopy = From1;
+        } else if (std::abs(To1) > std::abs(FromCopy)) {
+            Mul *= B;
+            ToCopy = To1;
+        } else {
+            Mul *= (ToCopy) / (FromCopy);
+            break;
+        }
+    }
+
+    return Mul;
+#else
+    return To / From;
+#endif
+}
+
+template<typename T, typename Tr>
+void svd(Array<T> &arrU, Array<Tr> &arrS, Array<T> &arrVT, Array<T> &arrA,
+         bool want_vectors = true) {
+    ONEAPI_NOT_SUPPORTED("");
+    dim4 idims    = arrA.dims();
+    dim4 istrides = arrA.strides();
+
+    const int m      = static_cast<int>(idims[0]);
+    const int n      = static_cast<int>(idims[1]);
+    const int ldda   = static_cast<int>(istrides[1]);
+    const int lda    = m;
+    const int min_mn = std::min(m, n);
+    const int ldu    = m;
+    const int ldvt   = n;
+
+    const int nb    = magma_get_gebrd_nb<T>(n);
+    const int lwork = (m + n) * nb;
+
+    cpu_lapack_lacpy_func<T> cpu_lapack_lacpy;
+    cpu_lapack_bdsqr_work_func<T> cpu_lapack_bdsqr_work;
+    cpu_lapack_ungbr_work_func<T> cpu_lapack_ungbr_work;
+    cpu_lapack_lamch_func<Tr> cpu_lapack_lamch;
+
+    // Get machine constants
+    static const double eps    = cpu_lapack_lamch('P');
+    static const double smlnum = std::sqrt(cpu_lapack_lamch('S')) / eps;
+    static const double bignum = 1. / smlnum;
+
+    Tr anrm = abs(getScalar<T>(reduce_all<af_max_t, T, T>(arrA)));
+
+    T scale                = scalar<T>(1);
+    static const int ione  = 1;
+    static const int izero = 0;
+
+    bool iscl = false;
+    if (anrm > 0. && anrm < smlnum) {
+        iscl  = true;
+        scale = scalar<T>(calc_scale<Tr>(anrm, smlnum));
+    } else if (anrm > bignum) {
+        iscl  = true;
+        scale = scalar<T>(calc_scale<Tr>(anrm, bignum));
+    }
+
+    if (iscl == 1) { multiply_inplace(arrA, abs(scale)); }
+
+    int nru  = 0;
+    int ncvt = 0;
+
+    // Instead of copying U, S, VT, and A to the host and copying the results
+    // back to the device, create a pointer that's mapped to device memory where
+    // the computation can directly happen
+    T *mappedA = static_cast<T *>(getQueue().enqueueMapBuffer(
+        *arrA.get(), CL_FALSE, CL_MAP_READ, sizeof(T) * arrA.getOffset(),
+        sizeof(T) * arrA.elements()));
+    std::vector<T> tauq(min_mn), taup(min_mn);
+    std::vector<T> work(lwork);
+    Tr *mappedS0 = (Tr *)getQueue().enqueueMapBuffer(
+        *arrS.get(), CL_TRUE, CL_MAP_WRITE, sizeof(Tr) * arrS.getOffset(),
+        sizeof(Tr) * arrS.elements());
+    std::vector<Tr> s1(min_mn - 1);
+    std::vector<Tr> rwork(5 * min_mn);
+
+    int info = 0;
+
+    // Bidiagonalize A
+    // (CWorkspace: need 2*N + M, prefer 2*N + (M + N)*NB)
+    // (RWorkspace: need N)
+    magma_gebrd_hybrid<T>(m, n, mappedA, lda, (*arrA.get())(), arrA.getOffset(),
+                          ldda, (void *)mappedS0, static_cast<void *>(&s1[0]),
+                          &tauq[0], &taup[0], &work[0], lwork, getQueue()(),
+                          &info, false);
+
+    T *mappedU = nullptr, *mappedVT = nullptr;
+    std::vector<T> cdummy(1);
+
+    if (want_vectors) {
+        mappedU  = static_cast<T *>(getQueue().enqueueMapBuffer(
+            *arrU.get(), CL_FALSE, CL_MAP_WRITE, sizeof(T) * arrU.getOffset(),
+            sizeof(T) * arrU.elements()));
+        mappedVT = static_cast<T *>(getQueue().enqueueMapBuffer(
+            *arrVT.get(), CL_TRUE, CL_MAP_WRITE, sizeof(T) * arrVT.getOffset(),
+            sizeof(T) * arrVT.elements()));
+
+        // If left singular vectors desired in U, copy result to U
+        // and generate left bidiagonalizing vectors in U
+        // (CWorkspace: need 2*N + NCU, prefer 2*N + NCU*NB)
+        // (RWorkspace: 0)
+        LAPACKE_CHECK(cpu_lapack_lacpy('L', m, n, mappedA, lda, mappedU, ldu));
+
+        int ncu = m;
+        LAPACKE_CHECK(cpu_lapack_ungbr_work('Q', m, ncu, n, mappedU, ldu,
+                                            &tauq[0], &work[0], lwork));
+
+        // If right singular vectors desired in VT, copy result to
+        // VT and generate right bidiagonalizing vectors in VT
+        // (CWorkspace: need 3*N-1, prefer 2*N + (N-1)*NB)
+        // (RWorkspace: 0)
+        LAPACKE_CHECK(
+            cpu_lapack_lacpy('U', n, n, mappedA, lda, mappedVT, ldvt));
+        LAPACKE_CHECK(cpu_lapack_ungbr_work('P', n, n, n, mappedVT, ldvt,
+                                            &taup[0], &work[0], lwork));
+
+        nru  = m;
+        ncvt = n;
+    }
+    getQueue().enqueueUnmapMemObject(*arrA.get(), mappedA);
+
+    // Perform bidiagonal QR iteration, if desired, computing
+    // left singular vectors in U and computing right singular
+    // vectors in VT
+    // (CWorkspace: need 0)
+    // (RWorkspace: need BDSPAC)
+    LAPACKE_CHECK(cpu_lapack_bdsqr_work('U', n, ncvt, nru, izero, mappedS0,
+                                        &s1[0], mappedVT, ldvt, mappedU, ldu,
+                                        &cdummy[0], ione, &rwork[0]));
+
+    if (want_vectors) {
+        getQueue().enqueueUnmapMemObject(*arrU.get(), mappedU);
+        getQueue().enqueueUnmapMemObject(*arrVT.get(), mappedVT);
+    }
+
+    getQueue().enqueueUnmapMemObject(*arrS.get(), mappedS0);
+
+    if (iscl == 1) {
+        Tr rscale = scalar<Tr>(1);
+        if (anrm > bignum) {
+            rscale = calc_scale<Tr>(bignum, anrm);
+        } else if (anrm < smlnum) {
+            rscale = calc_scale<Tr>(smlnum, anrm);
+        }
+        multiply_inplace(arrS, rscale);
+    }
+}
+
+template<typename T, typename Tr>
+void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+    // if (OpenCLCPUOffload()) { return cpu::svdInPlace(s, u, vt, in); }
+
+    // svd<T, Tr>(u, s, vt, in, true);
+}
+
+template<typename T, typename Tr>
+void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+
+    // if (OpenCLCPUOffload()) { return cpu::svd(s, u, vt, in); }
+
+    // dim4 iDims = in.dims();
+    // int M      = iDims[0];
+    // int N      = iDims[1];
+
+    // if (M >= N) {
+    //     Array<T> in_copy = copyArray(in);
+    //     svdInPlace(s, u, vt, in_copy);
+    // } else {
+    //     Array<T> in_trans = transpose(in, true);
+    //     svdInPlace(s, vt, u, in_trans);
+    //     transpose_inplace(u, true);
+    //     transpose_inplace(vt, true);
+    // }
+}
+
+#define INSTANTIATE(T, Tr)                                               \
+    template void svd<T, Tr>(Array<Tr> & s, Array<T> & u, Array<T> & vt, \
+                             const Array<T> &in);                        \
+    template void svdInPlace<T, Tr>(Array<Tr> & s, Array<T> & u,         \
+                                    Array<T> & vt, Array<T> & in);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+INSTANTIATE(cfloat, float)
+INSTANTIATE(cdouble, double)
+
+}  // namespace opencl
+
+#else  // WITH_LINEAR_ALGEBRA
+
+namespace oneapi {
+
+template<typename T, typename Tr>
+void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+    AF_ERROR("Linear Algebra is disabled on OneAPI", AF_ERR_NOT_CONFIGURED);
+}
+
+template<typename T, typename Tr>
+void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in) {
+    ONEAPI_NOT_SUPPORTED("");
+    AF_ERROR("Linear Algebra is disabled on OneAPI", AF_ERR_NOT_CONFIGURED);
+}
+
+#define INSTANTIATE(T, Tr)                                               \
+    template void svd<T, Tr>(Array<Tr> & s, Array<T> & u, Array<T> & vt, \
+                             const Array<T> &in);                        \
+    template void svdInPlace<T, Tr>(Array<Tr> & s, Array<T> & u,         \
+                                    Array<T> & vt, Array<T> & in);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+INSTANTIATE(cfloat, float)
+INSTANTIATE(cdouble, double)
+
+}  // namespace oneapi
+
+#endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/svd.hpp b/src/backend/oneapi/svd.hpp
new file mode 100644
index 0000000000..297c899be6
--- /dev/null
+++ b/src/backend/oneapi/svd.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T, typename Tr>
+void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
+
+template<typename T, typename Tr>
+void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/tile.cpp b/src/backend/oneapi/tile.cpp
new file mode 100644
index 0000000000..5aac53265b
--- /dev/null
+++ b/src/backend/oneapi/tile.cpp
@@ -0,0 +1,51 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+//#include <kernel/tile.hpp>
+#include <tile.hpp>
+#include <err_oneapi.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <stdexcept>
+
+using common::half;
+
+namespace oneapi {
+template<typename T>
+Array<T> tile(const Array<T> &in, const af::dim4 &tileDims) {
+    const af::dim4 &iDims = in.dims();
+    af::dim4 oDims        = iDims;
+    oDims *= tileDims;
+
+    Array<T> out = createEmptyArray<T>(oDims);
+
+    ONEAPI_NOT_SUPPORTED("tile Not supported");
+    // kernel::tile<T>(out, in);
+
+    return out;
+}
+
+#define INSTANTIATE(T) \
+    template Array<T> tile<T>(const Array<T> &in, const af::dim4 &tileDims);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/tile.hpp b/src/backend/oneapi/tile.hpp
new file mode 100644
index 0000000000..0ad5a9869a
--- /dev/null
+++ b/src/backend/oneapi/tile.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> tile(const Array<T> &in, const af::dim4 &tileDims);
+}
diff --git a/src/backend/oneapi/topk.cpp b/src/backend/oneapi/topk.cpp
new file mode 100644
index 0000000000..06d4218221
--- /dev/null
+++ b/src/backend/oneapi/topk.cpp
@@ -0,0 +1,182 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <index.hpp>
+#include <sort.hpp>
+#include <sort_index.hpp>
+#include <types.hpp>
+
+#include <algorithm>
+#include <cmath>
+#include <numeric>
+#include <vector>
+
+// using cl::Buffer;
+// using cl::Event;
+using common::half;
+
+using std::iota;
+using std::min;
+using std::partial_sort_copy;
+using std::transform;
+using std::vector;
+
+namespace oneapi {
+vector<af_index_t> indexForTopK(const int k) {
+    af_index_t idx;
+    idx.idx.seq = af_seq{0.0, static_cast<double>(k) - 1.0, 1.0};
+    idx.isSeq   = true;
+    idx.isBatch = false;
+
+    af_index_t sp;
+    sp.idx.seq = af_span;
+    sp.isSeq   = true;
+    sp.isBatch = false;
+
+    return vector<af_index_t>({idx, sp, sp, sp});
+}
+
+template<typename T>
+void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
+          const int k, const int dim, const af::topkFunction order) {
+
+    ONEAPI_NOT_SUPPORTED("topk Not supported");
+
+    // if (getDeviceType() == CL_DEVICE_TYPE_CPU) {
+    //     // This branch optimizes for CPU devices by first mapping the buffer
+    //     // and calling partial sort on the buffer
+
+    //     // TODO(umar): implement this in the kernel namespace
+
+    //     // The out_dims is of size k along the dimension of the topk operation
+    //     // and the same as the input dimension otherwise.
+    //     dim4 out_dims(1);
+    //     int ndims = in.dims().ndims();
+    //     for (int i = 0; i < ndims; i++) {
+    //         if (i == dim) {
+    //             out_dims[i] = min(k, (int)in.dims()[i]);
+    //         } else {
+    //             out_dims[i] = in.dims()[i];
+    //         }
+    //     }
+
+    //     auto values          = createEmptyArray<T>(out_dims);
+    //     auto indices         = createEmptyArray<unsigned>(out_dims);
+    //     const Buffer* in_buf = in.get();
+    //     Buffer* ibuf         = indices.get();
+    //     Buffer* vbuf         = values.get();
+
+    //     cl::Event ev_in, ev_val, ev_ind;
+
+    //     T* ptr     = static_cast<T*>(getQueue().enqueueMapBuffer(
+    //         *in_buf, CL_FALSE, CL_MAP_READ, 0, in.elements() * sizeof(T),
+    //         nullptr, &ev_in));
+    //     uint* iptr = static_cast<uint*>(getQueue().enqueueMapBuffer(
+    //         *ibuf, CL_FALSE, CL_MAP_READ | CL_MAP_WRITE, 0, k * sizeof(uint),
+    //         nullptr, &ev_ind));
+    //     T* vptr    = static_cast<T*>(getQueue().enqueueMapBuffer(
+    //         *vbuf, CL_FALSE, CL_MAP_WRITE, 0, k * sizeof(T), nullptr, &ev_val));
+
+    //     vector<uint> idx(in.elements());
+
+    //     // Create a linear index
+    //     iota(begin(idx), end(idx), 0);
+    //     cl::Event::waitForEvents({ev_in, ev_ind});
+
+    //     int iter = in.dims()[1] * in.dims()[2] * in.dims()[3];
+    //     for (int i = 0; i < iter; i++) {
+    //         auto idx_itr = begin(idx) + i * in.strides()[1];
+    //         auto kiptr   = iptr + k * i;
+
+    //         if (order & AF_TOPK_MIN) {
+    //             if (order & AF_TOPK_STABLE) {
+    //                 partial_sort_copy(
+    //                     idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+    //                     [ptr](const uint lhs, const uint rhs) -> bool {
+    //                         return (compute_t<T>(ptr[lhs]) <
+    //                                 compute_t<T>(ptr[rhs]))
+    //                                    ? true
+    //                                : compute_t<T>(ptr[lhs]) ==
+    //                                        compute_t<T>(ptr[rhs])
+    //                                    ? (lhs < rhs)
+    //                                    : false;
+    //                     });
+    //             } else {
+    //                 // Sort the top k values in each column
+    //                 partial_sort_copy(
+    //                     idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+    //                     [ptr](const uint lhs, const uint rhs) -> bool {
+    //                         return compute_t<T>(ptr[lhs]) <
+    //                                compute_t<T>(ptr[rhs]);
+    //                     });
+    //             }
+    //         } else {
+    //             if (order & AF_TOPK_STABLE) {
+    //                 partial_sort_copy(
+    //                     idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+    //                     [ptr](const uint lhs, const uint rhs) -> bool {
+    //                         return (compute_t<T>(ptr[lhs]) >
+    //                                 compute_t<T>(ptr[rhs]))
+    //                                    ? true
+    //                                : compute_t<T>(ptr[lhs]) ==
+    //                                        compute_t<T>(ptr[rhs])
+    //                                    ? (lhs < rhs)
+    //                                    : false;
+    //                     });
+    //             } else {
+    //                 partial_sort_copy(
+    //                     idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
+    //                     [ptr](const uint lhs, const uint rhs) -> bool {
+    //                         return compute_t<T>(ptr[lhs]) >
+    //                                compute_t<T>(ptr[rhs]);
+    //                     });
+    //             }
+    //         }
+    //         ev_val.wait();
+
+    //         auto kvptr = vptr + k * i;
+    //         for (int j = 0; j < k; j++) {
+    //             // Update the value arrays with the original values
+    //             kvptr[j] = ptr[kiptr[j]];
+    //             // Convert linear indices back to column indices
+    //             kiptr[j] -= i * in.strides()[1];
+    //         }
+    //     }
+
+    //     getQueue().enqueueUnmapMemObject(*ibuf, iptr);
+    //     getQueue().enqueueUnmapMemObject(*vbuf, vptr);
+    //     getQueue().enqueueUnmapMemObject(*in_buf, ptr);
+
+    //     vals = values;
+    //     idxs = indices;
+    // } else {
+    //     auto values  = createEmptyArray<T>(in.dims());
+    //     auto indices = createEmptyArray<unsigned>(in.dims());
+    //     sort_index(values, indices, in, dim, order & AF_TOPK_MIN);
+    //     auto indVec = indexForTopK(k);
+    //     vals        = index<T>(values, indVec.data());
+    //     idxs        = index<unsigned>(indices, indVec.data());
+    // }
+}
+
+#define INSTANTIATE(T)                                                  \
+    template void topk<T>(Array<T>&, Array<unsigned>&, const Array<T>&, \
+                          const int, const int, const af::topkFunction);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(long long)
+INSTANTIATE(unsigned long long)
+INSTANTIATE(half)
+}  // namespace oneapi
diff --git a/src/backend/oneapi/topk.hpp b/src/backend/oneapi/topk.hpp
new file mode 100644
index 0000000000..8390733751
--- /dev/null
+++ b/src/backend/oneapi/topk.hpp
@@ -0,0 +1,14 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+namespace oneapi {
+template<typename T>
+void topk(Array<T>& keys, Array<unsigned>& vals, const Array<T>& in,
+          const int k, const int dim, const af::topkFunction order);
+}
diff --git a/src/backend/oneapi/traits.hpp b/src/backend/oneapi/traits.hpp
new file mode 100644
index 0000000000..61fab0663c
--- /dev/null
+++ b/src/backend/oneapi/traits.hpp
@@ -0,0 +1,56 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/defines.hpp>
+#include <common/traits.hpp>
+#include <types.hpp>
+
+#include <sstream>
+#include <string>
+
+namespace af {
+
+template<typename T>
+static bool iscplx() {
+    return false;
+}
+template<>
+inline bool iscplx<oneapi::cfloat>() {
+    return true;
+}
+template<>
+inline bool iscplx<oneapi::cdouble>() {
+    return true;
+}
+
+template<typename T>
+inline std::string scalar_to_option(const T &val) {
+    using namespace common;
+    using namespace std;
+    return to_string(+val);
+}
+
+template<>
+inline std::string scalar_to_option<cl_float2>(const cl_float2 &val) {
+    std::ostringstream ss;
+    ss << val.s[0] << "," << val.s[1];
+    return ss.str();
+}
+
+template<>
+inline std::string scalar_to_option<cl_double2>(const cl_double2 &val) {
+    std::ostringstream ss;
+    ss << val.s[0] << "," << val.s[1];
+    return ss.str();
+}
+}  // namespace af
+
+using af::dtype_traits;
diff --git a/src/backend/oneapi/transform.cpp b/src/backend/oneapi/transform.cpp
new file mode 100644
index 0000000000..79cb584264
--- /dev/null
+++ b/src/backend/oneapi/transform.cpp
@@ -0,0 +1,58 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <transform.hpp>
+
+// #include <kernel/transform.hpp>
+#include <err_oneapi.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
+               const af_interp_type method, const bool inverse,
+               const bool perspective) {
+    ONEAPI_NOT_SUPPORTED("transform Not supported");
+    switch (method) {
+        case AF_INTERP_NEAREST:
+        case AF_INTERP_LOWER:
+            // kernel::transform<T>(out, in, tf, inverse, perspective, method, 1);
+            break;
+        case AF_INTERP_BILINEAR:
+        case AF_INTERP_BILINEAR_COSINE:
+            // kernel::transform<T>(out, in, tf, inverse, perspective, method, 2);
+            break;
+        case AF_INTERP_BICUBIC:
+        case AF_INTERP_BICUBIC_SPLINE:
+            // kernel::transform<T>(out, in, tf, inverse, perspective, method, 3);
+            break;
+        default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG);
+    }
+}
+
+#define INSTANTIATE(T)                                                       \
+    template void transform(Array<T> &out, const Array<T> &in,               \
+                            const Array<float> &tf,                          \
+                            const af_interp_type method, const bool inverse, \
+                            const bool perspective);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/transform.hpp b/src/backend/oneapi/transform.hpp
new file mode 100644
index 0000000000..4433518055
--- /dev/null
+++ b/src/backend/oneapi/transform.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
+               const af_interp_type method, const bool inverse,
+               const bool perspective);
+}
diff --git a/src/backend/oneapi/transpose.cpp b/src/backend/oneapi/transpose.cpp
new file mode 100644
index 0000000000..8384a6bfa1
--- /dev/null
+++ b/src/backend/oneapi/transpose.cpp
@@ -0,0 +1,54 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+// #include <kernel/transpose.hpp>
+#include <transpose.hpp>
+#include <err_oneapi.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+using common::half;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> transpose(const Array<T> &in, const bool conjugate) {
+    const dim4 &inDims = in.dims();
+    dim4 outDims       = dim4(inDims[1], inDims[0], inDims[2], inDims[3]);
+    Array<T> out       = createEmptyArray<T>(outDims);
+
+    // const bool is32multiple =
+    //     inDims[0] % kernel::TILE_DIM == 0 && inDims[1] % kernel::TILE_DIM == 0;
+
+    ONEAPI_NOT_SUPPORTED("transpose Not supported");
+    // kernel::transpose<T>(out, in, getQueue(), conjugate, is32multiple);
+
+    return out;
+}
+
+#define INSTANTIATE(T) \
+    template Array<T> transpose(const Array<T> &in, const bool conjugate);
+
+INSTANTIATE(float)
+INSTANTIATE(cfloat)
+INSTANTIATE(double)
+INSTANTIATE(cdouble)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/transpose.hpp b/src/backend/oneapi/transpose.hpp
new file mode 100644
index 0000000000..16056bb6c5
--- /dev/null
+++ b/src/backend/oneapi/transpose.hpp
@@ -0,0 +1,20 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+template<typename T>
+Array<T> transpose(const Array<T> &in, const bool conjugate);
+
+template<typename T>
+void transpose_inplace(Array<T> &in, const bool conjugate);
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/transpose_inplace.cpp b/src/backend/oneapi/transpose_inplace.cpp
new file mode 100644
index 0000000000..2792a4200b
--- /dev/null
+++ b/src/backend/oneapi/transpose_inplace.cpp
@@ -0,0 +1,44 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+//#include <kernel/transpose_inplace.hpp>
+#include <transpose.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+using common::half;
+
+namespace oneapi {
+
+template<typename T>
+void transpose_inplace(Array<T> &in, const bool conjugate) {
+    ONEAPI_NOT_SUPPORTED("");
+}
+
+#define INSTANTIATE(T) \
+    template void transpose_inplace(Array<T> &in, const bool conjugate);
+
+INSTANTIATE(float)
+INSTANTIATE(cfloat)
+INSTANTIATE(double)
+INSTANTIATE(cdouble)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(uchar)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/triangle.cpp b/src/backend/oneapi/triangle.cpp
new file mode 100644
index 0000000000..ad22dcaa6c
--- /dev/null
+++ b/src/backend/oneapi/triangle.cpp
@@ -0,0 +1,56 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+// #include <kernel/triangle.hpp>
+#include <triangle.hpp>
+#include <err_oneapi.hpp>
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <af/dim4.hpp>
+
+using af::dim4;
+using common::half;
+
+namespace oneapi {
+
+template<typename T>
+void triangle(Array<T> &out, const Array<T> &in, const bool is_upper,
+              const bool is_unit_diag) {
+    ONEAPI_NOT_SUPPORTED("triangle Not supported");
+    // kernel::triangle<T>(out, in, is_upper, is_unit_diag);
+}
+
+template<typename T>
+Array<T> triangle(const Array<T> &in, const bool is_upper,
+                  const bool is_unit_diag) {
+    Array<T> out = createEmptyArray<T>(in.dims());
+    triangle<T>(out, in, is_upper, is_unit_diag);
+    return out;
+}
+
+#define INSTANTIATE(T)                                                  \
+    template void triangle<T>(Array<T> &, const Array<T> &, const bool, \
+                              const bool);                              \
+    template Array<T> triangle<T>(const Array<T> &, const bool, const bool);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(char)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+
+}  // namespace opencl
diff --git a/src/backend/oneapi/triangle.hpp b/src/backend/oneapi/triangle.hpp
new file mode 100644
index 0000000000..0dc1a48a11
--- /dev/null
+++ b/src/backend/oneapi/triangle.hpp
@@ -0,0 +1,20 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+void triangle(Array<T> &out, const Array<T> &in, const bool is_upper,
+              const bool is_unit_diag);
+
+template<typename T>
+Array<T> triangle(const Array<T> &in, const bool is_upper,
+                  const bool is_unit_diag);
+}  // namespace oneapi
diff --git a/src/backend/oneapi/types.hpp b/src/backend/oneapi/types.hpp
new file mode 100644
index 0000000000..945d1366c7
--- /dev/null
+++ b/src/backend/oneapi/types.hpp
@@ -0,0 +1,163 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <CL/sycl.hpp>
+#include <common/kernel_type.hpp>
+#include <common/traits.hpp>
+#include <af/compilers.h>
+#include <af/traits.hpp>
+
+#include <algorithm>
+#include <array>
+#include <complex>
+#include <string>
+
+namespace common {
+/// This is a CPU based half which need to be converted into floats before they
+/// are used
+template<>
+struct kernel_type<common::half> {
+    using data = common::half;
+
+    // These are the types within a kernel
+    using native = float;
+
+    using compute = float;
+};
+}  // namespace common
+
+namespace oneapi {
+using cdouble = std::complex<double>;
+using cfloat  = std::complex<float>;
+using intl    = long long;
+using uchar   = cl_uchar;
+using uint    = cl_uint;
+using uintl   = unsigned long long;
+using ushort  = cl_ushort;
+
+template<typename T>
+using compute_t = typename common::kernel_type<T>::compute;
+
+template<typename T>
+using data_t = typename common::kernel_type<T>::data;
+
+template<typename T>
+struct ToNumStr {
+    std::string operator()(T val);
+    template<typename CONVERSION_TYPE>
+    std::string operator()(CONVERSION_TYPE val);
+};
+
+namespace {
+template<typename T>
+inline const char *shortname(bool caps = false) {
+    return caps ? "X" : "x";
+}
+
+template<>
+inline const char *shortname<float>(bool caps) {
+    return caps ? "S" : "s";
+}
+template<>
+inline const char *shortname<double>(bool caps) {
+    return caps ? "D" : "d";
+}
+template<>
+inline const char *shortname<cfloat>(bool caps) {
+    return caps ? "C" : "c";
+}
+template<>
+inline const char *shortname<cdouble>(bool caps) {
+    return caps ? "Z" : "z";
+}
+template<>
+inline const char *shortname<int>(bool caps) {
+    return caps ? "I" : "i";
+}
+template<>
+inline const char *shortname<uint>(bool caps) {
+    return caps ? "U" : "u";
+}
+template<>
+inline const char *shortname<char>(bool caps) {
+    return caps ? "J" : "j";
+}
+template<>
+inline const char *shortname<uchar>(bool caps) {
+    return caps ? "V" : "v";
+}
+template<>
+inline const char *shortname<intl>(bool caps) {
+    return caps ? "L" : "l";
+}
+template<>
+inline const char *shortname<uintl>(bool caps) {
+    return caps ? "K" : "k";
+}
+template<>
+inline const char *shortname<short>(bool caps) {
+    return caps ? "P" : "p";
+}
+template<>
+inline const char *shortname<ushort>(bool caps) {
+    return caps ? "Q" : "q";
+}
+
+template<typename T>
+inline const char *getFullName() {
+    return af::dtype_traits<T>::getName();
+}
+
+template<>
+inline const char *getFullName<cfloat>() {
+    return "float2";
+}
+
+template<>
+inline const char *getFullName<cdouble>() {
+    return "double2";
+}
+}  // namespace
+
+#if 0
+template<typename... ARGS>
+AF_CONSTEXPR const char *getTypeBuildDefinition() {
+    using common::half;
+    using std::any_of;
+    using std::array;
+    using std::begin;
+    using std::end;
+    using std::is_same;
+    array<bool, sizeof...(ARGS)> is_half    = {is_same<ARGS, half>::value...};
+    array<bool, sizeof...(ARGS)> is_double  = {is_same<ARGS, double>::value...};
+    array<bool, sizeof...(ARGS)> is_cdouble = {
+        is_same<ARGS, cdouble>::value...};
+
+    bool half_def =
+        any_of(begin(is_half), end(is_half), [](bool val) { return val; });
+    bool double_def =
+        any_of(begin(is_double), end(is_double), [](bool val) { return val; });
+    bool cdouble_def = any_of(begin(is_cdouble), end(is_cdouble),
+                              [](bool val) { return val; });
+
+    if (half_def && (double_def || cdouble_def)) {
+        return " -D USE_HALF -D USE_DOUBLE";
+    } else if (half_def) {
+        return " -D USE_HALF";
+    } else if (double_def || cdouble_def) {
+        return " -D USE_DOUBLE";
+    } else {
+        return "";
+    }
+}
+#endif
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/unary.hpp b/src/backend/oneapi/unary.hpp
new file mode 100644
index 0000000000..0e8a267c07
--- /dev/null
+++ b/src/backend/oneapi/unary.hpp
@@ -0,0 +1,111 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Array.hpp>
+#include <common/jit/UnaryNode.hpp>
+#include <math.hpp>
+#include <optypes.hpp>
+#include <af/traits.hpp>
+
+namespace oneapi {
+
+template<af_op_t op>
+static const char *unaryName();
+
+#define UNARY_DECL(OP, FNAME)                     \
+    template<>                                    \
+    inline const char *unaryName<af_##OP##_t>() { \
+        return FNAME;                             \
+    }
+
+#define UNARY_FN(OP) UNARY_DECL(OP, #OP)
+
+UNARY_FN(sin)
+UNARY_FN(cos)
+UNARY_FN(tan)
+
+UNARY_FN(asin)
+UNARY_FN(acos)
+UNARY_FN(atan)
+
+UNARY_FN(sinh)
+UNARY_FN(cosh)
+UNARY_FN(tanh)
+
+UNARY_FN(asinh)
+UNARY_FN(acosh)
+UNARY_FN(atanh)
+
+UNARY_FN(exp)
+UNARY_DECL(sigmoid, "__sigmoid")
+UNARY_FN(expm1)
+UNARY_FN(erf)
+UNARY_FN(erfc)
+
+UNARY_FN(tgamma)
+UNARY_FN(lgamma)
+
+UNARY_FN(log)
+UNARY_FN(log1p)
+UNARY_FN(log10)
+UNARY_FN(log2)
+
+UNARY_FN(sqrt)
+UNARY_FN(rsqrt)
+UNARY_FN(cbrt)
+
+UNARY_FN(trunc)
+UNARY_FN(round)
+UNARY_FN(signbit)
+UNARY_FN(ceil)
+UNARY_FN(floor)
+
+UNARY_FN(isinf)
+UNARY_FN(isnan)
+UNARY_FN(iszero)
+UNARY_DECL(noop, "__noop")
+
+UNARY_DECL(bitnot, "__bitnot")
+
+#undef UNARY_FN
+
+template<typename T, af_op_t op>
+Array<T> unaryOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
+    using common::Node;
+    using common::Node_ptr;
+    using std::array;
+
+    auto createUnary = [](array<Node_ptr, 1> &operands) {
+        return common::Node_ptr(new common::UnaryNode(
+            static_cast<af::dtype>(af::dtype_traits<T>::af_type),
+            unaryName<op>(), operands[0], op));
+    };
+
+    if (outDim == dim4(-1, -1, -1, -1)) { outDim = in.dims(); }
+    Node_ptr out = common::createNaryNode<T, 1>(outDim, createUnary, {&in});
+    return createNodeArray<T>(outDim, out);
+}
+
+template<typename T, af_op_t op>
+Array<char> checkOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
+    using common::Node_ptr;
+
+    auto createUnary = [](std::array<Node_ptr, 1> &operands) {
+        return Node_ptr(new common::UnaryNode(
+            static_cast<af::dtype>(af::dtype_traits<char>::af_type),
+            unaryName<op>(), operands[0], op));
+    };
+
+    if (outDim == dim4(-1, -1, -1, -1)) { outDim = in.dims(); }
+    Node_ptr out = common::createNaryNode<T, 1>(outDim, createUnary, {&in});
+    return createNodeArray<char>(outDim, out);
+}
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/unwrap.cpp b/src/backend/oneapi/unwrap.cpp
new file mode 100644
index 0000000000..200da9d307
--- /dev/null
+++ b/src/backend/oneapi/unwrap.cpp
@@ -0,0 +1,63 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/unwrap.hpp>
+#include <unwrap.hpp>
+#include <stdexcept>
+
+using common::half;
+
+namespace oneapi {
+
+template<typename T>
+Array<T> unwrap(const Array<T> &in, const dim_t wx, const dim_t wy,
+                const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
+                const dim_t dx, const dim_t dy, const bool is_column) {
+    af::dim4 idims = in.dims();
+
+    dim_t nx = 1 + (idims[0] + 2 * px - (((wx - 1) * dx) + 1)) / sx;
+    dim_t ny = 1 + (idims[1] + 2 * py - (((wy - 1) * dy) + 1)) / sy;
+
+    af::dim4 odims(wx * wy, nx * ny, idims[2], idims[3]);
+
+    if (!is_column) { std::swap(odims[0], odims[1]); }
+
+    Array<T> outArray = createEmptyArray<T>(odims);
+    ONEAPI_NOT_SUPPORTED("unwrap Not supported");
+    // kernel::unwrap<T>(outArray, in, wx, wy, sx, sy, px, py, dx, dy, nx,
+    //                   is_column);
+
+    return outArray;
+}
+
+#define INSTANTIATE(T)                                                      \
+    template Array<T> unwrap<T>(                                            \
+        const Array<T> &in, const dim_t wx, const dim_t wy, const dim_t sx, \
+        const dim_t sy, const dim_t px, const dim_t py, const dim_t dx,     \
+        const dim_t dy, const bool is_column);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(half)
+#undef INSTANTIATE
+
+}  // namespace opencl
diff --git a/src/backend/oneapi/unwrap.hpp b/src/backend/oneapi/unwrap.hpp
new file mode 100644
index 0000000000..beab1dca4c
--- /dev/null
+++ b/src/backend/oneapi/unwrap.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<T> unwrap(const Array<T> &in, const dim_t wx, const dim_t wy,
+                const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
+                const dim_t dx, const dim_t dy, const bool is_column);
+}
diff --git a/src/backend/oneapi/vector_field.cpp b/src/backend/oneapi/vector_field.cpp
new file mode 100644
index 0000000000..40c7be146d
--- /dev/null
+++ b/src/backend/oneapi/vector_field.cpp
@@ -0,0 +1,36 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <GraphicsResourceManager.hpp>
+#include <err_oneapi.hpp>
+#include <vector_field.hpp>
+
+using af::dim4;
+
+namespace oneapi {
+
+template<typename T>
+void copy_vector_field(const Array<T> &points, const Array<T> &directions,
+                       fg_vector_field vfield) {
+}
+
+#define INSTANTIATE(T)                                                     \
+    template void copy_vector_field<T>(const Array<T> &, const Array<T> &, \
+                                       fg_vector_field);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+INSTANTIATE(uchar)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/vector_field.hpp b/src/backend/oneapi/vector_field.hpp
new file mode 100644
index 0000000000..2c2a9b565b
--- /dev/null
+++ b/src/backend/oneapi/vector_field.hpp
@@ -0,0 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/graphics_common.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void copy_vector_field(const Array<T> &points, const Array<T> &directions,
+                       fg_vector_field vfield);
+}
diff --git a/src/backend/oneapi/where.cpp b/src/backend/oneapi/where.cpp
new file mode 100644
index 0000000000..4dc3e42565
--- /dev/null
+++ b/src/backend/oneapi/where.cpp
@@ -0,0 +1,44 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/where.hpp>
+#include <where.hpp>
+#include <af/dim4.hpp>
+#include <complex>
+
+namespace oneapi {
+
+template<typename T>
+Array<uint> where(const Array<T> &in) {
+    //Param<uint> Out;
+    // Param<T> In = in;
+    ONEAPI_NOT_SUPPORTED("where Not supported");
+    // kernel::where<T>(Out, In);
+    //return createParamArray<uint>(Out, true);
+    return createEmptyArray<uint>(af::dim4(1));
+}
+
+#define INSTANTIATE(T) template Array<uint> where<T>(const Array<T> &in);
+
+INSTANTIATE(float)
+INSTANTIATE(cfloat)
+INSTANTIATE(double)
+INSTANTIATE(cdouble)
+INSTANTIATE(char)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+
+}  // namespace opencl
diff --git a/src/backend/oneapi/where.hpp b/src/backend/oneapi/where.hpp
new file mode 100644
index 0000000000..a63ca73cb9
--- /dev/null
+++ b/src/backend/oneapi/where.hpp
@@ -0,0 +1,15 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+template<typename T>
+Array<uint> where(const Array<T>& in);
+}
diff --git a/src/backend/oneapi/wrap.cpp b/src/backend/oneapi/wrap.cpp
new file mode 100644
index 0000000000..5dd0d7d78f
--- /dev/null
+++ b/src/backend/oneapi/wrap.cpp
@@ -0,0 +1,76 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+#include <common/dispatch.hpp>
+#include <common/half.hpp>
+#include <err_oneapi.hpp>
+// #include <kernel/wrap.hpp>
+#include <math.hpp>
+#include <wrap.hpp>
+#include <stdexcept>
+
+using common::half;
+
+namespace oneapi {
+
+template<typename T>
+void wrap(Array<T> &out, const Array<T> &in, const dim_t wx, const dim_t wy,
+          const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
+          const bool is_column) {
+  ONEAPI_NOT_SUPPORTED("wrap Not supported");
+  // kernel::wrap<T>(out, in, wx, wy, sx, sy, px, py, is_column);
+}
+
+#define INSTANTIATE(T)                                                        \
+    template void wrap<T>(Array<T> & out, const Array<T> &in, const dim_t wx, \
+                          const dim_t wy, const dim_t sx, const dim_t sy,     \
+                          const dim_t px, const dim_t py,                     \
+                          const bool is_column);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(cfloat)
+INSTANTIATE(cdouble)
+INSTANTIATE(int)
+INSTANTIATE(uint)
+INSTANTIATE(intl)
+INSTANTIATE(uintl)
+INSTANTIATE(uchar)
+INSTANTIATE(char)
+INSTANTIATE(short)
+INSTANTIATE(ushort)
+#undef INSTANTIATE
+
+template<typename T>
+Array<T> wrap_dilated(const Array<T> &in, const dim_t ox, const dim_t oy,
+                      const dim_t wx, const dim_t wy, const dim_t sx,
+                      const dim_t sy, const dim_t px, const dim_t py,
+                      const dim_t dx, const dim_t dy, const bool is_column) {
+    af::dim4 idims = in.dims();
+    af::dim4 odims(ox, oy, idims[2], idims[3]);
+    Array<T> out = createValueArray<T>(odims, scalar<T>(0));
+
+    // kernel::wrap_dilated<T>(out, in, wx, wy, sx, sy, px, py, dx, dy, is_column);
+    ONEAPI_NOT_SUPPORTED("wrap_dilated Not supported");
+    return out;
+}
+
+#define INSTANTIATE(T)                                                      \
+    template Array<T> wrap_dilated<T>(                                      \
+        const Array<T> &in, const dim_t ox, const dim_t oy, const dim_t wx, \
+        const dim_t wy, const dim_t sx, const dim_t sy, const dim_t px,     \
+        const dim_t py, const dim_t dx, const dim_t dy, const bool is_column);
+
+INSTANTIATE(float)
+INSTANTIATE(double)
+INSTANTIATE(half)
+#undef INSTANTIATE
+
+}  // namespace opencl
diff --git a/src/backend/oneapi/wrap.hpp b/src/backend/oneapi/wrap.hpp
new file mode 100644
index 0000000000..ae831a9bb1
--- /dev/null
+++ b/src/backend/oneapi/wrap.hpp
@@ -0,0 +1,24 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Array.hpp>
+
+namespace oneapi {
+
+template<typename T>
+void wrap(Array<T> &out, const Array<T> &in, const dim_t wx, const dim_t wy,
+          const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
+          const bool is_column);
+
+template<typename T>
+Array<T> wrap_dilated(const Array<T> &in, const dim_t ox, const dim_t oy,
+                      const dim_t wx, const dim_t wy, const dim_t sx,
+                      const dim_t sy, const dim_t px, const dim_t py,
+                      const dim_t dx, const dim_t dy, const bool is_column);
+}  // namespace oneapi
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index d1bbebbdeb..e2a580a1c1 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -95,6 +95,10 @@ if(AF_BUILD_OPENCL)
   list(APPEND enabled_backends "opencl")
 endif(AF_BUILD_OPENCL)
 
+if(AF_BUILD_ONEAPI)
+  list(APPEND enabled_backends "oneapi")
+endif(AF_BUILD_ONEAPI)
+
 if(AF_BUILD_UNIFIED)
   list(APPEND enabled_backends "unified")
 endif(AF_BUILD_UNIFIED)

From acc2a9db37fee228990103d5d759ee1d1d2ee78d Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 13 Sep 2022 22:17:35 -0400
Subject: [PATCH 093/473] basic implementation of device_manager

---
 include/af/defines.h                  |   3 +-
 include/af/oneapi.h                   | 443 ++++++++++++++++++++++++++
 src/backend/oneapi/device_manager.cpp | 155 ++++++++-
 src/backend/oneapi/device_manager.hpp |  11 +-
 src/backend/oneapi/memory.hpp         |  60 ++--
 src/backend/oneapi/platform.cpp       |  38 ++-
 src/backend/oneapi/platform.hpp       |   2 +-
 7 files changed, 648 insertions(+), 64 deletions(-)
 create mode 100644 include/af/oneapi.h

diff --git a/include/af/defines.h b/include/af/defines.h
index 611a025375..da6c5591de 100644
--- a/include/af/defines.h
+++ b/include/af/defines.h
@@ -414,7 +414,8 @@ typedef enum {
     AF_BACKEND_DEFAULT = 0,  ///< Default backend order: OpenCL -> CUDA -> CPU
     AF_BACKEND_CPU     = 1,  ///< CPU a.k.a sequential algorithms
     AF_BACKEND_CUDA    = 2,  ///< CUDA Compute Backend
-    AF_BACKEND_OPENCL  = 4   ///< OpenCL Compute Backend
+    AF_BACKEND_OPENCL  = 4,  ///< OpenCL Compute Backend
+    AF_BACKEND_ONEAPI  = 8   ///< OneAPI Compute Backend
 } af_backend;
 #endif
 
diff --git a/include/af/oneapi.h b/include/af/oneapi.h
new file mode 100644
index 0000000000..5400a34d1a
--- /dev/null
+++ b/include/af/oneapi.h
@@ -0,0 +1,443 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <CL/sycl.hpp>
+#include <af/defines.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if AF_API_VERSION >= 39
+typedef enum
+{
+    //AF_ONEAPI_DEVICE_TYPE_CPU     = sycl::info::device_type::cpu,
+    //AF_ONEAPI_DEVICE_TYPE_GPU     = sycl::info::device_type::gpu,
+    //AF_ONEAPI_DEVICE_TYPE_ACC     = sycl::info::device_type::accelerator
+    //AF_ONEAPI_DEVICE_TYPE_UNKNOWN = -1
+    AF_ONEAPI_DEVICE_TYPE_CPU     = 0,
+    AF_ONEAPI_DEVICE_TYPE_GPU     = 1,
+    AF_ONEAPI_DEVICE_TYPE_ACC     = 2,
+    AF_ONEAPI_DEVICE_TYPE_UNKNOWN = -1
+} af_oneapi_device_type;
+#endif
+
+#if AF_API_VERSION >= 39
+typedef enum
+{
+    AF_ONEAPI_PLATFORM_AMD     = 0,
+    AF_ONEAPI_PLATFORM_APPLE   = 1,
+    AF_ONEAPI_PLATFORM_INTEL   = 2,
+    AF_ONEAPI_PLATFORM_NVIDIA  = 3,
+    AF_ONEAPI_PLATFORM_BEIGNET = 4,
+    AF_ONEAPI_PLATFORM_POCL    = 5,
+    AF_ONEAPI_PLATFORM_UNKNOWN = -1
+} af_oneapi_platform;
+#endif
+
+#if 0
+/**
+    \ingroup opencl_mat
+    @{
+*/
+/**
+  Get a handle to ArrayFire's OpenCL context
+
+  \param[out] ctx the current context being used by ArrayFire
+  \param[in] retain if true calls clRetainContext prior to returning the context
+  \returns \ref af_err error code
+
+  \note Set \p retain to true if this value will be passed to a cl::Context constructor
+*/
+AFAPI af_err afcl_get_context(cl_context *ctx, const bool retain);
+
+/**
+  Get a handle to ArrayFire's OpenCL command queue
+
+  \param[out] queue the current command queue being used by ArrayFire
+  \param[in] retain if true calls clRetainCommandQueue prior to returning the context
+  \returns \ref af_err error code
+
+  \note Set \p retain to true if this value will be passed to a cl::CommandQueue constructor
+*/
+AFAPI af_err afcl_get_queue(cl_command_queue *queue, const bool retain);
+
+/**
+   Get the device ID for ArrayFire's current active device
+
+   \param[out] id the cl_device_id of the current device
+   \returns \ref af_err error code
+*/
+AFAPI af_err afcl_get_device_id(cl_device_id *id);
+
+#if AF_API_VERSION >= 39
+/**
+   Set ArrayFire's active device based on \p id of type cl_device_id
+
+   \param[in] id the cl_device_id of the device to be set as active device
+   \returns \ref af_err error code
+*/
+AFAPI af_err afcl_set_device_id(cl_device_id id);
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Push user provided device control constructs into the ArrayFire device manager pool
+
+   This function should be used only when the user would like ArrayFire to use an
+   user generated OpenCL context and related objects for ArrayFire operations.
+
+   \param[in] dev is the OpenCL device for which user provided context will be used by ArrayFire
+   \param[in] ctx is the user provided OpenCL cl_context to be used by ArrayFire
+   \param[in] que is the user provided OpenCL cl_command_queue to be used by ArrayFire. If this
+                  parameter is NULL, then we create a command queue for the user using the OpenCL
+                  context they provided us.
+
+   \note ArrayFire does not take control of releasing the objects passed to it. The user needs to release them appropriately.
+*/
+AFAPI af_err afcl_add_device_context(cl_device_id dev, cl_context ctx, cl_command_queue que);
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Set active device using cl_context and cl_device_id
+
+   \param[in] dev is the OpenCL device id that is to be set as Active device inside ArrayFire
+   \param[in] ctx is the OpenCL cl_context being used by ArrayFire
+*/
+AFAPI af_err afcl_set_device_context(cl_device_id dev, cl_context ctx);
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Remove the user provided device control constructs from the ArrayFire device manager pool
+
+   This function should be used only when the user would like ArrayFire to remove an already
+   pushed user generated OpenCL context and related objects.
+
+   \param[in] dev is the OpenCL device id that has to be popped
+   \param[in] ctx is the cl_context object to be removed from ArrayFire pool
+
+   \note ArrayFire does not take control of releasing the objects passed to it. The user needs to release them appropriately.
+*/
+AFAPI af_err afcl_delete_device_context(cl_device_id dev, cl_context ctx);
+#endif
+
+#if AF_API_VERSION >= 39
+ Ge
+  t the type of the current device
+*/
+AFAPI af_err afcl_get_device_type(afcl_device_type *res);
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Get the platform of the current device
+*/
+AFAPI af_err afcl_get_platform(afcl_platform *res);
+#endif
+
+/**
+  @}
+*/
+#endif //if 0 comment
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+
+#include <af/array.h>
+#include <af/dim4.hpp>
+#include <af/exception.h>
+#include <af/device.h>
+#include <stdio.h>
+
+namespace afoneapi
+{
+
+#if 0
+ /**
+     \addtogroup opencl_mat
+     @{
+ */
+
+ /**
+ Get a handle to ArrayFire's OpenCL context
+
+ \param[in] retain if true calls clRetainContext prior to returning the context
+ \returns the current context being used by ArrayFire
+
+ \note Set \p retain to true if this value will be passed to a cl::Context constructor
+ */
+ static inline cl_context getContext(bool retain = false)
+ {
+     cl_context ctx;
+     af_err err = afcl_get_context(&ctx, retain);
+     if (err != AF_SUCCESS) throw af::exception("Failed to get OpenCL context from arrayfire");
+     return ctx;
+ }
+
+ /**
+ Get a handle to ArrayFire's OpenCL command queue
+
+ \param[in] retain if true calls clRetainCommandQueue prior to returning the context
+ \returns the current command queue being used by ArrayFire
+
+ \note Set \p retain to true if this value will be passed to a cl::CommandQueue constructor
+ */
+ static inline cl_command_queue getQueue(bool retain = false)
+ {
+     cl_command_queue queue;
+     af_err err = afcl_get_queue(&queue, retain);
+     if (err != AF_SUCCESS) throw af::exception("Failed to get OpenCL command queue from arrayfire");
+     return queue;
+ }
+
+ /**
+    Get the device ID for ArrayFire's current active device
+    \returns the cl_device_id of the current device
+ */
+ static inline cl_device_id getDeviceId()
+ {
+     cl_device_id id;
+     af_err err = afcl_get_device_id(&id);
+     if (err != AF_SUCCESS) throw af::exception("Failed to get OpenCL device ID");
+
+     return id;
+ }
+
+#if AF_API_VERSION >= 39
+ /**
+   Set ArrayFire's active device based on \p id of type cl_device_id
+
+   \param[in] id the cl_device_id of the device to be set as active device
+ */
+ static inline void setDeviceId(cl_device_id id)
+ {
+     af_err err = afcl_set_device_id(id);
+     if (err != AF_SUCCESS) throw af::exception("Failed to set OpenCL device as active device");
+ }
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Push user provided device control constructs into the ArrayFire device manager pool
+
+   This function should be used only when the user would like ArrayFire to use an
+   user generated OpenCL context and related objects for ArrayFire operations.
+
+   \param[in] dev is the OpenCL device for which user provided context will be used by ArrayFire
+   \param[in] ctx is the user provided OpenCL cl_context to be used by ArrayFire
+   \param[in] que is the user provided OpenCL cl_command_queue to be used by ArrayFire. If this
+                  parameter is NULL, then we create a command queue for the user using the OpenCL
+                  context they provided us.
+
+   \note ArrayFire does not take control of releasing the objects passed to it. The user needs to release them appropriately.
+*/
+static inline void addDevice(cl_device_id dev, cl_context ctx, cl_command_queue que)
+{
+    af_err err = afcl_add_device_context(dev, ctx, que);
+    if (err!=AF_SUCCESS) throw af::exception("Failed to push user provided device/context to ArrayFire pool");
+}
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Set active device using cl_context and cl_device_id
+
+   \param[in] dev is the OpenCL device id that is to be set as Active device inside ArrayFire
+   \param[in] ctx is the OpenCL cl_context being used by ArrayFire
+*/
+static inline void setDevice(cl_device_id dev, cl_context ctx)
+{
+    af_err err = afcl_set_device_context(dev, ctx);
+    if (err!=AF_SUCCESS) throw af::exception("Failed to set device based on cl_device_id & cl_context");
+}
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Remove the user provided device control constructs from the ArrayFire device manager pool
+
+   This function should be used only when the user would like ArrayFire to remove an already
+   pushed user generated OpenCL context and related objects.
+
+   \param[in] dev is the OpenCL device id that has to be popped
+   \param[in] ctx is the cl_context object to be removed from ArrayFire pool
+
+   \note ArrayFire does not take control of releasing the objects passed to it. The user needs to release them appropriately.
+*/
+static inline void deleteDevice(cl_device_id dev, cl_context ctx)
+{
+    af_err err = afcl_delete_device_context(dev, ctx);
+    if (err!=AF_SUCCESS) throw af::exception("Failed to remove the requested device from ArrayFire device pool");
+}
+#endif
+
+
+#if AF_API_VERSION >= 39
+ typedef afcl_device_type deviceType;
+ typedef afcl_platform platform;
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Get the type of the current device
+*/
+static inline deviceType getDeviceType()
+{
+    afcl_device_type res = AFCL_DEVICE_TYPE_UNKNOWN;
+    af_err err = afcl_get_device_type(&res);
+    if (err!=AF_SUCCESS) throw af::exception("Failed to get OpenCL device type");
+    return res;
+}
+#endif
+
+#if AF_API_VERSION >= 39
+/**
+   Get a vendor enumeration for the current platform
+*/
+static inline platform getPlatform()
+{
+    afcl_platform res = AFCL_PLATFORM_UNKNOWN;
+    af_err err = afcl_get_platform(&res);
+    if (err!=AF_SUCCESS) throw af::exception("Failed to get OpenCL platform");
+    return res;
+}
+#endif
+
+ /**
+ Create an af::array object from an OpenCL cl_mem buffer
+
+ \param[in] idims the dimensions of the buffer
+ \param[in] buf the OpenCL memory object
+ \param[in] type the data type contained in the buffer
+ \param[in] retain if true, instructs ArrayFire to retain the memory object
+ \returns an array object created from the OpenCL buffer
+
+ \note Set \p retain to true if the memory originates from a cl::Buffer object
+  */
+ static inline af::array array(af::dim4 idims, cl_mem buf, af::dtype type, bool retain=false)
+ {
+     const unsigned ndims = (unsigned)idims.ndims();
+     const dim_t *dims = idims.get();
+
+     cl_context context;
+     cl_int clerr = clGetMemObjectInfo(buf, CL_MEM_CONTEXT, sizeof(cl_context), &context, NULL);
+     if (clerr != CL_SUCCESS) {
+         throw af::exception("Failed to get context from cl_mem object \"buf\" ");
+     }
+
+     if (context != getContext()) {
+         throw(af::exception("Context mismatch between input \"buf\" and arrayfire"));
+     }
+
+
+     if (retain) clerr = clRetainMemObject(buf);
+
+     af_array out;
+     af_err err = af_device_array(&out, buf, ndims, dims, type);
+
+     if (err != AF_SUCCESS || clerr != CL_SUCCESS) {
+         if (retain && clerr == CL_SUCCESS) clReleaseMemObject(buf);
+         throw af::exception("Failed to create device array");
+     }
+
+     return af::array(out);
+ }
+
+ /**
+ Create an af::array object from an OpenCL cl_mem buffer
+
+ \param[in] dim0 the length of the first dimension of the buffer
+ \param[in] buf the OpenCL memory object
+ \param[in] type the data type contained in the buffer
+ \param[in] retain if true, instructs ArrayFire to retain the memory object
+ \returns an array object created from the OpenCL buffer
+
+ \note Set \p retain to true if the memory originates from a cl::Buffer object
+  */
+ static inline af::array array(dim_t dim0,
+                               cl_mem buf, af::dtype type, bool retain=false)
+ {
+     return afcl::array(af::dim4(dim0), buf, type, retain);
+ }
+
+ /**
+ Create an af::array object from an OpenCL cl_mem buffer
+
+ \param[in] dim0 the length of the first dimension of the buffer
+ \param[in] dim1 the length of the second dimension of the buffer
+ \param[in] buf the OpenCL memory object
+ \param[in] type the data type contained in the buffer
+ \param[in] retain if true, instructs ArrayFire to retain the memory object
+ \returns an array object created from the OpenCL buffer
+
+ \note Set \p retain to true if the memory originates from a cl::Buffer object
+  */
+ static inline af::array array(dim_t dim0, dim_t dim1,
+                               cl_mem buf, af::dtype type, bool retain=false)
+ {
+     return afcl::array(af::dim4(dim0, dim1), buf, type, retain);
+ }
+
+ /**
+ Create an af::array object from an OpenCL cl_mem buffer
+
+ \param[in] dim0 the length of the first dimension of the buffer
+ \param[in] dim1 the length of the second dimension of the buffer
+ \param[in] dim2 the length of the third dimension of the buffer
+ \param[in] buf the OpenCL memory object
+ \param[in] type the data type contained in the buffer
+ \param[in] retain if true, instructs ArrayFire to retain the memory object
+ \returns an array object created from the OpenCL buffer
+
+ \note Set \p retain to true if the memory originates from a cl::Buffer object
+  */
+ static inline af::array array(dim_t dim0, dim_t dim1,
+                               dim_t dim2,
+                               cl_mem buf, af::dtype type, bool retain=false)
+ {
+     return afcl::array(af::dim4(dim0, dim1, dim2), buf, type, retain);
+ }
+
+ /**
+ Create an af::array object from an OpenCL cl_mem buffer
+
+ \param[in] dim0 the length of the first dimension of the buffer
+ \param[in] dim1 the length of the second dimension of the buffer
+ \param[in] dim2 the length of the third dimension of the buffer
+ \param[in] dim3 the length of the fourth dimension of the buffer
+ \param[in] buf the OpenCL memory object
+ \param[in] type the data type contained in the buffer
+ \param[in] retain if true, instructs ArrayFire to retain the memory object
+ \returns an array object created from the OpenCL buffer
+
+ \note Set \p retain to true if the memory originates from a cl::Buffer object
+  */
+ static inline af::array array(dim_t dim0, dim_t dim1,
+                               dim_t dim2, dim_t dim3,
+                               cl_mem buf, af::dtype type, bool retain=false)
+ {
+     return afcl::array(af::dim4(dim0, dim1, dim2, dim3), buf, type, retain);
+ }
+
+/**
+   @}
+*/
+#endif //#IF 0 tmp comment
+
+}
+
+
+#endif
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index 5ef59d2682..d4750defae 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -10,8 +10,8 @@
 #include <common/graphics_common.hpp>
 
 #include <GraphicsResourceManager.hpp>
+#include <platform.hpp> //TODO: blas.hpp? y tho, also Array.hpp 
 #include <common/DefaultMemoryManager.hpp>
-#include <common/MemoryManagerBase.hpp>
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
 #include <common/host_memory.hpp>
@@ -20,13 +20,9 @@
 #include <err_oneapi.hpp>
 //#include <errorcodes.hpp>
 #include <version.hpp>
-//#include <af/oneapi.h>
+#include <af/oneapi.h>
 #include <af/version.h>
-#include <memory>
-
-#ifdef OS_MAC
-#include <OpenGL/CGLCurrent.h>
-#endif
+#include <memory.hpp>
 
 #include <algorithm>
 #include <iterator>
@@ -43,48 +39,175 @@ using std::stringstream;
 using std::unique_ptr;
 using std::vector;
 using sycl::device;
+using sycl::platform;
 
 namespace oneapi {
 
-bool checkExtnAvailability(const device& pDevice, const string& pName) {
-    ONEAPI_NOT_SUPPORTED("");
-    return false;
+static inline bool compare_default(const unique_ptr<sycl::device>& ldev,
+                                   const unique_ptr<sycl::device>& rdev) {
+    //TODO: update sorting criteria
+    //select according to something applicable to oneapi backend
+    auto l_mem = ldev->get_info<sycl::info::device::local_mem_size>();
+    auto r_mem = rdev->get_info<sycl::info::device::local_mem_size>();
+    return l_mem > r_mem;
 }
 
 DeviceManager::DeviceManager()
     : logger(common::loggerFactory("platform"))
     , mUserDeviceOffset(0)
     , fgMngr(nullptr) {
+    vector<sycl::platform> platforms;
+    try {
+        platforms = sycl::platform::get_platforms();
+    } catch (sycl::exception& err) {
+        AF_ERROR(
+            "No sycl platforms found on this system. Ensure you have "
+            "installed the device driver as well as the runtime.",
+            AF_ERR_RUNTIME);
+    }
+
+    fgMngr = std::make_unique<graphics::ForgeManager>();
+
+    AF_TRACE("Found {} sycl platforms", platforms.size());
+    // Iterate through platforms, get all available devices and store them
+    for (auto& platform : platforms) {
+        vector<sycl::device> current_devices;
+        try {
+            current_devices = platform.get_devices();
+        } catch(sycl::exception& err) {
+            printf("DeviceManager::DeviceManager() exception: %s\n", err.what());
+            throw;
+        }
+        AF_TRACE("Found {} devices on platform {}", current_devices.size(),
+                 platform.get_info<sycl::info::platform::name>());
+
+        for (auto& dev : current_devices) {
+            mDevices.emplace_back(make_unique<sycl::device>(dev));
+            AF_TRACE("Found device {} on platform {}",
+                     dev.get_info<sycl::info::device::name>(),
+                     platform.get_info<sycl::info::platform::name>());
+        }
+    }
+
+    int nDevices = mDevices.size();
+    AF_TRACE("Found {} sycl devices", nDevices);
+
+    if (nDevices == 0) { AF_ERROR("No sycl devices found", AF_ERR_RUNTIME); }
+
+    // Sort sycl devices based on default criteria
+    stable_sort(mDevices.begin(), mDevices.end(), compare_default);
+
+    auto devices = move(mDevices);
+    mDevices.clear();
+
+    // Create contexts and queues once the sort is done
+    for (int i = 0; i < nDevices; i++) {
+        try{
+            mContexts.push_back(make_unique<sycl::context>(*devices[i]));
+            mQueues.push_back(make_unique<sycl::queue>(
+                *mContexts.back(), *devices[i]));
+            mIsGLSharingOn.push_back(false);
+            //TODO:
+            //mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
+            //mPlatforms.push_back(getPlatformEnum(*devices[i]));
+            mDevices.emplace_back(std::move(devices[i]));
+        } catch (sycl::exception& err) {
+            AF_TRACE("Error creating context for device {} with error {}\n",
+                     devices[i]->get_info<sycl::info::device::name>(), err.what());
+        }
+    }
+    nDevices = mDevices.size();
+
+    bool default_device_set = false;
+    string deviceENV        = getEnvVar("AF_ONEAPI_DEFAULT_DEVICE");
+    if (!deviceENV.empty()) {
+        //TODO: handle default device from env variable
+    }
+
+    deviceENV = getEnvVar("AF_OPENCL_DEFAULT_DEVICE_TYPE");
+    if (!default_device_set && !deviceENV.empty()) {
+        //TODO: handle default device by type env variable
+    }
+
+    // Define AF_DISABLE_GRAPHICS with any value to disable initialization
+    string noGraphicsENV = getEnvVar("AF_DISABLE_GRAPHICS");
+    if (fgMngr->plugin().isLoaded() && noGraphicsENV.empty()) {
+        //TODO: handle forge shared contexts
+    }
+
+    mUserDeviceOffset = mDevices.size();
+
+    // TODO: init other needed libraries?
+    // blas? program cache?
+    // AF_TRACE("Default device: {}", getActiveDeviceId());
 }
 
 spdlog::logger* DeviceManager::getLogger() { return logger.get(); }
 
 DeviceManager& DeviceManager::getInstance() {
-    ONEAPI_NOT_SUPPORTED("");
     static auto* my_instance = new DeviceManager();
     return *my_instance;
 }
 
 void DeviceManager::setMemoryManager(
     std::unique_ptr<MemoryManagerBase> newMgr) {
-    ONEAPI_NOT_SUPPORTED("");
+    std::lock_guard<std::mutex> l(mutex);
+    // It's possible we're setting a memory manager and the default memory
+    // manager still hasn't been initialized, so initialize it anyways so we
+    // don't inadvertently reset to it when we first call memoryManager()
+    memoryManager();
+    // Calls shutdown() on the existing memory manager.
+    if (memManager) { memManager->shutdownAllocator(); }
+    memManager = std::move(newMgr);
+    // Set the backend memory manager for this new manager to register native
+    // functions correctly.
+    std::unique_ptr<oneapi::Allocator> deviceMemoryManager(
+        new oneapi::Allocator());
+    memManager->setAllocator(std::move(deviceMemoryManager));
+    memManager->initialize();
 }
 
 void DeviceManager::resetMemoryManager() {
-    ONEAPI_NOT_SUPPORTED("");
+    // Replace with default memory manager
+    std::unique_ptr<MemoryManagerBase> mgr(
+        new common::DefaultMemoryManager(getDeviceCount(), common::MAX_BUFFERS,
+                                         AF_MEM_DEBUG || AF_ONEAPI_MEM_DEBUG));
+    setMemoryManager(std::move(mgr));
 }
 
 void DeviceManager::setMemoryManagerPinned(
     std::unique_ptr<MemoryManagerBase> newMgr) {
-    ONEAPI_NOT_SUPPORTED("");
+    std::lock_guard<std::mutex> l(mutex);
+    // It's possible we're setting a pinned memory manager and the default
+    // memory manager still hasn't been initialized, so initialize it anyways so
+    // we don't inadvertently reset to it when we first call
+    // pinnedMemoryManager()
+    pinnedMemoryManager();
+    // Calls shutdown() on the existing memory manager.
+    if (pinnedMemManager) { pinnedMemManager->shutdownAllocator(); }
+    // Set the backend pinned memory manager for this new manager to register
+    // native functions correctly.
+    pinnedMemManager = std::move(newMgr);
+    std::unique_ptr<oneapi::AllocatorPinned> deviceMemoryManager(
+        new oneapi::AllocatorPinned());
+    pinnedMemManager->setAllocator(std::move(deviceMemoryManager));
+    pinnedMemManager->initialize();
 }
 
 void DeviceManager::resetMemoryManagerPinned() {
-    ONEAPI_NOT_SUPPORTED("");
+    // Replace with default memory manager
+    std::unique_ptr<MemoryManagerBase> mgr(
+        new common::DefaultMemoryManager(getDeviceCount(), common::MAX_BUFFERS,
+                                         AF_MEM_DEBUG || AF_ONEAPI_MEM_DEBUG));
+    setMemoryManagerPinned(std::move(mgr));
 }
 
 DeviceManager::~DeviceManager() {
-    ONEAPI_NOT_SUPPORTED("");
+    for (int i = 0; i < getDeviceCount(); ++i) { gfxManagers[i] = nullptr; }
+    memManager       = nullptr;
+    pinnedMemManager = nullptr;
+
+    // TODO: cleanup mQueues, mContexts, mDevices??
 }
 
 void DeviceManager::markDeviceForInterop(const int device,
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
index b4f291afc2..ab6804789a 100644
--- a/src/backend/oneapi/device_manager.hpp
+++ b/src/backend/oneapi/device_manager.hpp
@@ -15,15 +15,10 @@
 #include <string>
 #include <vector>
 
-#ifndef AF_OPENCL_MEM_DEBUG
-#define AF_OPENCL_MEM_DEBUG 0
+#ifndef AF_ONEAPI_MEM_DEBUG
+#define AF_ONEAPI_MEM_DEBUG 0
 #endif
 
-namespace boost {
-template<typename T>
-class shared_ptr;
-}  // namespace boost
-
 namespace spdlog {
 class logger;
 }
@@ -71,8 +66,6 @@ class DeviceManager {
 
     friend GraphicsResourceManager& interopManager();
 
-    //friend PlanCache& fftManager();
-
     friend void addKernelToCache(int device, const std::string& key,
                                  const kc_entry_t entry);
 
diff --git a/src/backend/oneapi/memory.hpp b/src/backend/oneapi/memory.hpp
index 2e18a13ae4..bb0e9f181e 100644
--- a/src/backend/oneapi/memory.hpp
+++ b/src/backend/oneapi/memory.hpp
@@ -45,36 +45,36 @@ void memLock(const sycl::buffer<T> *ptr);
 template<typename T>
 void memUnlock(const sycl::buffer<T> *ptr);
 
-  bool isLocked(const void *ptr);
-
-  template<typename T>
-  T *pinnedAlloc(const size_t &elements);
-  template<typename T>
-  void pinnedFree(T *ptr);
-
-  void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
-                        size_t *lock_bytes, size_t *lock_buffers);
-  void signalMemoryCleanup();
-  void shutdownMemoryManager();
-  void pinnedGarbageCollect();
-
-  void printMemInfo(const char *msg, const int device);
-
-  float getMemoryPressure();
-  float getMemoryPressureThreshold();
-  bool jitTreeExceedsMemoryPressure(size_t bytes);
-  void setMemStepSize(size_t step_bytes);
-  size_t getMemStepSize(void);
-
-  class Allocator final : public common::memory::AllocatorInterface {
-     public:
-      Allocator();
-      ~Allocator() = default;
-      void shutdown() override;
-      int getActiveDeviceId() override;
-      size_t getMaxMemorySize(int id) override;
-      void *nativeAlloc(const size_t bytes) override;
-      void nativeFree(void *ptr) override;
+bool isLocked(const void *ptr);
+
+template<typename T>
+T *pinnedAlloc(const size_t &elements);
+template<typename T>
+void pinnedFree(T *ptr);
+
+void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
+                      size_t *lock_bytes, size_t *lock_buffers);
+void signalMemoryCleanup();
+void shutdownMemoryManager();
+void pinnedGarbageCollect();
+
+void printMemInfo(const char *msg, const int device);
+
+float getMemoryPressure();
+float getMemoryPressureThreshold();
+bool jitTreeExceedsMemoryPressure(size_t bytes);
+void setMemStepSize(size_t step_bytes);
+size_t getMemStepSize(void);
+
+class Allocator final : public common::memory::AllocatorInterface {
+    public:
+    Allocator();
+    ~Allocator() = default;
+    void shutdown() override;
+    int getActiveDeviceId() override;
+    size_t getMaxMemorySize(int id) override;
+    void *nativeAlloc(const size_t bytes) override;
+    void nativeFree(void *ptr) override;
 };
 
 class AllocatorPinned final : public common::memory::AllocatorInterface {
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index ef28dadbdb..31e32117f5 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -40,6 +40,7 @@ using sycl::queue;
 using sycl::context;
 using sycl::device;
 using sycl::platform;
+
 using std::begin;
 using std::call_once;
 using std::end;
@@ -77,8 +78,9 @@ static string get_system() {
 #endif
 }
 
-int getBackend() { return AF_BACKEND_OPENCL; }
+int getBackend() { return AF_BACKEND_ONEAPI; }
 
+/*
 bool verify_present(const string& pname, const string ref) {
     auto iter =
         search(begin(pname), end(pname), begin(ref), end(ref),
@@ -109,6 +111,7 @@ static string platformMap(string& platStr) {
         return idx->second;
     }
 }
+*/
 
 /*
 afcl::platform getPlatformEnum(cl::Device dev) {
@@ -153,8 +156,15 @@ void setActiveContext(int device) {
     tlocalActiveDeviceId() = make_pair(device, device);
 }
 
-int getDeviceCount() noexcept {
-    ONEAPI_NOT_SUPPORTED("");
+int getDeviceCount() noexcept try {
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+    return static_cast<int>(devMngr.mQueues.size());
+} catch (const AfError& err) {
+    UNUSED(err);
+    // If device manager threw an error then return 0 because no platforms
+    // were found
     return 0;
 }
 
@@ -339,7 +349,7 @@ MemoryManagerBase& memoryManager() {
         // By default, create an instance of the default memory manager
         inst.memManager = make_unique<common::DefaultMemoryManager>(
             getDeviceCount(), common::MAX_BUFFERS,
-            AF_MEM_DEBUG || AF_OPENCL_MEM_DEBUG);
+            AF_MEM_DEBUG || AF_ONEAPI_MEM_DEBUG);
         // Set the memory manager's device memory manager
         unique_ptr<Allocator> deviceMemoryManager;
         deviceMemoryManager = make_unique<Allocator>();
@@ -350,11 +360,25 @@ MemoryManagerBase& memoryManager() {
     return *(inst.memManager.get());
 }
 
-/*
 MemoryManagerBase& pinnedMemoryManager() {
-    ONEAPI_NOT_SUPPORTED("");
+    static once_flag flag;
+
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    call_once(flag, [&]() {
+        // By default, create an instance of the default memory manager
+        inst.pinnedMemManager = make_unique<common::DefaultMemoryManager>(
+            getDeviceCount(), common::MAX_BUFFERS,
+            AF_MEM_DEBUG || AF_ONEAPI_MEM_DEBUG);
+        // Set the memory manager's device memory manager
+        unique_ptr<AllocatorPinned> deviceMemoryManager;
+        deviceMemoryManager = make_unique<AllocatorPinned>();
+        inst.pinnedMemManager->setAllocator(move(deviceMemoryManager));
+        inst.pinnedMemManager->initialize();
+    });
+
+    return *(inst.pinnedMemManager.get());
 }
-*/
 
 void setMemoryManager(unique_ptr<MemoryManagerBase> mgr) {
     ONEAPI_NOT_SUPPORTED("");
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index da33f35690..c1eea64837 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -10,7 +10,7 @@
 #pragma once
 
 #include <CL/sycl.hpp>
-//#include <af/oneapi.h>
+#include <af/oneapi.h>
 
 #include <memory>
 #include <string>

From 2aa02581787e0bbc3ece5d197f49dce5c4ce4d8d Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 15 Sep 2022 16:54:48 -0400
Subject: [PATCH 094/473] basic platform.cpp implementation

---
 include/af/oneapi.h                   |  11 +-
 src/backend/oneapi/device_manager.hpp |  10 +-
 src/backend/oneapi/platform.cpp       | 349 ++++++++++++++++++++------
 src/backend/oneapi/platform.hpp       |   2 +-
 4 files changed, 285 insertions(+), 87 deletions(-)

diff --git a/include/af/oneapi.h b/include/af/oneapi.h
index 5400a34d1a..baf28bf73b 100644
--- a/include/af/oneapi.h
+++ b/include/af/oneapi.h
@@ -19,13 +19,9 @@ extern "C" {
 #if AF_API_VERSION >= 39
 typedef enum
 {
-    //AF_ONEAPI_DEVICE_TYPE_CPU     = sycl::info::device_type::cpu,
-    //AF_ONEAPI_DEVICE_TYPE_GPU     = sycl::info::device_type::gpu,
-    //AF_ONEAPI_DEVICE_TYPE_ACC     = sycl::info::device_type::accelerator
-    //AF_ONEAPI_DEVICE_TYPE_UNKNOWN = -1
-    AF_ONEAPI_DEVICE_TYPE_CPU     = 0,
-    AF_ONEAPI_DEVICE_TYPE_GPU     = 1,
-    AF_ONEAPI_DEVICE_TYPE_ACC     = 2,
+    AF_ONEAPI_DEVICE_TYPE_CPU     = (int)sycl::info::device_type::cpu,
+    AF_ONEAPI_DEVICE_TYPE_GPU     = (int)sycl::info::device_type::gpu,
+    AF_ONEAPI_DEVICE_TYPE_ACC     = (int)sycl::info::device_type::accelerator,
     AF_ONEAPI_DEVICE_TYPE_UNKNOWN = -1
 } af_oneapi_device_type;
 #endif
@@ -33,6 +29,7 @@ typedef enum
 #if AF_API_VERSION >= 39
 typedef enum
 {
+    //TODO: update? are these relevant in sycl
     AF_ONEAPI_PLATFORM_AMD     = 0,
     AF_ONEAPI_PLATFORM_APPLE   = 1,
     AF_ONEAPI_PLATFORM_INTEL   = 2,
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
index ab6804789a..f6530dcbd9 100644
--- a/src/backend/oneapi/device_manager.hpp
+++ b/src/backend/oneapi/device_manager.hpp
@@ -98,14 +98,12 @@ class DeviceManager {
 
     friend int setDevice(int device);
 
-/*
-    friend void addDeviceContext(cl_device_id dev, cl_context ctx,
-                                 cl_command_queue que);
+    friend void addDeviceContext(sycl::device dev, sycl::context ctx,
+                                 sycl::queue que);
 
-    friend void setDeviceContext(cl_device_id dev, cl_context ctx);
+    friend void setDeviceContext(sycl::device dev, sycl::context ctx);
 
-    friend void removeDeviceContext(cl_device_id dev, cl_context ctx);
-*/
+    friend void removeDeviceContext(sycl::device dev, sycl::context ctx);
 
     friend int getActiveDeviceType();
 
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index 31e32117f5..c466ff60af 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -80,7 +80,6 @@ static string get_system() {
 
 int getBackend() { return AF_BACKEND_ONEAPI; }
 
-/*
 bool verify_present(const string& pname, const string ref) {
     auto iter =
         search(begin(pname), end(pname), begin(ref), end(ref),
@@ -91,6 +90,7 @@ bool verify_present(const string& pname, const string ref) {
     return iter != end(pname);
 }
 
+//TODO: update to new platforms?
 static string platformMap(string& platStr) {
     using strmap_t                = map<string, string>;
     static const strmap_t platMap = {
@@ -111,35 +111,80 @@ static string platformMap(string& platStr) {
         return idx->second;
     }
 }
-*/
 
-/*
-afcl::platform getPlatformEnum(cl::Device dev) {
+af_oneapi_platform getPlatformEnum(sycl::device dev) {
     string pname = getPlatformName(dev);
     if (verify_present(pname, "AMD"))
-        return AFCL_PLATFORM_AMD;
+        return AF_ONEAPI_PLATFORM_AMD;
     else if (verify_present(pname, "NVIDIA"))
-        return AFCL_PLATFORM_NVIDIA;
+        return AF_ONEAPI_PLATFORM_NVIDIA;
     else if (verify_present(pname, "INTEL"))
-        return AFCL_PLATFORM_INTEL;
+        return AF_ONEAPI_PLATFORM_INTEL;
     else if (verify_present(pname, "APPLE"))
-        return AFCL_PLATFORM_APPLE;
+        return AF_ONEAPI_PLATFORM_APPLE;
     else if (verify_present(pname, "BEIGNET"))
-        return AFCL_PLATFORM_BEIGNET;
+        return AF_ONEAPI_PLATFORM_BEIGNET;
     else if (verify_present(pname, "POCL"))
-        return AFCL_PLATFORM_POCL;
-    return AFCL_PLATFORM_UNKNOWN;
+        return AF_ONEAPI_PLATFORM_POCL;
+    return AF_ONEAPI_PLATFORM_UNKNOWN;
 }
-*/
 
 string getDeviceInfo() noexcept {
-    ONEAPI_NOT_SUPPORTED("");
-    return "";
+    ostringstream info;
+    info << "ArrayFire v" << AF_VERSION << " (OpenCL, " << get_system()
+         << ", build " << AF_REVISION << ")\n";
+
+    vector<sycl::device*> devices;
+    try {
+        DeviceManager& devMngr = DeviceManager::getInstance();
+
+        common::lock_guard_t lock(devMngr.deviceMutex);
+        unsigned nDevices = 0;
+        for (auto& device : devMngr.mDevices) {
+            //const Platform platform(device->getInfo<CL_DEVICE_PLATFORM>());
+
+            string dstr = device->get_info<sycl::info::device::name>();
+            bool show_braces =
+                (static_cast<unsigned>(getActiveDeviceId()) == nDevices);
+
+            string id = (show_braces ? string("[") : "-") +
+                        to_string(nDevices) + (show_braces ? string("]") : "-");
+
+            size_t msize = device->get_info<sycl::info::device::global_mem_size>();
+            info << id << " " << getPlatformName(*device) << ": " << ltrim(dstr)
+                 << ", " << msize / 1048576 << " MB";
+#ifndef NDEBUG
+            info << " -- ";
+            string devVersion = device->get_info<sycl::info::device::version>();
+            string driVersion = device->get_info<sycl::info::device::driver_version>();
+            info << devVersion;
+            info << " -- Device driver " << driVersion;
+            info
+                << " -- FP64 Support: "
+                << (device->get_info<sycl::info::device::preferred_vector_width_double>() >
+                            0
+                        ? "True"
+                        : "False");
+            info << " -- Unified Memory ("
+                 << (isHostUnifiedMemory(*device) ? "True" : "False") << ")";
+#endif
+            info << endl;
+
+            nDevices++;
+        }
+    } catch (const AfError& err) {
+        UNUSED(err);
+        info << "No platforms found.\n";
+        // Don't throw an exception here. Info should pass even if the system
+        // doesn't have the correct drivers installed.
+    }
+    return info.str();
 }
 
 string getPlatformName(const sycl::device& device) {
-    ONEAPI_NOT_SUPPORTED("");
-    return "";
+    std::string platStr = device.get_platform().get_info<sycl::info::platform::name>();
+    //return platformMap(platStr);
+    return platStr;
 }
 
 typedef pair<unsigned, unsigned> device_id_t;
@@ -169,12 +214,14 @@ int getDeviceCount() noexcept try {
 }
 
 void init() {
-    ONEAPI_NOT_SUPPORTED("");
+    thread_local const DeviceManager& devMngr = DeviceManager::getInstance();
+    UNUSED(devMngr);
 }
 
 unsigned getActiveDeviceId() {
-    ONEAPI_NOT_SUPPORTED("");
-    return 0;
+    // Second element is the queue id, which is
+    // what we mean by active device id in opencl backend
+    return get<1>(tlocalActiveDeviceId());
 }
 
 /*
@@ -186,27 +233,35 @@ int getDeviceIdFromNativeId(cl_device_id id) {
     int nDevices = static_cast<int>(devMngr.mDevices.size());
     int devId    = 0;
     for (devId = 0; devId < nDevices; ++devId) {
-        if (id == devMngr.mDevices[devId]->operator()()) { break; }
+        //TODO: how to get cl_device_id from sycl::device
+        if (id == devMngr.mDevices[devId]->get()) { return devId; }
     }
-
-    return devId;
+    // TODO: reasonable if no match??
+    return -1;
 }
 */
 
 int getActiveDeviceType() {
-    ONEAPI_NOT_SUPPORTED("");
-    return 0;
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mDeviceTypes[get<1>(devId)];
 }
 
 int getActivePlatform() {
-    ONEAPI_NOT_SUPPORTED("");
-    return 0;
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mPlatforms[get<1>(devId)];
 }
-const context& getContext() {
-    ONEAPI_NOT_SUPPORTED("");
-    sycl::context c;
-    return c;
-    /*
+
+const sycl::context& getContext() {
     device_id_t& devId = tlocalActiveDeviceId();
 
     DeviceManager& devMngr = DeviceManager::getInstance();
@@ -214,13 +269,9 @@ const context& getContext() {
     common::lock_guard_t lock(devMngr.deviceMutex);
 
     return *(devMngr.mContexts[get<0>(devId)]);
-    */
 }
 
 sycl::queue& getQueue() {
-    sycl::queue q;
-    return q; 
-    /*
     device_id_t& devId = tlocalActiveDeviceId();
 
     DeviceManager& devMngr = DeviceManager::getInstance();
@@ -228,13 +279,9 @@ sycl::queue& getQueue() {
     common::lock_guard_t lock(devMngr.deviceMutex);
 
     return *(devMngr.mQueues[get<1>(devId)]);
-    */
 }
 
 const sycl::device& getDevice(int id) {
-    sycl::device d;
-    return d;
-    /*
     device_id_t& devId = tlocalActiveDeviceId();
 
     if (id == -1) { id = get<1>(devId); }
@@ -243,47 +290,87 @@ const sycl::device& getDevice(int id) {
 
     common::lock_guard_t lock(devMngr.deviceMutex);
     return *(devMngr.mDevices[id]);
-    */
 }
 
 size_t getDeviceMemorySize(int device) {
-    ONEAPI_NOT_SUPPORTED("");
-    return 0;
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    sycl::device dev;
+    {
+        common::lock_guard_t lock(devMngr.deviceMutex);
+        // Assuming devices don't deallocate or are invalidated during execution
+        dev = *devMngr.mDevices[device];
+    }
+    size_t msize = dev.get_info<sycl::info::device::global_mem_size>();
+    return msize;
 }
 
 size_t getHostMemorySize() { return common::getHostMemorySize(); }
 
-/*
-cl_device_type getDeviceType() {
+sycl::info::device_type getDeviceType() {
     const sycl::device& device = getDevice();
-    cl_device_type type        = device.getInfo<CL_DEVICE_TYPE>();
+    sycl::info::device_type type = device.get_info<sycl::info::device::device_type>();
     return type;
 }
-*/
 
 bool isHostUnifiedMemory(const sycl::device& device) {
-    ONEAPI_NOT_SUPPORTED("");
-    return false;
-}
-
-bool OpenCLCPUOffload(bool forceOffloadOSX) {
-    ONEAPI_NOT_SUPPORTED("");
-    return false;
+    return device.get_info<sycl::info::device::host_unified_memory>();
+}
+
+bool OneAPICPUOffload(bool forceOffloadOSX) {
+    static const bool offloadEnv = getEnvVar("AF_ONEAPI_CPU_OFFLOAD") != "0";
+    bool offload                 = false;
+    if (offloadEnv) { offload = isHostUnifiedMemory(getDevice()); }
+#if OS_MAC
+    // FORCED OFFLOAD FOR LAPACK FUNCTIONS ON OSX UNIFIED MEMORY DEVICES
+    //
+    // On OSX Unified Memory devices (Intel), always offload LAPACK but not GEMM
+    // irrespective of the AF_OPENCL_CPU_OFFLOAD value
+    // From GEMM, OpenCLCPUOffload(false) is called which will render the
+    // variable inconsequential to the returned result.
+    //
+    // Issue https://github.com/arrayfire/arrayfire/issues/662
+    //
+    // Make sure device has unified memory
+    bool osx_offload = isHostUnifiedMemory(getDevice());
+    // Force condition
+    offload = osx_offload && (offload || forceOffloadOSX);
+#else
+    UNUSED(forceOffloadOSX);
+#endif
+    return offload;
 }
 
 bool isGLSharingSupported() {
-    ONEAPI_NOT_SUPPORTED("");
-    return false;
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mIsGLSharingOn[get<1>(devId)];
 }
 
 bool isDoubleSupported(unsigned device) {
-    ONEAPI_NOT_SUPPORTED("");
-    return false;
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    sycl::device dev;
+    {
+        common::lock_guard_t lock(devMngr.deviceMutex);
+        dev = *devMngr.mDevices[device];
+    }
+    return dev.has(sycl::aspect::fp64);
 }
 
 bool isHalfSupported(unsigned device) {
-    ONEAPI_NOT_SUPPORTED("");
-    return false;
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    sycl::device dev;
+    {
+        common::lock_guard_t lock(devMngr.deviceMutex);
+        dev = *devMngr.mDevices[device];
+    }
+    return dev.has(sycl::aspect::fp16);
 }
 
 void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute) {
@@ -291,28 +378,133 @@ void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute) {
 }
 
 int setDevice(int device) {
-    ONEAPI_NOT_SUPPORTED("");
-    return 0;
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    if (device >= static_cast<int>(devMngr.mQueues.size()) ||
+        device >= static_cast<int>(DeviceManager::MAX_DEVICES)) {
+        return -1;
+    } else {
+        int old = getActiveDeviceId();
+        setActiveContext(device);
+        return old;
+    }
 }
 
 void sync(int device) {
-    ONEAPI_NOT_SUPPORTED("");
+    int currDevice = getActiveDeviceId();
+    setDevice(device);
+    getQueue().wait();
+    setDevice(currDevice);
 }
 
 void addDeviceContext(sycl::device dev, sycl::context ctx, sycl::queue que) {
-    ONEAPI_NOT_SUPPORTED("");
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    int nDevices = 0;
+    {
+        common::lock_guard_t lock(devMngr.deviceMutex);
+
+        auto tDevice  = make_unique<sycl::device>(dev);
+        auto tContext = make_unique<sycl::context>(ctx);
+        // queue atleast has implicit context and device if created
+        auto tQueue   = make_unique<sycl::queue>(que);
+
+        devMngr.mPlatforms.push_back(getPlatformEnum(*tDevice));
+        // FIXME: add OpenGL Interop for user provided contexts later
+        devMngr.mIsGLSharingOn.push_back(false);
+        devMngr.mDeviceTypes.push_back(
+            static_cast<int>(tDevice->get_info<sycl::info::device::device_type>()));
+
+        devMngr.mDevices.push_back(move(tDevice));
+        devMngr.mContexts.push_back(move(tContext));
+        devMngr.mQueues.push_back(move(tQueue));
+        nDevices = static_cast<int>(devMngr.mDevices.size()) - 1;
+
+        //TODO: cache?
+    }
+
+    // Last/newly added device needs memory management
+    memoryManager().addMemoryManagement(nDevices);
 }
 
 void setDeviceContext(sycl::device dev, sycl::context ctx) {
-    ONEAPI_NOT_SUPPORTED("");
+    // FIXME: add OpenGL Interop for user provided contexts later
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    const int dCount = static_cast<int>(devMngr.mDevices.size());
+    for (int i = 0; i < dCount; ++i) {
+        if (*devMngr.mDevices[i] == dev &&
+            *devMngr.mContexts[i] == ctx) {
+            setActiveContext(i);
+            return;
+        }
+    }
+    AF_ERROR("No matching device found", AF_ERR_ARG);
 }
 
 void removeDeviceContext(sycl::device dev, sycl::context ctx) {
-    ONEAPI_NOT_SUPPORTED("");
+    if (getDevice() == dev && getContext() == ctx) {
+        AF_ERROR("Cannot pop the device currently in use", AF_ERR_ARG);
+    }
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    int deleteIdx = -1;
+    {
+        common::lock_guard_t lock(devMngr.deviceMutex);
+
+        const int dCount = static_cast<int>(devMngr.mDevices.size());
+        for (int i = 0; i < dCount; ++i) {
+            if (*devMngr.mDevices[i] == dev &&
+                *devMngr.mContexts[i] == ctx) {
+                deleteIdx = i;
+                break;
+            }
+        }
+    }
+
+    if (deleteIdx < static_cast<int>(devMngr.mUserDeviceOffset)) {
+        AF_ERROR("Cannot pop ArrayFire internal devices", AF_ERR_ARG);
+    } else if (deleteIdx == -1) {
+        AF_ERROR("No matching device found", AF_ERR_ARG);
+    } else {
+        // remove memory management for device added by user outside of the lock
+        memoryManager().removeMemoryManagement(deleteIdx);
+
+        common::lock_guard_t lock(devMngr.deviceMutex);
+        // FIXME: this case can potentially cause issues due to the
+        // modification of the device pool stl containers.
+
+        // IF the current active device is enumerated at a position
+        // that lies ahead of the device that has been requested
+        // to be removed. We just pop the entries from pool since it
+        // has no side effects.
+        devMngr.mDevices.erase(devMngr.mDevices.begin() + deleteIdx);
+        devMngr.mContexts.erase(devMngr.mContexts.begin() + deleteIdx);
+        devMngr.mQueues.erase(devMngr.mQueues.begin() + deleteIdx);
+        devMngr.mPlatforms.erase(devMngr.mPlatforms.begin() + deleteIdx);
+
+        // FIXME: add OpenGL Interop for user provided contexts later
+        devMngr.mIsGLSharingOn.erase(devMngr.mIsGLSharingOn.begin() +
+                                     deleteIdx);
+
+        // OTHERWISE, update(decrement) the thread local active device ids
+        device_id_t& devId = tlocalActiveDeviceId();
+
+        if (deleteIdx < static_cast<int>(devId.first)) {
+            device_id_t newVals = make_pair(devId.first - 1, devId.second - 1);
+            devId               = newVals;
+        }
+    }
 }
 
 bool synchronize_calls() {
-    return false;
+    static const bool sync = getEnvVar("AF_SYNCHRONOUS_CALLS") == "1";
+    return sync;
 }
 
 int& getMaxJitSize() {
@@ -335,7 +527,6 @@ int& getMaxJitSize() {
 }
 
 bool& evalFlag() {
-    ONEAPI_NOT_SUPPORTED("");
     thread_local bool flag = true;
     return flag;
 }
@@ -381,32 +572,44 @@ MemoryManagerBase& pinnedMemoryManager() {
 }
 
 void setMemoryManager(unique_ptr<MemoryManagerBase> mgr) {
-    ONEAPI_NOT_SUPPORTED("");
+    return DeviceManager::getInstance().setMemoryManager(move(mgr));
 }
 
 void resetMemoryManager() {
-    ONEAPI_NOT_SUPPORTED("");
+    return DeviceManager::getInstance().resetMemoryManagerPinned();
 }
 
 void setMemoryManagerPinned(unique_ptr<MemoryManagerBase> mgr) {
-    ONEAPI_NOT_SUPPORTED("");
+    return DeviceManager::getInstance().setMemoryManagerPinned(move(mgr));
 }
 
 void resetMemoryManagerPinned() {
-    ONEAPI_NOT_SUPPORTED("");
+    return DeviceManager::getInstance().resetMemoryManagerPinned();
 }
 
 graphics::ForgeManager& forgeManager() {
-    ONEAPI_NOT_SUPPORTED("");
+    return *(DeviceManager::getInstance().fgMngr);
 }
 
 GraphicsResourceManager& interopManager() {
-    ONEAPI_NOT_SUPPORTED("");
+    static once_flag initFlags[DeviceManager::MAX_DEVICES];
+
+    int id = getActiveDeviceId();
+
+    DeviceManager& inst = DeviceManager::getInstance();
+
+    call_once(initFlags[id], [&] {
+        inst.gfxManagers[id] = make_unique<GraphicsResourceManager>();
+    });
+
+    return *(inst.gfxManagers[id].get());
 }
 
 }  // namespace oneapi
 
 /*
+//TODO: select which external api functions to expose and add to header+implement
+
 using namespace oneapi;
 
 af_err afcl_get_device_type(afcl_device_type* res) {
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index c1eea64837..d82868454e 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -63,7 +63,7 @@ size_t getDeviceMemorySize(int device);
 size_t getHostMemorySize();
 
 //sycl::device::is_cpu,is_gpu,is_accelerator
-//cl_device_type getDeviceType();
+sycl::info::device_type getDeviceType();
 
 bool isHostUnifiedMemory(const sycl::device& device);
 

From 37a0b4cd4625296572597e27a20c3b7192bf1c54 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 19 Sep 2022 20:40:38 -0400
Subject: [PATCH 095/473] adds more methods to Array.cpp implementation

---
 src/backend/oneapi/Array.cpp      | 116 ++++++++++++++++--------------
 src/backend/oneapi/CMakeLists.txt |   1 +
 src/backend/oneapi/Param.cpp      |  30 ++++++++
 src/backend/oneapi/Param.hpp      |   6 +-
 src/backend/oneapi/memory.cpp     |   4 +-
 test/CMakeLists.txt               |   6 +-
 6 files changed, 103 insertions(+), 60 deletions(-)
 create mode 100644 src/backend/oneapi/Param.cpp

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 5f53e37052..b62bc8ea3e 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -122,11 +122,14 @@ Array<T>::Array(const dim4 &dims, const T *const in_data)
     static_assert(
         offsetof(Array<T>, info) == 0,
         "Array<T>::info must be the first member variable of Array<T>");
-    // TODO(oneapi): Copy to buffer
     //getQueue().enqueueWriteBuffer(*data.get(), CL_TRUE, 0,
-    //sizeof(T) * info.elements(), in_data);
+                                  //sizeof(T) * info.elements(), in_data);
+    getQueue().submit([&] (sycl::handler &h) {
+        h.copy(in_data, data->get_access(h));
+    }).wait();
 }
 
+
 template<typename T>
 Array<T>::Array(const af::dim4 &dims, buffer<T> *const mem, size_t offset,
                 bool copy)
@@ -139,12 +142,9 @@ Array<T>::Array(const af::dim4 &dims, buffer<T> *const mem, size_t offset,
     , node()
     , owner(true) {
     if (copy) {
-      //clRetainMemObject(mem);
-      //buffer src_buf = buffer(mem);
-        // TODO(oneapi): copy buffer
-        ONEAPI_NOT_SUPPORTED("Buffer constructor not implamented");
-        //getQueue().enqueueCopyBuffer(src_buf, *data.get(), src_offset, 0,
-        //sizeof(T) * info.elements());
+        getQueue().submit([&] (sycl::handler &h) {
+            h.copy(mem->get_access(h), data->get_access(h));
+        }).wait();
     }
 }
 
@@ -180,16 +180,16 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
     : info(getActiveDeviceId(), dims, offset_, strides,
            static_cast<af_dtype>(dtype_traits<T>::af_type))
     , data(is_device ? (new buffer<T>(*reinterpret_cast<buffer<T>*>(
-                                                                    const_cast<T *>(in_data))))
+                                        const_cast<T *>(in_data))))
                      : (memAlloc<T>(info.elements()).release()),
            bufferFree<T>)
     , data_dims(dims)
     , node()
     , owner(true) {
     if (!is_device) {
-      ONEAPI_NOT_SUPPORTED("Write to buffer from Host");
-          //getQueue().enqueueWriteBuffer(*data.get(), CL_TRUE, 0,
-          //sizeof(T) * info.total(), in_data);
+        getQueue().submit([&] (sycl::handler &h) {
+            h.copy(in_data, data->get_access(h));
+        }).wait();
     }
 }
 
@@ -198,18 +198,20 @@ void Array<T>::eval() {
     if (isReady()) { return; }
 
     this->setId(getActiveDeviceId());
-    data = std::shared_ptr<buffer<T>>(memAlloc<T>(info.elements()).release(),
+    data = std::shared_ptr<sycl::buffer<T>>(memAlloc<T>(info.elements()).release(),
                                        bufferFree<T>);
 
-    ONEAPI_NOT_SUPPORTED("JIT Not supported");
     // Do not replace this with cast operator
-    Param<T> info; //= {{dims()[0], dims()[1], dims()[2], dims()[3]},
-                  // {strides()[0], strides()[1], strides()[2], strides()[3]},
-                  // 0};
+    KParam info = {{dims()[0], dims()[1], dims()[2], dims()[3]},
+                   {strides()[0], strides()[1], strides()[2], strides()[3]},
+                   0};
+
+    Param<T> res{data.get(), info};
 
-    Param<T> res;// = {data.get(), info};
 
-    evalNodes(res, getNode().get());
+    //TODO: implement
+    ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
+    //evalNodes(res, getNode().get());
     node.reset();
 }
 
@@ -232,43 +234,44 @@ void evalMultiple(vector<Array<T> *> arrays) {
     vector<Array<T> *> output_arrays;
     vector<Node *> nodes;
 
-    ONEAPI_NOT_SUPPORTED("JIT Not supported");
-    // // Check if all the arrays have the same dimension
-    // auto it = std::adjacent_find(begin(arrays), end(arrays),
-    //                              [](const Array<T> *l, const Array<T> *r) {
-    //                                  return l->dims() != r->dims();
-    //                              });
-
-    // // If they are not the same. eval individually
-    // if (it != end(arrays)) {
-    //     for (auto ptr : arrays) { ptr->eval(); }
-    //     return;
-    // }
+    // Check if all the arrays have the same dimension
+    auto it = std::adjacent_find(begin(arrays), end(arrays),
+                                 [](const Array<T> *l, const Array<T> *r) {
+                                     return l->dims() != r->dims();
+                                 });
+
+    // If they are not the same. eval individually
+    if (it != end(arrays)) {
+        for (auto ptr : arrays) { ptr->eval(); }
+        return;
+    }
 
-    // for (Array<T> *array : arrays) {
-    //     if (array->isReady()) { continue; }
+    for (Array<T> *array : arrays) {
+        if (array->isReady()) { continue; }
 
-    //     const ArrayInfo info = array->info;
+        const ArrayInfo info = array->info;
 
-    //     array->setId(getActiveDeviceId());
-    //     array->data = std::shared_ptr<buffer<T>>(
-    //         memAlloc<T>(info.elements()).release(), bufferFree<T>);
+        array->setId(getActiveDeviceId());
+        array->data = std::shared_ptr<buffer<T>>(
+            memAlloc<T>(info.elements()).release(), bufferFree<T>);
 
-    //     // Do not replace this with cast operator
-    //     Param<T> kInfo = {
-    //         {info.dims()[0], info.dims()[1], info.dims()[2], info.dims()[3]},
-    //         {info.strides()[0], info.strides()[1], info.strides()[2],
-    //          info.strides()[3]},
-    //         0};
+        // Do not replace this with cast operator
+        KParam kInfo = {
+            {info.dims()[0], info.dims()[1], info.dims()[2], info.dims()[3]},
+            {info.strides()[0], info.strides()[1], info.strides()[2],
+             info.strides()[3]},
+            0};
 
-    //     outputs.emplace_back(array->data.get(), kInfo);
-    //     output_arrays.push_back(array);
-    //     nodes.push_back(array->getNode().get());
-    // }
+        outputs.emplace_back(array->data.get(), kInfo);
+        output_arrays.push_back(array);
+        nodes.push_back(array->getNode().get());
+    }
 
-    // evalNodes(outputs, nodes);
+    //TODO: implement
+    ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
+    //evalNodes(outputs, nodes);
 
-    // for (Array<T> *array : output_arrays) { array->node.reset(); }
+    for (Array<T> *array : output_arrays) { array->node.reset(); }
 }
 
 template<typename T>
@@ -383,12 +386,16 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     return kJITHeuristics::Pass;
 }
 
+//Doesn't make sense with sycl::buffer
+//TODO: accessors? or return sycl::buffer?
+//TODO: return accessor.get_pointer() for access::target::global_buffer or (host_buffer?)
 template<typename T>
 void *getDevicePtr(const Array<T> &arr) {
     const buffer<T> *buf = arr.device();
     //if (!buf) { return NULL; }
     //memLock(buf);
     //cl_mem mem = (*buf)();
+    ONEAPI_NOT_SUPPORTED("pointer to sycl::buffer should be accessor");
     return (void *)buf;
 }
 
@@ -474,8 +481,12 @@ template<typename T>
 void writeHostDataArray(Array<T> &arr, const T *const data,
                         const size_t bytes) {
     if (!arr.isOwner()) { arr = copyArray<T>(arr); }
-
-    ONEAPI_NOT_SUPPORTED("writeHostDataArray Not supported");
+    getQueue().submit([&] (sycl::handler &h) {
+        buffer<T> &buf = *arr.get();
+        //auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>)
+        auto offset_acc = buf.get_access(h);
+        h.copy(data, offset_acc);
+    }).wait();
     //getQueue().enqueueWriteBuffer(*arr.get(), CL_TRUE, arr.getOffset(), bytes,
     //data);
 }
@@ -505,14 +516,11 @@ void Array<T>::setDataDims(const dim4 &new_dims) {
 
 template<typename T>
 size_t Array<T>::getAllocatedBytes() const {
-    return 0;
-    /*
     if (!isReady()) { return 0; }
     size_t bytes = memoryManager().allocated(data.get());
     // External device pointer
     if (bytes == 0 && data.get()) { return data_dims.elements() * sizeof(T); }
     return bytes;
-    */
 }
 
 #define INSTANTIATE(T)                                                        \
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 61ce0f1eae..ed95713b67 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -16,6 +16,7 @@ add_library(afoneapi
   GraphicsResourceManager.cpp
   GraphicsResourceManager.hpp
   Module.hpp
+  Param.cpp
   Param.hpp
   all.cpp
   anisotropic_diffusion.cpp
diff --git a/src/backend/oneapi/Param.cpp b/src/backend/oneapi/Param.cpp
new file mode 100644
index 0000000000..c5d2b16762
--- /dev/null
+++ b/src/backend/oneapi/Param.cpp
@@ -0,0 +1,30 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Param.hpp>
+#include <kernel/KParam.hpp>
+#include <platform.hpp>
+#include <af/defines.h>
+
+namespace oneapi {
+
+template<typename T>
+Param<T> makeParam(sycl::buffer<T> &mem, int off, const int dims[4],
+                const int strides[4]) {
+    Param<T> out;
+    out.data        = &mem;
+    out.info.offset = off;
+    for (int i = 0; i < 4; i++) {
+        out.info.dims[i]    = dims[i];
+        out.info.strides[i] = strides[i];
+    }
+    return out;
+}
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index 0536d3dc0c..b65e28f2e7 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -23,9 +23,11 @@ struct Param {
     Param(Param&& other)                 = default;
 
     // AF_DEPRECATED("Use Array<T>")
-    Param();
+    Param() : data(nullptr), info{{0, 0, 0, 0}, {0, 0, 0, 0}, 0} {}
+
     // AF_DEPRECATED("Use Array<T>")
-    Param(sycl::buffer<T>* data_, KParam info_);
+    Param(sycl::buffer<T> *data_, KParam info_) : data(data_), info(info_) {}
+
     ~Param() = default;
 };
 
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index 2f869d3147..0eca0c9d7a 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -58,9 +58,7 @@ template<typename T>
 //unique_ptr<int, function<void(int *)>> memAlloc(
 std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>> memAlloc(
     const size_t &elements) {
-    ONEAPI_NOT_SUPPORTED("memAlloc Not supported");
-    //return unique_ptr<int, function<void(int *)>>();
-    return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>();
+    return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>(new sycl::buffer<T>(sycl::range(elements)), bufferFree<T>);
     // // TODO: make memAlloc aware of array shapes
     // if (elements) {
     //     dim4 dims(elements);
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index e2a580a1c1..aa46bdaebb 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -20,7 +20,7 @@ if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
 elseif(NOT TARGET GTest::gtest)
   af_dep_check_and_populate(${gtest_prefix}
     URI https://github.com/google/googletest.git
-    REF release-1.8.1
+    REF release-1.12.1
   )
 
   # gtest targets cmake version 2.6 which throws warnings for policy CMP0042 on
@@ -32,6 +32,7 @@ elseif(NOT TARGET GTest::gtest)
     set(BUILD_SHARED_LIBS OFF)
   endif()
 
+  add_definitions(-DGTEST_HAS_SEH=OFF)
   add_subdirectory(${${gtest_prefix}_SOURCE_DIR} ${${gtest_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
   set_target_properties(gtest gtest_main
     PROPERTIES
@@ -44,6 +45,9 @@ elseif(NOT TARGET GTest::gtest)
       target_compile_options(gtest_main PRIVATE -Wno-maybe-uninitialized)
     endif()
   endif()
+  if(WIN32)
+      target_compile_options(gtest PRIVATE -Wno-error=ignored-attributes)
+  endif()
 
   # Hide gtest project variables
   mark_as_advanced(

From b31309dff54ba4e6670d6f05b04d0c796030650a Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 20 Sep 2022 19:06:30 -0400
Subject: [PATCH 096/473] remove exception in bufferFree

---
 src/backend/oneapi/Array.cpp  |  3 ++-
 src/backend/oneapi/copy.cpp   | 33 ++++++++++++++++++++++++++++++++-
 src/backend/oneapi/memory.cpp |  6 +++---
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index b62bc8ea3e..f9d8e8e3e7 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -128,7 +128,7 @@ Array<T>::Array(const dim4 &dims, const T *const in_data)
         h.copy(in_data, data->get_access(h));
     }).wait();
 }
-
+   
 
 template<typename T>
 Array<T>::Array(const af::dim4 &dims, buffer<T> *const mem, size_t offset,
@@ -484,6 +484,7 @@ void writeHostDataArray(Array<T> &arr, const T *const data,
     getQueue().submit([&] (sycl::handler &h) {
         buffer<T> &buf = *arr.get();
         //auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>)
+        //TODO: offset accessor
         auto offset_acc = buf.get_access(h);
         h.copy(data, offset_acc);
     }).wait();
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index 5e708bb593..6ffd6bd05c 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -21,7 +21,38 @@ namespace oneapi {
 
 template<typename T>
 void copyData(T *data, const Array<T> &A) {
-    ONEAPI_NOT_SUPPORTED("");
+    /*
+    if (A.elements() == 0) { return; }
+
+    // FIXME: Merge this with copyArray
+    A.eval();
+
+    dim_t offset = 0;
+    sycl::buffer<T>* buf;
+    Array<T> out = A;
+
+    if (A.isLinear() ||  // No offsets, No strides
+        A.ndims() == 1   // Simple offset, no strides.
+    ) {
+        buf    = A.get();
+        offset = A.getOffset();
+    } else {
+        // FIXME: Think about implementing eval
+        out    = copyArray(A);
+        buf    = out.get();
+        offset = 0;
+    }sycl::access::target::device>
+
+    // FIXME: Add checks
+    getQueue().submit([&] (sycl::handler &h) {
+        //auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>)
+        //TODO: offset accessor
+        auto offset_acc = buf->get_access(h);
+        h.copy(offset_acc, data);
+    }).wait();
+    //getQueue().enqueueReadBuffer(buf, CL_TRUE, sizeof(T) * offset,
+                                 //sizeof(T) * A.elements(), data);
+    */
 }
 
 template<typename T>
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index 0eca0c9d7a..add529c8cc 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -124,9 +124,9 @@ sycl::buffer<T> *bufferAlloc(const size_t &bytes) {
 
 template<typename T>
 void bufferFree(sycl::buffer<T> *buf) {
-
-    ONEAPI_NOT_SUPPORTED("bufferFree Not supported");
-
+    if(buf) {
+        delete buf;
+    }
     // if (buf) {
     //     cl_mem mem = (*buf)();
     //     delete buf;

From bb5de0ae2e4a8eca153483381c93451d89a82068 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 21 Sep 2022 20:43:57 -0400
Subject: [PATCH 097/473] array tests passing for non-JIT non-device operations

---
 src/backend/oneapi/copy.cpp          | 91 ++++++++++++++++++++++------
 src/backend/oneapi/random_engine.cpp |  2 +-
 2 files changed, 75 insertions(+), 18 deletions(-)

diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index 6ffd6bd05c..474dfe849f 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -21,14 +21,13 @@ namespace oneapi {
 
 template<typename T>
 void copyData(T *data, const Array<T> &A) {
-    /*
     if (A.elements() == 0) { return; }
 
     // FIXME: Merge this with copyArray
     A.eval();
 
     dim_t offset = 0;
-    sycl::buffer<T>* buf;
+    const sycl::buffer<T>* buf;
     Array<T> out = A;
 
     if (A.isLinear() ||  // No offsets, No strides
@@ -41,43 +40,92 @@ void copyData(T *data, const Array<T> &A) {
         out    = copyArray(A);
         buf    = out.get();
         offset = 0;
-    }sycl::access::target::device>
+    }
 
     // FIXME: Add checks
-    getQueue().submit([&] (sycl::handler &h) {
-        //auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>)
-        //TODO: offset accessor
-        auto offset_acc = buf->get_access(h);
+    getQueue().submit([=] (sycl::handler &h) {
+        sycl::range rr(A.elements());
+        sycl::id offset_id(offset);
+        auto offset_acc = const_cast<sycl::buffer<T>*>(buf)->get_access(h, rr, offset_id);
         h.copy(offset_acc, data);
     }).wait();
-    //getQueue().enqueueReadBuffer(buf, CL_TRUE, sizeof(T) * offset,
-                                 //sizeof(T) * A.elements(), data);
-    */
 }
 
 template<typename T>
 Array<T> copyArray(const Array<T> &A) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> out = createEmptyArray<T>(dim4(1));
+    Array<T> out = createEmptyArray<T>(A.dims());
+    if (A.elements() == 0) { return out; }
+
+    dim_t offset = A.getOffset();
+    if (A.isLinear()) {
+        // FIXME: Add checks
+
+        const sycl::buffer<T>* A_buf = A.get();
+        sycl::buffer<T>* out_buf = out.get();
+
+        getQueue().submit([=] (sycl::handler &h) {
+            sycl::range rr(A.elements());
+            sycl::id offset_id(offset);
+            auto offset_acc_A  = const_cast<sycl::buffer<T>*>(A_buf)->get_access(h, rr, offset_id);
+            auto acc_out = out_buf->get_access(h);
+
+            h.copy(offset_acc_A, acc_out);
+        }).wait();
+    } else {
+        ONEAPI_NOT_SUPPORTED("");
+        /*
+        TODO:
+        kernel::memcopy<T>(*out.get(), out.strides().get(), *A.get(),
+                           A.dims().get(), A.strides().get(), offset,
+                           (uint)A.ndims());
+        */
+    }
     return out;
 }
 
 template<typename T>
 void multiply_inplace(Array<T> &in, double val) {
     ONEAPI_NOT_SUPPORTED("");
+    //TODO:
+    //kernel::copy<T, T>(in, in, in.ndims(), scalar<T>(0), val, true);
 }
 
 template<typename inType, typename outType>
 struct copyWrapper {
     void operator()(Array<outType> &out, Array<inType> const &in) {
-        ONEAPI_NOT_SUPPORTED("");
+        //TODO:
+        //kernel::copy<inType, outType>(out, in, in.ndims(), scalar<outType>(0),
+                                      //1, in.dims() == out.dims());
     }
 };
 
 template<typename T>
 struct copyWrapper<T, T> {
     void operator()(Array<T> &out, Array<T> const &in) {
-        ONEAPI_NOT_SUPPORTED("");
+        if (out.isLinear() && in.isLinear() &&
+            out.elements() == in.elements()) {
+
+            dim_t in_offset  = in.getOffset() * sizeof(T);
+            dim_t out_offset = out.getOffset() * sizeof(T);
+
+            const sycl::buffer<T>* in_buf = in.get();
+            sycl::buffer<T>* out_buf = out.get();
+
+            getQueue().submit([=] (sycl::handler &h) {
+                sycl::range rr(in.elements());
+                sycl::id in_offset_id(in_offset);
+                sycl::id out_offset_id(out_offset);
+
+                auto offset_acc_in  = const_cast<sycl::buffer<T>*>(in_buf)->get_access(h, rr, in_offset_id);
+                auto offset_acc_out = out_buf->get_access(h, rr, out_offset_id);
+
+                h.copy(offset_acc_in, offset_acc_out);
+            }).wait();
+        } else {
+            //TODO:
+            //kernel::copy<T, T>(out, in, in.ndims(), scalar<T>(0), 1,
+                               //in.dims() == out.dims());
+        }
     }
 };
 
@@ -85,7 +133,8 @@ template<typename inType, typename outType>
 void copyArray(Array<outType> &out, Array<inType> const &in) {
     static_assert(!(is_complex<inType>::value && !is_complex<outType>::value),
                   "Cannot copy from complex value to a non complex value");
-    ONEAPI_NOT_SUPPORTED("");
+    copyWrapper<inType, outType> copyFn;
+    copyFn(out, in);
 }
 
 #define INSTANTIATE(T)                                         \
@@ -158,8 +207,16 @@ INSTANTIATE_COPY_ARRAY_COMPLEX(cdouble)
 
 template<typename T>
 T getScalar(const Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("");
-    return (T)0;
+    T retVal{};
+
+    getQueue().submit([=] (sycl::handler &h) {
+        sycl::range rr(1);
+        sycl::id offset_id(in.getOffset());
+        auto acc_in  = const_cast<sycl::buffer<T>*>(in.get())->get_access(h, rr, offset_id);
+        h.copy(acc_in, (void*)&retVal);
+    }).wait();
+
+    return retVal;
 }
 
 #define INSTANTIATE_GETSCALAR(T) template T getScalar(const Array<T> &in);
diff --git a/src/backend/oneapi/random_engine.cpp b/src/backend/oneapi/random_engine.cpp
index 9e9e7ba305..db56d21638 100644
--- a/src/backend/oneapi/random_engine.cpp
+++ b/src/backend/oneapi/random_engine.cpp
@@ -29,7 +29,7 @@ Array<T> uniformDistribution(const af::dim4 &dims,
                              const af_random_engine_type type,
                              const uintl &seed, uintl &counter) {
 
-    ONEAPI_NOT_SUPPORTED("uniformDistribution Not supported");
+    //ONEAPI_NOT_SUPPORTED("uniformDistribution Not supported");
 
     Array<T> out = createEmptyArray<T>(dims);
     // kernel::uniformDistributionCBRNG<T>(*out.get(), out.elements(), type, seed,

From 4bfac8e8677e3d375a9665842910357822488f69 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 21 Sep 2022 21:02:06 -0400
Subject: [PATCH 098/473] turn off oneapi backend by default

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 60df46c5a3..72b2ca4317 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -76,7 +76,7 @@ include(config_ccache)
 option(AF_BUILD_CPU      "Build ArrayFire with a CPU backend"        ON)
 option(AF_BUILD_CUDA     "Build ArrayFire with a CUDA backend"       ${CUDA_FOUND})
 option(AF_BUILD_OPENCL   "Build ArrayFire with a OpenCL backend"     ${OpenCL_FOUND})
-option(AF_BUILD_ONEAPI   "Build ArrayFire with a oneAPI backend"     ${IntelDPCPP_FOUND})
+option(AF_BUILD_ONEAPI   "Build ArrayFire with a oneAPI backend"     OFF)
 option(AF_BUILD_UNIFIED  "Build Backend-Independent ArrayFire API"   ON)
 option(AF_BUILD_DOCS     "Create ArrayFire Documentation"            ${DOXYGEN_FOUND})
 option(AF_BUILD_EXAMPLES "Build Examples"                            ON)

From 4d65e6c8cb28fd29e29e915ed72015fa1866f05b Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 28 Sep 2022 21:23:31 -0400
Subject: [PATCH 099/473] adds first pass copy, iota, range kernels

---
 src/backend/oneapi/CMakeLists.txt     |   7 +
 src/backend/oneapi/copy.cpp           |  21 +-
 src/backend/oneapi/iota.cpp           |  12 +-
 src/backend/oneapi/kernel/iota.hpp    | 109 +++++++++
 src/backend/oneapi/kernel/memcopy.hpp | 318 ++++++++++++++++++++++++++
 src/backend/oneapi/kernel/range.hpp   | 119 ++++++++++
 src/backend/oneapi/range.cpp          |   9 +-
 7 files changed, 574 insertions(+), 21 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/iota.hpp
 create mode 100644 src/backend/oneapi/kernel/memcopy.hpp
 create mode 100644 src/backend/oneapi/kernel/range.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index ed95713b67..8cf9384b9c 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -202,6 +202,13 @@ add_library(afoneapi
   wrap.hpp
   )
 
+target_sources(afoneapi
+  PRIVATE
+    kernel/KParam.hpp
+    kernel/iota.hpp
+    kernel/memcopy.hpp
+)
+
 add_library(ArrayFire::afoneapi ALIAS afoneapi)
 
 arrayfire_set_default_cxx_flags(afoneapi)
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index 474dfe849f..622268eb91 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -9,6 +9,7 @@
 #include <copy.hpp>
 
 #include <Array.hpp>
+#include <kernel/memcopy.hpp>
 #include <common/complex.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
@@ -72,30 +73,23 @@ Array<T> copyArray(const Array<T> &A) {
             h.copy(offset_acc_A, acc_out);
         }).wait();
     } else {
-        ONEAPI_NOT_SUPPORTED("");
-        /*
-        TODO:
-        kernel::memcopy<T>(*out.get(), out.strides().get(), *A.get(),
+        kernel::memcopy<T>(out.get(), out.strides().get(), A.get(),
                            A.dims().get(), A.strides().get(), offset,
                            (uint)A.ndims());
-        */
     }
     return out;
 }
 
 template<typename T>
 void multiply_inplace(Array<T> &in, double val) {
-    ONEAPI_NOT_SUPPORTED("");
-    //TODO:
-    //kernel::copy<T, T>(in, in, in.ndims(), scalar<T>(0), val, true);
+    kernel::copy<T, T>(in, in, in.ndims(), scalar<T>(0), val, true);
 }
 
 template<typename inType, typename outType>
 struct copyWrapper {
     void operator()(Array<outType> &out, Array<inType> const &in) {
-        //TODO:
-        //kernel::copy<inType, outType>(out, in, in.ndims(), scalar<outType>(0),
-                                      //1, in.dims() == out.dims());
+        kernel::copy<inType, outType>(out, in, in.ndims(), scalar<outType>(0),
+                                      1, in.dims() == out.dims());
     }
 };
 
@@ -122,9 +116,8 @@ struct copyWrapper<T, T> {
                 h.copy(offset_acc_in, offset_acc_out);
             }).wait();
         } else {
-            //TODO:
-            //kernel::copy<T, T>(out, in, in.ndims(), scalar<T>(0), 1,
-                               //in.dims() == out.dims());
+            kernel::copy<T, T>(out, in, in.ndims(), scalar<T>(0), 1,
+                               in.dims() == out.dims());
         }
     }
 };
diff --git a/src/backend/oneapi/iota.cpp b/src/backend/oneapi/iota.cpp
index 92fbbd2ede..bb6380993b 100644
--- a/src/backend/oneapi/iota.cpp
+++ b/src/backend/oneapi/iota.cpp
@@ -7,6 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 #include <iota.hpp>
+#include <kernel/iota.hpp>
 
 #include <Array.hpp>
 #include <common/half.hpp>
@@ -20,10 +21,19 @@ using common::half;
 namespace oneapi {
 template<typename T>
 Array<T> iota(const dim4 &dims, const dim4 &tile_dims) {
-    ONEAPI_NOT_SUPPORTED("");
     dim4 outdims = dims * tile_dims;
 
     Array<T> out = createEmptyArray<T>(outdims);
+    kernel::iota<T>(out, dims);
+    return out;
+}
+
+template<>
+Array<half> iota(const dim4 &dims, const dim4 &tile_dims) {
+    ONEAPI_NOT_SUPPORTED("");
+    dim4 outdims = dims * tile_dims;
+
+    Array<half> out = createEmptyArray<half>(outdims);
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
new file mode 100644
index 0000000000..223a990d34
--- /dev/null
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -0,0 +1,109 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/half.hpp>
+#include <common/kernel_cache.hpp>
+#include <traits.hpp>
+#include <af/dim4.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+class iotaKernel {
+public:
+    iotaKernel(sycl::accessor<T> out, KParam oinfo,
+               const int s0, const int s1, const int s2, const int s3,
+               const int blocksPerMatX, const int blocksPerMatY, 
+               sycl::stream debug) :
+        out_(out), oinfo_(oinfo),
+        s0_(s0), s1_(s1), s2_(s2), s3_(s3),
+        blocksPerMatX_(blocksPerMatX), blocksPerMatY_(blocksPerMatY),
+        debug_(debug) {}
+
+    void operator() (sycl::nd_item<2> it) const {
+        //printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
+        //debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) << "]" << sycl::stream_manipulator::endl;
+
+        sycl::group gg = it.get_group();
+        const int oz = gg.get_group_id(0) / blocksPerMatX_;
+        const int ow = gg.get_group_id(1) / blocksPerMatY_;
+
+        const int blockIdx_x = gg.get_group_id(0) - oz * blocksPerMatX_;
+        const int blockIdx_y = gg.get_group_id(1) - ow * blocksPerMatY_;
+
+        const int xx = it.get_local_id(0) + blockIdx_x * gg.get_local_range(0);
+        const int yy = it.get_local_id(1) + blockIdx_y * gg.get_local_range(1);
+
+        if (xx >= oinfo_.dims[0] || yy >= oinfo_.dims[1] || oz >= oinfo_.dims[2] ||
+            ow >= oinfo_.dims[3])
+            return;
+
+        const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
+
+        T val = static_cast<T>((ow % s3_) * s2_ * s1_ * s0_);
+        val +=  static_cast<T>((oz % s2_) * s1_ * s0_);
+
+        const int incy = blocksPerMatY_ * gg.get_local_range(1);
+        const int incx = blocksPerMatX_ * gg.get_local_range(0);
+
+        for (int oy = yy; oy < oinfo_.dims[1]; oy += incy) {
+            T valY   = val + (oy % s1_) * s0_;
+            int oyzw = ozw + oy * oinfo_.strides[1];
+            for (int ox = xx; ox < oinfo_.dims[0]; ox += incx) {
+                int oidx = oyzw + ox;
+                out_[oidx] = valY + (ox % s0_);
+            }
+        }
+    }
+
+protected:
+    sycl::accessor<T> out_;
+    KParam oinfo_;
+    int s0_, s1_, s2_, s3_;
+    int blocksPerMatX_, blocksPerMatY_;
+    sycl::stream debug_;
+};
+
+template<typename T>
+void iota(Param<T> out, const af::dim4& sdims) {
+    constexpr int IOTA_TX = 32;
+    constexpr int IOTA_TY = 8;
+    constexpr int TILEX   = 512;
+    constexpr int TILEY   = 32;
+
+    sycl::range<2> local(IOTA_TX, IOTA_TY);
+
+    int blocksPerMatX = divup(out.info.dims[0], TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], TILEY);
+    sycl::range<2> global(local[0] * blocksPerMatX * out.info.dims[2],
+                          local[1] * blocksPerMatY * out.info.dims[3]);
+    sycl::nd_range<2> ndrange(global, local);
+
+    getQueue().submit([=] (sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+
+        sycl::stream debug_stream(2048, 128, h);
+
+        h.parallel_for(ndrange, iotaKernel<T>(out_acc, out.info,
+            static_cast<int>(sdims[0]), static_cast<int>(sdims[1]),
+            static_cast<int>(sdims[2]), static_cast<int>(sdims[3]),
+            blocksPerMatX, blocksPerMatY, debug_stream));
+    });
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
new file mode 100644
index 0000000000..2fae4238b2
--- /dev/null
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -0,0 +1,318 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <CL/sycl.hpp>
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/half.hpp>
+//#include <common/kernel_cache.hpp>
+#include <common/traits.hpp>
+#include <traits.hpp>
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+typedef struct {
+    dim_t dim[4];
+} dims_t;
+
+template<typename T>
+class memCopy {
+public:
+    memCopy(sycl::accessor<T> out, dims_t ostrides, 
+            sycl::accessor<T> in, dims_t idims, dims_t istrides,
+            int offset, int groups_0, int groups_1, sycl::stream debug) :
+                out_(out), ostrides_(ostrides), in_(in), idims_(idims), istrides_(istrides),
+                offset_(offset), groups_0_(groups_0), groups_1_(groups_1), debug_(debug) {}
+
+    void operator() (sycl::nd_item<2> it) const {
+        //printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
+        //debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) << "]" << sycl::stream_manipulator::endl;
+        const int lid0 = it.get_local_id(0);
+        const int lid1 = it.get_local_id(1);
+
+        sycl::group gg = it.get_group();
+        const int id2        = gg.get_group_id(0) / groups_0_;
+        const int id3        = gg.get_group_id(1) / groups_1_;
+        const int group_id_0 = gg.get_group_id(0) - groups_0_ * id2;
+        const int group_id_1 = gg.get_group_id(1) - groups_1_ * id3;
+        const int id0        = group_id_0 * gg.get_local_range(0) + lid0;
+        const int id1        = group_id_1 * gg.get_local_range(1) + lid1;
+
+        debug_ << "[" << id0 <<  "," << id1 <<  "," << id2 <<  "," << id3 <<  "]" << sycl::stream_manipulator::endl;
+
+        T* iptr = in_.get_pointer();
+        iptr += offset_;
+        // FIXME: Do more work per work group
+
+        T* optr = out_.get_pointer();
+        optr +=
+            id3 * ostrides_.dim[3] + id2 * ostrides_.dim[2] + id1 * ostrides_.dim[1];
+        iptr += id3 * istrides_.dim[3] + id2 * istrides_.dim[2] + id1 * istrides_.dim[1];
+
+        int istride0 = istrides_.dim[0];
+        if (id0 < idims_.dim[0] && id1 < idims_.dim[1] && id2 < idims_.dim[2] &&
+            id3 < idims_.dim[3]) {
+            optr[id0] = iptr[id0 * istride0];
+        }
+    }
+
+protected:
+    sycl::accessor<T> out_, in_;
+    dims_t ostrides_, idims_, istrides_;
+    int offset_, groups_0_, groups_1_;
+    sycl::stream debug_;
+};
+
+
+constexpr uint DIM0 = 32;
+constexpr uint DIM1 = 8;
+
+template<typename T>
+void memcopy(sycl::buffer<T>* out, const dim_t *ostrides, const sycl::buffer<T>* in,
+             const dim_t *idims, const dim_t *istrides, int offset,
+             uint ndims) {
+    
+    dims_t _ostrides = {{ostrides[0], ostrides[1], ostrides[2], ostrides[3]}};
+    dims_t _istrides = {{istrides[0], istrides[1], istrides[2], istrides[3]}};
+    dims_t _idims    = {{idims[0], idims[1], idims[2], idims[3]}};
+
+    size_t local_size[2] = { DIM0, DIM1 };
+    if (ndims == 1) {
+        local_size[0] *= local_size[1];
+        local_size[1] = 1;
+    }
+
+    int groups_0 = divup(idims[0], local_size[0]);
+    int groups_1 = divup(idims[1], local_size[1]);
+
+    sycl::range<2> local(local_size[0], local_size[1]);
+    sycl::range<2> global(groups_0 * idims[2] * local_size[0],
+                          groups_1 * idims[3] * local_size[1]);
+    sycl::nd_range<2> ndrange(global, local);
+
+    printf("<%d, %d> <%d, %d>\n", ndrange.get_global_range().get(0), ndrange.get_global_range().get(1), ndrange.get_local_range().get(0), ndrange.get_local_range().get(1));
+    printf("<%d, %d> ", ndrange.get_group_range().get(0), ndrange.get_group_range().get(1));
+    getQueue().submit([=] (sycl::handler &h) {
+        auto out_acc = out->get_access(h);
+        auto in_acc  = const_cast<sycl::buffer<T>*>(in)->get_access(h);
+
+        sycl::stream debug_stream(2048, 128, h);
+
+        h.parallel_for(ndrange, memCopy<T>(
+            out_acc, _ostrides, 
+            in_acc, _idims, _istrides,
+            offset, groups_0, groups_1, debug_stream));
+    });
+}
+
+template<typename T>
+static T scale(T value, double factor) {
+    return (T)(double(value) * factor);
+}
+
+template<>
+cfloat scale<cfloat>(cfloat value, double factor) {
+    return (cfloat)(value.real() * factor, value.imag() * factor);
+}
+
+template<>
+cdouble scale<cdouble>(cdouble value, double factor) {
+    return (cdouble)(value.real() * factor, value.imag() * factor);
+}
+
+template<typename inType, typename outType>
+outType convertType(inType value) {
+    return static_cast<outType>(value);
+}
+
+template<>
+char convertType<compute_t<common::half>, char>(
+    compute_t<common::half> value) {
+    return (char)((short)value);
+}
+
+template<>
+compute_t<common::half>
+convertType<char, compute_t<common::half>>(char value) {
+    return compute_t<common::half>(value);
+}
+
+template<>
+unsigned char
+convertType<compute_t<common::half>, unsigned char>(
+    compute_t<common::half> value) {
+    return (unsigned char)((short)value);
+}
+
+template<>
+compute_t<common::half>
+convertType<unsigned char, compute_t<common::half>>(unsigned char value) {
+    return compute_t<common::half>(value);
+}
+
+template<>
+cdouble convertType<cfloat, cdouble>(cfloat value) {
+    return cdouble(value.real(), value.imag());
+}
+
+template<>
+cfloat convertType<cdouble, cfloat>(cdouble value) {
+    return cfloat(value.real(), value.imag());
+}
+
+#define OTHER_SPECIALIZATIONS(IN_T)                         \
+    template<>                                              \
+    cfloat convertType<IN_T, cfloat>(IN_T value) {          \
+        return cfloat(static_cast<float>(value), 0.0f);     \
+    }                                                       \
+                                                            \
+    template<>                                              \
+    cdouble convertType<IN_T, cdouble>(IN_T value) {        \
+        return cdouble(static_cast<double>(value), 0.0);    \
+    }
+
+OTHER_SPECIALIZATIONS(float)
+OTHER_SPECIALIZATIONS(double)
+OTHER_SPECIALIZATIONS(int)
+OTHER_SPECIALIZATIONS(uint)
+OTHER_SPECIALIZATIONS(intl)
+OTHER_SPECIALIZATIONS(uintl)
+OTHER_SPECIALIZATIONS(short)
+OTHER_SPECIALIZATIONS(ushort)
+OTHER_SPECIALIZATIONS(uchar)
+OTHER_SPECIALIZATIONS(char)
+OTHER_SPECIALIZATIONS(common::half)
+
+template<typename inType, typename outType, bool SAMEDIMS>
+class reshapeCopy {
+public:
+    reshapeCopy(sycl::accessor<outType> dst, KParam oInfo, 
+                sycl::accessor<inType> src, KParam iInfo,
+                outType default_value, float factor, dims_t trgt,
+                int blk_x, int blk_y, sycl::stream debug) : 
+            dst_(dst), oInfo_(oInfo), src_(src), iInfo_(iInfo),
+            default_value_(default_value), factor_(factor), trgt_(trgt),
+            blk_x_(blk_x), blk_y_(blk_y), debug_(debug) {}
+
+    void operator() (sycl::nd_item<2> it) const {
+
+        const uint lx = it.get_local_id(0);
+        const uint ly = it.get_local_id(1);
+
+        sycl::group gg = it.get_group();
+        uint gz         = gg.get_group_id(0) / blk_x_;
+        uint gw         = gg.get_group_id(1) / blk_y_;
+        uint blockIdx_x = gg.get_group_id(0) - (blk_x_)*gz;
+        uint blockIdx_y = gg.get_group_id(1) - (blk_y_)*gw;
+        uint gx         = blockIdx_x * gg.get_local_range(0) + lx;
+        uint gy         = blockIdx_y * gg.get_local_range(1) + ly;
+
+        const inType* srcptr  = src_.get_pointer();
+        outType* dstptr = dst_.get_pointer();
+
+        const inType *in =
+            srcptr + (gw * iInfo_.strides[3] + gz * iInfo_.strides[2] +
+                      gy * iInfo_.strides[1] + iInfo_.offset);
+        outType *out = dstptr + (gw * oInfo_.strides[3] + gz * oInfo_.strides[2] +
+                                 gy * oInfo_.strides[1] + oInfo_.offset);
+
+        uint istride0 = iInfo_.strides[0];
+        uint ostride0 = oInfo_.strides[0];
+
+        if (gy < oInfo_.dims[1] && gz < oInfo_.dims[2] && gw < oInfo_.dims[3]) {
+            int loop_offset = gg.get_local_range(0) * blk_x_;
+            bool cond = gy < trgt_.dim[1] && gz < trgt_.dim[2] && gw < trgt_.dim[3];
+            for (int rep = gx; rep < oInfo_.dims[0]; rep += loop_offset) {
+                outType temp = default_value_;
+                if (SAMEDIMS || (rep < trgt_.dim[0] && cond)) {
+                    temp = convertType<inType, outType>(
+                        scale<inType>(in[rep * istride0], factor_));
+                }
+                out[rep * ostride0] = temp;
+            }
+        }
+    }
+
+protected:
+    sycl::accessor<outType> dst_;
+    sycl::accessor<inType> src_;
+    KParam oInfo_, iInfo_;
+    outType default_value_;
+    float factor_;
+    dims_t trgt_;
+    int blk_x_, blk_y_;
+    sycl::stream debug_;
+};
+
+template<typename inType, typename outType>
+void copy(Param<outType> dst, const Param<inType> src, const int ndims,
+          const outType default_value, const double factor,
+          const bool same_dims) {
+    using std::string;
+    
+    sycl::range<2> local(DIM0, DIM1);
+    size_t local_size[] = {DIM0, DIM1};
+
+    local_size[0] *= local_size[1];
+    if (ndims == 1) { local_size[1] = 1; }
+
+    int blk_x = divup(dst.info.dims[0], local_size[0]);
+    int blk_y = divup(dst.info.dims[1], local_size[1]);
+
+    sycl::range<2> global(blk_x * dst.info.dims[2] * DIM0,
+                          blk_y * dst.info.dims[3] * DIM1);
+
+    sycl::nd_range<2> ndrange(global, local);
+    printf("reshape wat?\n");
+    printf("<%d, %d> <%d, %d>\n", ndrange.get_global_range().get(0), ndrange.get_global_range().get(1), ndrange.get_local_range().get(0), ndrange.get_local_range().get(1));
+    printf("<%d, %d> ", ndrange.get_group_range().get(0), ndrange.get_group_range().get(1));
+
+    dims_t trgt_dims;
+    if (same_dims) {
+        trgt_dims = {{dst.info.dims[0], dst.info.dims[1], dst.info.dims[2],
+                      dst.info.dims[3]}};
+    } else {
+        dim_t trgt_l = std::min(dst.info.dims[3], src.info.dims[3]);
+        dim_t trgt_k = std::min(dst.info.dims[2], src.info.dims[2]);
+        dim_t trgt_j = std::min(dst.info.dims[1], src.info.dims[1]);
+        dim_t trgt_i = std::min(dst.info.dims[0], src.info.dims[0]);
+        trgt_dims    = {{trgt_i, trgt_j, trgt_k, trgt_l}};
+    }
+
+    getQueue().submit([=] (sycl::handler &h) {
+        auto dst_acc = dst.data->get_access(h);
+        auto src_acc  = const_cast<sycl::buffer<inType>*>(src.data)->get_access(h);
+
+        sycl::stream debug_stream(2048, 128, h);
+
+        if(same_dims) {
+            h.parallel_for(ndrange, reshapeCopy<inType, outType, true>(
+                    dst_acc, dst.info,
+                    src_acc, src.info,
+                    default_value, (float)factor, trgt_dims,
+                    blk_x, blk_y, debug_stream));
+        } else {
+            h.parallel_for(ndrange, reshapeCopy<inType, outType, false>(
+                    dst_acc, dst.info,
+                    src_acc, src.info,
+                    default_value, (float)factor, trgt_dims,
+                    blk_x, blk_y, debug_stream));
+        }
+    });
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
new file mode 100644
index 0000000000..0ad4797730
--- /dev/null
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -0,0 +1,119 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/half.hpp>
+#include <common/kernel_cache.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+#include <af/dim4.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+class rangeOp {
+public:
+    rangeOp(sycl::accessor<T> out, KParam oinfo, const int dim,
+               const int blocksPerMatX, const int blocksPerMatY, 
+               sycl::stream debug) :
+        out_(out), oinfo_(oinfo), dim_(dim),
+        blocksPerMatX_(blocksPerMatX), blocksPerMatY_(blocksPerMatY),
+        debug_(debug) {}
+
+    void operator() (sycl::nd_item<2> it) const {
+        //printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
+        //debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) << "]" << sycl::stream_manipulator::endl;
+
+        const int mul0 = (dim_ == 0);
+        const int mul1 = (dim_ == 1);
+        const int mul2 = (dim_ == 2);
+        const int mul3 = (dim_ == 3);
+
+        sycl::group g = it.get_group();
+        const int oz = g.get_group_id(0) / blocksPerMatX_;
+        const int ow = g.get_group_id(1) / blocksPerMatY_;
+
+        const int blockIdx_x = g.get_group_id(0) - oz * blocksPerMatX_;
+        const int blockIdx_y = g.get_group_id(1) - ow * blocksPerMatY_;
+
+        const int xx = it.get_local_id(0) + blockIdx_x * it.get_local_range(0);
+        const int yy = it.get_local_id(1) + blockIdx_y * it.get_local_range(1);
+
+        if (xx >= oinfo_.dims[0] || yy >= oinfo_.dims[1] || oz >= oinfo_.dims[2] ||
+            ow >= oinfo_.dims[3])
+            return;
+
+        const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
+
+        const int incy = blocksPerMatY_ * g.get_local_range(1);
+        const int incx = blocksPerMatX_ * g.get_local_range(0);
+
+        T valZW = (mul3 * ow) + (mul2 * oz);
+
+        T* optr = out_.get_pointer();
+        for (int oy = yy; oy < oinfo_.dims[1]; oy += incy) {
+           T valYZW = valZW + (mul1 * oy);
+            int oyzw = ozw + oy * oinfo_.strides[1];
+            for (int ox = xx; ox < oinfo_.dims[0]; ox += incx) {
+                int oidx = oyzw + ox;
+                T val    = valYZW + (mul0 * ox);
+
+                optr[oidx] = val;
+            }
+        }
+    }
+
+protected:
+    sycl::accessor<T> out_;
+    KParam oinfo_;
+    int dim_;
+    int blocksPerMatX_, blocksPerMatY_;
+    sycl::stream debug_;
+};
+
+
+template<typename T>
+void range(Param<T> out, const int dim) {
+    constexpr int RANGE_TX    = 32;
+    constexpr int RANGE_TY    = 8;
+    constexpr int RANGE_TILEX = 512;
+    constexpr int RANGE_TILEY = 32;
+
+    sycl::range<2> local(RANGE_TX, RANGE_TY);
+
+    int blocksPerMatX = divup(out.info.dims[0], RANGE_TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], RANGE_TILEY);
+    sycl::range<2> global(local[0] * blocksPerMatX * out.info.dims[2],
+                          local[1] * blocksPerMatY * out.info.dims[3]);
+    sycl::nd_range<2> ndrange(global, local);
+
+    getQueue().submit([=] (sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+
+        sycl::stream debug_stream(2048, 128, h);
+
+        h.parallel_for(ndrange, rangeOp<T>(out_acc, out.info,
+            dim, blocksPerMatX, blocksPerMatY, debug_stream));
+    });
+}
+
+template<>
+void range(Param<common::half> out, const int dim) {
+    ONEAPI_NOT_SUPPORTED("TODO: fix common::half support");
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/range.cpp b/src/backend/oneapi/range.cpp
index e47a9cc664..015ae955db 100644
--- a/src/backend/oneapi/range.cpp
+++ b/src/backend/oneapi/range.cpp
@@ -6,14 +6,14 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
-// #include <kernel/range.hpp>
 #include <range.hpp>
-#include <err_oneapi.hpp>
+#include <kernel/range.hpp>
 
 #include <Array.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
 #include <math.hpp>
+
 #include <stdexcept>
 
 using common::half;
@@ -21,9 +21,6 @@ using common::half;
 namespace oneapi {
 template<typename T>
 Array<T> range(const dim4& dim, const int seq_dim) {
-
-    ONEAPI_NOT_SUPPORTED("range Not supported");
-
     // Set dimension along which the sequence should be
     // Other dimensions are simply tiled
     int _seq_dim = seq_dim;
@@ -36,7 +33,7 @@ Array<T> range(const dim4& dim, const int seq_dim) {
     }
 
     Array<T> out = createEmptyArray<T>(dim);
-    // kernel::range<T>(out, _seq_dim);
+    kernel::range<T>(out, _seq_dim);
 
     return out;
 }

From d35f77c4955dd26b05488f317bf5f3dac07f0bee Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Thu, 29 Sep 2022 13:36:18 -0400
Subject: [PATCH 100/473] oneapi/transpose: kernels passes all but 5 tests (123
 pass)

missing uniform random function elsewhere in arrayfire. maxdims fails
like everywhere else. gfor fails.
---
 src/backend/oneapi/CMakeLists.txt       |   2 +
 src/backend/oneapi/kernel/transpose.hpp | 154 ++++++++++++++++++++++++
 src/backend/oneapi/transpose.cpp        |  10 +-
 3 files changed, 160 insertions(+), 6 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/transpose.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 8cf9384b9c..a511d05077 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -207,6 +207,8 @@ target_sources(afoneapi
     kernel/KParam.hpp
     kernel/iota.hpp
     kernel/memcopy.hpp
+    kernel/range.hpp
+    kernel/transpose.hpp
 )
 
 add_library(ArrayFire::afoneapi ALIAS afoneapi)
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
new file mode 100644
index 0000000000..8cc0c66fa5
--- /dev/null
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -0,0 +1,154 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+constexpr int TILE_DIM  = 32;
+constexpr int THREADS_X = TILE_DIM;
+constexpr int THREADS_Y = 256 / TILE_DIM;
+
+template<typename T>
+T getConjugate(const T &in) {
+    // For non-complex types return same
+    return in;
+}
+
+template<>
+cfloat getConjugate(const cfloat &in) {
+    return std::conj(in);
+}
+
+template<>
+cdouble getConjugate(const cdouble &in) {
+    return std::conj(in);
+}
+
+template <typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write, sycl::access::target::local>;
+
+template <typename T>
+class transposeKernel {
+public:
+  transposeKernel(sycl::accessor<T> oData, const KParam out,
+                  const sycl::accessor<T> iData, const KParam in,
+                  const int blocksPerMatX, const int blocksPerMatY,
+                  const bool conjugate, const bool IS32MULTIPLE,
+                  local_accessor<T, 1> shrdMem,
+                  sycl::stream debugStream) :
+    oData_(oData), out_(out), iData_(iData), in_(in), blocksPerMatX_(blocksPerMatX),
+    blocksPerMatY_(blocksPerMatY), conjugate_(conjugate), IS32MULTIPLE_(IS32MULTIPLE), shrdMem_(shrdMem), debugStream_(debugStream) {}
+  void operator() (sycl::nd_item<2> it) const {
+      const int shrdStride = TILE_DIM + 1;
+
+      const int oDim0 = out_.dims[0];
+      const int oDim1 = out_.dims[1];
+      const int iDim0 = in_.dims[0];
+      const int iDim1 = in_.dims[1];
+
+      // calculate strides
+      const int oStride1 = out_.strides[1];
+      const int iStride1 = in_.strides[1];
+
+      const int lx = it.get_local_id(0);
+      const int ly = it.get_local_id(1);
+
+      // batch based block Id
+      sycl::group g = it.get_group();
+      const int batchId_x  = g.get_group_id(0) / blocksPerMatX_;
+      const int blockIdx_x = (g.get_group_id(0) - batchId_x * blocksPerMatX_);
+
+      const int batchId_y  = g.get_group_id(1) / blocksPerMatY_;
+      const int blockIdx_y = (g.get_group_id(1) - batchId_y * blocksPerMatY_);
+
+      const int x0 = TILE_DIM * blockIdx_x;
+      const int y0 = TILE_DIM * blockIdx_y;
+
+      // calculate global in_dices
+      int gx = lx + x0;
+      int gy = ly + y0;
+
+      // offset in_ and out_ based on batch id
+      // also add the subBuffer offsets
+      T *iDataPtr = iData_.get_pointer(), *oDataPtr = oData_.get_pointer();
+      iDataPtr += batchId_x * in_.strides[2] + batchId_y * in_.strides[3] + in_.offset;
+      oDataPtr +=
+        batchId_x * out_.strides[2] + batchId_y * out_.strides[3] + out_.offset;
+
+      for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+        int gy_ = gy + repeat;
+        if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+          shrdMem_[(ly + repeat) * shrdStride + lx] =
+            iDataPtr[gy_ * iStride1 + gx];
+      }
+      it.barrier();
+
+      gx = lx + y0;
+      gy = ly + x0;
+
+      for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+        int gy_ = gy + repeat;
+        if (IS32MULTIPLE_ || (gx < oDim0 && gy_ < oDim1)) {
+          const T val = shrdMem_[lx * shrdStride + ly + repeat];
+          oDataPtr[gy_ * oStride1 + gx] = conjugate_ ? getConjugate(val) : val;
+        }
+      }
+  }
+private:
+  sycl::accessor<T> oData_;
+  KParam out_;
+  sycl::accessor<T> iData_;
+  KParam in_;
+  int blocksPerMatX_;
+  int blocksPerMatY_;
+  sycl::stream debugStream_;
+  bool conjugate_;
+  bool IS32MULTIPLE_;
+  local_accessor<T, 1> shrdMem_;
+};
+
+template<typename T>
+void transpose(Param<T> out, const Param<T> in, const bool conjugate, const bool IS32MULTIPLE) {
+    auto local = sycl::range{THREADS_X, THREADS_Y};
+
+    const int blk_x = divup(in.info.dims[0], TILE_DIM);
+    const int blk_y = divup(in.info.dims[1], TILE_DIM);
+
+    auto global = sycl::range{blk_x * local[0] * in.info.dims[2],
+                              blk_y * local[1] * in.info.dims[3]};
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto r = in.data->get_access(h);
+        auto q = out.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        auto shrdMem = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       transposeKernel<T>(q, out.info,
+                                          r, in.info,
+                                          blk_x, blk_y,
+                                          conjugate, IS32MULTIPLE,
+                                          shrdMem, debugStream));
+    }).wait();
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/transpose.cpp b/src/backend/oneapi/transpose.cpp
index 8384a6bfa1..0985bc48fa 100644
--- a/src/backend/oneapi/transpose.cpp
+++ b/src/backend/oneapi/transpose.cpp
@@ -6,7 +6,7 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
-// #include <kernel/transpose.hpp>
+#include <kernel/transpose.hpp>
 #include <transpose.hpp>
 #include <err_oneapi.hpp>
 
@@ -25,11 +25,9 @@ Array<T> transpose(const Array<T> &in, const bool conjugate) {
     dim4 outDims       = dim4(inDims[1], inDims[0], inDims[2], inDims[3]);
     Array<T> out       = createEmptyArray<T>(outDims);
 
-    // const bool is32multiple =
-    //     inDims[0] % kernel::TILE_DIM == 0 && inDims[1] % kernel::TILE_DIM == 0;
-
-    ONEAPI_NOT_SUPPORTED("transpose Not supported");
-    // kernel::transpose<T>(out, in, getQueue(), conjugate, is32multiple);
+    const bool is32multiple =
+        inDims[0] % kernel::TILE_DIM == 0 && inDims[1] % kernel::TILE_DIM == 0;
+    kernel::transpose<T>(out, in, conjugate, is32multiple);
 
     return out;
 }

From d06eb6024a0c41f84b39c1f420d22070852649aa Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 29 Sep 2022 17:46:40 -0400
Subject: [PATCH 101/473] adds ONEAPI_DEBUG_FINISH(q)

---
 src/backend/oneapi/debug_oneapi.hpp     | 25 +++++++++++++++++++++++++
 src/backend/oneapi/kernel/iota.hpp      |  2 ++
 src/backend/oneapi/kernel/memcopy.hpp   |  3 +++
 src/backend/oneapi/kernel/range.hpp     |  2 ++
 src/backend/oneapi/kernel/transpose.hpp |  4 +++-
 5 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 src/backend/oneapi/debug_oneapi.hpp

diff --git a/src/backend/oneapi/debug_oneapi.hpp b/src/backend/oneapi/debug_oneapi.hpp
new file mode 100644
index 0000000000..ea7cf992ee
--- /dev/null
+++ b/src/backend/oneapi/debug_oneapi.hpp
@@ -0,0 +1,25 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <platform.hpp>
+
+#ifndef NDEBUG
+
+#define ONEAPI_DEBUG_FINISH(Q) Q.wait_and_throw()
+
+#else
+
+#define ONEAPI_DEBUG_FINISH(Q)                                   \
+    do {                                                         \
+        if (oneapi::synchronize_calls()) { Q.wait_and_throw(); } \
+    } while (false);
+
+#endif
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index 223a990d34..5141726cdb 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
 #include <traits.hpp>
 #include <af/dim4.hpp>
 
@@ -103,6 +104,7 @@ void iota(Param<T> out, const af::dim4& sdims) {
             static_cast<int>(sdims[2]), static_cast<int>(sdims[3]),
             blocksPerMatX, blocksPerMatY, debug_stream));
     });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 }  // namespace kernel
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 2fae4238b2..4376ae0121 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -15,6 +15,7 @@
 #include <common/half.hpp>
 //#include <common/kernel_cache.hpp>
 #include <common/traits.hpp>
+#include <debug_oneapi.hpp>
 #include <traits.hpp>
 
 #include <algorithm>
@@ -116,6 +117,7 @@ void memcopy(sycl::buffer<T>* out, const dim_t *ostrides, const sycl::buffer<T>*
             in_acc, _idims, _istrides,
             offset, groups_0, groups_1, debug_stream));
     });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 template<typename T>
@@ -312,6 +314,7 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
                     blk_x, blk_y, debug_stream));
         }
     });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 }  // namespace kernel
diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
index 0ad4797730..3a0c447035 100644
--- a/src/backend/oneapi/kernel/range.hpp
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 #include <af/dim4.hpp>
@@ -108,6 +109,7 @@ void range(Param<T> out, const int dim) {
         h.parallel_for(ndrange, rangeOp<T>(out_acc, out.info,
             dim, blocksPerMatX, blocksPerMatY, debug_stream));
     });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 template<>
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index 8cc0c66fa5..ef87bc77b2 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <err_oneapi.hpp>
+#include <debug_oneapi.hpp>
 #include <traits.hpp>
 
 #include <string>
@@ -147,7 +148,8 @@ void transpose(Param<T> out, const Param<T> in, const bool conjugate, const bool
                                           blk_x, blk_y,
                                           conjugate, IS32MULTIPLE,
                                           shrdMem, debugStream));
-    }).wait();
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 }  // namespace kernel

From 870a7c79f184f1caf21bad252cd33188e99787f6 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 30 Sep 2022 13:52:34 -0400
Subject: [PATCH 102/473] adds assign kernel to oneapi backend

---
 src/backend/oneapi/CMakeLists.txt    |   1 +
 src/backend/oneapi/assign.cpp        |  42 ++++++++-
 src/backend/oneapi/kernel/assign.hpp | 136 +++++++++++++++++++++++++++
 3 files changed, 178 insertions(+), 1 deletion(-)
 create mode 100644 src/backend/oneapi/kernel/assign.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index a511d05077..ee3798f503 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -205,6 +205,7 @@ add_library(afoneapi
 target_sources(afoneapi
   PRIVATE
     kernel/KParam.hpp
+    kernel/assign.hpp
     kernel/iota.hpp
     kernel/memcopy.hpp
     kernel/range.hpp
diff --git a/src/backend/oneapi/assign.cpp b/src/backend/oneapi/assign.cpp
index 06e0f63abf..a41365101c 100644
--- a/src/backend/oneapi/assign.cpp
+++ b/src/backend/oneapi/assign.cpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include <assign.hpp>
+#include <kernel/assign.hpp>
 
 #include <Array.hpp>
 #include <common/half.hpp>
@@ -23,7 +24,46 @@ namespace oneapi {
 
 template<typename T>
 void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs) {
-    ONEAPI_NOT_SUPPORTED("");
+    kernel::AssignKernelParam_t p;
+    std::vector<af_seq> seqs(4, af_span);
+    // create seq vector to retrieve output
+    // dimensions, offsets & offsets
+    for (dim_t x = 0; x < 4; ++x) {
+        if (idxrs[x].isSeq) { seqs[x] = idxrs[x].idx.seq; }
+    }
+
+    // retrieve dimensions, strides and offsets
+    const dim4& dDims = out.dims();
+    // retrieve dimensions & strides for array
+    // to which rhs is being copied to
+    dim4 dstOffs  = toOffset(seqs, dDims);
+    dim4 dstStrds = toStride(seqs, dDims);
+
+    for (dim_t i = 0; i < 4; ++i) {
+        p.isSeq[i] = idxrs[i].isSeq;
+        p.offs[i]  = dstOffs[i];
+        p.strds[i] = dstStrds[i];
+    }
+
+    sycl::buffer<uint>* bPtrs[4];
+
+    std::vector<Array<uint>> idxArrs(4, createEmptyArray<uint>(dim4()));
+    // look through indexs to read af_array indexs
+    for (dim_t x = 0; x < 4; ++x) {
+        // set index pointers were applicable
+        if (!p.isSeq[x]) {
+            idxArrs[x] = castArray<uint>(idxrs[x].idx.arr);
+            bPtrs[x]   = idxArrs[x].get();
+        } else {
+            // alloc an 1-element buffer to avoid OpenCL from failing using
+            // direct buffer allocation as opposed to mem manager to avoid
+            // reference count desprepancies between different backends
+            static auto* empty = new sycl::buffer<uint>(sycl::range{1});
+            bPtrs[x] = empty;
+        }
+    }
+
+    kernel::assign<T>(out, rhs, p, bPtrs);
     return;
 }
 
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
new file mode 100644
index 0000000000..9896306cac
--- /dev/null
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -0,0 +1,136 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+typedef struct {
+    int offs[4];
+    int strds[4];
+    char isSeq[4];
+} AssignKernelParam_t;
+
+static int trimIndex(int idx, const int len) {
+    int ret_val = idx;
+    if (ret_val < 0) {
+        int offset = (abs(ret_val) - 1) % len;
+        ret_val    = offset;
+    } else if (ret_val >= len) {
+        int offset = abs(ret_val) % len;
+        ret_val    = len - offset - 1;
+    }
+    return ret_val;
+}
+
+template<typename T>
+class assignKernel {
+public:
+    assignKernel(sycl::accessor<T> out, KParam oInfo,
+                sycl::accessor<T> in, KParam iInfo, AssignKernelParam_t p,
+                sycl::accessor<uint> ptr0, sycl::accessor<uint> ptr1, 
+                sycl::accessor<uint> ptr2, sycl::accessor<uint> ptr3,
+                const int nBBS0, const int nBBS1, sycl::stream debug) :
+        out_(out), oInfo_(oInfo), in_(in), iInfo_(iInfo), p_(p),
+        ptr0_(ptr0), ptr1_(ptr1), ptr2_(ptr2), ptr3_(ptr3),
+        nBBS0_(nBBS0), nBBS1_(nBBS1), debug_(debug) {}
+
+
+    void operator() (sycl::nd_item<2> it) const {
+        // retrive booleans that tell us which index to use
+        const bool s0 = p_.isSeq[0];
+        const bool s1 = p_.isSeq[1];
+        const bool s2 = p_.isSeq[2];
+        const bool s3 = p_.isSeq[3];
+
+        sycl::group g = it.get_group();
+        const int gz = g.get_group_id(0) / nBBS0_;
+        const int gw = g.get_group_id(1) / nBBS1_;
+        const int gx =
+            g.get_local_range(0) * (g.get_group_id(0) - gz * nBBS0_) + it.get_local_id(0);
+        const int gy =
+            g.get_local_range(1) * (g.get_group_id(1) - gw * nBBS1_) + it.get_local_id(1);
+        if (gx < iInfo_.dims[0] && gy < iInfo_.dims[1] && gz < iInfo_.dims[2] &&
+            gw < iInfo_.dims[3]) {
+            // calculate pointer offsets for input
+            int i = p_.strds[0] *
+                    trimIndex(s0 ? gx + p_.offs[0] : ptr0_[gx], oInfo_.dims[0]);
+            int j = p_.strds[1] *
+                    trimIndex(s1 ? gy + p_.offs[1] : ptr1_[gy], oInfo_.dims[1]);
+            int k = p_.strds[2] *
+                    trimIndex(s2 ? gz + p_.offs[2] : ptr2_[gz], oInfo_.dims[2]);
+            int l = p_.strds[3] *
+                    trimIndex(s3 ? gw + p_.offs[3] : ptr3_[gw], oInfo_.dims[3]);
+            
+            T* iptr = in_.get_pointer();
+            // offset input and output pointers
+            const T* src =
+                iptr +
+                (gx * iInfo_.strides[0] + gy * iInfo_.strides[1] +
+                 gz * iInfo_.strides[2] + gw * iInfo_.strides[3] + iInfo_.offset);
+
+            T* optr = out_.get_pointer();
+            T* dst = optr + (i + j + k + l) + oInfo_.offset;
+            // set the output
+            dst[0] = src[0];
+        }
+    }
+
+protected:
+    sycl::accessor<T> out_, in_;
+    KParam oInfo_, iInfo_;
+    AssignKernelParam_t p_;
+    sycl::accessor<uint> ptr0_, ptr1_, ptr2_, ptr3_;
+    const int nBBS0_, nBBS1_;
+    sycl::stream debug_;
+};
+
+template<typename T>
+void assign(Param<T> out, const Param<T> in, const AssignKernelParam_t& p,
+            sycl::buffer<uint>* bPtr[4]) {
+    constexpr int THREADS_X = 32;
+    constexpr int THREADS_Y = 8;
+
+    sycl::range<2> local(THREADS_X, THREADS_Y);
+
+    int blk_x = divup(in.info.dims[0], THREADS_X);
+    int blk_y = divup(in.info.dims[1], THREADS_Y);
+
+    sycl::range<2> global(blk_x * in.info.dims[2] * THREADS_X,
+                          blk_y * in.info.dims[3] * THREADS_Y);
+
+    getQueue().submit([=] (sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto in_acc  = in.data->get_access(h);
+
+        auto bptr0 = bPtr[0]->get_access(h);
+        auto bptr1 = bPtr[1]->get_access(h);
+        auto bptr2 = bPtr[2]->get_access(h);
+        auto bptr3 = bPtr[3]->get_access(h);
+
+        sycl::stream debug_stream(2048, 128, h);
+
+        h.parallel_for(sycl::nd_range<2>(global, local), assignKernel<T>(
+            out_acc, out.info, in_acc, in.info, 
+            p, bptr0, bptr1, bptr2, bptr3,
+            blk_x, blk_y, debug_stream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+}  // namespace kernel
+}  // namespace oneapi

From b0d33df27598c3f1864578abbe03e787756b24ff Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 3 Oct 2022 21:38:37 -0400
Subject: [PATCH 103/473] adds RNG, needs half support

---
 src/backend/oneapi/Array.hpp                  |   6 +-
 src/backend/oneapi/CMakeLists.txt             |   5 +
 src/backend/oneapi/kernel/random_engine.hpp   | 205 +++++
 .../oneapi/kernel/random_engine_mersenne.hpp  | 337 ++++++++
 .../oneapi/kernel/random_engine_philox.hpp    | 183 ++++
 .../oneapi/kernel/random_engine_threefry.hpp  | 248 ++++++
 .../oneapi/kernel/random_engine_write.hpp     | 804 ++++++++++++++++++
 src/backend/oneapi/random_engine.cpp          |  86 +-
 src/backend/oneapi/types.hpp                  |   6 +-
 9 files changed, 1805 insertions(+), 75 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/random_engine.hpp
 create mode 100644 src/backend/oneapi/kernel/random_engine_mersenne.hpp
 create mode 100644 src/backend/oneapi/kernel/random_engine_philox.hpp
 create mode 100644 src/backend/oneapi/kernel/random_engine_threefry.hpp
 create mode 100644 src/backend/oneapi/kernel/random_engine_write.hpp

diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index eb010385d4..47c3c8bc7d 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -18,9 +18,9 @@
 //#include <err_opencl.hpp>
 //#include <jit/BufferNode.hpp>
 //#include <memory.hpp>
-//#include <platform.hpp>
-//#include <traits.hpp>
-//#include <types.hpp>
+#include <platform.hpp>
+#include <traits.hpp>
+#include <types.hpp>
 
 //#include <af/dim4.hpp>
 
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index ee3798f503..adb97f30ff 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -208,6 +208,11 @@ target_sources(afoneapi
     kernel/assign.hpp
     kernel/iota.hpp
     kernel/memcopy.hpp
+    kernel/random_engine.hpp
+    kernel/random_engine_write.hpp
+    kernel/random_engine_mersenne.hpp
+    kernel/random_engine_philox.hpp
+    kernel/random_engine_threefry.hpp
     kernel/range.hpp
     kernel/transpose.hpp
 )
diff --git a/src/backend/oneapi/kernel/random_engine.hpp b/src/backend/oneapi/kernel/random_engine.hpp
new file mode 100644
index 0000000000..4597b33a3a
--- /dev/null
+++ b/src/backend/oneapi/kernel/random_engine.hpp
@@ -0,0 +1,205 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <af/defines.h>
+#include <common/dispatch.hpp>
+#include <common/half.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <random_engine.hpp>
+#include <kernel/random_engine_write.hpp>
+#include <kernel/random_engine_mersenne.hpp>
+#include <kernel/random_engine_philox.hpp>
+#include <kernel/random_engine_threefry.hpp>
+#include <traits.hpp>
+#include <types.hpp>
+
+#include <functional>
+#include <string>
+#include <vector>
+
+static const int N          = 351;
+static const int TABLE_SIZE = 16;
+static const int MAX_BLOCKS = 32;
+static const int STATE_SIZE = (256 * 3);
+
+namespace oneapi {
+namespace kernel {
+
+static const uint THREADS = 256;
+static const uint THREADS_PER_GROUP = 256;
+static const uint THREADS_X         = 32;
+static const uint THREADS_Y         = THREADS_PER_GROUP / THREADS_X;
+static const uint REPEAT            = 32;
+
+template<typename T>
+void uniformDistributionCBRNG(Param<T> out,  const size_t elements,
+                              const af_random_engine_type type,
+                              const uintl &seed, uintl &counter) {
+    int threads          = THREADS;
+    int elementsPerBlock = threads * 4 * sizeof(uint) / sizeof(T);
+    int blocks           = divup(elements, elementsPerBlock);
+    uint hi              = seed >> 32;
+    uint lo              = seed;
+    uint hic             = counter >> 32;
+    uint loc             = counter;
+    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads), sycl::range<1>(threads));
+    switch (type) {
+        case AF_RANDOM_ENGINE_PHILOX_4X32_10:
+            getQueue().submit([=] (sycl::handler &h) {
+                auto out_acc = out.data->get_access(h);
+
+                sycl::stream debug_stream(2048, 128, h);
+                h.parallel_for(ndrange, 
+                    uniformPhilox<T>(out_acc,
+                                     hi, lo, hic, loc,
+                                     elementsPerBlock, elements,
+                                     debug_stream));
+            });
+            ONEAPI_DEBUG_FINISH(getQueue());
+            break;
+        case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
+            getQueue().submit([=] (sycl::handler &h) {
+                auto out_acc = out.data->get_access(h);
+
+                sycl::stream debug_stream(2048, 128, h);
+                h.parallel_for(ndrange, 
+                    uniformThreefry<T>(out_acc,
+                                       hi, lo, hic, loc,
+                                       elementsPerBlock, elements,
+                                       debug_stream));
+            });
+            ONEAPI_DEBUG_FINISH(getQueue());
+            break;
+        default:
+            AF_ERROR("Random Engine Type Not Supported", AF_ERR_NOT_SUPPORTED);
+    }
+    counter += elements;
+}
+
+template<typename T>
+void normalDistributionCBRNG(Param<T> out, const size_t elements,
+                             const af_random_engine_type type,
+                             const uintl &seed, uintl &counter) {
+    int threads          = THREADS;
+    int elementsPerBlock = threads * 4 * sizeof(uint) / sizeof(T);
+    int blocks           = divup(elements, elementsPerBlock);
+    uint hi              = seed >> 32;
+    uint lo              = seed;
+    uint hic             = counter >> 32;
+    uint loc             = counter;
+    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads), sycl::range<1>(threads));
+    switch (type) {
+        case AF_RANDOM_ENGINE_PHILOX_4X32_10:
+            getQueue().submit([=] (sycl::handler &h) {
+                auto out_acc = out.data->get_access(h);
+
+                sycl::stream debug_stream(2048, 128, h);
+                h.parallel_for(ndrange, 
+                    normalPhilox<T>(out_acc,
+                                    hi, lo, hic, loc,
+                                    elementsPerBlock, elements,
+                                    debug_stream));
+            });
+            break;
+        case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
+            getQueue().submit([=] (sycl::handler &h) {
+                auto out_acc = out.data->get_access(h);
+
+                sycl::stream debug_stream(2048, 128, h);
+                h.parallel_for(ndrange, 
+                    normalThreefry<T>(out_acc,
+                                      hi, lo, hic, loc,
+                                      elementsPerBlock, elements,
+                                      debug_stream));
+            });
+            break;
+        default:
+            AF_ERROR("Random Engine Type Not Supported", AF_ERR_NOT_SUPPORTED);
+    }
+    counter += elements;
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+void uniformDistributionMT(Param<T> out, const size_t elements,
+                           Param<uint> state, Param<uint> pos, Param<uint> sh1,
+                           Param<uint> sh2, const uint mask,
+                           Param<uint> recursion_table,
+                           Param<uint> temper_table) {
+    int threads                = THREADS;
+    int min_elements_per_block = 32 * threads * 4 * sizeof(uint) / sizeof(T);
+    int blocks                 = divup(elements, min_elements_per_block);
+    blocks                     = (blocks > BLOCKS) ? BLOCKS : blocks;
+    uint elementsPerBlock      = divup(elements, blocks);
+
+    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads), sycl::range<1>(threads));
+    getQueue().submit([=] (sycl::handler &h) {
+        auto out_acc       = out.data->get_access(h);
+        auto state_acc     = state.data->get_access(h);
+        auto pos_acc       = pos.data->get_access(h);
+        auto sh1_acc       = sh1.data->get_access(h);
+        auto sh2_acc       = sh2.data->get_access(h);
+        auto recursion_acc = sh2.data->get_access(h);
+        auto temper_acc    = sh2.data->get_access(h);
+
+        auto lstate_acc     = local_accessor<uint, 1>(STATE_SIZE, h);
+        auto lrecursion_acc = local_accessor<uint, 1>(TABLE_SIZE, h);
+        auto ltemper_acc    = local_accessor<uint, 1>(TABLE_SIZE, h);
+
+        sycl::stream debug_stream(2048, 128, h);
+        h.parallel_for(ndrange, 
+            uniformMersenne<T>(out_acc,
+                               state_acc, pos_acc, sh1_acc, sh2_acc, mask, 
+                               recursion_acc, temper_acc,
+                               lstate_acc, lrecursion_acc, ltemper_acc,
+                               elementsPerBlock, elements, debug_stream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+void normalDistributionMT(Param<T> out, const size_t elements,
+                          Param<uint> state, Param<uint> pos, Param<uint> sh1,
+                          Param<uint> sh2, const uint mask,
+                          Param<uint> recursion_table, Param<uint> temper_table) {
+    int threads                = THREADS;
+    int min_elements_per_block = 32 * threads * 4 * sizeof(uint) / sizeof(T);
+    int blocks                 = divup(elements, min_elements_per_block);
+    blocks                     = (blocks > BLOCKS) ? BLOCKS : blocks;
+    uint elementsPerBlock      = divup(elements, blocks);
+
+    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads), sycl::range<1>(threads));
+    getQueue().submit([=] (sycl::handler &h) {
+        auto out_acc       = out.data->get_access(h);
+        auto state_acc     = state.data->get_access(h);
+        auto pos_acc       = pos.data->get_access(h);
+        auto sh1_acc       = sh1.data->get_access(h);
+        auto sh2_acc       = sh2.data->get_access(h);
+        auto recursion_acc = sh2.data->get_access(h);
+        auto temper_acc    = sh2.data->get_access(h);
+
+        auto lstate_acc     = local_accessor<uint, 1>(STATE_SIZE, h);
+        auto lrecursion_acc = local_accessor<uint, 1>(TABLE_SIZE, h);
+        auto ltemper_acc    = local_accessor<uint, 1>(TABLE_SIZE, h);
+
+        sycl::stream debug_stream(2048, 128, h);
+        h.parallel_for(ndrange, 
+            normalMersenne<T>(out_acc,
+                              state_acc, pos_acc, sh1_acc, sh2_acc, mask, 
+                              recursion_acc, temper_acc,
+                              lstate_acc, lrecursion_acc, ltemper_acc,
+                              elementsPerBlock, elements, debug_stream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/random_engine_mersenne.hpp b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
new file mode 100644
index 0000000000..9fd8985ccf
--- /dev/null
+++ b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
@@ -0,0 +1,337 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+/********************************************************
+ * Copyright (c) 2009, 2010 Mutsuo Saito, Makoto Matsumoto and Hiroshima
+ * University.
+ * Copyright (c) 2011, 2012 Mutsuo Saito, Makoto Matsumoto, Hiroshima
+ * University and University of Tokyo.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ *       copyright notice, this list of conditions and the following
+ *       disclaimer in the documentation and/or other materials provided
+ *       with the distribution.
+ *     * Neither the name of the Hiroshima University, The Uinversity
+ *       of Tokyo nor the names of its contributors may be used to
+ *       endorse or promote products derived from this software without
+ *       specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *******************************************************/
+#pragma once
+#include <kernel/random_engine_write.hpp>
+
+namespace oneapi {
+namespace kernel {
+
+constexpr int N          = 351;
+constexpr int BLOCKS     = 32;
+constexpr int STATE_SIZE = (256 * 3);
+constexpr int TABLE_SIZE = 16;
+
+template <typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write, sycl::access::target::local>;
+
+
+// Utils
+static inline void read_table(uint *const sharedTable,
+                              const uint *const table,
+                              size_t groupId, size_t localId) {
+    const uint *const t = table + (groupId * TABLE_SIZE);
+    if (localId < TABLE_SIZE) { sharedTable[localId] = t[localId]; }
+}
+
+static inline void state_read(uint *const state,
+                              const uint *const gState,
+                              size_t groupRange, size_t groupId, size_t localId) {
+    const uint *const g                 = gState + (groupId * N);
+    state[STATE_SIZE - N + localId] = g[localId];
+    if (localId < N - groupRange) {
+        state[STATE_SIZE - N + groupRange + localId] =
+            g[groupRange + localId];
+    }
+}
+
+static inline void state_write(uint *const gState,
+                               const uint *const state,
+                               size_t groupRange, size_t groupId, size_t localId) {
+    uint *const g = gState + (groupId * N);
+    g[localId] = state[STATE_SIZE - N + localId];
+    if (localId < N - groupRange) {
+        g[groupRange + localId] =
+            state[STATE_SIZE - N + groupRange + localId];
+    }
+}
+
+static inline uint recursion(const uint *const recursion_table,
+                             const uint mask, const uint sh1,
+                             const uint sh2, const uint x1,
+                             const uint x2, uint y) {
+    uint x = (x1 & mask) ^ x2;
+    x ^= x << sh1;
+    y        = x ^ (y >> sh2);
+    uint mat = recursion_table[y & 0x0f];
+    return y ^ mat;
+}
+
+static inline uint temper(const uint *const temper_table,
+                          const uint v, uint t) {
+    t ^= t >> 16;
+    t ^= t >> 8;
+    uint mat = temper_table[t & 0x0f];
+    return v ^ mat;
+}
+
+// Initialization
+class initMersenneKernel {
+public:
+    initMersenneKernel(sycl::accessor<uint> state,
+                       sycl::accessor<uint> tbl,
+                       local_accessor<uint, 1> lstate,
+                       uintl seed, sycl::stream debug_stream) :
+            state_(state), tbl_(tbl), lstate_(lstate), seed_(seed), debug_(debug_stream) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+
+        const uint *ltbl = tbl_.get_pointer() + (TABLE_SIZE * g.get_group_id(0));
+        uint hidden_seed = ltbl[4] ^ (ltbl[8] << 16);
+        uint tmp         = hidden_seed;
+        tmp += tmp >> 16;
+        tmp += tmp >> 8;
+        tmp &= 0xff;
+        tmp |= tmp << 8;
+        tmp |= tmp << 16;
+        lstate_[it.get_local_id(0)] = tmp;
+        it.barrier();
+        if (it.get_local_id(0) == 0) {
+            lstate_[0] = seed_;
+            lstate_[1] = hidden_seed;
+            for (int i = 1; i < N; ++i) {
+                lstate_[i] ^=
+                    ((uint)(1812433253) * (lstate_[i - 1] ^ (lstate_[i - 1] >> 30)) + i);
+            }
+        }
+        it.barrier();
+        state_[N * g.get_group_id(0) + it.get_local_id(0)] = lstate_[it.get_local_id(0)];
+    }
+
+protected:
+    sycl::accessor<uint> state_, tbl_;
+    local_accessor<uint, 1> lstate_;
+    uintl seed_;
+    sycl::stream debug_;
+};
+
+void initMersenneState(Param<uint> state, const Param<uint> tbl, uintl seed) {
+    sycl::nd_range<1> ndrange({BLOCKS * N}, {N});
+    getQueue().submit([=] (sycl::handler &h) {
+        auto state_acc = state.data->get_access(h);
+        auto tbl_acc = tbl.data->get_access(h);
+        auto lstate_acc = local_accessor<uint, 1>(N, h);
+
+        sycl::stream debug_stream(2048, 128, h);
+        h.parallel_for(ndrange, 
+            initMersenneKernel(state_acc,
+                               tbl_acc, lstate_acc,
+                               seed, debug_stream));
+    });
+    //TODO: do we need to sync before using Mersenne generators?
+    //force wait() here?
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+
+
+template<typename T>
+class uniformMersenne {
+public:
+    uniformMersenne(sycl::accessor<T> out, sycl::accessor<uint> gState,
+                    sycl::accessor<uint> pos_tbl,
+                    sycl::accessor<uint> sh1_tbl,
+                    sycl::accessor<uint> sh2_tbl, uint mask,
+                    sycl::accessor<uint> g_recursion_table,
+                    sycl::accessor<uint> g_temper_table,
+                    //local memory caches of global state
+                    local_accessor<uint, 1> state,
+                    local_accessor<uint, 1> recursion_table,
+                    local_accessor<uint, 1> temper_table,
+                    uint elementsPerBlock, size_t elements,
+                    sycl::stream debug) :
+            out_(out), gState_(gState),
+            pos_tbl_(pos_tbl), sh1_tbl_(sh1_tbl), sh2_tbl_(sh2_tbl), mask_(mask),
+            g_recursion_table_(g_recursion_table), g_temper_table_(g_temper_table),
+            state_(state), recursion_table_(recursion_table), temper_table_(temper_table),
+            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+        uint start                    = g.get_group_id(0) * elementsPerBlock_;
+        uint end                      = start + elementsPerBlock_;
+        end                           = (end > elements_) ? elements_ : end;
+        int elementsPerBlockIteration = (g.get_local_range(0) * 4 * sizeof(uint)) / sizeof(T);
+        int iter = divup((end - start), elementsPerBlockIteration);
+
+        uint pos = pos_tbl_[it.get_group(0)];
+        uint sh1 = sh1_tbl_[it.get_group(0)];
+        uint sh2 = sh2_tbl_[it.get_group(0)];
+        state_read(state_.get_pointer(), gState_.get_pointer(), g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
+        read_table(recursion_table_.get_pointer(), g_recursion_table_.get_pointer(), g.get_group_id(0), it.get_local_id(0));
+        read_table(temper_table_.get_pointer(), g_temper_table_.get_pointer(), g.get_group_id(0), it.get_local_id(0));
+        it.barrier();
+
+        uint index = start;
+        uint o[4];
+        int offsetX1 = (STATE_SIZE - N + it.get_local_id(0)) % STATE_SIZE;
+        int offsetX2 = (STATE_SIZE - N + it.get_local_id(0) + 1) % STATE_SIZE;
+        int offsetY  = (STATE_SIZE - N + it.get_local_id(0) + pos) % STATE_SIZE;
+        int offsetT  = (STATE_SIZE - N + it.get_local_id(0) + pos - 1) % STATE_SIZE;
+        int offsetO  = it.get_local_id(0);
+
+        for (int i = 0; i < iter; ++i) {
+            for (int ii = 0; ii < 4; ++ii) {
+                uint r = recursion(recursion_table_.get_pointer(), mask_, sh1, sh2, state_[offsetX1],
+                                   state_[offsetX2], state_[offsetY]);
+                state_[offsetO] = r;
+                o[ii]          = temper(temper_table_.get_pointer(), r, state_[offsetT]);
+                offsetX1       = (offsetX1 + g.get_local_range(0)) % STATE_SIZE;
+                offsetX2       = (offsetX2 + g.get_local_range(0)) % STATE_SIZE;
+                offsetY        = (offsetY + g.get_local_range(0)) % STATE_SIZE;
+                offsetT        = (offsetT + g.get_local_range(0)) % STATE_SIZE;
+                offsetO        = (offsetO + g.get_local_range(0)) % STATE_SIZE;
+                it.barrier();
+            }
+            if (i == iter - 1) {
+                partialWriteOut128Bytes(out_.get_pointer(), index + it.get_local_id(0), g.get_local_range(0),
+                                        o[0], o[1], o[2], o[3], elements_);
+            } else {
+                writeOut128Bytes(out_.get_pointer(), index + it.get_local_id(0),
+                                 g.get_local_range(0), o[0], o[1], o[2], o[3]);
+            }
+            index += elementsPerBlockIteration;
+        }
+        state_write(gState_.get_pointer(), state_.get_pointer(), 
+                    g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
+    }
+
+protected:
+    sycl::accessor<T> out_;
+    sycl::accessor<uint> gState_;
+    sycl::accessor<uint> pos_tbl_, sh1_tbl_, sh2_tbl_;
+    uint mask_;
+    sycl::accessor<uint> g_recursion_table_, g_temper_table_;
+    local_accessor<uint, 1> state_, recursion_table_, temper_table_;
+    uint elementsPerBlock_;
+    size_t elements_;
+    sycl::stream debug_;
+};
+
+template<typename T>
+class normalMersenne {
+public:
+    normalMersenne(sycl::accessor<T> out, sycl::accessor<uint> gState,
+                   sycl::accessor<uint> pos_tbl,
+                   sycl::accessor<uint> sh1_tbl,
+                   sycl::accessor<uint> sh2_tbl, uint mask,
+                   sycl::accessor<uint> g_recursion_table,
+                   sycl::accessor<uint> g_temper_table,
+                   //local memory caches of global state
+                   local_accessor<uint, 1> state,
+                   local_accessor<uint, 1> recursion_table,
+                   local_accessor<uint, 1> temper_table,
+                   uint elementsPerBlock, size_t elements,
+                   sycl::stream debug) :
+            out_(out), gState_(gState),
+            pos_tbl_(pos_tbl), sh1_tbl_(sh1_tbl), sh2_tbl_(sh2_tbl), mask_(mask),
+            g_recursion_table_(g_recursion_table), g_temper_table_(g_temper_table),
+            state_(state), recursion_table_(recursion_table), temper_table_(temper_table),
+            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+        uint start                    = g.get_group_id(0) * elementsPerBlock_;
+        uint end                      = start + elementsPerBlock_;
+        end                           = (end > elements_) ? elements_ : end;
+        int elementsPerBlockIteration = (g.get_local_range(0) * 4 * sizeof(uint)) / sizeof(T);
+        int iter = divup((end - start), elementsPerBlockIteration);
+
+        uint pos = pos_tbl_[it.get_group(0)];
+        uint sh1 = sh1_tbl_[it.get_group(0)];
+        uint sh2 = sh2_tbl_[it.get_group(0)];
+        state_read(state_.get_pointer(), gState_.get_pointer(), g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
+        read_table(recursion_table_.get_pointer(), g_recursion_table_.get_pointer(), g.get_group_id(0), it.get_local_id(0));
+        read_table(temper_table_.get_pointer(), g_temper_table_.get_pointer(), g.get_group_id(0), it.get_local_id(0));
+        it.barrier();
+
+        uint index = start;
+        uint o[4];
+        int offsetX1 = (STATE_SIZE - N + it.get_local_id(0)) % STATE_SIZE;
+        int offsetX2 = (STATE_SIZE - N + it.get_local_id(0) + 1) % STATE_SIZE;
+        int offsetY  = (STATE_SIZE - N + it.get_local_id(0) + pos) % STATE_SIZE;
+        int offsetT  = (STATE_SIZE - N + it.get_local_id(0) + pos - 1) % STATE_SIZE;
+        int offsetO  = it.get_local_id(0);
+
+        for (int i = 0; i < iter; ++i) {
+            for (int ii = 0; ii < 4; ++ii) {
+                uint r = recursion(recursion_table_.get_pointer(), mask_, sh1, sh2, state_[offsetX1],
+                                   state_[offsetX2], state_[offsetY]);
+                state_[offsetO] = r;
+                o[ii]          = temper(temper_table_.get_pointer(), r, state_[offsetT]);
+                offsetX1       = (offsetX1 + g.get_local_range(0)) % STATE_SIZE;
+                offsetX2       = (offsetX2 + g.get_local_range(0)) % STATE_SIZE;
+                offsetY        = (offsetY + g.get_local_range(0)) % STATE_SIZE;
+                offsetT        = (offsetT + g.get_local_range(0)) % STATE_SIZE;
+                offsetO        = (offsetO + g.get_local_range(0)) % STATE_SIZE;
+                it.barrier();
+            }
+            if (i == iter - 1) {
+                partialBoxMullerWriteOut128Bytes(out_.get_pointer(), index + it.get_local_id(0), 
+                                        g.get_local_range(0), o[0], o[1], o[2], o[3], elements_);
+            } else {
+                boxMullerWriteOut128Bytes(out_.get_pointer(), index + it.get_local_id(0),
+                                          g.get_local_range(0), o[0], o[1], o[2], o[3]);
+            }
+            index += elementsPerBlockIteration;
+        }
+        state_write(gState_.get_pointer(), state_.get_pointer(), 
+                    g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
+    }
+
+protected:
+    sycl::accessor<T> out_;
+    sycl::accessor<uint> gState_;
+    sycl::accessor<uint> pos_tbl_, sh1_tbl_, sh2_tbl_;
+    uint mask_;
+    sycl::accessor<uint> g_recursion_table_, g_temper_table_;
+    local_accessor<uint, 1> state_, recursion_table_, temper_table_;
+    uint elementsPerBlock_;
+    size_t elements_;
+    sycl::stream debug_;
+};
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/random_engine_philox.hpp b/src/backend/oneapi/kernel/random_engine_philox.hpp
new file mode 100644
index 0000000000..3cb3dbd95b
--- /dev/null
+++ b/src/backend/oneapi/kernel/random_engine_philox.hpp
@@ -0,0 +1,183 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+/*******************************************************
+ * Modified version of Random123 library:
+ * https://www.deshawresearch.com/downloads/download_random123.cgi/
+ * The original copyright can be seen here:
+ *
+ * RANDOM123 LICENSE AGREEMENT
+ *
+ * Copyright 2010-2011, D. E. Shaw Research. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions, and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions, and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of D. E. Shaw Research nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *********************************************************/
+
+#pragma once
+#include <kernel/random_engine_write.hpp>
+
+namespace oneapi {
+namespace kernel {
+// Utils
+// Source of these constants :
+// github.com/DEShawResearch/Random123-Boost/blob/master/boost/random/philox.hpp
+
+constexpr uint m4x32_0 = 0xD2511F53;
+constexpr uint m4x32_1 = 0xCD9E8D57;
+constexpr uint w32_0   = 0x9E3779B9;
+constexpr uint w32_1   = 0xBB67AE85;
+
+static inline void mulhilo(uint a, uint b, uint &hi, uint &lo) {
+    hi = sycl::mul_hi(a, b);
+    lo = a * b;
+}
+
+static inline void philoxBump(uint k[2]) {
+    k[0] += w32_0;
+    k[1] += w32_1;
+}
+
+static inline void philoxRound(const uint m0, const uint m1,
+                               const uint k[2], uint c[4]) {
+    uint hi0, lo0, hi1, lo1;
+    mulhilo(m0, c[0], hi0, lo0);
+    mulhilo(m1, c[2], hi1, lo1);
+    c[0] = hi1 ^ c[1] ^ k[0];
+    c[1] = lo1;
+    c[2] = hi0 ^ c[3] ^ k[1];
+    c[3] = lo0;
+}
+
+static inline void philox(uint key[2], uint ctr[4]) {
+    // 10 Rounds
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+    philoxBump(key);
+    philoxRound(m4x32_0, m4x32_1, key, ctr);
+}
+
+template<typename T>
+class uniformPhilox {
+public:
+    uniformPhilox(sycl::accessor<T> out,
+                  uint hi, uint lo, uint hic, uint loc,
+                  uint elementsPerBlock, uint elements,
+                  sycl::stream debug_stream) :
+            out_(out), hi_(hi), lo_(lo), hic_(hic), loc_(loc),
+            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug_stream) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+
+        //debug_ << "<" << g.get_group_id(0)  << ":" << it.get_local_id(0) << "/" << g.get_group_range(0) << sycl::stream_manipulator::endl;
+        uint index  = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
+        uint key[2] = {lo_, hi_};
+        uint ctr[4] = {loc_, hic_, 0, 0};
+        ctr[0] += index;
+        ctr[1] += (ctr[0] < loc_);
+        ctr[2] += (ctr[1] < hic_);
+        T* optr = out_.get_pointer();
+        if (g.get_group_id(0) != (g.get_group_range(0) - 1)) {
+            philox(key, ctr);
+            writeOut128Bytes(optr, index, g.get_local_range(0), ctr[0], ctr[1], ctr[2], ctr[3]);
+        } else {
+            philox(key, ctr);
+            partialWriteOut128Bytes(optr, index, g.get_local_range(0), ctr[0], ctr[1], ctr[2], ctr[3],
+                                    elements_);
+        }
+    }
+
+protected:
+    sycl::accessor<T> out_;
+    uint hi_, lo_, hic_, loc_;
+    uint elementsPerBlock_, elements_;
+    sycl::stream debug_;
+};
+
+template<typename T>
+class normalPhilox {
+public:
+    normalPhilox(sycl::accessor<T> out,
+                  uint hi, uint lo, uint hic, uint loc,
+                  uint elementsPerBlock, uint elements,
+                  sycl::stream debug_stream) :
+            out_(out), hi_(hi), lo_(lo), hic_(hic), loc_(loc),
+            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug_stream) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+        //debug_ << "<" << g.get_group_id(0)  << ":" << it.get_local_id(0) << "/" << g.get_group_range(0) << sycl::stream_manipulator::endl;
+
+        uint index  = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
+        uint key[2] = {lo_, hi_};
+        uint ctr[4] = {loc_, hic_, 0, 0};
+        ctr[0] += index;
+        ctr[1] += (ctr[0] < loc_);
+        ctr[2] += (ctr[1] < hic_);
+
+        philox(key, ctr);
+
+        T* optr = out_.get_pointer();
+        if (g.get_group_id(0) != (g.get_group_range(0) - 1)) {
+            boxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), ctr[0], ctr[1], ctr[2], ctr[3]);
+        } else {
+            partialBoxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), 
+                                             ctr[0], ctr[1], ctr[2], ctr[3], elements_);
+        }
+    }
+
+protected:
+    sycl::accessor<T> out_;
+    uint hi_, lo_, hic_, loc_;
+    uint elementsPerBlock_, elements_;
+    sycl::stream debug_;
+};
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/random_engine_threefry.hpp b/src/backend/oneapi/kernel/random_engine_threefry.hpp
new file mode 100644
index 0000000000..bb93e299bc
--- /dev/null
+++ b/src/backend/oneapi/kernel/random_engine_threefry.hpp
@@ -0,0 +1,248 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+/*******************************************************
+ * Modified version of Random123 library:
+ * https://www.deshawresearch.com/downloads/download_random123.cgi/
+ * The original copyright can be seen here:
+ *
+ * RANDOM123 LICENSE AGREEMENT
+ *
+ * Copyright 2010-2011, D. E. Shaw Research. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions, and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions, and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of D. E. Shaw Research nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *********************************************************/
+
+#pragma once
+#include <kernel/random_engine_write.hpp>
+
+namespace oneapi {
+namespace kernel {
+// Utils
+// Source of these constants :
+// github.com/DEShawResearch/Random123-Boost/blob/master/boost/random/threefry.hpp
+
+static const uint SKEIN_KS_PARITY32 = 0x1BD11BDA;
+
+static const uint R0 = 13;
+static const uint R1 = 15;
+static const uint R2 = 26;
+static const uint R3 = 6;
+static const uint R4 = 17;
+static const uint R5 = 29;
+static const uint R6 = 16;
+static const uint R7 = 24;
+
+
+static inline void setSkeinParity(uint *ptr) {
+    *ptr = SKEIN_KS_PARITY32;
+}
+
+static inline uint rotL(uint x, uint N) {
+    return (x << (N & 31)) | (x >> ((32 - N) & 31));
+}
+
+void threefry(uint k[2], uint c[2], uint X[2]) {
+    uint ks[3];
+
+    setSkeinParity(&ks[2]);
+    ks[0] = k[0];
+    X[0]  = c[0];
+    ks[2] ^= k[0];
+    ks[1] = k[1];
+    X[1]  = c[1];
+    ks[2] ^= k[1];
+
+    X[0] += ks[0];
+    X[1] += ks[1];
+
+    X[0] += X[1];
+    X[1] = rotL(X[1], R0);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R1);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R2);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R3);
+    X[1] ^= X[0];
+
+    /* InjectKey(r=1) */
+    X[0] += ks[1];
+    X[1] += ks[2];
+    X[1] += 1; /* X[2-1] += r  */
+
+    X[0] += X[1];
+    X[1] = rotL(X[1], R4);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R5);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R6);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R7);
+    X[1] ^= X[0];
+
+    /* InjectKey(r=2) */
+    X[0] += ks[2];
+    X[1] += ks[0];
+    X[1] += 2;
+
+    X[0] += X[1];
+    X[1] = rotL(X[1], R0);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R1);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R2);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R3);
+    X[1] ^= X[0];
+
+    /* InjectKey(r=3) */
+    X[0] += ks[0];
+    X[1] += ks[1];
+    X[1] += 3;
+
+    X[0] += X[1];
+    X[1] = rotL(X[1], R4);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R5);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R6);
+    X[1] ^= X[0];
+    X[0] += X[1];
+    X[1] = rotL(X[1], R7);
+    X[1] ^= X[0];
+
+    /* InjectKey(r=4) */
+    X[0] += ks[1];
+    X[1] += ks[2];
+    X[1] += 4;
+}
+
+template<typename T>
+class uniformThreefry {
+public:
+    uniformThreefry(sycl::accessor<T> out,
+                    uint hi, uint lo, uint hic, uint loc,
+                    uint elementsPerBlock, uint elements,
+                    sycl::stream debug_stream) :
+            out_(out), hi_(hi), lo_(lo), hic_(hic), loc_(loc),
+            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug_stream) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+        uint index  = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
+
+        uint key[2] = {lo_, hi_};
+        uint ctr[4] = {loc_, hic_, 0, 0};
+        ctr[0] += index;
+        ctr[1] += (ctr[0] < loc_);
+        uint o[4];
+
+        threefry(key, ctr, o);
+        uint step = elementsPerBlock_ / 2;
+        ctr[0] += step;
+        ctr[1] += (ctr[0] < step);
+        threefry(key, ctr, o + 2);
+
+        T* optr = out_.get_pointer();
+        if (g.get_group_id(0) != (g.get_group_range(0) - 1)) {
+            writeOut128Bytes(optr, index, g.get_local_range(0), o[0], o[1], o[2], o[3]);
+        } else {
+            partialWriteOut128Bytes(optr, index, g.get_local_range(0), 
+                                    o[0], o[1], o[2], o[3], elements_);
+        }
+    }
+
+protected:
+    sycl::accessor<T> out_;
+    uint hi_, lo_, hic_, loc_;
+    uint elementsPerBlock_, elements_;
+    sycl::stream debug_;
+};
+
+template<typename T>
+class normalThreefry {
+public:
+    normalThreefry(sycl::accessor<T> out,
+                    uint hi, uint lo, uint hic, uint loc,
+                    uint elementsPerBlock, uint elements,
+                    sycl::stream debug_stream) :
+            out_(out), hi_(hi), lo_(lo), hic_(hic), loc_(loc),
+            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug_stream) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+        uint index  = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
+
+        uint key[2] = {lo_, hi_};
+        uint ctr[4] = {loc_, hic_, 0, 0};
+        ctr[0] += index;
+        ctr[1] += (ctr[0] < loc_);
+        uint o[4];
+
+        threefry(key, ctr, o);
+        uint step = elementsPerBlock_ / 2;
+        ctr[0] += step;
+        ctr[1] += (ctr[0] < step);
+        threefry(key, ctr, o + 2);
+
+        T* optr = out_.get_pointer();
+        if (g.get_group_id(0) != (g.get_group_range(0) - 1)) {
+            boxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), o[0], o[1], o[2], o[3]);
+        } else {
+            partialBoxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), 
+                                             o[0], o[1], o[2], o[3], elements_);
+        }
+    }
+
+protected:
+    sycl::accessor<T> out_;
+    uint hi_, lo_, hic_, loc_;
+    uint elementsPerBlock_, elements_;
+    sycl::stream debug_;
+};
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
new file mode 100644
index 0000000000..9e943eaad2
--- /dev/null
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -0,0 +1,804 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+#include <CL/sycl.hpp>
+
+namespace oneapi {
+namespace kernel {
+
+//TODO: !!!! half functions still need to be ported !!!!
+
+
+//// Conversion to half adapted from Random123
+//// #define HALF_FACTOR (1.0f) / (std::numeric_limits<ushort>::max() + (1.0f))
+//// #define HALF_HALF_FACTOR ((0.5f) * HALF_FACTOR)
+////
+//// NOTE: The following constants for half were calculated using the formulas
+//// above. This is done so that we can avoid unnecessary computations because the
+//// __half datatype is not a constexprable type. This prevents the compiler from
+//// peforming these operations at compile time.
+//#define HALF_FACTOR __ushort_as_half(0x100u)
+//#define HALF_HALF_FACTOR __ushort_as_half(0x80)
+//
+//// Conversion to half adapted from Random123
+////#define SIGNED_HALF_FACTOR                                \
+//    //((1.0f) / (std::numeric_limits<short>::max() + (1.0f)))
+////#define SIGNED_HALF_HALF_FACTOR ((0.5f) * SIGNED_HALF_FACTOR)
+////
+//// NOTE: The following constants for half were calculated using the formulas
+//// above. This is done so that we can avoid unnecessary computations because the
+//// __half datatype is not a constexprable type. This prevents the compiler from
+//// peforming these operations at compile time
+//#define SIGNED_HALF_FACTOR __ushort_as_half(0x200u)
+//#define SIGNED_HALF_HALF_FACTOR __ushort_as_half(0x100u)
+//
+///// This is the largest integer representable by fp16. We need to
+///// make sure that the value converted from ushort is smaller than this
+///// value to avoid generating infinity
+//constexpr ushort max_int_before_infinity = 65504;
+//
+//// Generates rationals in (0, 1]
+//__device__ static __half oneMinusGetHalf01(uint num) {
+//    // convert to ushort before the min operation
+//    ushort v = min(max_int_before_infinity, ushort(num));
+//#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 530
+//    return (1.0f - __half2float(__hfma(__ushort2half_rn(v), HALF_FACTOR,
+//                                       HALF_HALF_FACTOR)));
+//#else
+//    __half out = __ushort_as_half(0x3c00u) /*1.0h*/ -
+//                 __hfma(__ushort2half_rn(v), HALF_FACTOR, HALF_HALF_FACTOR);
+//    if (__hisinf(out)) printf("val: %d ushort: %d\n", num, v);
+//    return out;
+//#endif
+//}
+//
+//// Generates rationals in (0, 1]
+//__device__ static __half getHalf01(uint num) {
+//    // convert to ushort before the min operation
+//    ushort v = min(max_int_before_infinity, ushort(num));
+//    return __hfma(__ushort2half_rn(v), HALF_FACTOR, HALF_HALF_FACTOR);
+//}
+//
+//// Generates rationals in (-1, 1]
+//__device__ static __half getHalfNegative11(uint num) {
+//    // convert to ushort before the min operation
+//    ushort v = min(max_int_before_infinity, ushort(num));
+//    return __hfma(__ushort2half_rn(v), SIGNED_HALF_FACTOR,
+//                  SIGNED_HALF_HALF_FACTOR);
+//}
+//
+// Generates rationals in (0, 1]
+static float getFloat01(uint num) {
+    // Conversion to floats adapted from Random123
+    constexpr float factor =
+        ((1.0f) /
+         (static_cast<float>(std::numeric_limits<unsigned int>::max()) +
+          (1.0f)));
+    constexpr float half_factor = ((0.5f) * factor);
+
+    return sycl::fma(static_cast<float>(num), factor, half_factor);
+}
+
+// Generates rationals in (-1, 1]
+static float getFloatNegative11(uint num) {
+    // Conversion to floats adapted from Random123
+    constexpr float factor =
+        ((1.0) /
+         (static_cast<double>(std::numeric_limits<int>::max()) + (1.0)));
+    constexpr float half_factor = ((0.5f) * factor);
+
+    return sycl::fma(static_cast<float>(num), factor, half_factor);
+}
+
+// Generates rationals in (0, 1]
+static double getDouble01(uint num1, uint num2) {
+    uint64_t n1 = num1;
+    uint64_t n2 = num2;
+    n1 <<= 32;
+    uint64_t num = n1 | n2;
+    constexpr double factor =
+        ((1.0) / (std::numeric_limits<unsigned long long>::max() +
+                  static_cast<double>(1.0)));
+    constexpr double half_factor((0.5) * factor);
+
+    return sycl::fma(static_cast<double>(num), factor, half_factor);
+}
+
+// Conversion to doubles adapted from Random123
+constexpr double signed_factor =
+    ((1.0l) / (std::numeric_limits<long long>::max() + (1.0l)));
+constexpr double half_factor = ((0.5) * signed_factor);
+
+// Generates rationals in (-1, 1]
+static double getDoubleNegative11(uint num1, uint num2) {
+    uint32_t arr[2] = {num2, num1};
+    uint64_t num;
+
+    memcpy(&num, arr, sizeof(uint64_t));
+    return sycl::fma(static_cast<double>(num), signed_factor, half_factor);
+}
+
+namespace {
+//
+//#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
+//#define HALF_MATH_FUNC(OP, HALF_OP)    \
+//    template<>                         \
+//    __device__ __half OP(__half val) { \
+//        return ::HALF_OP(val);         \
+//    }
+//#else
+//#define HALF_MATH_FUNC(OP, HALF_OP)     \
+//    template<>                          \
+//    __device__ __half OP(__half val) {  \
+//        float fval = __half2float(val); \
+//        return __float2half(OP(fval));  \
+//    }
+//#endif
+//
+//#define MATH_FUNC(OP, DOUBLE_OP, FLOAT_OP, HALF_OP) \
+//    template<typename T>                            \
+//    __device__ T OP(T val);                         \
+//    template<>                                      \
+//    __device__ double OP(double val) {              \
+//        return ::DOUBLE_OP(val);                    \
+//    }                                               \
+//    template<>                                      \
+//    __device__ float OP(float val) {                \
+//        return ::FLOAT_OP(val);                     \
+//    }                                               \
+//    HALF_MATH_FUNC(OP, HALF_OP)
+//
+//MATH_FUNC(log, log, logf, hlog)
+//MATH_FUNC(sqrt, sqrt, sqrtf, hsqrt)
+//MATH_FUNC(sin, sin, sinf, hsin)
+//MATH_FUNC(cos, cos, cosf, hcos)
+//
+//template<typename T>
+//__device__ void sincos(T val, T *sptr, T *cptr);
+//
+//template<>
+//__device__ void sincos(double val, double *sptr, double *cptr) {
+//    ::sincos(val, sptr, cptr);
+//}
+//
+//template<>
+//__device__ void sincos(float val, float *sptr, float *cptr) {
+//    sincosf(val, sptr, cptr);
+//}
+//
+//template<>
+//__device__ void sincos(__half val, __half *sptr, __half *cptr) {
+//#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
+//    *sptr = sin(val);
+//    *cptr = cos(val);
+//#else
+//    float s, c;
+//    float fval = __half2float(val);
+//    sincos(fval, &s, &c);
+//    *sptr = __float2half(s);
+//    *cptr = __float2half(c);
+//#endif
+//}
+//
+template<typename T>
+void sincospi(T val, T *sptr, T *cptr) {
+    *sptr = sycl::sinpi(val);
+    *cptr = sycl::cospi(val);
+}
+
+//template<>
+//__device__ void sincospi(__half val, __half *sptr, __half *cptr) {
+//    // CUDA cannot make __half into a constexpr as of CUDA 11 so we are
+//    // converting this offline
+//#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
+//    const __half pi_val = __ushort_as_half(0x4248);  // 0x4248 == 3.14062h
+//    val *= pi_val;
+//    *sptr = sin(val);
+//    *cptr = cos(val);
+//#else
+//    float fval = __half2float(val);
+//    float s, c;
+//    sincospi(fval, &s, &c);
+//    *sptr = __float2half(s);
+//    *cptr = __float2half(c);
+//#endif
+//}
+//
+}  // namespace
+//
+template<typename T>
+constexpr T neg_two() {
+    return -2.0;
+}
+//
+//template<typename T>
+//constexpr __device__ T two_pi() {
+//    return 2.0 * PI_VAL;
+//};
+//
+template<typename Tc>
+static void boxMullerTransform(cfloat *const cOut,
+                               const Tc &r1, const Tc &r2) {
+    /*
+     * The log of a real value x where 0 < x < 1 is negative.
+     */
+    Tc r = sycl::sqrt(neg_two<Tc>() * sycl::log(r2));
+    Tc s, c;
+
+    // Multiplying by PI instead of 2*PI seems to yeild a better distribution
+    // even though the original boxMuller algorithm calls for 2 * PI
+    // sincos(two_pi<Tc>() * r1, &s, &c);
+    sincospi(r1, &s, &c);
+    cOut->real(static_cast<float>(r * s));
+    cOut->imag(static_cast<float>(r * c));
+}
+
+template<typename Tc>
+static void boxMullerTransform(cdouble *const cOut,
+                               const Tc &r1, const Tc &r2) {
+    /*
+     * The log of a real value x where 0 < x < 1 is negative.
+     */
+    Tc r = sycl::sqrt(neg_two<Tc>() * sycl::log(r2));
+    Tc s, c;
+
+    // Multiplying by PI instead of 2*PI seems to yeild a better distribution
+    // even though the original boxMuller algorithm calls for 2 * PI
+    // sincos(two_pi<Tc>() * r1, &s, &c);
+    sincospi(r1, &s, &c);
+    cOut->real(static_cast<double>(r * s));
+    cOut->imag(static_cast<double>(r * c));
+}
+
+template<typename Td, typename Tc>
+static void boxMullerTransform(Td *const out1, Td *const out2,
+                               const Tc &r1, const Tc &r2) {
+    /*
+     * The log of a real value x where 0 < x < 1 is negative.
+     */
+    Tc r = sycl::sqrt(neg_two<Tc>() * sycl::log(r2));
+    Tc s, c;
+
+    // Multiplying by PI instead of 2*PI seems to yeild a better distribution
+    // even though the original boxMuller algorithm calls for 2 * PI
+    // sincos(two_pi<Tc>() * r1, &s, &c);
+    sincospi(r1, &s, &c);
+    *out1 = static_cast<Td>(r * s);
+    *out2 = static_cast<Td>(r * c);
+}
+//template<>
+//__device__ void boxMullerTransform<common::half, __half>(
+//    common::half *const out1, common::half *const out2, const __half &r1,
+//    const __half &r2) {
+//    float o1, o2;
+//    float fr1 = __half2float(r1);
+//    float fr2 = __half2float(r2);
+//    boxMullerTransform(&o1, &o2, fr1, fr2);
+//    *out1 = o1;
+//    *out2 = o2;
+//}
+
+// Writes without boundary checking
+static void writeOut128Bytes(uchar *out, const uint &index, const uint groupSz,
+                             const uint &r1, const uint &r2,
+                             const uint &r3, const uint &r4) {
+    out[index]                = r1;
+    out[index + groupSz]      = r1 >> 8;
+    out[index + 2 * groupSz]  = r1 >> 16;
+    out[index + 3 * groupSz]  = r1 >> 24;
+    out[index + 4 * groupSz]  = r2;
+    out[index + 5 * groupSz]  = r2 >> 8;
+    out[index + 6 * groupSz]  = r2 >> 16;
+    out[index + 7 * groupSz]  = r2 >> 24;
+    out[index + 8 * groupSz]  = r3;
+    out[index + 9 * groupSz]  = r3 >> 8;
+    out[index + 10 * groupSz] = r3 >> 16;
+    out[index + 11 * groupSz] = r3 >> 24;
+    out[index + 12 * groupSz] = r4;
+    out[index + 13 * groupSz] = r4 >> 8;
+    out[index + 14 * groupSz] = r4 >> 16;
+    out[index + 15 * groupSz] = r4 >> 24;
+}
+
+static void writeOut128Bytes(char *out, const uint &index, const uint groupSz,
+                             const uint &r1, const uint &r2,
+                             const uint &r3, const uint &r4) {
+    out[index]                = (r1)&0x1;
+    out[index + groupSz]      = (r1 >> 1) & 0x1;
+    out[index + 2 * groupSz]  = (r1 >> 2) & 0x1;
+    out[index + 3 * groupSz]  = (r1 >> 3) & 0x1;
+    out[index + 4 * groupSz]  = (r2)&0x1;
+    out[index + 5 * groupSz]  = (r2 >> 1) & 0x1;
+    out[index + 6 * groupSz]  = (r2 >> 2) & 0x1;
+    out[index + 7 * groupSz]  = (r2 >> 3) & 0x1;
+    out[index + 8 * groupSz]  = (r3)&0x1;
+    out[index + 9 * groupSz]  = (r3 >> 1) & 0x1;
+    out[index + 10 * groupSz] = (r3 >> 2) & 0x1;
+    out[index + 11 * groupSz] = (r3 >> 3) & 0x1;
+    out[index + 12 * groupSz] = (r4)&0x1;
+    out[index + 13 * groupSz] = (r4 >> 1) & 0x1;
+    out[index + 14 * groupSz] = (r4 >> 2) & 0x1;
+    out[index + 15 * groupSz] = (r4 >> 3) & 0x1;
+}
+
+static void writeOut128Bytes(short *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    out[index]               = r1;
+    out[index + groupSz]     = r1 >> 16;
+    out[index + 2 * groupSz] = r2;
+    out[index + 3 * groupSz] = r2 >> 16;
+    out[index + 4 * groupSz] = r3;
+    out[index + 5 * groupSz] = r3 >> 16;
+    out[index + 6 * groupSz] = r4;
+    out[index + 7 * groupSz] = r4 >> 16;
+}
+
+static void writeOut128Bytes(ushort *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    writeOut128Bytes((short *)(out), index, groupSz, r1, r2, r3, r4);
+}
+
+static void writeOut128Bytes(int *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    out[index]               = r1;
+    out[index + groupSz]     = r2;
+    out[index + 2 * groupSz] = r3;
+    out[index + 3 * groupSz] = r4;
+}
+
+static void writeOut128Bytes(uint *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    writeOut128Bytes((int *)(out), index, groupSz, r1, r2, r3, r4);
+}
+
+static void writeOut128Bytes(intl *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    intl c1              = r2;
+    c1                   = (c1 << 32) | r1;
+    intl c2              = r4;
+    c2                   = (c2 << 32) | r3;
+    out[index]           = c1;
+    out[index + groupSz] = c2;
+}
+
+static void writeOut128Bytes(uintl *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    writeOut128Bytes((intl *)(out), index, groupSz, r1, r2, r3, r4);
+}
+
+static void writeOut128Bytes(float *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    out[index]               = 1.f - getFloat01(r1);
+    out[index + groupSz]     = 1.f - getFloat01(r2);
+    out[index + 2 * groupSz] = 1.f - getFloat01(r3);
+    out[index + 3 * groupSz] = 1.f - getFloat01(r4);
+}
+
+static void writeOut128Bytes(cfloat *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    out[index] = {1.f - getFloat01(r1), 1.f - getFloat01(r2)};
+    out[index + groupSz] = {1.f - getFloat01(r3), 1.f - getFloat01(r4)};
+}
+
+static void writeOut128Bytes(double *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    out[index]           = 1.0 - getDouble01(r1, r2);
+    out[index + groupSz] = 1.0 - getDouble01(r3, r4);
+}
+
+static void writeOut128Bytes(cdouble *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    out[index] = {1.0 - getDouble01(r1, r2), 1.0 - getDouble01(r3, r4)};
+}
+
+static void writeOut128Bytes(common::half *out, const uint &index, const uint groupSz,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+   //out[index]               = oneMinusGetHalf01(r1);
+   //out[index + groupSz]     = oneMinusGetHalf01(r1 >> 16);
+   //out[index + 2 * groupSz] = oneMinusGetHalf01(r2);
+   //out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16);
+   //out[index + 4 * groupSz] = oneMinusGetHalf01(r3);
+   //out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16);
+   //out[index + 6 * groupSz] = oneMinusGetHalf01(r4);
+   //out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16);
+}
+
+// Normalized writes without boundary checking
+
+static void boxMullerWriteOut128Bytes(float *out, const uint &index, const uint groupSz,
+                                                 const uint &r1, const uint &r2,
+                                                 const uint &r3,
+                                                 const uint &r4) {
+    boxMullerTransform(&out[index], &out[index + groupSz],
+                       getFloatNegative11(r1), getFloat01(r2));
+    boxMullerTransform(&out[index + 2 * groupSz],
+                       &out[index + 3 * groupSz],
+                       getFloatNegative11(r3),
+                       getFloat01(r4));
+}
+
+static void boxMullerWriteOut128Bytes(cfloat *out, const uint &index, const uint groupSz,
+                                                 const uint &r1, const uint &r2,
+                                                 const uint &r3,
+                                                 const uint &r4) {
+    boxMullerTransform(&out[index], getFloatNegative11(r1), getFloat01(r2));
+    boxMullerTransform(&out[index + groupSz], getFloatNegative11(r3), getFloat01(r4));
+}
+
+static void boxMullerWriteOut128Bytes(double *out, const uint &index, const uint groupSz,
+                                                 const uint &r1, const uint &r2,
+                                                 const uint &r3,
+                                                 const uint &r4) {
+    boxMullerTransform(&out[index], &out[index + groupSz],
+                       getDoubleNegative11(r1, r2), getDouble01(r3, r4));
+}
+
+static void boxMullerWriteOut128Bytes(cdouble *out,
+                                                 const uint &index, const uint groupSz,
+                                                 const uint &r1, const uint &r2,
+                                                 const uint &r3,
+                                                 const uint &r4) {
+    boxMullerTransform(&out[index], getDoubleNegative11(r1, r2), getDouble01(r3, r4));
+}
+
+static void boxMullerWriteOut128Bytes(common::half *out,
+                                                 const uint &index, const uint groupSz,
+                                                 const uint &r1, const uint &r2,
+                                                 const uint &r3,
+                                                 const uint &r4) {
+//   boxMullerTransform(&out[index], &out[index + groupSz],
+//                      getHalfNegative11(r1), getHalf01(r1 >> 16));
+//   boxMullerTransform(&out[index + 2 * groupSz],
+//                      &out[index + 3 * groupSz], getHalfNegative11(r2),
+//                      getHalf01(r2 >> 16));
+//   boxMullerTransform(&out[index + 4 * groupSz],
+//                      &out[index + 5 * groupSz], getHalfNegative11(r3),
+//                      getHalf01(r3 >> 16));
+//   boxMullerTransform(&out[index + 6 * groupSz],
+//                      &out[index + 7 * groupSz], getHalfNegative11(r4),
+//                      getHalf01(r4 >> 16));
+}
+
+// Writes with boundary checking
+
+static void partialWriteOut128Bytes(uchar *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    if (index < elements) { out[index] = r1; }
+    if (index + groupSz < elements) { out[index + groupSz] = r1 >> 8; }
+    if (index + 2 * groupSz < elements) {
+        out[index + 2 * groupSz] = r1 >> 16;
+    }
+    if (index + 3 * groupSz < elements) {
+        out[index + 3 * groupSz] = r1 >> 24;
+    }
+    if (index + 4 * groupSz < elements) { out[index + 4 * groupSz] = r2; }
+    if (index + 5 * groupSz < elements) {
+        out[index + 5 * groupSz] = r2 >> 8;
+    }
+    if (index + 6 * groupSz < elements) {
+        out[index + 6 * groupSz] = r2 >> 16;
+    }
+    if (index + 7 * groupSz < elements) {
+        out[index + 7 * groupSz] = r2 >> 24;
+    }
+    if (index + 8 * groupSz < elements) { out[index + 8 * groupSz] = r3; }
+    if (index + 9 * groupSz < elements) {
+        out[index + 9 * groupSz] = r3 >> 8;
+    }
+    if (index + 10 * groupSz < elements) {
+        out[index + 10 * groupSz] = r3 >> 16;
+    }
+    if (index + 11 * groupSz < elements) {
+        out[index + 11 * groupSz] = r3 >> 24;
+    }
+    if (index + 12 * groupSz < elements) {
+        out[index + 12 * groupSz] = r4;
+    }
+    if (index + 13 * groupSz < elements) {
+        out[index + 13 * groupSz] = r4 >> 8;
+    }
+    if (index + 14 * groupSz < elements) {
+        out[index + 14 * groupSz] = r4 >> 16;
+    }
+    if (index + 15 * groupSz < elements) {
+        out[index + 15 * groupSz] = r4 >> 24;
+    }
+}
+
+static void partialWriteOut128Bytes(char *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    if (index < elements) { out[index] = (r1)&0x1; }
+    if (index + groupSz < elements) {
+        out[index + groupSz] = (r1 >> 1) & 0x1;
+    }
+    if (index + 2 * groupSz < elements) {
+        out[index + 2 * groupSz] = (r1 >> 2) & 0x1;
+    }
+    if (index + 3 * groupSz < elements) {
+        out[index + 3 * groupSz] = (r1 >> 3) & 0x1;
+    }
+    if (index + 4 * groupSz < elements) {
+        out[index + 4 * groupSz] = (r2)&0x1;
+    }
+    if (index + 5 * groupSz < elements) {
+        out[index + 5 * groupSz] = (r2 >> 1) & 0x1;
+    }
+    if (index + 6 * groupSz < elements) {
+        out[index + 6 * groupSz] = (r2 >> 2) & 0x1;
+    }
+    if (index + 7 * groupSz < elements) {
+        out[index + 7 * groupSz] = (r2 >> 3) & 0x1;
+    }
+    if (index + 8 * groupSz < elements) {
+        out[index + 8 * groupSz] = (r3)&0x1;
+    }
+    if (index + 9 * groupSz < elements) {
+        out[index + 9 * groupSz] = (r3 >> 1) & 0x1;
+    }
+    if (index + 10 * groupSz < elements) {
+        out[index + 10 * groupSz] = (r3 >> 2) & 0x1;
+    }
+    if (index + 11 * groupSz < elements) {
+        out[index + 11 * groupSz] = (r3 >> 3) & 0x1;
+    }
+    if (index + 12 * groupSz < elements) {
+        out[index + 12 * groupSz] = (r4)&0x1;
+    }
+    if (index + 13 * groupSz < elements) {
+        out[index + 13 * groupSz] = (r4 >> 1) & 0x1;
+    }
+    if (index + 14 * groupSz < elements) {
+        out[index + 14 * groupSz] = (r4 >> 2) & 0x1;
+    }
+    if (index + 15 * groupSz < elements) {
+        out[index + 15 * groupSz] = (r4 >> 3) & 0x1;
+    }
+}
+
+static void partialWriteOut128Bytes(short *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    if (index < elements) { out[index] = r1; }
+    if (index + groupSz < elements) { out[index + groupSz] = r1 >> 16; }
+    if (index + 2 * groupSz < elements) { out[index + 2 * groupSz] = r2; }
+    if (index + 3 * groupSz < elements) {
+        out[index + 3 * groupSz] = r2 >> 16;
+    }
+    if (index + 4 * groupSz < elements) { out[index + 4 * groupSz] = r3; }
+    if (index + 5 * groupSz < elements) {
+        out[index + 5 * groupSz] = r3 >> 16;
+    }
+    if (index + 6 * groupSz < elements) { out[index + 6 * groupSz] = r4; }
+    if (index + 7 * groupSz < elements) {
+        out[index + 7 * groupSz] = r4 >> 16;
+    }
+}
+
+static void partialWriteOut128Bytes(ushort *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    partialWriteOut128Bytes((short *)(out), index, groupSz, r1, r2, r3, r4, elements);
+}
+
+static void partialWriteOut128Bytes(int *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    if (index < elements) { out[index] = r1; }
+    if (index + groupSz < elements) { out[index + groupSz] = r2; }
+    if (index + 2 * groupSz < elements) { out[index + 2 * groupSz] = r3; }
+    if (index + 3 * groupSz < elements) { out[index + 3 * groupSz] = r4; }
+}
+
+static void partialWriteOut128Bytes(uint *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    partialWriteOut128Bytes((int *)(out), index, groupSz, r1, r2, r3, r4, elements);
+}
+
+static void partialWriteOut128Bytes(intl *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    intl c1 = r2;
+    c1      = (c1 << 32) | r1;
+    intl c2 = r4;
+    c2      = (c2 << 32) | r3;
+    if (index < elements) { out[index] = c1; }
+    if (index + groupSz < elements) { out[index + groupSz] = c2; }
+}
+
+static void partialWriteOut128Bytes(uintl *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    partialWriteOut128Bytes((intl *)(out), index, groupSz, r1, r2, r3, r4, elements);
+}
+
+static void partialWriteOut128Bytes(float *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    if (index < elements) { out[index] = 1.f - getFloat01(r1); }
+    if (index + groupSz < elements) {
+        out[index + groupSz] = 1.f - getFloat01(r2);
+    }
+    if (index + 2 * groupSz < elements) {
+        out[index + 2 * groupSz] = 1.f - getFloat01(r3);
+    }
+    if (index + 3 * groupSz < elements) {
+        out[index + 3 * groupSz] = 1.f - getFloat01(r4);
+    }
+}
+
+static void partialWriteOut128Bytes(cfloat *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    if (index < elements) {
+        out[index] = {1.f - getFloat01(r1), 1.f - getFloat01(r2)};
+    }
+    if (index + groupSz < elements) {
+        out[index + groupSz] = {1.f - getFloat01(r3), 1.f - getFloat01(r4)};
+    }
+}
+
+static void partialWriteOut128Bytes(double *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    if (index < elements) { out[index] = 1.0 - getDouble01(r1, r2); }
+    if (index + groupSz < elements) {
+        out[index + groupSz] = 1.0 - getDouble01(r3, r4);
+    }
+}
+
+static void partialWriteOut128Bytes(cdouble *out, const uint &index, const uint groupSz,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    if (index < elements) {
+        out[index] = {1.0 - getDouble01(r1, r2), 1.0 - getDouble01(r3, r4)};
+    }
+}
+
+// Normalized writes with boundary checking
+static void partialBoxMullerWriteOut128Bytes(
+    float *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2,
+    const uint &r3, const uint &r4, const uint &elements) {
+    float n1, n2, n3, n4;
+    boxMullerTransform(&n1, &n2, getFloatNegative11(r1), getFloat01(r2));
+    boxMullerTransform(&n3, &n4, getFloatNegative11(r3), getFloat01(r4));
+    if (index < elements) { out[index] = n1; }
+    if (index + groupSz < elements) { out[index + groupSz] = n2; }
+    if (index + 2 * groupSz < elements) { out[index + 2 * groupSz] = n3; }
+    if (index + 3 * groupSz < elements) { out[index + 3 * groupSz] = n4; }
+}
+
+static void partialBoxMullerWriteOut128Bytes(
+    cfloat *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2,
+    const uint &r3, const uint &r4, const uint &elements) {
+    float n1, n2, n3, n4;
+    boxMullerTransform(&n1, &n2, getFloatNegative11(r1), getFloat01(r2));
+    boxMullerTransform(&n3, &n4, getFloatNegative11(r3), getFloat01(r4));
+    if (index < elements) {
+        out[index] = {n1, n2};
+    }
+    if (index + groupSz < elements) {
+        out[index + groupSz] = {n3, n4};
+    }
+}
+
+static void partialBoxMullerWriteOut128Bytes(
+    double *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2,
+    const uint &r3, const uint &r4, const uint &elements) {
+    double n1, n2;
+    boxMullerTransform(&n1, &n2, getDoubleNegative11(r1, r2),
+                       getDouble01(r3, r4));
+    if (index < elements) { out[index] = n1; }
+    if (index + groupSz < elements) { out[index + groupSz] = n2; }
+}
+
+static void partialBoxMullerWriteOut128Bytes(
+    cdouble *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2,
+    const uint &r3, const uint &r4, const uint &elements) {
+    double n1, n2;
+    boxMullerTransform(&n1, &n2, getDoubleNegative11(r1, r2),
+                       getDouble01(r3, r4));
+    if (index < elements) {
+        out[index] = {n1, n2};
+    }
+}
+
+static void partialWriteOut128Bytes(common::half *out, 
+                                             const uint &index, const uint groupSz,
+                                             const uint &r1, const uint &r2,
+                                             const uint &r3, const uint &r4,
+                                             const uint &elements) {
+//  if (index < elements) { out[index] = oneMinusGetHalf01(r1); }
+//  if (index + groupSz < elements) {
+//      out[index + groupSz] = oneMinusGetHalf01(r1 >> 16);
+//  }
+//  if (index + 2 * groupSz < elements) {
+//      out[index + 2 * groupSz] = oneMinusGetHalf01(r2);
+//  }
+//  if (index + 3 * groupSz < elements) {
+//      out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16);
+//  }
+//  if (index + 4 * groupSz < elements) {
+//      out[index + 4 * groupSz] = oneMinusGetHalf01(r3);
+//  }
+//  if (index + 5 * groupSz < elements) {
+//      out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16);
+//  }
+//  if (index + 6 * groupSz < elements) {
+//      out[index + 6 * groupSz] = oneMinusGetHalf01(r4);
+//  }
+//  if (index + 7 * groupSz < elements) {
+//      out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16);
+//  }
+}
+
+
+// Normalized writes with boundary checking
+static void partialBoxMullerWriteOut128Bytes(
+    common::half *out, const uint &index, const uint groupSz,
+    const uint &r1, const uint &r2,
+    const uint &r3, const uint &r4, const uint &elements) {
+//    common::half n[8];
+//    boxMullerTransform(n + 0, n + 1, getHalfNegative11(r1),
+//                       getHalf01(r1 >> 16));
+//    boxMullerTransform(n + 2, n + 3, getHalfNegative11(r2),
+//                       getHalf01(r2 >> 16));
+//    boxMullerTransform(n + 4, n + 5, getHalfNegative11(r3),
+//                       getHalf01(r3 >> 16));
+//    boxMullerTransform(n + 6, n + 7, getHalfNegative11(r4),
+//                       getHalf01(r4 >> 16));
+//    if (index < elements) { out[index] = n[0]; }
+//    if (index + groupSz < elements) { out[index + groupSz] = n[1]; }
+//    if (index + 2 * groupSz < elements) {
+//        out[index + 2 * groupSz] = n[2];
+//    }
+//    if (index + 3 * groupSz < elements) {
+//        out[index + 3 * groupSz] = n[3];
+//    }
+//    if (index + 4 * groupSz < elements) {
+//        out[index + 4 * groupSz] = n[4];
+//    }
+//    if (index + 5 * groupSz < elements) {
+//        out[index + 5 * groupSz] = n[5];
+//    }
+//    if (index + 6 * groupSz < elements) {
+//        out[index + 6 * groupSz] = n[6];
+//    }
+//    if (index + 7 * groupSz < elements) {
+//        out[index + 7 * groupSz] = n[7];
+//    }
+}
+
+} // namespace kernel
+} // namespace oneapi
diff --git a/src/backend/oneapi/random_engine.cpp b/src/backend/oneapi/random_engine.cpp
index db56d21638..5f8231706e 100644
--- a/src/backend/oneapi/random_engine.cpp
+++ b/src/backend/oneapi/random_engine.cpp
@@ -10,18 +10,16 @@
 #include <Array.hpp>
 #include <err_oneapi.hpp>
 #include <common/half.hpp>
-// #include <kernel/random_engine.hpp>
+#include <af/defines.h>
 #include <af/dim4.hpp>
+#include <kernel/random_engine.hpp>
 
 using common::half;
 
 namespace oneapi {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl) {
-
-    ONEAPI_NOT_SUPPORTED("initMersenneState Not supported");
-
-    // kernel::initMersenneState(*state.get(), *tbl.get(), seed);
+     kernel::initMersenneState(state, tbl, seed);
 }
 
 template<typename T>
@@ -29,11 +27,9 @@ Array<T> uniformDistribution(const af::dim4 &dims,
                              const af_random_engine_type type,
                              const uintl &seed, uintl &counter) {
 
-    //ONEAPI_NOT_SUPPORTED("uniformDistribution Not supported");
-
     Array<T> out = createEmptyArray<T>(dims);
-    // kernel::uniformDistributionCBRNG<T>(*out.get(), out.elements(), type, seed,
-    //                                     counter);
+    kernel::uniformDistributionCBRNG<T>(out, out.elements(), type, seed,
+                                        counter);
     return out;
 }
 
@@ -41,12 +37,9 @@ template<typename T>
 Array<T> normalDistribution(const af::dim4 &dims,
                             const af_random_engine_type type, const uintl &seed,
                             uintl &counter) {
-
-    ONEAPI_NOT_SUPPORTED("normalDistribution Not supported");
-
     Array<T> out = createEmptyArray<T>(dims);
-    // kernel::normalDistributionCBRNG<T>(*out.get(), out.elements(), type, seed,
-    //                                    counter);
+    kernel::normalDistributionCBRNG<T>(out, out.elements(), type, seed,
+                                       counter);
     return out;
 }
 
@@ -55,13 +48,10 @@ Array<T> uniformDistribution(const af::dim4 &dims, Array<uint> pos,
                              Array<uint> sh1, Array<uint> sh2, uint mask,
                              Array<uint> recursion_table,
                              Array<uint> temper_table, Array<uint> state) {
-
-    ONEAPI_NOT_SUPPORTED("uniformDistribution Not supported");
-
     Array<T> out = createEmptyArray<T>(dims);
-    // kernel::uniformDistributionMT<T>(
-    //     *out.get(), out.elements(), *state.get(), *pos.get(), *sh1.get(),
-    //     *sh2.get(), mask, *recursion_table.get(), *temper_table.get());
+    kernel::uniformDistributionMT<T>(
+        out, out.elements(), state, pos, sh1,
+        sh2, mask, recursion_table, temper_table);
     return out;
 }
 
@@ -70,13 +60,10 @@ Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
                             Array<uint> sh1, Array<uint> sh2, uint mask,
                             Array<uint> recursion_table,
                             Array<uint> temper_table, Array<uint> state) {
-
-    ONEAPI_NOT_SUPPORTED("normalDistribution Not supported");
-
     Array<T> out = createEmptyArray<T>(dims);
-    // kernel::normalDistributionMT<T>(
-    //     *out.get(), out.elements(), *state.get(), *pos.get(), *sh1.get(),
-    //     *sh2.get(), mask, *recursion_table.get(), *temper_table.get());
+    kernel::normalDistributionMT<T>(
+        out, out.elements(), state, pos, sh1,
+        sh2, mask, recursion_table, temper_table);
     return out;
 }
 
@@ -98,45 +85,10 @@ Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
         Array<uint> sh2, uint mask, Array<uint> recursion_table, \
         Array<uint> temper_table, Array<uint> state);
 
-#define COMPLEX_UNIFORM_DISTRIBUTION(T, TR)                                    \
-    template<>                                                                 \
-    Array<T> uniformDistribution<T>(const af::dim4 &dims,                      \
-                                    const af_random_engine_type type,          \
-                                    const uintl &seed, uintl &counter) {       \
-        ONEAPI_NOT_SUPPORTED("uniformDistribution Not supported");             \
-        Array<T> out    = createEmptyArray<T>(dims);                           \
-        return out;                                                            \
-    }                                                                          \
-    template<>                                                                 \
-    Array<T> uniformDistribution<T>(                                           \
-        const af::dim4 &dims, Array<uint> pos, Array<uint> sh1,                \
-        Array<uint> sh2, uint mask, Array<uint> recursion_table,               \
-        Array<uint> temper_table, Array<uint> state) {                         \
-        Array<T> out    = createEmptyArray<T>(dims);                           \
-        return out;                                                            \
-    }
-
-#define COMPLEX_NORMAL_DISTRIBUTION(T, TR)                                    \
-    template<>                                                                \
-    Array<T> normalDistribution<T>(const af::dim4 &dims,                      \
-                                   const af_random_engine_type type,          \
-                                   const uintl &seed, uintl &counter) {       \
-        ONEAPI_NOT_SUPPORTED("normalDistribution Not supported");             \
-        Array<T> out    = createEmptyArray<T>(dims);                          \
-        return out;                                                           \
-    }                                                                         \
-    template<>                                                                \
-    Array<T> normalDistribution<T>(                                           \
-        const af::dim4 &dims, Array<uint> pos, Array<uint> sh1,               \
-        Array<uint> sh2, uint mask, Array<uint> recursion_table,              \
-        Array<uint> temper_table, Array<uint> state) {                        \
-        ONEAPI_NOT_SUPPORTED("normalDistribution Not supported");             \
-        Array<T> out    = createEmptyArray<T>(dims);                          \
-        return out;                                                           \
-    }
-
 INSTANTIATE_UNIFORM(float)
 INSTANTIATE_UNIFORM(double)
+INSTANTIATE_UNIFORM(cfloat)
+INSTANTIATE_UNIFORM(cdouble)
 INSTANTIATE_UNIFORM(int)
 INSTANTIATE_UNIFORM(uint)
 INSTANTIATE_UNIFORM(intl)
@@ -149,12 +101,8 @@ INSTANTIATE_UNIFORM(half)
 
 INSTANTIATE_NORMAL(float)
 INSTANTIATE_NORMAL(double)
+INSTANTIATE_NORMAL(cdouble)
+INSTANTIATE_NORMAL(cfloat)
 INSTANTIATE_NORMAL(half)
 
-COMPLEX_UNIFORM_DISTRIBUTION(cdouble, double)
-COMPLEX_UNIFORM_DISTRIBUTION(cfloat, float)
-
-COMPLEX_NORMAL_DISTRIBUTION(cdouble, double)
-COMPLEX_NORMAL_DISTRIBUTION(cfloat, float)
-
 }  // namespace oneapi
diff --git a/src/backend/oneapi/types.hpp b/src/backend/oneapi/types.hpp
index 945d1366c7..10bd0e64c7 100644
--- a/src/backend/oneapi/types.hpp
+++ b/src/backend/oneapi/types.hpp
@@ -38,10 +38,10 @@ namespace oneapi {
 using cdouble = std::complex<double>;
 using cfloat  = std::complex<float>;
 using intl    = long long;
-using uchar   = cl_uchar;
-using uint    = cl_uint;
+using uchar   = unsigned char;
+using uint    = unsigned int;
 using uintl   = unsigned long long;
-using ushort  = cl_ushort;
+using ushort  = unsigned short;
 
 template<typename T>
 using compute_t = typename common::kernel_type<T>::compute;

From 17ec326ab154078e12a1b35f55267a3727349b53 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Wed, 5 Oct 2022 08:46:50 -0400
Subject: [PATCH 104/473] transpose_inplace ported. passes all. see below.

test Transpose/0.TranposeIP_10 takes a much longer time to run
that
    cpu. investigate
---
 src/backend/oneapi/CMakeLists.txt             |   1 +
 .../oneapi/kernel/transpose_inplace.hpp       | 189 ++++++++++++++++++
 src/backend/oneapi/transpose_inplace.cpp      |   9 +-
 3 files changed, 197 insertions(+), 2 deletions(-)
 create mode 100755 src/backend/oneapi/kernel/transpose_inplace.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index adb97f30ff..f069b6a433 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -215,6 +215,7 @@ target_sources(afoneapi
     kernel/random_engine_threefry.hpp
     kernel/range.hpp
     kernel/transpose.hpp
+    kernel/transpose_inplace.hpp
 )
 
 add_library(ArrayFire::afoneapi ALIAS afoneapi)
diff --git a/src/backend/oneapi/kernel/transpose_inplace.hpp b/src/backend/oneapi/kernel/transpose_inplace.hpp
new file mode 100755
index 0000000000..c5230f364e
--- /dev/null
+++ b/src/backend/oneapi/kernel/transpose_inplace.hpp
@@ -0,0 +1,189 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <err_oneapi.hpp>
+#include <debug_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+T static getConjugate(const T &in) {
+    // For non-complex types return same
+    return in;
+}
+
+template<>
+cfloat static getConjugate(const cfloat &in) {
+    return std::conj(in);
+}
+
+template<>
+cdouble static getConjugate(const cdouble &in) {
+    return std::conj(in);
+}
+
+#define doOp(v) (conjugate_ ? getConjugate((v)) : (v))
+
+constexpr int TILE_DIM  = 16;
+constexpr int THREADS_X = TILE_DIM;
+constexpr int THREADS_Y = 256 / TILE_DIM;
+
+template <typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write, sycl::access::target::local>;
+
+template <typename T>
+class transposeInPlaceKernel {
+public:
+  transposeInPlaceKernel(const sycl::accessor<T> iData, const KParam in,
+                         const int blocksPerMatX, const int blocksPerMatY,
+                         const bool conjugate, const bool IS32MULTIPLE,
+                         local_accessor<T, 1> shrdMem_s, local_accessor<T, 1> shrdMem_d,
+                         sycl::stream debugStream) :
+    iData_(iData), in_(in), blocksPerMatX_(blocksPerMatX),
+    blocksPerMatY_(blocksPerMatY), conjugate_(conjugate), IS32MULTIPLE_(IS32MULTIPLE), shrdMem_s_(shrdMem_s), shrdMem_d_(shrdMem_d), debugStream_(debugStream) {}
+  void operator() (sycl::nd_item<2> it) const {
+    const int shrdStride = TILE_DIM + 1;
+
+    // create variables to hold output dimensions
+    const int iDim0 = in_.dims[0];
+    const int iDim1 = in_.dims[1];
+
+    // calculate strides
+    const int iStride1 = in_.strides[1];
+
+    const int lx = it.get_local_id(0);
+    const int ly = it.get_local_id(1);
+
+    // batch based block Id
+    sycl::group g = it.get_group();
+    const int batchId_x  = g.get_group_id(0) / blocksPerMatX_;
+    const int blockIdx_x = (g.get_group_id(0) - batchId_x * blocksPerMatX_);
+
+    const int batchId_y  = g.get_group_id(1) / blocksPerMatY_;
+    const int blockIdx_y = (g.get_group_id(1) - batchId_y * blocksPerMatY_);
+
+    const int x0 = TILE_DIM * blockIdx_x;
+    const int y0 = TILE_DIM * blockIdx_y;
+
+    T *iDataPtr = iData_.get_pointer();
+    iDataPtr += batchId_x * in_.strides[2] + batchId_y * in_.strides[3] + in_.offset;
+
+    if (blockIdx_y > blockIdx_x) {
+        // calculate global indices
+        int gx = lx + x0;
+        int gy = ly + y0;
+        int dx = lx + y0;
+        int dy = ly + x0;
+
+        // Copy to shared memory
+        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+            int gy_ = gy + repeat;
+            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+              shrdMem_s_[(ly + repeat) * shrdStride + lx] =
+                iDataPtr[gy_ * iStride1 + gx];
+
+            int dy_ = dy + repeat;
+            if (IS32MULTIPLE_ || (dx < iDim0 && dy_ < iDim1))
+                shrdMem_d_[(ly + repeat) * shrdStride + lx] =
+                    iDataPtr[dy_ * iStride1 + dx];
+        }
+
+        it.barrier();
+
+        // Copy from shared memory to global memory
+        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+            int dy_ = dy + repeat;
+            if (IS32MULTIPLE_ || (dx < iDim0 && dy_ < iDim1))
+                iDataPtr[dy_ * iStride1 + dx] =
+                    doOp(shrdMem_s_[(ly + repeat) + (shrdStride * lx)]);
+
+            int gy_ = gy + repeat;
+            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+                iDataPtr[gy_ * iStride1 + gx] =
+                    doOp(shrdMem_d_[(ly + repeat) + (shrdStride * lx)]);
+        }
+
+    } else if (blockIdx_y == blockIdx_x) {
+        // calculate global indices
+        int gx = lx + x0;
+        int gy = ly + y0;
+
+        // Copy to shared memory
+        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+            int gy_ = gy + repeat;
+            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+                shrdMem_s_[(ly + repeat) * shrdStride + lx] =
+                    iDataPtr[gy_ * iStride1 + gx];
+        }
+
+        it.barrier();
+
+        // Copy from shared memory to global memory
+        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+            int gy_ = gy + repeat;
+            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+                iDataPtr[gy_ * iStride1 + gx] =
+                    doOp(shrdMem_s_[(ly + repeat) + (shrdStride * lx)]);
+        }
+    }
+  }
+private:
+  sycl::accessor<T> iData_;
+  KParam in_;
+  int blocksPerMatX_;
+  int blocksPerMatY_;
+  sycl::stream debugStream_;
+  bool conjugate_;
+  bool IS32MULTIPLE_;
+  local_accessor<T, 1> shrdMem_s_;
+  local_accessor<T, 1> shrdMem_d_;
+};
+
+template<typename T>
+void transpose_inplace(Param<T> in, const bool conjugate, const bool IS32MULTIPLE)
+{
+    auto local = sycl::range{THREADS_X, THREADS_Y};
+
+    int blk_x = divup(in.info.dims[0], TILE_DIM);
+    int blk_y = divup(in.info.dims[1], TILE_DIM);
+
+    auto global = sycl::range{blk_x * local[0] * in.info.dims[2],
+                              blk_y * local[1] * in.info.dims[3]};
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto r = in.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        auto shrdMem_s = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
+        auto shrdMem_d = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       transposeInPlaceKernel<T>(r, in.info,
+                                                 blk_x, blk_y,
+                                                 conjugate, IS32MULTIPLE,
+                                                 shrdMem_s, shrdMem_d,
+                                                 debugStream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/transpose_inplace.cpp b/src/backend/oneapi/transpose_inplace.cpp
index 2792a4200b..52a62d7837 100644
--- a/src/backend/oneapi/transpose_inplace.cpp
+++ b/src/backend/oneapi/transpose_inplace.cpp
@@ -10,7 +10,7 @@
 #include <Array.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
-//#include <kernel/transpose_inplace.hpp>
+#include <kernel/transpose_inplace.hpp>
 #include <transpose.hpp>
 #include <af/dim4.hpp>
 
@@ -21,7 +21,12 @@ namespace oneapi {
 
 template<typename T>
 void transpose_inplace(Array<T> &in, const bool conjugate) {
-    ONEAPI_NOT_SUPPORTED("");
+    const dim4 &inDims = in.dims();
+
+    const bool is32multiple =
+        inDims[0] % kernel::TILE_DIM == 0 && inDims[1] % kernel::TILE_DIM == 0;
+
+    kernel::transpose_inplace<T>(in, conjugate, is32multiple);
 }
 
 #define INSTANTIATE(T) \

From edc57407a23373f78849862ecaa624ff399e5db9 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Wed, 5 Oct 2022 11:29:40 -0400
Subject: [PATCH 105/473] triangle ported. passes all but gfor. see below.

---
 src/backend/oneapi/CMakeLists.txt      |   1 +
 src/backend/oneapi/kernel/triangle.hpp | 111 +++++++++++++++++++++++++
 src/backend/oneapi/triangle.cpp        |   5 +-
 3 files changed, 114 insertions(+), 3 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/triangle.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index f069b6a433..4559aa9292 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -216,6 +216,7 @@ target_sources(afoneapi
     kernel/range.hpp
     kernel/transpose.hpp
     kernel/transpose_inplace.hpp
+    kernel/triangle.hpp
 )
 
 add_library(ArrayFire::afoneapi ALIAS afoneapi)
diff --git a/src/backend/oneapi/kernel/triangle.hpp b/src/backend/oneapi/kernel/triangle.hpp
new file mode 100644
index 0000000000..4f71ce1243
--- /dev/null
+++ b/src/backend/oneapi/kernel/triangle.hpp
@@ -0,0 +1,111 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <err_oneapi.hpp>
+#include <debug_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template <typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write, sycl::access::target::local>;
+
+template <typename T>
+class triangleKernel {
+public:
+  triangleKernel(sycl::accessor<T> rAcc, KParam rinfo, sycl::accessor<T> iAcc,
+                 KParam iinfo, const int groups_x, const int groups_y,
+                 const bool is_upper, const bool is_unit_diag) :
+    rAcc_(rAcc), rinfo_(rinfo), iAcc_(iAcc), iinfo_(iinfo), groups_x_(groups_x), groups_y_(groups_y), is_upper_(is_upper), is_unit_diag_(is_unit_diag) {}
+  void operator() (sycl::nd_item<2> it) const {
+    sycl::group g = it.get_group();
+    const int oz = g.get_group_id(0) / groups_x_;
+    const int ow = g.get_group_id(1) / groups_y_;
+
+    const int groupId_0 = g.get_group_id(0) - oz * groups_x_;
+    const int groupId_1 = g.get_group_id(1) - ow * groups_y_;
+
+    const int xx = it.get_local_id(0) + groupId_0 * it.get_local_range(0);
+    const int yy = it.get_local_id(1) + groupId_1 * it.get_local_range(1);
+
+    const int incy = groups_y_ * it.get_local_range(1);
+    const int incx = groups_x_ * it.get_local_range(0);
+
+    T *d_r = rAcc_.get_pointer();
+    const T *d_i = iAcc_.get_pointer() + iinfo_.offset;
+
+    if (oz < rinfo_.dims[2] && ow < rinfo_.dims[3]) {
+        d_i = d_i + oz * iinfo_.strides[2] + ow * iinfo_.strides[3];
+        d_r = d_r + oz * rinfo_.strides[2] + ow * rinfo_.strides[3];
+
+        for (int oy = yy; oy < rinfo_.dims[1]; oy += incy) {
+            const T *Yd_i = d_i + oy * iinfo_.strides[1];
+            T *Yd_r       = d_r + oy * rinfo_.strides[1];
+
+            for (int ox = xx; ox < rinfo_.dims[0]; ox += incx) {
+                bool cond         = is_upper_ ? (oy >= ox) : (oy <= ox);
+                bool do_unit_diag = is_unit_diag_ && (oy == ox);
+                if (cond) {
+                    Yd_r[ox] = do_unit_diag ? (T)(1) : Yd_i[ox];
+                } else {
+                    Yd_r[ox] = (T)(0);
+                }
+            }
+        }
+    }
+  }
+private:
+  sycl::accessor<T> rAcc_;
+  KParam rinfo_;
+  sycl::accessor<T> iAcc_;
+  KParam iinfo_;
+  const int groups_x_;
+  const int groups_y_;
+  const bool is_upper_;
+  const bool is_unit_diag_;
+};
+
+template<typename T>
+void triangle(Param<T> out, const Param<T> in, bool is_upper, bool is_unit_diag) {
+    constexpr unsigned TX    = 32;
+    constexpr unsigned TY    = 8;
+    constexpr unsigned TILEX = 128;
+    constexpr unsigned TILEY = 32;
+
+    auto local = sycl::range{TX, TY};
+
+    int groups_x = divup(out.info.dims[0], TILEX);
+    int groups_y = divup(out.info.dims[1], TILEY);
+
+    auto global = sycl::range{groups_x * out.info.dims[2] * local[0],
+                              groups_y * out.info.dims[3] * local[1]};
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto iAcc = in.data->get_access(h);
+        auto rAcc = out.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       triangleKernel<T>(rAcc, out.info, iAcc, in.info, groups_x, groups_y,
+                                         is_upper, is_unit_diag));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/triangle.cpp b/src/backend/oneapi/triangle.cpp
index ad22dcaa6c..f514b8d64b 100644
--- a/src/backend/oneapi/triangle.cpp
+++ b/src/backend/oneapi/triangle.cpp
@@ -6,7 +6,7 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
-// #include <kernel/triangle.hpp>
+#include <kernel/triangle.hpp>
 #include <triangle.hpp>
 #include <err_oneapi.hpp>
 
@@ -22,8 +22,7 @@ namespace oneapi {
 template<typename T>
 void triangle(Array<T> &out, const Array<T> &in, const bool is_upper,
               const bool is_unit_diag) {
-    ONEAPI_NOT_SUPPORTED("triangle Not supported");
-    // kernel::triangle<T>(out, in, is_upper, is_unit_diag);
+    kernel::triangle<T>(out, in, is_upper, is_unit_diag);
 }
 
 template<typename T>

From ef9898dd557b5c03e7720d1770524a1ddb926a61 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 5 Oct 2022 11:23:52 -0400
Subject: [PATCH 106/473] Apply clang-format to oneapi backend

---
 src/api/c/det.cpp                             |   2 +-
 src/backend/common/EventBase.hpp              |   4 +-
 src/backend/common/Logger.hpp                 |   1 -
 src/backend/common/forge_loader.hpp           |   1 -
 src/backend/common/jit/BufferNodeBase.hpp     |   2 +-
 src/backend/common/jit/Node.hpp               |  48 +-
 src/backend/oneapi/Array.cpp                  | 113 ++--
 src/backend/oneapi/Array.hpp                  |  81 +--
 src/backend/oneapi/Event.cpp                  |  20 +-
 src/backend/oneapi/Event.hpp                  |  18 +-
 .../oneapi/GraphicsResourceManager.hpp        |   1 -
 src/backend/oneapi/Kernel.hpp                 |  17 +-
 src/backend/oneapi/Module.hpp                 |  11 +-
 src/backend/oneapi/Param.cpp                  |   2 +-
 src/backend/oneapi/Param.hpp                  |   2 +-
 src/backend/oneapi/anisotropic_diffusion.cpp  |   2 +-
 src/backend/oneapi/approx.cpp                 |  25 +-
 src/backend/oneapi/assign.cpp                 |   2 +-
 src/backend/oneapi/bilateral.cpp              |   3 +-
 src/backend/oneapi/blas.cpp                   |   6 +-
 src/backend/oneapi/compile_module.cpp         |  21 +-
 src/backend/oneapi/copy.cpp                   |  86 +--
 src/backend/oneapi/copy.hpp                   |   2 +-
 src/backend/oneapi/device_manager.cpp         |  34 +-
 src/backend/oneapi/device_manager.hpp         |   6 +-
 src/backend/oneapi/diff.cpp                   |   2 +-
 src/backend/oneapi/errorcodes.cpp             |   5 +-
 src/backend/oneapi/exampleFunction.cpp        |   7 +-
 src/backend/oneapi/fft.cpp                    |   3 +-
 src/backend/oneapi/fftconvolve.cpp            |   2 +-
 src/backend/oneapi/gradient.cpp               |   2 +-
 src/backend/oneapi/homography.cpp             |   2 +-
 src/backend/oneapi/inverse.cpp                |   3 +-
 src/backend/oneapi/jit/BufferNode.hpp         |   7 +-
 src/backend/oneapi/jit/kernel_generators.hpp  |  17 +-
 src/backend/oneapi/kernel/assign.hpp          |  64 ++-
 src/backend/oneapi/kernel/iota.hpp            |  57 +-
 src/backend/oneapi/kernel/memcopy.hpp         | 179 +++---
 src/backend/oneapi/kernel/random_engine.hpp   |  95 ++--
 .../oneapi/kernel/random_engine_mersenne.hpp  | 260 +++++----
 .../oneapi/kernel/random_engine_philox.hpp    |  71 ++-
 .../oneapi/kernel/random_engine_threefry.hpp  |  61 +-
 .../oneapi/kernel/random_engine_write.hpp     | 536 +++++++++---------
 src/backend/oneapi/kernel/range.hpp           |  44 +-
 src/backend/oneapi/kernel/transpose.hpp       | 182 +++---
 .../oneapi/kernel/transpose_inplace.hpp       | 236 ++++----
 src/backend/oneapi/kernel/triangle.hpp        | 121 ++--
 src/backend/oneapi/lu.cpp                     |   6 +-
 src/backend/oneapi/math.cpp                   |  20 +-
 src/backend/oneapi/math.hpp                   |   1 -
 src/backend/oneapi/mean.cpp                   |   6 +-
 src/backend/oneapi/meanshift.cpp              |   4 +-
 src/backend/oneapi/medfilt.cpp                |   2 -
 src/backend/oneapi/memory.cpp                 | 109 ++--
 src/backend/oneapi/memory.hpp                 |   5 +-
 src/backend/oneapi/moments.cpp                |   1 -
 src/backend/oneapi/morph.cpp                  |   2 -
 src/backend/oneapi/nearest_neighbour.cpp      |   1 -
 src/backend/oneapi/orb.cpp                    |   1 -
 src/backend/oneapi/platform.cpp               |  52 +-
 src/backend/oneapi/platform.hpp               |   4 +-
 src/backend/oneapi/plot.cpp                   |   9 +-
 src/backend/oneapi/random_engine.cpp          |  17 +-
 src/backend/oneapi/range.cpp                  |   2 +-
 src/backend/oneapi/reduce_impl.hpp            |   1 -
 src/backend/oneapi/regions.cpp                |   1 -
 src/backend/oneapi/reorder.cpp                |   1 -
 src/backend/oneapi/reshape.cpp                |   1 -
 src/backend/oneapi/rotate.cpp                 |   3 +-
 src/backend/oneapi/scan.cpp                   |   3 +-
 src/backend/oneapi/scan_by_key.cpp            |   7 +-
 src/backend/oneapi/select.cpp                 |   8 +-
 src/backend/oneapi/set.cpp                    |  23 +-
 src/backend/oneapi/shift.cpp                  |   2 +-
 src/backend/oneapi/sift.cpp                   |   1 -
 src/backend/oneapi/sobel.cpp                  |   1 -
 src/backend/oneapi/solve.cpp                  |   3 -
 src/backend/oneapi/sort.cpp                   |   1 -
 src/backend/oneapi/sort_index.cpp             |  11 +-
 src/backend/oneapi/sparse.cpp                 |   8 +-
 src/backend/oneapi/sparse_arith.cpp           |   8 +-
 src/backend/oneapi/sparse_blas.cpp            |   2 +-
 src/backend/oneapi/surface.cpp                |   9 +-
 src/backend/oneapi/susan.cpp                  |   5 +-
 src/backend/oneapi/svd.cpp                    |   6 +-
 src/backend/oneapi/tile.cpp                   |   2 +-
 src/backend/oneapi/topk.cpp                   |   7 +-
 src/backend/oneapi/transform.cpp              |   9 +-
 src/backend/oneapi/transpose.cpp              |   2 +-
 src/backend/oneapi/triangle.cpp               |   5 +-
 src/backend/oneapi/unwrap.cpp                 |   2 +-
 src/backend/oneapi/vector_field.cpp           |   3 +-
 src/backend/oneapi/where.cpp                  |   6 +-
 src/backend/oneapi/wrap.cpp                   |   9 +-
 94 files changed, 1476 insertions(+), 1382 deletions(-)
 mode change 100755 => 100644 src/backend/oneapi/kernel/transpose_inplace.hpp

diff --git a/src/api/c/det.cpp b/src/api/c/det.cpp
index 0d0e5cc1d7..8507675b85 100644
--- a/src/api/c/det.cpp
+++ b/src/api/c/det.cpp
@@ -24,9 +24,9 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::createEmptyArray;
-using detail::scalar;
 using detail::imag;
 using detail::real;
+using detail::scalar;
 
 template<typename T>
 T det(const af_array a) {
diff --git a/src/backend/common/EventBase.hpp b/src/backend/common/EventBase.hpp
index 874ec5b6c6..82ad049061 100644
--- a/src/backend/common/EventBase.hpp
+++ b/src/backend/common/EventBase.hpp
@@ -36,8 +36,8 @@ class EventBase {
 
     /// \brief Event destructor. Calls the destroy event call on the native API
     ~EventBase() noexcept {
-      //if (e_)
-          NativeEventPolicy::destroyEvent(&e_);
+        // if (e_)
+        NativeEventPolicy::destroyEvent(&e_);
     }
 
     /// \brief Creates the event object by calling the native create API
diff --git a/src/backend/common/Logger.hpp b/src/backend/common/Logger.hpp
index 50e74ae03b..5241dc9126 100644
--- a/src/backend/common/Logger.hpp
+++ b/src/backend/common/Logger.hpp
@@ -46,7 +46,6 @@
 /* Other */
 #endif
 
-
 namespace common {
 std::shared_ptr<spdlog::logger> loggerFactory(const std::string& name);
 std::string bytesToString(size_t bytes);
diff --git a/src/backend/common/forge_loader.hpp b/src/backend/common/forge_loader.hpp
index 1e3edc7125..c87e98690c 100644
--- a/src/backend/common/forge_loader.hpp
+++ b/src/backend/common/forge_loader.hpp
@@ -43,7 +43,6 @@
 /* Other */
 #endif
 
-
 class ForgeModule : public common::DependencyModule {
    public:
     ForgeModule();
diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index 6b3d56162b..a7d6747036 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -12,8 +12,8 @@
 #include <common/jit/Node.hpp>
 #include <jit/kernel_generators.hpp>
 
-#include <sstream>
 #include <cstring>
+#include <sstream>
 
 namespace common {
 
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 3062935909..bbe3fcb859 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -71,18 +71,18 @@ using Node_ptr = std::shared_ptr<Node>;
 
 static const char *getFullName(af::dtype type) {
     switch (type) {
-    case f32: return detail::getFullName<float>();
-    case f64: return detail::getFullName<double>();
-    case c32: return detail::getFullName<detail::cfloat>();
-    case c64: return detail::getFullName<detail::cdouble>();
-    case u32: return detail::getFullName<unsigned>();
-    case s32: return detail::getFullName<int>();
-    case u64: return detail::getFullName<unsigned long long>();
-    case s64: return detail::getFullName<long long>();
-    case u16: return detail::getFullName<unsigned short>();
-    case s16: return detail::getFullName<short>();
-    case b8: return  detail::getFullName<char>();
-    case u8: return  detail::getFullName<unsigned char>();
+        case f32: return detail::getFullName<float>();
+        case f64: return detail::getFullName<double>();
+        case c32: return detail::getFullName<detail::cfloat>();
+        case c64: return detail::getFullName<detail::cdouble>();
+        case u32: return detail::getFullName<unsigned>();
+        case s32: return detail::getFullName<int>();
+        case u64: return detail::getFullName<unsigned long long>();
+        case s64: return detail::getFullName<long long>();
+        case u16: return detail::getFullName<unsigned short>();
+        case s16: return detail::getFullName<short>();
+        case b8: return detail::getFullName<char>();
+        case u8: return detail::getFullName<unsigned char>();
         case f16: return "half";
     }
     return "";
@@ -90,18 +90,18 @@ static const char *getFullName(af::dtype type) {
 
 static const char *getShortName(af::dtype type) {
     switch (type) {
-    case f32: return detail::shortname<float>();
-    case f64: return detail::shortname<double>();
-    case c32: return detail::shortname<detail::cfloat>();
-    case c64: return detail::shortname<detail::cdouble>();
-    case u32: return detail::shortname<unsigned>();
-    case s32: return detail::shortname<int>();
-    case u64: return detail::shortname<unsigned long long>();
-    case s64: return detail::shortname<long long>();
-    case u16: return detail::shortname<unsigned short>();
-    case s16: return detail::shortname<short>();
-    case b8: return  detail::shortname<char>();
-    case u8: return  detail::shortname<unsigned char>();
+        case f32: return detail::shortname<float>();
+        case f64: return detail::shortname<double>();
+        case c32: return detail::shortname<detail::cfloat>();
+        case c64: return detail::shortname<detail::cdouble>();
+        case u32: return detail::shortname<unsigned>();
+        case s32: return detail::shortname<int>();
+        case u64: return detail::shortname<unsigned long long>();
+        case s64: return detail::shortname<long long>();
+        case u16: return detail::shortname<unsigned short>();
+        case s16: return detail::shortname<short>();
+        case b8: return detail::shortname<char>();
+        case u8: return detail::shortname<unsigned char>();
         case f16: return "h";
     }
     return "";
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index f9d8e8e3e7..db4bce10e3 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -12,10 +12,10 @@
 #include <common/half.hpp>
 #include <common/jit/NodeIterator.hpp>
 #include <common/jit/ScalarNode.hpp>
-#include <jit/BufferNode.hpp>
 #include <common/util.hpp>
-#include <err_oneapi.hpp>
 #include <copy.hpp>
+#include <err_oneapi.hpp>
+#include <jit/BufferNode.hpp>
 #include <memory.hpp>
 #include <platform.hpp>
 #include <scalar.hpp>
@@ -36,11 +36,11 @@
 using af::dim4;
 using af::dtype_traits;
 
-using oneapi::jit::BufferNode;
 using common::half;
 using common::Node;
 using common::Node_ptr;
 using common::NodeIterator;
+using oneapi::jit::BufferNode;
 
 using nonstd::span;
 using std::accumulate;
@@ -122,29 +122,29 @@ Array<T>::Array(const dim4 &dims, const T *const in_data)
     static_assert(
         offsetof(Array<T>, info) == 0,
         "Array<T>::info must be the first member variable of Array<T>");
-    //getQueue().enqueueWriteBuffer(*data.get(), CL_TRUE, 0,
-                                  //sizeof(T) * info.elements(), in_data);
-    getQueue().submit([&] (sycl::handler &h) {
-        h.copy(in_data, data->get_access(h));
-    }).wait();
+    // getQueue().enqueueWriteBuffer(*data.get(), CL_TRUE, 0,
+    // sizeof(T) * info.elements(), in_data);
+    getQueue()
+        .submit([&](sycl::handler &h) { h.copy(in_data, data->get_access(h)); })
+        .wait();
 }
-   
 
 template<typename T>
 Array<T>::Array(const af::dim4 &dims, buffer<T> *const mem, size_t offset,
                 bool copy)
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(
-           copy ? memAlloc<T>(info.elements()).release() : new buffer<T>(*mem),
-          bufferFree<T>)
+    , data(copy ? memAlloc<T>(info.elements()).release() : new buffer<T>(*mem),
+           bufferFree<T>)
     , data_dims(dims)
     , node()
     , owner(true) {
     if (copy) {
-        getQueue().submit([&] (sycl::handler &h) {
-            h.copy(mem->get_access(h), data->get_access(h));
-        }).wait();
+        getQueue()
+            .submit([&](sycl::handler &h) {
+                h.copy(mem->get_access(h), data->get_access(h));
+            })
+            .wait();
     }
 }
 
@@ -168,7 +168,7 @@ Array<T>::Array(Param<T> &tmp, bool owner_)
                 tmp.info.strides[3]),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
     , data(
-           tmp.data, owner_ ? bufferFree<T> : [](buffer<T> * /*unused*/) {})
+          tmp.data, owner_ ? bufferFree<T> : [](buffer<T> * /*unused*/) {})
     , data_dims(dim4(tmp.info.dims[0], tmp.info.dims[1], tmp.info.dims[2],
                      tmp.info.dims[3]))
     , node()
@@ -179,17 +179,18 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
                 const T *const in_data, bool is_device)
     : info(getActiveDeviceId(), dims, offset_, strides,
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(is_device ? (new buffer<T>(*reinterpret_cast<buffer<T>*>(
-                                        const_cast<T *>(in_data))))
+    , data(is_device ? (new buffer<T>(*reinterpret_cast<buffer<T> *>(
+                           const_cast<T *>(in_data))))
                      : (memAlloc<T>(info.elements()).release()),
            bufferFree<T>)
     , data_dims(dims)
     , node()
     , owner(true) {
     if (!is_device) {
-        getQueue().submit([&] (sycl::handler &h) {
-            h.copy(in_data, data->get_access(h));
-        }).wait();
+        getQueue()
+            .submit(
+                [&](sycl::handler &h) { h.copy(in_data, data->get_access(h)); })
+            .wait();
     }
 }
 
@@ -198,8 +199,8 @@ void Array<T>::eval() {
     if (isReady()) { return; }
 
     this->setId(getActiveDeviceId());
-    data = std::shared_ptr<sycl::buffer<T>>(memAlloc<T>(info.elements()).release(),
-                                       bufferFree<T>);
+    data = std::shared_ptr<sycl::buffer<T>>(
+        memAlloc<T>(info.elements()).release(), bufferFree<T>);
 
     // Do not replace this with cast operator
     KParam info = {{dims()[0], dims()[1], dims()[2], dims()[3]},
@@ -208,10 +209,9 @@ void Array<T>::eval() {
 
     Param<T> res{data.get(), info};
 
-
-    //TODO: implement
+    // TODO: implement
     ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
-    //evalNodes(res, getNode().get());
+    // evalNodes(res, getNode().get());
     node.reset();
 }
 
@@ -267,9 +267,9 @@ void evalMultiple(vector<Array<T> *> arrays) {
         nodes.push_back(array->getNode().get());
     }
 
-    //TODO: implement
+    // TODO: implement
     ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
-    //evalNodes(outputs, nodes);
+    // evalNodes(outputs, nodes);
 
     for (Array<T> *array : output_arrays) { array->node.reset(); }
 }
@@ -342,7 +342,8 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     //         (3 * sizeof(uint));
 
     //     const cl::Device &device = getDevice();
-    //     size_t max_param_size = device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>();
+    //     size_t max_param_size =
+    //     device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>();
     //     // typical values:
     //     //   NVIDIA     = 4096
     //     //   AMD        = 3520  (AMD A10 iGPU = 1024)
@@ -375,7 +376,8 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     //     }
     //     isBufferLimit = jitTreeExceedsMemoryPressure(info.total_buffer_size);
 
-    //     size_t param_size = (info.num_buffers * (sizeof(Param<T>) + sizeof(T *)) +
+    //     size_t param_size = (info.num_buffers * (sizeof(Param<T>) + sizeof(T
+    //     *)) +
     //                          info.param_scalar_size);
 
     //     bool isParamLimit = param_size >= max_param_size;
@@ -386,15 +388,16 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     return kJITHeuristics::Pass;
 }
 
-//Doesn't make sense with sycl::buffer
-//TODO: accessors? or return sycl::buffer?
-//TODO: return accessor.get_pointer() for access::target::global_buffer or (host_buffer?)
+// Doesn't make sense with sycl::buffer
+// TODO: accessors? or return sycl::buffer?
+// TODO: return accessor.get_pointer() for access::target::global_buffer or
+// (host_buffer?)
 template<typename T>
 void *getDevicePtr(const Array<T> &arr) {
     const buffer<T> *buf = arr.device();
-    //if (!buf) { return NULL; }
-    //memLock(buf);
-    //cl_mem mem = (*buf)();
+    // if (!buf) { return NULL; }
+    // memLock(buf);
+    // cl_mem mem = (*buf)();
     ONEAPI_NOT_SUPPORTED("pointer to sycl::buffer should be accessor");
     return (void *)buf;
 }
@@ -451,7 +454,7 @@ Array<T> createDeviceDataArray(const dim4 &dims, void *data) {
     verifyTypeSupport<T>();
 
     bool copy_device = false;
-    return Array<T>(dims, static_cast<buffer<T>*>(data), 0, copy_device);
+    return Array<T>(dims, static_cast<buffer<T> *>(data), 0, copy_device);
 }
 
 template<typename T>
@@ -481,15 +484,17 @@ template<typename T>
 void writeHostDataArray(Array<T> &arr, const T *const data,
                         const size_t bytes) {
     if (!arr.isOwner()) { arr = copyArray<T>(arr); }
-    getQueue().submit([&] (sycl::handler &h) {
-        buffer<T> &buf = *arr.get();
-        //auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>)
-        //TODO: offset accessor
-        auto offset_acc = buf.get_access(h);
-        h.copy(data, offset_acc);
-    }).wait();
-    //getQueue().enqueueWriteBuffer(*arr.get(), CL_TRUE, arr.getOffset(), bytes,
-    //data);
+    getQueue()
+        .submit([&](sycl::handler &h) {
+            buffer<T> &buf = *arr.get();
+            // auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>)
+            // TODO: offset accessor
+            auto offset_acc = buf.get_access(h);
+            h.copy(data, offset_acc);
+        })
+        .wait();
+    // getQueue().enqueueWriteBuffer(*arr.get(), CL_TRUE, arr.getOffset(),
+    // bytes, data);
 }
 
 template<typename T>
@@ -499,14 +504,14 @@ void writeDeviceDataArray(Array<T> &arr, const void *const data,
 
     buffer<T> &buf = *arr.get();
 
-    //clRetainMemObject(
+    // clRetainMemObject(
     //    reinterpret_cast<buffer<T> *>(const_cast<void *>(data)));
-    //buffer<T> data_buf =
+    // buffer<T> data_buf =
     //  buffer<T>(reinterpret_cast<buffer<T>*>(const_cast<void *>(data)));
 
     ONEAPI_NOT_SUPPORTED("writeDeviceDataArray not supported");
-    //getQueue().enqueueCopyBuffer(data_buf, buf, 0,
-    //static_cast<size_t>(arr.getOffset()), bytes);
+    // getQueue().enqueueCopyBuffer(data_buf, buf, 0,
+    // static_cast<size_t>(arr.getOffset()), bytes);
 }
 
 template<typename T>
@@ -530,7 +535,7 @@ size_t Array<T>::getAllocatedBytes() const {
     template Array<T> createDeviceDataArray<T>(const dim4 &dims, void *data); \
     template Array<T> createValueArray<T>(const dim4 &dims, const T &value);  \
     template Array<T> createEmptyArray<T>(const dim4 &dims);                  \
-    template Array<T> createParamArray<T>(Param<T> & tmp, bool owner);           \
+    template Array<T> createParamArray<T>(Param<T> & tmp, bool owner);        \
     template Array<T> createSubArray<T>(                                      \
         const Array<T> &parent, const vector<af_seq> &index, bool copy);      \
     template void destroyArray<T>(Array<T> * A);                              \
@@ -538,13 +543,13 @@ size_t Array<T>::getAllocatedBytes() const {
     template Array<T>::Array(const dim4 &dims, const dim4 &strides,           \
                              dim_t offset, const T *const in_data,            \
                              bool is_device);                                 \
-    template Array<T>::Array(const dim4 &dims, buffer<T>* mem, size_t src_offset, \
-                             bool copy);                                      \
+    template Array<T>::Array(const dim4 &dims, buffer<T> *mem,                \
+                             size_t src_offset, bool copy);                   \
     template Node_ptr Array<T>::getNode();                                    \
     template Node_ptr Array<T>::getNode() const;                              \
     template void Array<T>::eval();                                           \
     template void Array<T>::eval() const;                                     \
-    template buffer<T> *Array<T>::device();                             \
+    template buffer<T> *Array<T>::device();                                   \
     template void writeHostDataArray<T>(Array<T> & arr, const T *const data,  \
                                         const size_t bytes);                  \
     template void writeDeviceDataArray<T>(                                    \
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index 47c3c8bc7d..ae7234fb02 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -46,33 +46,33 @@ class Array;
 template<typename T>
 void evalMultiple(std::vector<Array<T> *> arrays);
 
-  template<typename T>
+template<typename T>
 void evalNodes(Param<T> &out, common::Node *node);
 
-  template<typename T>
-  void evalNodes(std::vector<Param<T>> &outputs,
-                 const std::vector<common::Node *> &nodes);
+template<typename T>
+void evalNodes(std::vector<Param<T>> &outputs,
+               const std::vector<common::Node *> &nodes);
 
-  /// Creates a new Array object on the heap and returns a reference to it.
-  template<typename T>
-  Array<T> createNodeArray(const af::dim4 &dims, common::Node_ptr node);
+/// Creates a new Array object on the heap and returns a reference to it.
+template<typename T>
+Array<T> createNodeArray(const af::dim4 &dims, common::Node_ptr node);
 
-  /// Creates a new Array object on the heap and returns a reference to it.
-  template<typename T>
-  Array<T> createValueArray(const af::dim4 &dims, const T &value);
+/// Creates a new Array object on the heap and returns a reference to it.
+template<typename T>
+Array<T> createValueArray(const af::dim4 &dims, const T &value);
 
-  /// Creates a new Array object on the heap and returns a reference to it.
-  template<typename T>
-  Array<T> createHostDataArray(const af::dim4 &dims, const T *const data);
+/// Creates a new Array object on the heap and returns a reference to it.
+template<typename T>
+Array<T> createHostDataArray(const af::dim4 &dims, const T *const data);
 
-  template<typename T>
-  Array<T> createDeviceDataArray(const af::dim4 &dims, void *data);
+template<typename T>
+Array<T> createDeviceDataArray(const af::dim4 &dims, void *data);
 
-  template<typename T>
-  Array<T> createStridedArray(const af::dim4 &dims, const af::dim4 &strides,
-                              dim_t offset, const T *const in_data,
-                              bool is_device) {
-      return Array<T>(dims, strides, offset, in_data, is_device);
+template<typename T>
+Array<T> createStridedArray(const af::dim4 &dims, const af::dim4 &strides,
+                            dim_t offset, const T *const in_data,
+                            bool is_device) {
+    return Array<T>(dims, strides, offset, in_data, is_device);
 }
 
 /// Copies data to an existing Array object from a host pointer
@@ -122,13 +122,13 @@ void *getDevicePtr(const Array<T> &arr);
 
 template<typename T>
 void *getRawPtr(const Array<T> &arr) {
-  //const sycl::buffer<T> *buf = arr.get();
-  //if (!buf) return NULL;
-  //cl_mem mem = (*buf)();
-  //return (void *)mem;
+    // const sycl::buffer<T> *buf = arr.get();
+    // if (!buf) return NULL;
+    // cl_mem mem = (*buf)();
+    // return (void *)mem;
 
-  // TODO:
-  return nullptr;
+    // TODO:
+    return nullptr;
 }
 
 template<typename T>
@@ -159,7 +159,8 @@ class Array {
     explicit Array(const af::dim4 &dims, common::Node_ptr n);
     explicit Array(const af::dim4 &dims, const T *const in_data);
 
-  explicit Array(const af::dim4 &dims, sycl::buffer<T>* const mem, size_t offset, bool copy);
+    explicit Array(const af::dim4 &dims, sycl::buffer<T> *const mem,
+                   size_t offset, bool copy);
 
    public:
     Array(const Array<T> &other) = default;
@@ -267,14 +268,14 @@ class Array {
         return out;
     }
 
-  operator KParam() const {
-      KParam kinfo = {
-          {dims()[0], dims()[1], dims()[2], dims()[3]},
-          {strides()[0], strides()[1], strides()[2], strides()[3]},
-          getOffset()};
+    operator KParam() const {
+        KParam kinfo = {
+            {dims()[0], dims()[1], dims()[2], dims()[3]},
+            {strides()[0], strides()[1], strides()[2], strides()[3]},
+            getOffset()};
 
-      return kinfo;
-  }
+        return kinfo;
+    }
 
     common::Node_ptr getNode() const;
     common::Node_ptr getNode();
@@ -285,16 +286,16 @@ class Array {
         if (!isReady()) eval();
         auto func = [data = data](void *ptr) {
             if (ptr != nullptr) {
-              //cl_int err = getQueue().enqueueUnmapMemObject(*data, ptr);
-              //UNUSED(err);
+                // cl_int err = getQueue().enqueueUnmapMemObject(*data, ptr);
+                // UNUSED(err);
                 ptr = nullptr;
             }
         };
 
-        //T *ptr = (T *)getQueue().enqueueMapBuffer(
-            //*static_cast<const sycl::buffer<T> *>(get()), CL_TRUE, map_flags,
-            //getOffset() * sizeof(T), elements() * sizeof(T), nullptr, nullptr,
-            //nullptr);
+        // T *ptr = (T *)getQueue().enqueueMapBuffer(
+        //*static_cast<const sycl::buffer<T> *>(get()), CL_TRUE, map_flags,
+        // getOffset() * sizeof(T), elements() * sizeof(T), nullptr, nullptr,
+        // nullptr);
 
         return mapped_ptr<T>(nullptr, func);
     }
diff --git a/src/backend/oneapi/Event.cpp b/src/backend/oneapi/Event.cpp
index 7e08c2fd44..a86d74f8ab 100644
--- a/src/backend/oneapi/Event.cpp
+++ b/src/backend/oneapi/Event.cpp
@@ -9,11 +9,11 @@
 
 #include <Event.hpp>
 
+#include <err_oneapi.hpp>
 #include <events.hpp>
 #include <platform.hpp>
 #include <af/event.h>
 #include <memory>
-#include <err_oneapi.hpp>
 
 #include <memory>
 
@@ -42,18 +42,18 @@ af_event createEvent() {
 
 void markEventOnActiveQueue(af_event eventHandle) {
     ONEAPI_NOT_SUPPORTED("");
-    //Event& event = getEvent(eventHandle);
+    // Event& event = getEvent(eventHandle);
     //// Use the currently-active stream
-    //if (event.mark(getQueue()()) != CL_SUCCESS) {
+    // if (event.mark(getQueue()()) != CL_SUCCESS) {
     //    AF_ERROR("Could not mark event on active queue", AF_ERR_RUNTIME);
     //}
 }
 
 void enqueueWaitOnActiveQueue(af_event eventHandle) {
     ONEAPI_NOT_SUPPORTED("");
-    //Event& event = getEvent(eventHandle);
+    // Event& event = getEvent(eventHandle);
     //// Use the currently-active stream
-    //if (event.enqueueWait(getQueue()()) != CL_SUCCESS) {
+    // if (event.enqueueWait(getQueue()()) != CL_SUCCESS) {
     //    AF_ERROR("Could not enqueue wait on active queue for event",
     //             AF_ERR_RUNTIME);
     //}
@@ -61,8 +61,8 @@ void enqueueWaitOnActiveQueue(af_event eventHandle) {
 
 void block(af_event eventHandle) {
     ONEAPI_NOT_SUPPORTED("");
-    //Event& event = getEvent(eventHandle);
-    //if (event.block() != CL_SUCCESS) {
+    // Event& event = getEvent(eventHandle);
+    // if (event.block() != CL_SUCCESS) {
     //    AF_ERROR("Could not block on active queue for event", AF_ERR_RUNTIME);
     //}
 }
@@ -70,9 +70,9 @@ void block(af_event eventHandle) {
 af_event createAndMarkEvent() {
     ONEAPI_NOT_SUPPORTED("");
     return 0;
-    //af_event handle = createEvent();
-    //markEventOnActiveQueue(handle);
-    //return handle;
+    // af_event handle = createEvent();
+    // markEventOnActiveQueue(handle);
+    // return handle;
 }
 
 }  // namespace oneapi
diff --git a/src/backend/oneapi/Event.hpp b/src/backend/oneapi/Event.hpp
index bc143283d0..ff600ebbcb 100644
--- a/src/backend/oneapi/Event.hpp
+++ b/src/backend/oneapi/Event.hpp
@@ -17,7 +17,7 @@ class OneAPIEventPolicy {
    public:
     using EventType = sycl::event;
     using QueueType = sycl::queue;
-    //using ErrorType = sycl::exception; //does this make sense
+    // using ErrorType = sycl::exception; //does this make sense
     using ErrorType = int;
 
     static ErrorType createAndMarkEvent(EventType *e) noexcept {
@@ -26,23 +26,23 @@ class OneAPIEventPolicy {
     }
 
     static ErrorType markEvent(EventType *e, QueueType stream) noexcept {
-        //return clEnqueueMarkerWithWaitList(stream, 0, nullptr, e);
-      return 0;
+        // return clEnqueueMarkerWithWaitList(stream, 0, nullptr, e);
+        return 0;
     }
 
     static ErrorType waitForEvent(EventType *e, QueueType stream) noexcept {
-        //return clEnqueueMarkerWithWaitList(stream, 1, e, nullptr);
-      return 0;
+        // return clEnqueueMarkerWithWaitList(stream, 1, e, nullptr);
+        return 0;
     }
 
     static ErrorType syncForEvent(EventType *e) noexcept {
-        //return clWaitForEvents(1, e);
-      return 0;
+        // return clWaitForEvents(1, e);
+        return 0;
     }
 
     static ErrorType destroyEvent(EventType *e) noexcept {
-        //return clReleaseEvent(*e);
-      return 0;
+        // return clReleaseEvent(*e);
+        return 0;
     }
 };
 
diff --git a/src/backend/oneapi/GraphicsResourceManager.hpp b/src/backend/oneapi/GraphicsResourceManager.hpp
index bdc889708a..6374f1ef7e 100644
--- a/src/backend/oneapi/GraphicsResourceManager.hpp
+++ b/src/backend/oneapi/GraphicsResourceManager.hpp
@@ -15,7 +15,6 @@
 #include <memory>
 #include <vector>
 
-
 namespace oneapi {
 class GraphicsResourceManager
     : public common::InteropManager<GraphicsResourceManager, std::byte> {
diff --git a/src/backend/oneapi/Kernel.hpp b/src/backend/oneapi/Kernel.hpp
index 823fc511ef..704237de24 100644
--- a/src/backend/oneapi/Kernel.hpp
+++ b/src/backend/oneapi/Kernel.hpp
@@ -12,8 +12,8 @@
 #include <common/KernelInterface.hpp>
 #include <common/Logger.hpp>
 
-#include <backend.hpp>
 #include <CL/sycl.hpp>
+#include <backend.hpp>
 #include <string>
 
 namespace oneapi {
@@ -44,7 +44,8 @@ class Kernel
     using KernelType = sycl::kernel;
     using DevPtrType<T> = sycl::buffer<T>*;
     using BaseClass =
-        common::KernelInterface<ModuleType, KernelType, Enqueuer, DevPtrType<T>>;
+        common::KernelInterface<ModuleType, KernelType, Enqueuer,
+DevPtrType<T>>;
 
     Kernel() : BaseClass("", nullptr, cl::Kernel{nullptr, false}) {}
     Kernel(std::string name, ModuleType mod, KernelType ker)
@@ -55,7 +56,8 @@ class Kernel
     DevPtrType<T> getDevPtr(const char* name) final;
     // clang-format on
 
-    void copyToReadOnly(DevPtrType<T> dst, DevPtrType<T> src, size_t bytes) final;
+    void copyToReadOnly(DevPtrType<T> dst, DevPtrType<T> src, size_t bytes)
+final;
 
     void setFlag(DevPtrType<T> dst, int* scalarValPtr,
                  const bool syncCopy = false) final;
@@ -66,12 +68,13 @@ class Kernel
 
 class Kernel {
    public:
-    using ModuleType = const sycl::kernel_bundle<sycl::bundle_state::executable> *;
+    using ModuleType =
+        const sycl::kernel_bundle<sycl::bundle_state::executable>*;
     using KernelType = sycl::kernel;
-  template<typename T>
+    template<typename T>
     using DevPtrType = sycl::buffer<T>*;
-    //using BaseClass =
-        //common::KernelInterface<ModuleType, KernelType, Enqueuer, DevPtrType<T>>;
+    // using BaseClass =
+    // common::KernelInterface<ModuleType, KernelType, Enqueuer, DevPtrType<T>>;
 
     Kernel() {}
     Kernel(std::string name, ModuleType mod, KernelType ker) {}
diff --git a/src/backend/oneapi/Module.hpp b/src/backend/oneapi/Module.hpp
index 1c34306d68..0aa1cc790d 100644
--- a/src/backend/oneapi/Module.hpp
+++ b/src/backend/oneapi/Module.hpp
@@ -9,14 +9,15 @@
 
 #pragma once
 
-#include <common/ModuleInterface.hpp>
 #include <CL/sycl.hpp>
-
+#include <common/ModuleInterface.hpp>
 
 namespace oneapi {
 
 /// oneapi backend wrapper for cl::Program object
-  class Module : public common::ModuleInterface<sycl::kernel_bundle<sycl::bundle_state::executable>> {
+class Module
+    : public common::ModuleInterface<
+          sycl::kernel_bundle<sycl::bundle_state::executable>> {
    public:
     using ModuleType = sycl::kernel_bundle<sycl::bundle_state::executable>;
     using BaseClass  = common::ModuleInterface<ModuleType>;
@@ -32,8 +33,8 @@ namespace oneapi {
 
     /// Unload the module
     void unload() final {
-      // TODO(oneapi): Unload kernel/program
-      ;
+        // TODO(oneapi): Unload kernel/program
+        ;
     }
 };
 
diff --git a/src/backend/oneapi/Param.cpp b/src/backend/oneapi/Param.cpp
index c5d2b16762..87a539ce67 100644
--- a/src/backend/oneapi/Param.cpp
+++ b/src/backend/oneapi/Param.cpp
@@ -16,7 +16,7 @@ namespace oneapi {
 
 template<typename T>
 Param<T> makeParam(sycl::buffer<T> &mem, int off, const int dims[4],
-                const int strides[4]) {
+                   const int strides[4]) {
     Param<T> out;
     out.data        = &mem;
     out.info.offset = off;
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index b65e28f2e7..4a0d6ff9cc 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -26,7 +26,7 @@ struct Param {
     Param() : data(nullptr), info{{0, 0, 0, 0}, {0, 0, 0, 0}, 0} {}
 
     // AF_DEPRECATED("Use Array<T>")
-    Param(sycl::buffer<T> *data_, KParam info_) : data(data_), info(info_) {}
+    Param(sycl::buffer<T>* data_, KParam info_) : data(data_), info(info_) {}
 
     ~Param() = default;
 };
diff --git a/src/backend/oneapi/anisotropic_diffusion.cpp b/src/backend/oneapi/anisotropic_diffusion.cpp
index c063736c21..a68b8aaa8f 100644
--- a/src/backend/oneapi/anisotropic_diffusion.cpp
+++ b/src/backend/oneapi/anisotropic_diffusion.cpp
@@ -9,8 +9,8 @@
 
 #include <Array.hpp>
 #include <anisotropic_diffusion.hpp>
-#include <err_oneapi.hpp>
 #include <copy.hpp>
+#include <err_oneapi.hpp>
 #include <af/dim4.hpp>
 
 namespace oneapi {
diff --git a/src/backend/oneapi/approx.cpp b/src/backend/oneapi/approx.cpp
index df22448704..e11216d00c 100644
--- a/src/backend/oneapi/approx.cpp
+++ b/src/backend/oneapi/approx.cpp
@@ -15,24 +15,23 @@ template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
              const int xdim, const Tp &xi_beg, const Tp &xi_step,
              const af_interp_type method, const float offGrid) {
-
     ONEAPI_NOT_SUPPORTED("");
     return;
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            //kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    //method, 1);
+            // kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step,
+            // offGrid, method, 1);
             break;
         case AF_INTERP_LINEAR:
         case AF_INTERP_LINEAR_COSINE:
-            //kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    //method, 2);
+            // kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step,
+            // offGrid, method, 2);
             break;
         case AF_INTERP_CUBIC:
         case AF_INTERP_CUBIC_SPLINE:
-            //kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    //method, 3);
+            // kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step,
+            // offGrid, method, 3);
             break;
         default: break;
     }
@@ -49,22 +48,22 @@ void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            //kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
-                                    //yi_beg, yi_step, offGrid, method, 1);
+            // kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+            // ydim, yi_beg, yi_step, offGrid, method, 1);
             break;
         case AF_INTERP_LINEAR:
         case AF_INTERP_BILINEAR:
         case AF_INTERP_LINEAR_COSINE:
         case AF_INTERP_BILINEAR_COSINE:
-            //kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
-                                    //yi_beg, yi_step, offGrid, method, 2);
+            // kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+            // ydim, yi_beg, yi_step, offGrid, method, 2);
             break;
         case AF_INTERP_CUBIC:
         case AF_INTERP_BICUBIC:
         case AF_INTERP_CUBIC_SPLINE:
         case AF_INTERP_BICUBIC_SPLINE:
-            //kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
-                                    //yi_beg, yi_step, offGrid, method, 3);
+            // kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+            // ydim, yi_beg, yi_step, offGrid, method, 3);
             break;
         default: break;
     }
diff --git a/src/backend/oneapi/assign.cpp b/src/backend/oneapi/assign.cpp
index a41365101c..5517793411 100644
--- a/src/backend/oneapi/assign.cpp
+++ b/src/backend/oneapi/assign.cpp
@@ -59,7 +59,7 @@ void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs) {
             // direct buffer allocation as opposed to mem manager to avoid
             // reference count desprepancies between different backends
             static auto* empty = new sycl::buffer<uint>(sycl::range{1});
-            bPtrs[x] = empty;
+            bPtrs[x]           = empty;
         }
     }
 
diff --git a/src/backend/oneapi/bilateral.cpp b/src/backend/oneapi/bilateral.cpp
index 4fef2afd5e..59b050d2bf 100644
--- a/src/backend/oneapi/bilateral.cpp
+++ b/src/backend/oneapi/bilateral.cpp
@@ -9,8 +9,8 @@
 
 #include <Array.hpp>
 #include <bilateral.hpp>
-#include <af/dim4.hpp>
 #include <err_oneapi.hpp>
+#include <af/dim4.hpp>
 
 using af::dim4;
 
@@ -22,7 +22,6 @@ Array<outType> bilateral(const Array<inType> &in, const float &sSigma,
     ONEAPI_NOT_SUPPORTED("");
     Array<outType> out = createEmptyArray<outType>(in.dims());
     return out;
-
 }
 
 #define INSTANTIATE(inT, outT)                                    \
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
index 852b277870..c8e8d69c98 100644
--- a/src/backend/oneapi/blas.cpp
+++ b/src/backend/oneapi/blas.cpp
@@ -26,9 +26,11 @@ using common::half;
 
 namespace oneapi {
 
-void initBlas() { /*gpu_blas_init();*/ }
+void initBlas() { /*gpu_blas_init();*/
+}
 
-void deInitBlas() { /*gpu_blas_deinit();*/ }
+void deInitBlas() { /*gpu_blas_deinit();*/
+}
 
 template<typename T>
 void gemm_fallback(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs,
diff --git a/src/backend/oneapi/compile_module.cpp b/src/backend/oneapi/compile_module.cpp
index a682ac7bfd..cc85d37005 100644
--- a/src/backend/oneapi/compile_module.cpp
+++ b/src/backend/oneapi/compile_module.cpp
@@ -30,13 +30,13 @@
 
 using common::loggerFactory;
 using fmt::format;
-//using oneapi::getActiveDeviceId;
-//using oneapi::getDevice;
-using sycl::kernel_bundle;
-using sycl::bundle_state;
+// using oneapi::getActiveDeviceId;
+// using oneapi::getDevice;
 using oneapi::Kernel;
 using oneapi::Module;
 using spdlog::logger;
+using sycl::bundle_state;
+using sycl::kernel_bundle;
 
 using std::begin;
 using std::end;
@@ -71,8 +71,8 @@ string getProgramBuildLog(const kernel_bundle<bundle_state::executable> &prog) {
 
 namespace oneapi {
 
-//const static string DEFAULT_MACROS_STR(
-    //"\n\
+// const static string DEFAULT_MACROS_STR(
+//"\n\
                                            //#ifdef USE_DOUBLE\n\
                                            //#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n\
                                            //#endif\n                     \
@@ -82,15 +82,16 @@ namespace oneapi {
                                            //#define half short\n          \
                                            //#endif\n                      \
                                            //#ifndef M_PI\n               \
-                                           //#define M_PI 3.1415926535897932384626433832795028841971693993751058209749445923078164\n \
+                                           //#define
+// M_PI 3.1415926535897932384626433832795028841971693993751058209749445923078164\n
+//\
                                            //#endif\n                     \
                                            //");
 
 /*
 get_kernel_bundle<>() needs sycl::context
-kernel_bundle<bundle_state::executable> buildProgram(const vector<string> &kernelSources,
-                           const vector<string> &compileOpts) {
-    ONEAPI_NOT_SUPPORTED("");
+kernel_bundle<bundle_state::executable> buildProgram(const vector<string>
+&kernelSources, const vector<string> &compileOpts) { ONEAPI_NOT_SUPPORTED("");
     kernel_bundle<bundle_state::executable> bb;
     return bb;
 }
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index 622268eb91..d852480342 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -9,10 +9,10 @@
 #include <copy.hpp>
 
 #include <Array.hpp>
-#include <kernel/memcopy.hpp>
 #include <common/complex.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/memcopy.hpp>
 #include <math.hpp>
 
 using common::half;
@@ -28,7 +28,7 @@ void copyData(T *data, const Array<T> &A) {
     A.eval();
 
     dim_t offset = 0;
-    const sycl::buffer<T>* buf;
+    const sycl::buffer<T> *buf;
     Array<T> out = A;
 
     if (A.isLinear() ||  // No offsets, No strides
@@ -44,12 +44,15 @@ void copyData(T *data, const Array<T> &A) {
     }
 
     // FIXME: Add checks
-    getQueue().submit([=] (sycl::handler &h) {
-        sycl::range rr(A.elements());
-        sycl::id offset_id(offset);
-        auto offset_acc = const_cast<sycl::buffer<T>*>(buf)->get_access(h, rr, offset_id);
-        h.copy(offset_acc, data);
-    }).wait();
+    getQueue()
+        .submit([=](sycl::handler &h) {
+            sycl::range rr(A.elements());
+            sycl::id offset_id(offset);
+            auto offset_acc = const_cast<sycl::buffer<T> *>(buf)->get_access(
+                h, rr, offset_id);
+            h.copy(offset_acc, data);
+        })
+        .wait();
 }
 
 template<typename T>
@@ -61,17 +64,21 @@ Array<T> copyArray(const Array<T> &A) {
     if (A.isLinear()) {
         // FIXME: Add checks
 
-        const sycl::buffer<T>* A_buf = A.get();
-        sycl::buffer<T>* out_buf = out.get();
-
-        getQueue().submit([=] (sycl::handler &h) {
-            sycl::range rr(A.elements());
-            sycl::id offset_id(offset);
-            auto offset_acc_A  = const_cast<sycl::buffer<T>*>(A_buf)->get_access(h, rr, offset_id);
-            auto acc_out = out_buf->get_access(h);
-
-            h.copy(offset_acc_A, acc_out);
-        }).wait();
+        const sycl::buffer<T> *A_buf = A.get();
+        sycl::buffer<T> *out_buf     = out.get();
+
+        getQueue()
+            .submit([=](sycl::handler &h) {
+                sycl::range rr(A.elements());
+                sycl::id offset_id(offset);
+                auto offset_acc_A =
+                    const_cast<sycl::buffer<T> *>(A_buf)->get_access(h, rr,
+                                                                     offset_id);
+                auto acc_out = out_buf->get_access(h);
+
+                h.copy(offset_acc_A, acc_out);
+            })
+            .wait();
     } else {
         kernel::memcopy<T>(out.get(), out.strides().get(), A.get(),
                            A.dims().get(), A.strides().get(), offset,
@@ -98,23 +105,27 @@ struct copyWrapper<T, T> {
     void operator()(Array<T> &out, Array<T> const &in) {
         if (out.isLinear() && in.isLinear() &&
             out.elements() == in.elements()) {
-
             dim_t in_offset  = in.getOffset() * sizeof(T);
             dim_t out_offset = out.getOffset() * sizeof(T);
 
-            const sycl::buffer<T>* in_buf = in.get();
-            sycl::buffer<T>* out_buf = out.get();
+            const sycl::buffer<T> *in_buf = in.get();
+            sycl::buffer<T> *out_buf      = out.get();
 
-            getQueue().submit([=] (sycl::handler &h) {
-                sycl::range rr(in.elements());
-                sycl::id in_offset_id(in_offset);
-                sycl::id out_offset_id(out_offset);
+            getQueue()
+                .submit([=](sycl::handler &h) {
+                    sycl::range rr(in.elements());
+                    sycl::id in_offset_id(in_offset);
+                    sycl::id out_offset_id(out_offset);
 
-                auto offset_acc_in  = const_cast<sycl::buffer<T>*>(in_buf)->get_access(h, rr, in_offset_id);
-                auto offset_acc_out = out_buf->get_access(h, rr, out_offset_id);
+                    auto offset_acc_in =
+                        const_cast<sycl::buffer<T> *>(in_buf)->get_access(
+                            h, rr, in_offset_id);
+                    auto offset_acc_out =
+                        out_buf->get_access(h, rr, out_offset_id);
 
-                h.copy(offset_acc_in, offset_acc_out);
-            }).wait();
+                    h.copy(offset_acc_in, offset_acc_out);
+                })
+                .wait();
         } else {
             kernel::copy<T, T>(out, in, in.ndims(), scalar<T>(0), 1,
                                in.dims() == out.dims());
@@ -202,12 +213,15 @@ template<typename T>
 T getScalar(const Array<T> &in) {
     T retVal{};
 
-    getQueue().submit([=] (sycl::handler &h) {
-        sycl::range rr(1);
-        sycl::id offset_id(in.getOffset());
-        auto acc_in  = const_cast<sycl::buffer<T>*>(in.get())->get_access(h, rr, offset_id);
-        h.copy(acc_in, (void*)&retVal);
-    }).wait();
+    getQueue()
+        .submit([=](sycl::handler &h) {
+            sycl::range rr(1);
+            sycl::id offset_id(in.getOffset());
+            auto acc_in = const_cast<sycl::buffer<T> *>(in.get())->get_access(
+                h, rr, offset_id);
+            h.copy(acc_in, (void *)&retVal);
+        })
+        .wait();
 
     return retVal;
 }
diff --git a/src/backend/oneapi/copy.hpp b/src/backend/oneapi/copy.hpp
index 00f01a8ac4..30d6196aa2 100644
--- a/src/backend/oneapi/copy.hpp
+++ b/src/backend/oneapi/copy.hpp
@@ -54,7 +54,7 @@ Array<T> padArrayBorders(Array<T> const &in, dim4 const &lowerBoundPadding,
 
     auto ret = createEmptyArray<T>(oDims);
 
-    //kernel::padBorders<T>(ret, in, lowerBoundPadding, btype);
+    // kernel::padBorders<T>(ret, in, lowerBoundPadding, btype);
 
     return ret;
 }
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index d4750defae..d8315eac38 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -10,7 +10,6 @@
 #include <common/graphics_common.hpp>
 
 #include <GraphicsResourceManager.hpp>
-#include <platform.hpp> //TODO: blas.hpp? y tho, also Array.hpp 
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
@@ -18,11 +17,12 @@
 #include <common/util.hpp>
 #include <device_manager.hpp>
 #include <err_oneapi.hpp>
+#include <platform.hpp>  //TODO: blas.hpp? y tho, also Array.hpp
 //#include <errorcodes.hpp>
+#include <memory.hpp>
 #include <version.hpp>
 #include <af/oneapi.h>
 #include <af/version.h>
-#include <memory.hpp>
 
 #include <algorithm>
 #include <iterator>
@@ -45,8 +45,8 @@ namespace oneapi {
 
 static inline bool compare_default(const unique_ptr<sycl::device>& ldev,
                                    const unique_ptr<sycl::device>& rdev) {
-    //TODO: update sorting criteria
-    //select according to something applicable to oneapi backend
+    // TODO: update sorting criteria
+    // select according to something applicable to oneapi backend
     auto l_mem = ldev->get_info<sycl::info::device::local_mem_size>();
     auto r_mem = rdev->get_info<sycl::info::device::local_mem_size>();
     return l_mem > r_mem;
@@ -74,8 +74,9 @@ DeviceManager::DeviceManager()
         vector<sycl::device> current_devices;
         try {
             current_devices = platform.get_devices();
-        } catch(sycl::exception& err) {
-            printf("DeviceManager::DeviceManager() exception: %s\n", err.what());
+        } catch (sycl::exception& err) {
+            printf("DeviceManager::DeviceManager() exception: %s\n",
+                   err.what());
             throw;
         }
         AF_TRACE("Found {} devices on platform {}", current_devices.size(),
@@ -102,18 +103,19 @@ DeviceManager::DeviceManager()
 
     // Create contexts and queues once the sort is done
     for (int i = 0; i < nDevices; i++) {
-        try{
+        try {
             mContexts.push_back(make_unique<sycl::context>(*devices[i]));
-            mQueues.push_back(make_unique<sycl::queue>(
-                *mContexts.back(), *devices[i]));
+            mQueues.push_back(
+                make_unique<sycl::queue>(*mContexts.back(), *devices[i]));
             mIsGLSharingOn.push_back(false);
-            //TODO:
-            //mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
-            //mPlatforms.push_back(getPlatformEnum(*devices[i]));
+            // TODO:
+            // mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
+            // mPlatforms.push_back(getPlatformEnum(*devices[i]));
             mDevices.emplace_back(std::move(devices[i]));
         } catch (sycl::exception& err) {
             AF_TRACE("Error creating context for device {} with error {}\n",
-                     devices[i]->get_info<sycl::info::device::name>(), err.what());
+                     devices[i]->get_info<sycl::info::device::name>(),
+                     err.what());
         }
     }
     nDevices = mDevices.size();
@@ -121,18 +123,18 @@ DeviceManager::DeviceManager()
     bool default_device_set = false;
     string deviceENV        = getEnvVar("AF_ONEAPI_DEFAULT_DEVICE");
     if (!deviceENV.empty()) {
-        //TODO: handle default device from env variable
+        // TODO: handle default device from env variable
     }
 
     deviceENV = getEnvVar("AF_OPENCL_DEFAULT_DEVICE_TYPE");
     if (!default_device_set && !deviceENV.empty()) {
-        //TODO: handle default device by type env variable
+        // TODO: handle default device by type env variable
     }
 
     // Define AF_DISABLE_GRAPHICS with any value to disable initialization
     string noGraphicsENV = getEnvVar("AF_DISABLE_GRAPHICS");
     if (fgMngr->plugin().isLoaded() && noGraphicsENV.empty()) {
-        //TODO: handle forge shared contexts
+        // TODO: handle forge shared contexts
     }
 
     mUserDeviceOffset = mDevices.size();
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
index f6530dcbd9..d84994226c 100644
--- a/src/backend/oneapi/device_manager.hpp
+++ b/src/backend/oneapi/device_manager.hpp
@@ -77,7 +77,7 @@ class DeviceManager {
 
     friend int getDeviceCount() noexcept;
 
-    //friend int getDeviceIdFromNativeId(cl_device_id id);
+    // friend int getDeviceIdFromNativeId(cl_device_id id);
 
     friend const sycl::context& getContext();
 
@@ -147,8 +147,8 @@ class DeviceManager {
     std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
     std::mutex mutex;
 
-  //using BoostProgCache = boost::shared_ptr<boost::compute::program_cache>;
-  //std::vector<BoostProgCache*> mBoostProgCacheVector;
+    // using BoostProgCache = boost::shared_ptr<boost::compute::program_cache>;
+    // std::vector<BoostProgCache*> mBoostProgCacheVector;
 };
 
 }  // namespace oneapi
diff --git a/src/backend/oneapi/diff.cpp b/src/backend/oneapi/diff.cpp
index 7dfffc1881..71e331a122 100644
--- a/src/backend/oneapi/diff.cpp
+++ b/src/backend/oneapi/diff.cpp
@@ -10,9 +10,9 @@
 #include <Array.hpp>
 #include <diff.hpp>
 //#include <kernel/diff.hpp>
+#include <err_oneapi.hpp>
 #include <af/dim4.hpp>
 #include <stdexcept>
-#include <err_oneapi.hpp>
 
 namespace oneapi {
 
diff --git a/src/backend/oneapi/errorcodes.cpp b/src/backend/oneapi/errorcodes.cpp
index 615bbb94e7..cf7152fa00 100644
--- a/src/backend/oneapi/errorcodes.cpp
+++ b/src/backend/oneapi/errorcodes.cpp
@@ -7,12 +7,11 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <errorcodes.hpp>
 #include <err_oneapi.hpp>
-
+#include <errorcodes.hpp>
 
 std::string getErrorMessage(int error_code) {
     ONEAPI_NOT_SUPPORTED("");
-    //return boost::compute::opencl_error::to_string(error_code);
+    // return boost::compute::opencl_error::to_string(error_code);
     return "";
 }
diff --git a/src/backend/oneapi/exampleFunction.cpp b/src/backend/oneapi/exampleFunction.cpp
index dc5c6a8680..bc5c52b031 100644
--- a/src/backend/oneapi/exampleFunction.cpp
+++ b/src/backend/oneapi/exampleFunction.cpp
@@ -16,8 +16,9 @@
 #include <err_oneapi.hpp>  // error check functions and Macros
                            // specific to oneapi backend
 
-//#include <kernel/exampleFunction.hpp>  // this header under the folder src/oneapi/kernel
-                                       // defines the OneAPI kernel wrapper
+//#include <kernel/exampleFunction.hpp>  // this header under the folder
+// src/oneapi/kernel
+// defines the OneAPI kernel wrapper
 // function to which the main computation of your
 // algorithm should be relayed to
 
@@ -41,7 +42,7 @@ Array<T> exampleFunction(const Array<T> &a, const Array<T> &b,
     // can create.
 
     // Relay the actual computation to OneAPI kernel wrapper
-    //kernel::exampleFunc<T>(out, a, b, method);
+    // kernel::exampleFunc<T>(out, a, b, method);
 
     return out;  // return the result
 }
diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
index 684cc860b7..1591e4b4cf 100644
--- a/src/backend/oneapi/fft.cpp
+++ b/src/backend/oneapi/fft.cpp
@@ -19,8 +19,7 @@ using af::dim4;
 
 namespace oneapi {
 
-void setFFTPlanCacheSize(size_t numPlans) {
-}
+void setFFTPlanCacheSize(size_t numPlans) {}
 
 /*
 template<typename T>
diff --git a/src/backend/oneapi/fftconvolve.cpp b/src/backend/oneapi/fftconvolve.cpp
index 5a2a64d869..dad10f492e 100644
--- a/src/backend/oneapi/fftconvolve.cpp
+++ b/src/backend/oneapi/fftconvolve.cpp
@@ -11,9 +11,9 @@
 
 #include <Array.hpp>
 #include <common/dispatch.hpp>
+#include <err_oneapi.hpp>
 #include <fft.hpp>
 #include <af/dim4.hpp>
-#include <err_oneapi.hpp>
 
 #include <cmath>
 #include <type_traits>
diff --git a/src/backend/oneapi/gradient.cpp b/src/backend/oneapi/gradient.cpp
index 0755b7a691..40b557a4ae 100644
--- a/src/backend/oneapi/gradient.cpp
+++ b/src/backend/oneapi/gradient.cpp
@@ -8,8 +8,8 @@
  ********************************************************/
 
 #include <Array.hpp>
-#include <gradient.hpp>
 #include <err_oneapi.hpp>
+#include <gradient.hpp>
 //#include <kernel/gradient.hpp>
 #include <math.hpp>
 #include <stdexcept>
diff --git a/src/backend/oneapi/homography.cpp b/src/backend/oneapi/homography.cpp
index e9b08cc475..5060cd50ae 100644
--- a/src/backend/oneapi/homography.cpp
+++ b/src/backend/oneapi/homography.cpp
@@ -10,8 +10,8 @@
 #include <homography.hpp>
 
 #include <arith.hpp>
-#include <af/dim4.hpp>
 #include <err_oneapi.hpp>
+#include <af/dim4.hpp>
 
 #include <algorithm>
 #include <limits>
diff --git a/src/backend/oneapi/inverse.cpp b/src/backend/oneapi/inverse.cpp
index 60026719db..079250d4f7 100644
--- a/src/backend/oneapi/inverse.cpp
+++ b/src/backend/oneapi/inverse.cpp
@@ -39,7 +39,8 @@ namespace oneapi {
 template<typename T>
 Array<T> inverse(const Array<T> &in) {
     ONEAPI_NOT_SUPPORTED("");
-    AF_ERROR("Linear Algebra is disabled on OneAPI backend", AF_ERR_NOT_CONFIGURED);
+    AF_ERROR("Linear Algebra is disabled on OneAPI backend",
+             AF_ERR_NOT_CONFIGURED);
 }
 
 #define INSTANTIATE(T) template Array<T> inverse<T>(const Array<T> &in);
diff --git a/src/backend/oneapi/jit/BufferNode.hpp b/src/backend/oneapi/jit/BufferNode.hpp
index 2e6ef7fe34..9925ec7211 100644
--- a/src/backend/oneapi/jit/BufferNode.hpp
+++ b/src/backend/oneapi/jit/BufferNode.hpp
@@ -14,10 +14,11 @@
 
 namespace oneapi {
 namespace jit {
-  template<typename T>
-  using BufferNode = common::BufferNodeBase<std::shared_ptr<sycl::buffer<T>>, KParam>;
+template<typename T>
+using BufferNode =
+    common::BufferNodeBase<std::shared_ptr<sycl::buffer<T>>, KParam>;
 }
-}  // namespace opencl
+}  // namespace oneapi
 
 namespace common {
 
diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
index 607d85ce98..a49b25de0c 100644
--- a/src/backend/oneapi/jit/kernel_generators.hpp
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -35,15 +35,16 @@ inline int setKernelArguments(
     int start_id, bool is_linear,
     std::function<void(int id, const void* ptr, size_t arg_size)>& setArg,
     const std::shared_ptr<sycl::buffer<T>>& ptr, const KParam& info) {
-  // TODO(oneapi)
-  ONEAPI_NOT_SUPPORTED("ERROR");
-  //setArg(start_id + 0, static_cast<const void*>(&ptr.get()->operator()()),
-  //sizeof(cl_mem));
+    // TODO(oneapi)
+    ONEAPI_NOT_SUPPORTED("ERROR");
+    // setArg(start_id + 0, static_cast<const void*>(&ptr.get()->operator()()),
+    // sizeof(cl_mem));
     if (is_linear) {
-      //setArg(start_id + 1, static_cast<const void*>(&info.offset),
-      //sizeof(dim_t));
+        // setArg(start_id + 1, static_cast<const void*>(&info.offset),
+        // sizeof(dim_t));
     } else {
-      //setArg(start_id + 1, static_cast<const void*>(&info), sizeof(KParam));
+        // setArg(start_id + 1, static_cast<const void*>(&info),
+        // sizeof(KParam));
     }
     return start_id + 2;
 }
@@ -109,4 +110,4 @@ inline void generateShiftNodeRead(std::stringstream& kerStream, int id,
               << "];\n";
 }
 }  // namespace
-}  // namespace opencl
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 9896306cac..7a75735f50 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -40,18 +40,26 @@ static int trimIndex(int idx, const int len) {
 
 template<typename T>
 class assignKernel {
-public:
-    assignKernel(sycl::accessor<T> out, KParam oInfo,
-                sycl::accessor<T> in, KParam iInfo, AssignKernelParam_t p,
-                sycl::accessor<uint> ptr0, sycl::accessor<uint> ptr1, 
-                sycl::accessor<uint> ptr2, sycl::accessor<uint> ptr3,
-                const int nBBS0, const int nBBS1, sycl::stream debug) :
-        out_(out), oInfo_(oInfo), in_(in), iInfo_(iInfo), p_(p),
-        ptr0_(ptr0), ptr1_(ptr1), ptr2_(ptr2), ptr3_(ptr3),
-        nBBS0_(nBBS0), nBBS1_(nBBS1), debug_(debug) {}
-
-
-    void operator() (sycl::nd_item<2> it) const {
+   public:
+    assignKernel(sycl::accessor<T> out, KParam oInfo, sycl::accessor<T> in,
+                 KParam iInfo, AssignKernelParam_t p, sycl::accessor<uint> ptr0,
+                 sycl::accessor<uint> ptr1, sycl::accessor<uint> ptr2,
+                 sycl::accessor<uint> ptr3, const int nBBS0, const int nBBS1,
+                 sycl::stream debug)
+        : out_(out)
+        , oInfo_(oInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , p_(p)
+        , ptr0_(ptr0)
+        , ptr1_(ptr1)
+        , ptr2_(ptr2)
+        , ptr3_(ptr3)
+        , nBBS0_(nBBS0)
+        , nBBS1_(nBBS1)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
         // retrive booleans that tell us which index to use
         const bool s0 = p_.isSeq[0];
         const bool s1 = p_.isSeq[1];
@@ -59,12 +67,14 @@ class assignKernel {
         const bool s3 = p_.isSeq[3];
 
         sycl::group g = it.get_group();
-        const int gz = g.get_group_id(0) / nBBS0_;
-        const int gw = g.get_group_id(1) / nBBS1_;
+        const int gz  = g.get_group_id(0) / nBBS0_;
+        const int gw  = g.get_group_id(1) / nBBS1_;
         const int gx =
-            g.get_local_range(0) * (g.get_group_id(0) - gz * nBBS0_) + it.get_local_id(0);
+            g.get_local_range(0) * (g.get_group_id(0) - gz * nBBS0_) +
+            it.get_local_id(0);
         const int gy =
-            g.get_local_range(1) * (g.get_group_id(1) - gw * nBBS1_) + it.get_local_id(1);
+            g.get_local_range(1) * (g.get_group_id(1) - gw * nBBS1_) +
+            it.get_local_id(1);
         if (gx < iInfo_.dims[0] && gy < iInfo_.dims[1] && gz < iInfo_.dims[2] &&
             gw < iInfo_.dims[3]) {
             // calculate pointer offsets for input
@@ -76,22 +86,22 @@ class assignKernel {
                     trimIndex(s2 ? gz + p_.offs[2] : ptr2_[gz], oInfo_.dims[2]);
             int l = p_.strds[3] *
                     trimIndex(s3 ? gw + p_.offs[3] : ptr3_[gw], oInfo_.dims[3]);
-            
+
             T* iptr = in_.get_pointer();
             // offset input and output pointers
             const T* src =
-                iptr +
-                (gx * iInfo_.strides[0] + gy * iInfo_.strides[1] +
-                 gz * iInfo_.strides[2] + gw * iInfo_.strides[3] + iInfo_.offset);
+                iptr + (gx * iInfo_.strides[0] + gy * iInfo_.strides[1] +
+                        gz * iInfo_.strides[2] + gw * iInfo_.strides[3] +
+                        iInfo_.offset);
 
             T* optr = out_.get_pointer();
-            T* dst = optr + (i + j + k + l) + oInfo_.offset;
+            T* dst  = optr + (i + j + k + l) + oInfo_.offset;
             // set the output
             dst[0] = src[0];
         }
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_, in_;
     KParam oInfo_, iInfo_;
     AssignKernelParam_t p_;
@@ -114,7 +124,7 @@ void assign(Param<T> out, const Param<T> in, const AssignKernelParam_t& p,
     sycl::range<2> global(blk_x * in.info.dims[2] * THREADS_X,
                           blk_y * in.info.dims[3] * THREADS_Y);
 
-    getQueue().submit([=] (sycl::handler &h) {
+    getQueue().submit([=](sycl::handler& h) {
         auto out_acc = out.data->get_access(h);
         auto in_acc  = in.data->get_access(h);
 
@@ -125,10 +135,10 @@ void assign(Param<T> out, const Param<T> in, const AssignKernelParam_t& p,
 
         sycl::stream debug_stream(2048, 128, h);
 
-        h.parallel_for(sycl::nd_range<2>(global, local), assignKernel<T>(
-            out_acc, out.info, in_acc, in.info, 
-            p, bptr0, bptr1, bptr2, bptr3,
-            blk_x, blk_y, debug_stream));
+        h.parallel_for(
+            sycl::nd_range<2>(global, local),
+            assignKernel<T>(out_acc, out.info, in_acc, in.info, p, bptr0, bptr1,
+                            bptr2, bptr3, blk_x, blk_y, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index 5141726cdb..d4672dfd0d 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -25,23 +25,28 @@ namespace kernel {
 
 template<typename T>
 class iotaKernel {
-public:
-    iotaKernel(sycl::accessor<T> out, KParam oinfo,
-               const int s0, const int s1, const int s2, const int s3,
-               const int blocksPerMatX, const int blocksPerMatY, 
-               sycl::stream debug) :
-        out_(out), oinfo_(oinfo),
-        s0_(s0), s1_(s1), s2_(s2), s3_(s3),
-        blocksPerMatX_(blocksPerMatX), blocksPerMatY_(blocksPerMatY),
-        debug_(debug) {}
-
-    void operator() (sycl::nd_item<2> it) const {
-        //printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
-        //debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) << "]" << sycl::stream_manipulator::endl;
+   public:
+    iotaKernel(sycl::accessor<T> out, KParam oinfo, const int s0, const int s1,
+               const int s2, const int s3, const int blocksPerMatX,
+               const int blocksPerMatY, sycl::stream debug)
+        : out_(out)
+        , oinfo_(oinfo)
+        , s0_(s0)
+        , s1_(s1)
+        , s2_(s2)
+        , s3_(s3)
+        , blocksPerMatX_(blocksPerMatX)
+        , blocksPerMatY_(blocksPerMatY)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        // printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
+        // debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) <<
+        // "]" << sycl::stream_manipulator::endl;
 
         sycl::group gg = it.get_group();
-        const int oz = gg.get_group_id(0) / blocksPerMatX_;
-        const int ow = gg.get_group_id(1) / blocksPerMatY_;
+        const int oz   = gg.get_group_id(0) / blocksPerMatX_;
+        const int ow   = gg.get_group_id(1) / blocksPerMatY_;
 
         const int blockIdx_x = gg.get_group_id(0) - oz * blocksPerMatX_;
         const int blockIdx_y = gg.get_group_id(1) - ow * blocksPerMatY_;
@@ -49,14 +54,14 @@ class iotaKernel {
         const int xx = it.get_local_id(0) + blockIdx_x * gg.get_local_range(0);
         const int yy = it.get_local_id(1) + blockIdx_y * gg.get_local_range(1);
 
-        if (xx >= oinfo_.dims[0] || yy >= oinfo_.dims[1] || oz >= oinfo_.dims[2] ||
-            ow >= oinfo_.dims[3])
+        if (xx >= oinfo_.dims[0] || yy >= oinfo_.dims[1] ||
+            oz >= oinfo_.dims[2] || ow >= oinfo_.dims[3])
             return;
 
         const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
 
         T val = static_cast<T>((ow % s3_) * s2_ * s1_ * s0_);
-        val +=  static_cast<T>((oz % s2_) * s1_ * s0_);
+        val += static_cast<T>((oz % s2_) * s1_ * s0_);
 
         const int incy = blocksPerMatY_ * gg.get_local_range(1);
         const int incx = blocksPerMatX_ * gg.get_local_range(0);
@@ -65,13 +70,13 @@ class iotaKernel {
             T valY   = val + (oy % s1_) * s0_;
             int oyzw = ozw + oy * oinfo_.strides[1];
             for (int ox = xx; ox < oinfo_.dims[0]; ox += incx) {
-                int oidx = oyzw + ox;
+                int oidx   = oyzw + ox;
                 out_[oidx] = valY + (ox % s0_);
             }
         }
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_;
     KParam oinfo_;
     int s0_, s1_, s2_, s3_;
@@ -94,15 +99,17 @@ void iota(Param<T> out, const af::dim4& sdims) {
                           local[1] * blocksPerMatY * out.info.dims[3]);
     sycl::nd_range<2> ndrange(global, local);
 
-    getQueue().submit([=] (sycl::handler &h) {
+    getQueue().submit([=](sycl::handler& h) {
         auto out_acc = out.data->get_access(h);
 
         sycl::stream debug_stream(2048, 128, h);
 
-        h.parallel_for(ndrange, iotaKernel<T>(out_acc, out.info,
-            static_cast<int>(sdims[0]), static_cast<int>(sdims[1]),
-            static_cast<int>(sdims[2]), static_cast<int>(sdims[3]),
-            blocksPerMatX, blocksPerMatY, debug_stream));
+        h.parallel_for(
+            ndrange, iotaKernel<T>(
+                         out_acc, out.info, static_cast<int>(sdims[0]),
+                         static_cast<int>(sdims[1]), static_cast<int>(sdims[2]),
+                         static_cast<int>(sdims[3]), blocksPerMatX,
+                         blocksPerMatY, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 4376ae0121..3f3fdce1ae 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -31,20 +31,28 @@ typedef struct {
 
 template<typename T>
 class memCopy {
-public:
-    memCopy(sycl::accessor<T> out, dims_t ostrides, 
-            sycl::accessor<T> in, dims_t idims, dims_t istrides,
-            int offset, int groups_0, int groups_1, sycl::stream debug) :
-                out_(out), ostrides_(ostrides), in_(in), idims_(idims), istrides_(istrides),
-                offset_(offset), groups_0_(groups_0), groups_1_(groups_1), debug_(debug) {}
-
-    void operator() (sycl::nd_item<2> it) const {
-        //printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
-        //debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) << "]" << sycl::stream_manipulator::endl;
+   public:
+    memCopy(sycl::accessor<T> out, dims_t ostrides, sycl::accessor<T> in,
+            dims_t idims, dims_t istrides, int offset, int groups_0,
+            int groups_1, sycl::stream debug)
+        : out_(out)
+        , ostrides_(ostrides)
+        , in_(in)
+        , idims_(idims)
+        , istrides_(istrides)
+        , offset_(offset)
+        , groups_0_(groups_0)
+        , groups_1_(groups_1)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        // printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
+        // debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) <<
+        // "]" << sycl::stream_manipulator::endl;
         const int lid0 = it.get_local_id(0);
         const int lid1 = it.get_local_id(1);
 
-        sycl::group gg = it.get_group();
+        sycl::group gg       = it.get_group();
         const int id2        = gg.get_group_id(0) / groups_0_;
         const int id3        = gg.get_group_id(1) / groups_1_;
         const int group_id_0 = gg.get_group_id(0) - groups_0_ * id2;
@@ -52,16 +60,18 @@ class memCopy {
         const int id0        = group_id_0 * gg.get_local_range(0) + lid0;
         const int id1        = group_id_1 * gg.get_local_range(1) + lid1;
 
-        debug_ << "[" << id0 <<  "," << id1 <<  "," << id2 <<  "," << id3 <<  "]" << sycl::stream_manipulator::endl;
+        debug_ << "[" << id0 << "," << id1 << "," << id2 << "," << id3 << "]"
+               << sycl::stream_manipulator::endl;
 
-        T* iptr = in_.get_pointer();
+        T *iptr = in_.get_pointer();
         iptr += offset_;
         // FIXME: Do more work per work group
 
-        T* optr = out_.get_pointer();
-        optr +=
-            id3 * ostrides_.dim[3] + id2 * ostrides_.dim[2] + id1 * ostrides_.dim[1];
-        iptr += id3 * istrides_.dim[3] + id2 * istrides_.dim[2] + id1 * istrides_.dim[1];
+        T *optr = out_.get_pointer();
+        optr += id3 * ostrides_.dim[3] + id2 * ostrides_.dim[2] +
+                id1 * ostrides_.dim[1];
+        iptr += id3 * istrides_.dim[3] + id2 * istrides_.dim[2] +
+                id1 * istrides_.dim[1];
 
         int istride0 = istrides_.dim[0];
         if (id0 < idims_.dim[0] && id1 < idims_.dim[1] && id2 < idims_.dim[2] &&
@@ -70,27 +80,25 @@ class memCopy {
         }
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_, in_;
     dims_t ostrides_, idims_, istrides_;
     int offset_, groups_0_, groups_1_;
     sycl::stream debug_;
 };
 
-
 constexpr uint DIM0 = 32;
 constexpr uint DIM1 = 8;
 
 template<typename T>
-void memcopy(sycl::buffer<T>* out, const dim_t *ostrides, const sycl::buffer<T>* in,
-             const dim_t *idims, const dim_t *istrides, int offset,
-             uint ndims) {
-    
+void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
+             const sycl::buffer<T> *in, const dim_t *idims,
+             const dim_t *istrides, int offset, uint ndims) {
     dims_t _ostrides = {{ostrides[0], ostrides[1], ostrides[2], ostrides[3]}};
     dims_t _istrides = {{istrides[0], istrides[1], istrides[2], istrides[3]}};
     dims_t _idims    = {{idims[0], idims[1], idims[2], idims[3]}};
 
-    size_t local_size[2] = { DIM0, DIM1 };
+    size_t local_size[2] = {DIM0, DIM1};
     if (ndims == 1) {
         local_size[0] *= local_size[1];
         local_size[1] = 1;
@@ -104,18 +112,20 @@ void memcopy(sycl::buffer<T>* out, const dim_t *ostrides, const sycl::buffer<T>*
                           groups_1 * idims[3] * local_size[1]);
     sycl::nd_range<2> ndrange(global, local);
 
-    printf("<%d, %d> <%d, %d>\n", ndrange.get_global_range().get(0), ndrange.get_global_range().get(1), ndrange.get_local_range().get(0), ndrange.get_local_range().get(1));
-    printf("<%d, %d> ", ndrange.get_group_range().get(0), ndrange.get_group_range().get(1));
-    getQueue().submit([=] (sycl::handler &h) {
+    printf("<%d, %d> <%d, %d>\n", ndrange.get_global_range().get(0),
+           ndrange.get_global_range().get(1), ndrange.get_local_range().get(0),
+           ndrange.get_local_range().get(1));
+    printf("<%d, %d> ", ndrange.get_group_range().get(0),
+           ndrange.get_group_range().get(1));
+    getQueue().submit([=](sycl::handler &h) {
         auto out_acc = out->get_access(h);
-        auto in_acc  = const_cast<sycl::buffer<T>*>(in)->get_access(h);
+        auto in_acc  = const_cast<sycl::buffer<T> *>(in)->get_access(h);
 
         sycl::stream debug_stream(2048, 128, h);
 
-        h.parallel_for(ndrange, memCopy<T>(
-            out_acc, _ostrides, 
-            in_acc, _idims, _istrides,
-            offset, groups_0, groups_1, debug_stream));
+        h.parallel_for(ndrange,
+                       memCopy<T>(out_acc, _ostrides, in_acc, _idims, _istrides,
+                                  offset, groups_0, groups_1, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
@@ -141,27 +151,24 @@ outType convertType(inType value) {
 }
 
 template<>
-char convertType<compute_t<common::half>, char>(
-    compute_t<common::half> value) {
+char convertType<compute_t<common::half>, char>(compute_t<common::half> value) {
     return (char)((short)value);
 }
 
 template<>
-compute_t<common::half>
-convertType<char, compute_t<common::half>>(char value) {
+compute_t<common::half> convertType<char, compute_t<common::half>>(char value) {
     return compute_t<common::half>(value);
 }
 
 template<>
-unsigned char
-convertType<compute_t<common::half>, unsigned char>(
+unsigned char convertType<compute_t<common::half>, unsigned char>(
     compute_t<common::half> value) {
     return (unsigned char)((short)value);
 }
 
 template<>
-compute_t<common::half>
-convertType<unsigned char, compute_t<common::half>>(unsigned char value) {
+compute_t<common::half> convertType<unsigned char, compute_t<common::half>>(
+    unsigned char value) {
     return compute_t<common::half>(value);
 }
 
@@ -175,15 +182,15 @@ cfloat convertType<cdouble, cfloat>(cdouble value) {
     return cfloat(value.real(), value.imag());
 }
 
-#define OTHER_SPECIALIZATIONS(IN_T)                         \
-    template<>                                              \
-    cfloat convertType<IN_T, cfloat>(IN_T value) {          \
-        return cfloat(static_cast<float>(value), 0.0f);     \
-    }                                                       \
-                                                            \
-    template<>                                              \
-    cdouble convertType<IN_T, cdouble>(IN_T value) {        \
-        return cdouble(static_cast<double>(value), 0.0);    \
+#define OTHER_SPECIALIZATIONS(IN_T)                      \
+    template<>                                           \
+    cfloat convertType<IN_T, cfloat>(IN_T value) {       \
+        return cfloat(static_cast<float>(value), 0.0f);  \
+    }                                                    \
+                                                         \
+    template<>                                           \
+    cdouble convertType<IN_T, cdouble>(IN_T value) {     \
+        return cdouble(static_cast<double>(value), 0.0); \
     }
 
 OTHER_SPECIALIZATIONS(float)
@@ -200,21 +207,27 @@ OTHER_SPECIALIZATIONS(common::half)
 
 template<typename inType, typename outType, bool SAMEDIMS>
 class reshapeCopy {
-public:
-    reshapeCopy(sycl::accessor<outType> dst, KParam oInfo, 
-                sycl::accessor<inType> src, KParam iInfo,
-                outType default_value, float factor, dims_t trgt,
-                int blk_x, int blk_y, sycl::stream debug) : 
-            dst_(dst), oInfo_(oInfo), src_(src), iInfo_(iInfo),
-            default_value_(default_value), factor_(factor), trgt_(trgt),
-            blk_x_(blk_x), blk_y_(blk_y), debug_(debug) {}
-
-    void operator() (sycl::nd_item<2> it) const {
-
+   public:
+    reshapeCopy(sycl::accessor<outType> dst, KParam oInfo,
+                sycl::accessor<inType> src, KParam iInfo, outType default_value,
+                float factor, dims_t trgt, int blk_x, int blk_y,
+                sycl::stream debug)
+        : dst_(dst)
+        , oInfo_(oInfo)
+        , src_(src)
+        , iInfo_(iInfo)
+        , default_value_(default_value)
+        , factor_(factor)
+        , trgt_(trgt)
+        , blk_x_(blk_x)
+        , blk_y_(blk_y)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
         const uint lx = it.get_local_id(0);
         const uint ly = it.get_local_id(1);
 
-        sycl::group gg = it.get_group();
+        sycl::group gg  = it.get_group();
         uint gz         = gg.get_group_id(0) / blk_x_;
         uint gw         = gg.get_group_id(1) / blk_y_;
         uint blockIdx_x = gg.get_group_id(0) - (blk_x_)*gz;
@@ -222,21 +235,23 @@ class reshapeCopy {
         uint gx         = blockIdx_x * gg.get_local_range(0) + lx;
         uint gy         = blockIdx_y * gg.get_local_range(1) + ly;
 
-        const inType* srcptr  = src_.get_pointer();
-        outType* dstptr = dst_.get_pointer();
+        const inType *srcptr = src_.get_pointer();
+        outType *dstptr      = dst_.get_pointer();
 
         const inType *in =
             srcptr + (gw * iInfo_.strides[3] + gz * iInfo_.strides[2] +
                       gy * iInfo_.strides[1] + iInfo_.offset);
-        outType *out = dstptr + (gw * oInfo_.strides[3] + gz * oInfo_.strides[2] +
-                                 gy * oInfo_.strides[1] + oInfo_.offset);
+        outType *out =
+            dstptr + (gw * oInfo_.strides[3] + gz * oInfo_.strides[2] +
+                      gy * oInfo_.strides[1] + oInfo_.offset);
 
         uint istride0 = iInfo_.strides[0];
         uint ostride0 = oInfo_.strides[0];
 
         if (gy < oInfo_.dims[1] && gz < oInfo_.dims[2] && gw < oInfo_.dims[3]) {
             int loop_offset = gg.get_local_range(0) * blk_x_;
-            bool cond = gy < trgt_.dim[1] && gz < trgt_.dim[2] && gw < trgt_.dim[3];
+            bool cond =
+                gy < trgt_.dim[1] && gz < trgt_.dim[2] && gw < trgt_.dim[3];
             for (int rep = gx; rep < oInfo_.dims[0]; rep += loop_offset) {
                 outType temp = default_value_;
                 if (SAMEDIMS || (rep < trgt_.dim[0] && cond)) {
@@ -248,7 +263,7 @@ class reshapeCopy {
         }
     }
 
-protected:
+   protected:
     sycl::accessor<outType> dst_;
     sycl::accessor<inType> src_;
     KParam oInfo_, iInfo_;
@@ -264,7 +279,7 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
           const outType default_value, const double factor,
           const bool same_dims) {
     using std::string;
-    
+
     sycl::range<2> local(DIM0, DIM1);
     size_t local_size[] = {DIM0, DIM1};
 
@@ -279,8 +294,11 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
 
     sycl::nd_range<2> ndrange(global, local);
     printf("reshape wat?\n");
-    printf("<%d, %d> <%d, %d>\n", ndrange.get_global_range().get(0), ndrange.get_global_range().get(1), ndrange.get_local_range().get(0), ndrange.get_local_range().get(1));
-    printf("<%d, %d> ", ndrange.get_group_range().get(0), ndrange.get_group_range().get(1));
+    printf("<%d, %d> <%d, %d>\n", ndrange.get_global_range().get(0),
+           ndrange.get_global_range().get(1), ndrange.get_local_range().get(0),
+           ndrange.get_local_range().get(1));
+    printf("<%d, %d> ", ndrange.get_group_range().get(0),
+           ndrange.get_group_range().get(1));
 
     dims_t trgt_dims;
     if (same_dims) {
@@ -294,24 +312,23 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
         trgt_dims    = {{trgt_i, trgt_j, trgt_k, trgt_l}};
     }
 
-    getQueue().submit([=] (sycl::handler &h) {
+    getQueue().submit([=](sycl::handler &h) {
         auto dst_acc = dst.data->get_access(h);
-        auto src_acc  = const_cast<sycl::buffer<inType>*>(src.data)->get_access(h);
+        auto src_acc =
+            const_cast<sycl::buffer<inType> *>(src.data)->get_access(h);
 
         sycl::stream debug_stream(2048, 128, h);
 
-        if(same_dims) {
+        if (same_dims) {
             h.parallel_for(ndrange, reshapeCopy<inType, outType, true>(
-                    dst_acc, dst.info,
-                    src_acc, src.info,
-                    default_value, (float)factor, trgt_dims,
-                    blk_x, blk_y, debug_stream));
+                                        dst_acc, dst.info, src_acc, src.info,
+                                        default_value, (float)factor, trgt_dims,
+                                        blk_x, blk_y, debug_stream));
         } else {
             h.parallel_for(ndrange, reshapeCopy<inType, outType, false>(
-                    dst_acc, dst.info,
-                    src_acc, src.info,
-                    default_value, (float)factor, trgt_dims,
-                    blk_x, blk_y, debug_stream));
+                                        dst_acc, dst.info, src_acc, src.info,
+                                        default_value, (float)factor, trgt_dims,
+                                        blk_x, blk_y, debug_stream));
         }
     });
     ONEAPI_DEBUG_FINISH(getQueue());
diff --git a/src/backend/oneapi/kernel/random_engine.hpp b/src/backend/oneapi/kernel/random_engine.hpp
index 4597b33a3a..8c9b5e9251 100644
--- a/src/backend/oneapi/kernel/random_engine.hpp
+++ b/src/backend/oneapi/kernel/random_engine.hpp
@@ -8,18 +8,18 @@
  ********************************************************/
 #pragma once
 
-#include <af/defines.h>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
-#include <random_engine.hpp>
-#include <kernel/random_engine_write.hpp>
 #include <kernel/random_engine_mersenne.hpp>
 #include <kernel/random_engine_philox.hpp>
 #include <kernel/random_engine_threefry.hpp>
+#include <kernel/random_engine_write.hpp>
+#include <random_engine.hpp>
 #include <traits.hpp>
 #include <types.hpp>
+#include <af/defines.h>
 
 #include <functional>
 #include <string>
@@ -33,14 +33,14 @@ static const int STATE_SIZE = (256 * 3);
 namespace oneapi {
 namespace kernel {
 
-static const uint THREADS = 256;
+static const uint THREADS           = 256;
 static const uint THREADS_PER_GROUP = 256;
 static const uint THREADS_X         = 32;
 static const uint THREADS_Y         = THREADS_PER_GROUP / THREADS_X;
 static const uint REPEAT            = 32;
 
 template<typename T>
-void uniformDistributionCBRNG(Param<T> out,  const size_t elements,
+void uniformDistributionCBRNG(Param<T> out, const size_t elements,
                               const af_random_engine_type type,
                               const uintl &seed, uintl &counter) {
     int threads          = THREADS;
@@ -50,31 +50,30 @@ void uniformDistributionCBRNG(Param<T> out,  const size_t elements,
     uint lo              = seed;
     uint hic             = counter >> 32;
     uint loc             = counter;
-    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads), sycl::range<1>(threads));
+    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads),
+                              sycl::range<1>(threads));
     switch (type) {
         case AF_RANDOM_ENGINE_PHILOX_4X32_10:
-            getQueue().submit([=] (sycl::handler &h) {
+            getQueue().submit([=](sycl::handler &h) {
                 auto out_acc = out.data->get_access(h);
 
                 sycl::stream debug_stream(2048, 128, h);
-                h.parallel_for(ndrange, 
-                    uniformPhilox<T>(out_acc,
-                                     hi, lo, hic, loc,
-                                     elementsPerBlock, elements,
-                                     debug_stream));
+                h.parallel_for(
+                    ndrange,
+                    uniformPhilox<T>(out_acc, hi, lo, hic, loc,
+                                     elementsPerBlock, elements, debug_stream));
             });
             ONEAPI_DEBUG_FINISH(getQueue());
             break;
         case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
-            getQueue().submit([=] (sycl::handler &h) {
+            getQueue().submit([=](sycl::handler &h) {
                 auto out_acc = out.data->get_access(h);
 
                 sycl::stream debug_stream(2048, 128, h);
-                h.parallel_for(ndrange, 
-                    uniformThreefry<T>(out_acc,
-                                       hi, lo, hic, loc,
-                                       elementsPerBlock, elements,
-                                       debug_stream));
+                h.parallel_for(ndrange,
+                               uniformThreefry<T>(out_acc, hi, lo, hic, loc,
+                                                  elementsPerBlock, elements,
+                                                  debug_stream));
             });
             ONEAPI_DEBUG_FINISH(getQueue());
             break;
@@ -95,30 +94,29 @@ void normalDistributionCBRNG(Param<T> out, const size_t elements,
     uint lo              = seed;
     uint hic             = counter >> 32;
     uint loc             = counter;
-    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads), sycl::range<1>(threads));
+    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads),
+                              sycl::range<1>(threads));
     switch (type) {
         case AF_RANDOM_ENGINE_PHILOX_4X32_10:
-            getQueue().submit([=] (sycl::handler &h) {
+            getQueue().submit([=](sycl::handler &h) {
                 auto out_acc = out.data->get_access(h);
 
                 sycl::stream debug_stream(2048, 128, h);
-                h.parallel_for(ndrange, 
-                    normalPhilox<T>(out_acc,
-                                    hi, lo, hic, loc,
-                                    elementsPerBlock, elements,
-                                    debug_stream));
+                h.parallel_for(
+                    ndrange,
+                    normalPhilox<T>(out_acc, hi, lo, hic, loc, elementsPerBlock,
+                                    elements, debug_stream));
             });
             break;
         case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
-            getQueue().submit([=] (sycl::handler &h) {
+            getQueue().submit([=](sycl::handler &h) {
                 auto out_acc = out.data->get_access(h);
 
                 sycl::stream debug_stream(2048, 128, h);
-                h.parallel_for(ndrange, 
-                    normalThreefry<T>(out_acc,
-                                      hi, lo, hic, loc,
-                                      elementsPerBlock, elements,
-                                      debug_stream));
+                h.parallel_for(ndrange,
+                               normalThreefry<T>(out_acc, hi, lo, hic, loc,
+                                                 elementsPerBlock, elements,
+                                                 debug_stream));
             });
             break;
         default:
@@ -140,8 +138,9 @@ void uniformDistributionMT(Param<T> out, const size_t elements,
     blocks                     = (blocks > BLOCKS) ? BLOCKS : blocks;
     uint elementsPerBlock      = divup(elements, blocks);
 
-    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads), sycl::range<1>(threads));
-    getQueue().submit([=] (sycl::handler &h) {
+    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads),
+                              sycl::range<1>(threads));
+    getQueue().submit([=](sycl::handler &h) {
         auto out_acc       = out.data->get_access(h);
         auto state_acc     = state.data->get_access(h);
         auto pos_acc       = pos.data->get_access(h);
@@ -155,12 +154,11 @@ void uniformDistributionMT(Param<T> out, const size_t elements,
         auto ltemper_acc    = local_accessor<uint, 1>(TABLE_SIZE, h);
 
         sycl::stream debug_stream(2048, 128, h);
-        h.parallel_for(ndrange, 
-            uniformMersenne<T>(out_acc,
-                               state_acc, pos_acc, sh1_acc, sh2_acc, mask, 
-                               recursion_acc, temper_acc,
-                               lstate_acc, lrecursion_acc, ltemper_acc,
-                               elementsPerBlock, elements, debug_stream));
+        h.parallel_for(ndrange, uniformMersenne<T>(
+                                    out_acc, state_acc, pos_acc, sh1_acc,
+                                    sh2_acc, mask, recursion_acc, temper_acc,
+                                    lstate_acc, lrecursion_acc, ltemper_acc,
+                                    elementsPerBlock, elements, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
@@ -169,15 +167,17 @@ template<typename T>
 void normalDistributionMT(Param<T> out, const size_t elements,
                           Param<uint> state, Param<uint> pos, Param<uint> sh1,
                           Param<uint> sh2, const uint mask,
-                          Param<uint> recursion_table, Param<uint> temper_table) {
+                          Param<uint> recursion_table,
+                          Param<uint> temper_table) {
     int threads                = THREADS;
     int min_elements_per_block = 32 * threads * 4 * sizeof(uint) / sizeof(T);
     int blocks                 = divup(elements, min_elements_per_block);
     blocks                     = (blocks > BLOCKS) ? BLOCKS : blocks;
     uint elementsPerBlock      = divup(elements, blocks);
 
-    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads), sycl::range<1>(threads));
-    getQueue().submit([=] (sycl::handler &h) {
+    sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads),
+                              sycl::range<1>(threads));
+    getQueue().submit([=](sycl::handler &h) {
         auto out_acc       = out.data->get_access(h);
         auto state_acc     = state.data->get_access(h);
         auto pos_acc       = pos.data->get_access(h);
@@ -191,12 +191,11 @@ void normalDistributionMT(Param<T> out, const size_t elements,
         auto ltemper_acc    = local_accessor<uint, 1>(TABLE_SIZE, h);
 
         sycl::stream debug_stream(2048, 128, h);
-        h.parallel_for(ndrange, 
-            normalMersenne<T>(out_acc,
-                              state_acc, pos_acc, sh1_acc, sh2_acc, mask, 
-                              recursion_acc, temper_acc,
-                              lstate_acc, lrecursion_acc, ltemper_acc,
-                              elementsPerBlock, elements, debug_stream));
+        h.parallel_for(ndrange, normalMersenne<T>(
+                                    out_acc, state_acc, pos_acc, sh1_acc,
+                                    sh2_acc, mask, recursion_acc, temper_acc,
+                                    lstate_acc, lrecursion_acc, ltemper_acc,
+                                    elementsPerBlock, elements, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/random_engine_mersenne.hpp b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
index 9fd8985ccf..6a429feee9 100644
--- a/src/backend/oneapi/kernel/random_engine_mersenne.hpp
+++ b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
@@ -52,44 +52,40 @@ constexpr int BLOCKS     = 32;
 constexpr int STATE_SIZE = (256 * 3);
 constexpr int TABLE_SIZE = 16;
 
-template <typename T, int dimensions>
+template<typename T, int dimensions>
 using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write, sycl::access::target::local>;
-
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
 
 // Utils
-static inline void read_table(uint *const sharedTable,
-                              const uint *const table,
+static inline void read_table(uint *const sharedTable, const uint *const table,
                               size_t groupId, size_t localId) {
     const uint *const t = table + (groupId * TABLE_SIZE);
     if (localId < TABLE_SIZE) { sharedTable[localId] = t[localId]; }
 }
 
-static inline void state_read(uint *const state,
-                              const uint *const gState,
-                              size_t groupRange, size_t groupId, size_t localId) {
-    const uint *const g                 = gState + (groupId * N);
+static inline void state_read(uint *const state, const uint *const gState,
+                              size_t groupRange, size_t groupId,
+                              size_t localId) {
+    const uint *const g             = gState + (groupId * N);
     state[STATE_SIZE - N + localId] = g[localId];
     if (localId < N - groupRange) {
-        state[STATE_SIZE - N + groupRange + localId] =
-            g[groupRange + localId];
+        state[STATE_SIZE - N + groupRange + localId] = g[groupRange + localId];
     }
 }
 
-static inline void state_write(uint *const gState,
-                               const uint *const state,
-                               size_t groupRange, size_t groupId, size_t localId) {
+static inline void state_write(uint *const gState, const uint *const state,
+                               size_t groupRange, size_t groupId,
+                               size_t localId) {
     uint *const g = gState + (groupId * N);
-    g[localId] = state[STATE_SIZE - N + localId];
+    g[localId]    = state[STATE_SIZE - N + localId];
     if (localId < N - groupRange) {
-        g[groupRange + localId] =
-            state[STATE_SIZE - N + groupRange + localId];
+        g[groupRange + localId] = state[STATE_SIZE - N + groupRange + localId];
     }
 }
 
-static inline uint recursion(const uint *const recursion_table,
-                             const uint mask, const uint sh1,
-                             const uint sh2, const uint x1,
+static inline uint recursion(const uint *const recursion_table, const uint mask,
+                             const uint sh1, const uint sh2, const uint x1,
                              const uint x2, uint y) {
     uint x = (x1 & mask) ^ x2;
     x ^= x << sh1;
@@ -98,8 +94,8 @@ static inline uint recursion(const uint *const recursion_table,
     return y ^ mat;
 }
 
-static inline uint temper(const uint *const temper_table,
-                          const uint v, uint t) {
+static inline uint temper(const uint *const temper_table, const uint v,
+                          uint t) {
     t ^= t >> 16;
     t ^= t >> 8;
     uint mat = temper_table[t & 0x0f];
@@ -108,17 +104,21 @@ static inline uint temper(const uint *const temper_table,
 
 // Initialization
 class initMersenneKernel {
-public:
-    initMersenneKernel(sycl::accessor<uint> state,
-                       sycl::accessor<uint> tbl,
-                       local_accessor<uint, 1> lstate,
-                       uintl seed, sycl::stream debug_stream) :
-            state_(state), tbl_(tbl), lstate_(lstate), seed_(seed), debug_(debug_stream) {}
+   public:
+    initMersenneKernel(sycl::accessor<uint> state, sycl::accessor<uint> tbl,
+                       local_accessor<uint, 1> lstate, uintl seed,
+                       sycl::stream debug_stream)
+        : state_(state)
+        , tbl_(tbl)
+        , lstate_(lstate)
+        , seed_(seed)
+        , debug_(debug_stream) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
 
-        const uint *ltbl = tbl_.get_pointer() + (TABLE_SIZE * g.get_group_id(0));
+        const uint *ltbl =
+            tbl_.get_pointer() + (TABLE_SIZE * g.get_group_id(0));
         uint hidden_seed = ltbl[4] ^ (ltbl[8] << 16);
         uint tmp         = hidden_seed;
         tmp += tmp >> 16;
@@ -132,15 +132,17 @@ class initMersenneKernel {
             lstate_[0] = seed_;
             lstate_[1] = hidden_seed;
             for (int i = 1; i < N; ++i) {
-                lstate_[i] ^=
-                    ((uint)(1812433253) * (lstate_[i - 1] ^ (lstate_[i - 1] >> 30)) + i);
+                lstate_[i] ^= ((uint)(1812433253) *
+                                   (lstate_[i - 1] ^ (lstate_[i - 1] >> 30)) +
+                               i);
             }
         }
         it.barrier();
-        state_[N * g.get_group_id(0) + it.get_local_id(0)] = lstate_[it.get_local_id(0)];
+        state_[N * g.get_group_id(0) + it.get_local_id(0)] =
+            lstate_[it.get_local_id(0)];
     }
 
-protected:
+   protected:
     sycl::accessor<uint> state_, tbl_;
     local_accessor<uint, 1> lstate_;
     uintl seed_;
@@ -149,59 +151,68 @@ class initMersenneKernel {
 
 void initMersenneState(Param<uint> state, const Param<uint> tbl, uintl seed) {
     sycl::nd_range<1> ndrange({BLOCKS * N}, {N});
-    getQueue().submit([=] (sycl::handler &h) {
-        auto state_acc = state.data->get_access(h);
-        auto tbl_acc = tbl.data->get_access(h);
+    getQueue().submit([=](sycl::handler &h) {
+        auto state_acc  = state.data->get_access(h);
+        auto tbl_acc    = tbl.data->get_access(h);
         auto lstate_acc = local_accessor<uint, 1>(N, h);
 
         sycl::stream debug_stream(2048, 128, h);
-        h.parallel_for(ndrange, 
-            initMersenneKernel(state_acc,
-                               tbl_acc, lstate_acc,
-                               seed, debug_stream));
+        h.parallel_for(ndrange,
+                       initMersenneKernel(state_acc, tbl_acc, lstate_acc, seed,
+                                          debug_stream));
     });
-    //TODO: do we need to sync before using Mersenne generators?
-    //force wait() here?
+    // TODO: do we need to sync before using Mersenne generators?
+    // force wait() here?
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 
-
-
 template<typename T>
 class uniformMersenne {
-public:
+   public:
     uniformMersenne(sycl::accessor<T> out, sycl::accessor<uint> gState,
-                    sycl::accessor<uint> pos_tbl,
-                    sycl::accessor<uint> sh1_tbl,
+                    sycl::accessor<uint> pos_tbl, sycl::accessor<uint> sh1_tbl,
                     sycl::accessor<uint> sh2_tbl, uint mask,
                     sycl::accessor<uint> g_recursion_table,
                     sycl::accessor<uint> g_temper_table,
-                    //local memory caches of global state
+                    // local memory caches of global state
                     local_accessor<uint, 1> state,
                     local_accessor<uint, 1> recursion_table,
-                    local_accessor<uint, 1> temper_table,
-                    uint elementsPerBlock, size_t elements,
-                    sycl::stream debug) :
-            out_(out), gState_(gState),
-            pos_tbl_(pos_tbl), sh1_tbl_(sh1_tbl), sh2_tbl_(sh2_tbl), mask_(mask),
-            g_recursion_table_(g_recursion_table), g_temper_table_(g_temper_table),
-            state_(state), recursion_table_(recursion_table), temper_table_(temper_table),
-            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug) {}
+                    local_accessor<uint, 1> temper_table, uint elementsPerBlock,
+                    size_t elements, sycl::stream debug)
+        : out_(out)
+        , gState_(gState)
+        , pos_tbl_(pos_tbl)
+        , sh1_tbl_(sh1_tbl)
+        , sh2_tbl_(sh2_tbl)
+        , mask_(mask)
+        , g_recursion_table_(g_recursion_table)
+        , g_temper_table_(g_temper_table)
+        , state_(state)
+        , recursion_table_(recursion_table)
+        , temper_table_(temper_table)
+        , elementsPerBlock_(elementsPerBlock)
+        , elements_(elements)
+        , debug_(debug) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
-        uint start                    = g.get_group_id(0) * elementsPerBlock_;
-        uint end                      = start + elementsPerBlock_;
-        end                           = (end > elements_) ? elements_ : end;
-        int elementsPerBlockIteration = (g.get_local_range(0) * 4 * sizeof(uint)) / sizeof(T);
+        uint start    = g.get_group_id(0) * elementsPerBlock_;
+        uint end      = start + elementsPerBlock_;
+        end           = (end > elements_) ? elements_ : end;
+        int elementsPerBlockIteration =
+            (g.get_local_range(0) * 4 * sizeof(uint)) / sizeof(T);
         int iter = divup((end - start), elementsPerBlockIteration);
 
         uint pos = pos_tbl_[it.get_group(0)];
         uint sh1 = sh1_tbl_[it.get_group(0)];
         uint sh2 = sh2_tbl_[it.get_group(0)];
-        state_read(state_.get_pointer(), gState_.get_pointer(), g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
-        read_table(recursion_table_.get_pointer(), g_recursion_table_.get_pointer(), g.get_group_id(0), it.get_local_id(0));
-        read_table(temper_table_.get_pointer(), g_temper_table_.get_pointer(), g.get_group_id(0), it.get_local_id(0));
+        state_read(state_.get_pointer(), gState_.get_pointer(),
+                   g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
+        read_table(recursion_table_.get_pointer(),
+                   g_recursion_table_.get_pointer(), g.get_group_id(0),
+                   it.get_local_id(0));
+        read_table(temper_table_.get_pointer(), g_temper_table_.get_pointer(),
+                   g.get_group_id(0), it.get_local_id(0));
         it.barrier();
 
         uint index = start;
@@ -209,36 +220,40 @@ class uniformMersenne {
         int offsetX1 = (STATE_SIZE - N + it.get_local_id(0)) % STATE_SIZE;
         int offsetX2 = (STATE_SIZE - N + it.get_local_id(0) + 1) % STATE_SIZE;
         int offsetY  = (STATE_SIZE - N + it.get_local_id(0) + pos) % STATE_SIZE;
-        int offsetT  = (STATE_SIZE - N + it.get_local_id(0) + pos - 1) % STATE_SIZE;
-        int offsetO  = it.get_local_id(0);
+        int offsetT =
+            (STATE_SIZE - N + it.get_local_id(0) + pos - 1) % STATE_SIZE;
+        int offsetO = it.get_local_id(0);
 
         for (int i = 0; i < iter; ++i) {
             for (int ii = 0; ii < 4; ++ii) {
-                uint r = recursion(recursion_table_.get_pointer(), mask_, sh1, sh2, state_[offsetX1],
-                                   state_[offsetX2], state_[offsetY]);
+                uint r = recursion(recursion_table_.get_pointer(), mask_, sh1,
+                                   sh2, state_[offsetX1], state_[offsetX2],
+                                   state_[offsetY]);
                 state_[offsetO] = r;
-                o[ii]          = temper(temper_table_.get_pointer(), r, state_[offsetT]);
-                offsetX1       = (offsetX1 + g.get_local_range(0)) % STATE_SIZE;
-                offsetX2       = (offsetX2 + g.get_local_range(0)) % STATE_SIZE;
-                offsetY        = (offsetY + g.get_local_range(0)) % STATE_SIZE;
-                offsetT        = (offsetT + g.get_local_range(0)) % STATE_SIZE;
-                offsetO        = (offsetO + g.get_local_range(0)) % STATE_SIZE;
+                o[ii] = temper(temper_table_.get_pointer(), r, state_[offsetT]);
+                offsetX1 = (offsetX1 + g.get_local_range(0)) % STATE_SIZE;
+                offsetX2 = (offsetX2 + g.get_local_range(0)) % STATE_SIZE;
+                offsetY  = (offsetY + g.get_local_range(0)) % STATE_SIZE;
+                offsetT  = (offsetT + g.get_local_range(0)) % STATE_SIZE;
+                offsetO  = (offsetO + g.get_local_range(0)) % STATE_SIZE;
                 it.barrier();
             }
             if (i == iter - 1) {
-                partialWriteOut128Bytes(out_.get_pointer(), index + it.get_local_id(0), g.get_local_range(0),
-                                        o[0], o[1], o[2], o[3], elements_);
+                partialWriteOut128Bytes(
+                    out_.get_pointer(), index + it.get_local_id(0),
+                    g.get_local_range(0), o[0], o[1], o[2], o[3], elements_);
             } else {
                 writeOut128Bytes(out_.get_pointer(), index + it.get_local_id(0),
                                  g.get_local_range(0), o[0], o[1], o[2], o[3]);
             }
             index += elementsPerBlockIteration;
         }
-        state_write(gState_.get_pointer(), state_.get_pointer(), 
-                    g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
+        state_write(gState_.get_pointer(), state_.get_pointer(),
+                    g.get_local_range(0), g.get_group_id(0),
+                    it.get_local_id(0));
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_;
     sycl::accessor<uint> gState_;
     sycl::accessor<uint> pos_tbl_, sh1_tbl_, sh2_tbl_;
@@ -252,39 +267,51 @@ class uniformMersenne {
 
 template<typename T>
 class normalMersenne {
-public:
+   public:
     normalMersenne(sycl::accessor<T> out, sycl::accessor<uint> gState,
-                   sycl::accessor<uint> pos_tbl,
-                   sycl::accessor<uint> sh1_tbl,
+                   sycl::accessor<uint> pos_tbl, sycl::accessor<uint> sh1_tbl,
                    sycl::accessor<uint> sh2_tbl, uint mask,
                    sycl::accessor<uint> g_recursion_table,
                    sycl::accessor<uint> g_temper_table,
-                   //local memory caches of global state
+                   // local memory caches of global state
                    local_accessor<uint, 1> state,
                    local_accessor<uint, 1> recursion_table,
-                   local_accessor<uint, 1> temper_table,
-                   uint elementsPerBlock, size_t elements,
-                   sycl::stream debug) :
-            out_(out), gState_(gState),
-            pos_tbl_(pos_tbl), sh1_tbl_(sh1_tbl), sh2_tbl_(sh2_tbl), mask_(mask),
-            g_recursion_table_(g_recursion_table), g_temper_table_(g_temper_table),
-            state_(state), recursion_table_(recursion_table), temper_table_(temper_table),
-            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug) {}
+                   local_accessor<uint, 1> temper_table, uint elementsPerBlock,
+                   size_t elements, sycl::stream debug)
+        : out_(out)
+        , gState_(gState)
+        , pos_tbl_(pos_tbl)
+        , sh1_tbl_(sh1_tbl)
+        , sh2_tbl_(sh2_tbl)
+        , mask_(mask)
+        , g_recursion_table_(g_recursion_table)
+        , g_temper_table_(g_temper_table)
+        , state_(state)
+        , recursion_table_(recursion_table)
+        , temper_table_(temper_table)
+        , elementsPerBlock_(elementsPerBlock)
+        , elements_(elements)
+        , debug_(debug) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
-        uint start                    = g.get_group_id(0) * elementsPerBlock_;
-        uint end                      = start + elementsPerBlock_;
-        end                           = (end > elements_) ? elements_ : end;
-        int elementsPerBlockIteration = (g.get_local_range(0) * 4 * sizeof(uint)) / sizeof(T);
+        uint start    = g.get_group_id(0) * elementsPerBlock_;
+        uint end      = start + elementsPerBlock_;
+        end           = (end > elements_) ? elements_ : end;
+        int elementsPerBlockIteration =
+            (g.get_local_range(0) * 4 * sizeof(uint)) / sizeof(T);
         int iter = divup((end - start), elementsPerBlockIteration);
 
         uint pos = pos_tbl_[it.get_group(0)];
         uint sh1 = sh1_tbl_[it.get_group(0)];
         uint sh2 = sh2_tbl_[it.get_group(0)];
-        state_read(state_.get_pointer(), gState_.get_pointer(), g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
-        read_table(recursion_table_.get_pointer(), g_recursion_table_.get_pointer(), g.get_group_id(0), it.get_local_id(0));
-        read_table(temper_table_.get_pointer(), g_temper_table_.get_pointer(), g.get_group_id(0), it.get_local_id(0));
+        state_read(state_.get_pointer(), gState_.get_pointer(),
+                   g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
+        read_table(recursion_table_.get_pointer(),
+                   g_recursion_table_.get_pointer(), g.get_group_id(0),
+                   it.get_local_id(0));
+        read_table(temper_table_.get_pointer(), g_temper_table_.get_pointer(),
+                   g.get_group_id(0), it.get_local_id(0));
         it.barrier();
 
         uint index = start;
@@ -292,36 +319,41 @@ class normalMersenne {
         int offsetX1 = (STATE_SIZE - N + it.get_local_id(0)) % STATE_SIZE;
         int offsetX2 = (STATE_SIZE - N + it.get_local_id(0) + 1) % STATE_SIZE;
         int offsetY  = (STATE_SIZE - N + it.get_local_id(0) + pos) % STATE_SIZE;
-        int offsetT  = (STATE_SIZE - N + it.get_local_id(0) + pos - 1) % STATE_SIZE;
-        int offsetO  = it.get_local_id(0);
+        int offsetT =
+            (STATE_SIZE - N + it.get_local_id(0) + pos - 1) % STATE_SIZE;
+        int offsetO = it.get_local_id(0);
 
         for (int i = 0; i < iter; ++i) {
             for (int ii = 0; ii < 4; ++ii) {
-                uint r = recursion(recursion_table_.get_pointer(), mask_, sh1, sh2, state_[offsetX1],
-                                   state_[offsetX2], state_[offsetY]);
+                uint r = recursion(recursion_table_.get_pointer(), mask_, sh1,
+                                   sh2, state_[offsetX1], state_[offsetX2],
+                                   state_[offsetY]);
                 state_[offsetO] = r;
-                o[ii]          = temper(temper_table_.get_pointer(), r, state_[offsetT]);
-                offsetX1       = (offsetX1 + g.get_local_range(0)) % STATE_SIZE;
-                offsetX2       = (offsetX2 + g.get_local_range(0)) % STATE_SIZE;
-                offsetY        = (offsetY + g.get_local_range(0)) % STATE_SIZE;
-                offsetT        = (offsetT + g.get_local_range(0)) % STATE_SIZE;
-                offsetO        = (offsetO + g.get_local_range(0)) % STATE_SIZE;
+                o[ii] = temper(temper_table_.get_pointer(), r, state_[offsetT]);
+                offsetX1 = (offsetX1 + g.get_local_range(0)) % STATE_SIZE;
+                offsetX2 = (offsetX2 + g.get_local_range(0)) % STATE_SIZE;
+                offsetY  = (offsetY + g.get_local_range(0)) % STATE_SIZE;
+                offsetT  = (offsetT + g.get_local_range(0)) % STATE_SIZE;
+                offsetO  = (offsetO + g.get_local_range(0)) % STATE_SIZE;
                 it.barrier();
             }
             if (i == iter - 1) {
-                partialBoxMullerWriteOut128Bytes(out_.get_pointer(), index + it.get_local_id(0), 
-                                        g.get_local_range(0), o[0], o[1], o[2], o[3], elements_);
+                partialBoxMullerWriteOut128Bytes(
+                    out_.get_pointer(), index + it.get_local_id(0),
+                    g.get_local_range(0), o[0], o[1], o[2], o[3], elements_);
             } else {
-                boxMullerWriteOut128Bytes(out_.get_pointer(), index + it.get_local_id(0),
-                                          g.get_local_range(0), o[0], o[1], o[2], o[3]);
+                boxMullerWriteOut128Bytes(
+                    out_.get_pointer(), index + it.get_local_id(0),
+                    g.get_local_range(0), o[0], o[1], o[2], o[3]);
             }
             index += elementsPerBlockIteration;
         }
-        state_write(gState_.get_pointer(), state_.get_pointer(), 
-                    g.get_local_range(0), g.get_group_id(0), it.get_local_id(0));
+        state_write(gState_.get_pointer(), state_.get_pointer(),
+                    g.get_local_range(0), g.get_group_id(0),
+                    it.get_local_id(0));
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_;
     sycl::accessor<uint> gState_;
     sycl::accessor<uint> pos_tbl_, sh1_tbl_, sh2_tbl_;
diff --git a/src/backend/oneapi/kernel/random_engine_philox.hpp b/src/backend/oneapi/kernel/random_engine_philox.hpp
index 3cb3dbd95b..e43cfa31e5 100644
--- a/src/backend/oneapi/kernel/random_engine_philox.hpp
+++ b/src/backend/oneapi/kernel/random_engine_philox.hpp
@@ -58,7 +58,7 @@ constexpr uint m4x32_1 = 0xCD9E8D57;
 constexpr uint w32_0   = 0x9E3779B9;
 constexpr uint w32_1   = 0xBB67AE85;
 
-static inline void mulhilo(uint a, uint b, uint &hi, uint &lo) {
+static inline void mulhilo(uint a, uint b, uint& hi, uint& lo) {
     hi = sycl::mul_hi(a, b);
     lo = a * b;
 }
@@ -68,8 +68,8 @@ static inline void philoxBump(uint k[2]) {
     k[1] += w32_1;
 }
 
-static inline void philoxRound(const uint m0, const uint m1,
-                               const uint k[2], uint c[4]) {
+static inline void philoxRound(const uint m0, const uint m1, const uint k[2],
+                               uint c[4]) {
     uint hi0, lo0, hi1, lo1;
     mulhilo(m0, c[0], hi0, lo0);
     mulhilo(m1, c[2], hi1, lo1);
@@ -104,19 +104,25 @@ static inline void philox(uint key[2], uint ctr[4]) {
 
 template<typename T>
 class uniformPhilox {
-public:
-    uniformPhilox(sycl::accessor<T> out,
-                  uint hi, uint lo, uint hic, uint loc,
+   public:
+    uniformPhilox(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
                   uint elementsPerBlock, uint elements,
-                  sycl::stream debug_stream) :
-            out_(out), hi_(hi), lo_(lo), hic_(hic), loc_(loc),
-            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug_stream) {}
+                  sycl::stream debug_stream)
+        : out_(out)
+        , hi_(hi)
+        , lo_(lo)
+        , hic_(hic)
+        , loc_(loc)
+        , elementsPerBlock_(elementsPerBlock)
+        , elements_(elements)
+        , debug_(debug_stream) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
 
-        //debug_ << "<" << g.get_group_id(0)  << ":" << it.get_local_id(0) << "/" << g.get_group_range(0) << sycl::stream_manipulator::endl;
-        uint index  = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
+        // debug_ << "<" << g.get_group_id(0)  << ":" << it.get_local_id(0) <<
+        // "/" << g.get_group_range(0) << sycl::stream_manipulator::endl;
+        uint index = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
         uint key[2] = {lo_, hi_};
         uint ctr[4] = {loc_, hic_, 0, 0};
         ctr[0] += index;
@@ -125,15 +131,16 @@ class uniformPhilox {
         T* optr = out_.get_pointer();
         if (g.get_group_id(0) != (g.get_group_range(0) - 1)) {
             philox(key, ctr);
-            writeOut128Bytes(optr, index, g.get_local_range(0), ctr[0], ctr[1], ctr[2], ctr[3]);
+            writeOut128Bytes(optr, index, g.get_local_range(0), ctr[0], ctr[1],
+                             ctr[2], ctr[3]);
         } else {
             philox(key, ctr);
-            partialWriteOut128Bytes(optr, index, g.get_local_range(0), ctr[0], ctr[1], ctr[2], ctr[3],
-                                    elements_);
+            partialWriteOut128Bytes(optr, index, g.get_local_range(0), ctr[0],
+                                    ctr[1], ctr[2], ctr[3], elements_);
         }
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
@@ -142,19 +149,25 @@ class uniformPhilox {
 
 template<typename T>
 class normalPhilox {
-public:
-    normalPhilox(sycl::accessor<T> out,
-                  uint hi, uint lo, uint hic, uint loc,
-                  uint elementsPerBlock, uint elements,
-                  sycl::stream debug_stream) :
-            out_(out), hi_(hi), lo_(lo), hic_(hic), loc_(loc),
-            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug_stream) {}
+   public:
+    normalPhilox(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
+                 uint elementsPerBlock, uint elements,
+                 sycl::stream debug_stream)
+        : out_(out)
+        , hi_(hi)
+        , lo_(lo)
+        , hic_(hic)
+        , loc_(loc)
+        , elementsPerBlock_(elementsPerBlock)
+        , elements_(elements)
+        , debug_(debug_stream) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
-        //debug_ << "<" << g.get_group_id(0)  << ":" << it.get_local_id(0) << "/" << g.get_group_range(0) << sycl::stream_manipulator::endl;
+        // debug_ << "<" << g.get_group_id(0)  << ":" << it.get_local_id(0) <<
+        // "/" << g.get_group_range(0) << sycl::stream_manipulator::endl;
 
-        uint index  = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
+        uint index = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
         uint key[2] = {lo_, hi_};
         uint ctr[4] = {loc_, hic_, 0, 0};
         ctr[0] += index;
@@ -165,14 +178,16 @@ class normalPhilox {
 
         T* optr = out_.get_pointer();
         if (g.get_group_id(0) != (g.get_group_range(0) - 1)) {
-            boxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), ctr[0], ctr[1], ctr[2], ctr[3]);
+            boxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), ctr[0],
+                                      ctr[1], ctr[2], ctr[3]);
         } else {
-            partialBoxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), 
-                                             ctr[0], ctr[1], ctr[2], ctr[3], elements_);
+            partialBoxMullerWriteOut128Bytes(optr, index, g.get_local_range(0),
+                                             ctr[0], ctr[1], ctr[2], ctr[3],
+                                             elements_);
         }
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
diff --git a/src/backend/oneapi/kernel/random_engine_threefry.hpp b/src/backend/oneapi/kernel/random_engine_threefry.hpp
index bb93e299bc..931e60ef63 100644
--- a/src/backend/oneapi/kernel/random_engine_threefry.hpp
+++ b/src/backend/oneapi/kernel/random_engine_threefry.hpp
@@ -64,10 +64,7 @@ static const uint R5 = 29;
 static const uint R6 = 16;
 static const uint R7 = 24;
 
-
-static inline void setSkeinParity(uint *ptr) {
-    *ptr = SKEIN_KS_PARITY32;
-}
+static inline void setSkeinParity(uint* ptr) { *ptr = SKEIN_KS_PARITY32; }
 
 static inline uint rotL(uint x, uint N) {
     return (x << (N & 31)) | (x >> ((32 - N) & 31));
@@ -162,17 +159,22 @@ void threefry(uint k[2], uint c[2], uint X[2]) {
 
 template<typename T>
 class uniformThreefry {
-public:
-    uniformThreefry(sycl::accessor<T> out,
-                    uint hi, uint lo, uint hic, uint loc,
+   public:
+    uniformThreefry(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
                     uint elementsPerBlock, uint elements,
-                    sycl::stream debug_stream) :
-            out_(out), hi_(hi), lo_(lo), hic_(hic), loc_(loc),
-            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug_stream) {}
+                    sycl::stream debug_stream)
+        : out_(out)
+        , hi_(hi)
+        , lo_(lo)
+        , hic_(hic)
+        , loc_(loc)
+        , elementsPerBlock_(elementsPerBlock)
+        , elements_(elements)
+        , debug_(debug_stream) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
-        uint index  = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
+        uint index = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
 
         uint key[2] = {lo_, hi_};
         uint ctr[4] = {loc_, hic_, 0, 0};
@@ -188,14 +190,15 @@ class uniformThreefry {
 
         T* optr = out_.get_pointer();
         if (g.get_group_id(0) != (g.get_group_range(0) - 1)) {
-            writeOut128Bytes(optr, index, g.get_local_range(0), o[0], o[1], o[2], o[3]);
+            writeOut128Bytes(optr, index, g.get_local_range(0), o[0], o[1],
+                             o[2], o[3]);
         } else {
-            partialWriteOut128Bytes(optr, index, g.get_local_range(0), 
-                                    o[0], o[1], o[2], o[3], elements_);
+            partialWriteOut128Bytes(optr, index, g.get_local_range(0), o[0],
+                                    o[1], o[2], o[3], elements_);
         }
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
@@ -204,17 +207,22 @@ class uniformThreefry {
 
 template<typename T>
 class normalThreefry {
-public:
-    normalThreefry(sycl::accessor<T> out,
-                    uint hi, uint lo, uint hic, uint loc,
-                    uint elementsPerBlock, uint elements,
-                    sycl::stream debug_stream) :
-            out_(out), hi_(hi), lo_(lo), hic_(hic), loc_(loc),
-            elementsPerBlock_(elementsPerBlock), elements_(elements), debug_(debug_stream) {}
+   public:
+    normalThreefry(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
+                   uint elementsPerBlock, uint elements,
+                   sycl::stream debug_stream)
+        : out_(out)
+        , hi_(hi)
+        , lo_(lo)
+        , hic_(hic)
+        , loc_(loc)
+        , elementsPerBlock_(elementsPerBlock)
+        , elements_(elements)
+        , debug_(debug_stream) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
-        uint index  = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
+        uint index = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
 
         uint key[2] = {lo_, hi_};
         uint ctr[4] = {loc_, hic_, 0, 0};
@@ -230,14 +238,15 @@ class normalThreefry {
 
         T* optr = out_.get_pointer();
         if (g.get_group_id(0) != (g.get_group_range(0) - 1)) {
-            boxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), o[0], o[1], o[2], o[3]);
+            boxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), o[0],
+                                      o[1], o[2], o[3]);
         } else {
-            partialBoxMullerWriteOut128Bytes(optr, index, g.get_local_range(0), 
+            partialBoxMullerWriteOut128Bytes(optr, index, g.get_local_range(0),
                                              o[0], o[1], o[2], o[3], elements_);
         }
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index 9e943eaad2..3b2857b92f 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -12,17 +12,16 @@
 namespace oneapi {
 namespace kernel {
 
-//TODO: !!!! half functions still need to be ported !!!!
-
+// TODO: !!!! half functions still need to be ported !!!!
 
 //// Conversion to half adapted from Random123
 //// #define HALF_FACTOR (1.0f) / (std::numeric_limits<ushort>::max() + (1.0f))
 //// #define HALF_HALF_FACTOR ((0.5f) * HALF_FACTOR)
 ////
 //// NOTE: The following constants for half were calculated using the formulas
-//// above. This is done so that we can avoid unnecessary computations because the
-//// __half datatype is not a constexprable type. This prevents the compiler from
-//// peforming these operations at compile time.
+//// above. This is done so that we can avoid unnecessary computations because
+/// the / __half datatype is not a constexprable type. This prevents the
+/// compiler from / peforming these operations at compile time.
 //#define HALF_FACTOR __ushort_as_half(0x100u)
 //#define HALF_HALF_FACTOR __ushort_as_half(0x80)
 //
@@ -32,16 +31,16 @@ namespace kernel {
 ////#define SIGNED_HALF_HALF_FACTOR ((0.5f) * SIGNED_HALF_FACTOR)
 ////
 //// NOTE: The following constants for half were calculated using the formulas
-//// above. This is done so that we can avoid unnecessary computations because the
-//// __half datatype is not a constexprable type. This prevents the compiler from
-//// peforming these operations at compile time
+//// above. This is done so that we can avoid unnecessary computations because
+/// the / __half datatype is not a constexprable type. This prevents the
+/// compiler from / peforming these operations at compile time
 //#define SIGNED_HALF_FACTOR __ushort_as_half(0x200u)
 //#define SIGNED_HALF_HALF_FACTOR __ushort_as_half(0x100u)
 //
 ///// This is the largest integer representable by fp16. We need to
 ///// make sure that the value converted from ushort is smaller than this
 ///// value to avoid generating infinity
-//constexpr ushort max_int_before_infinity = 65504;
+// constexpr ushort max_int_before_infinity = 65504;
 //
 //// Generates rationals in (0, 1]
 //__device__ static __half oneMinusGetHalf01(uint num) {
@@ -154,25 +153,25 @@ namespace {
 //    }                                               \
 //    HALF_MATH_FUNC(OP, HALF_OP)
 //
-//MATH_FUNC(log, log, logf, hlog)
-//MATH_FUNC(sqrt, sqrt, sqrtf, hsqrt)
-//MATH_FUNC(sin, sin, sinf, hsin)
-//MATH_FUNC(cos, cos, cosf, hcos)
+// MATH_FUNC(log, log, logf, hlog)
+// MATH_FUNC(sqrt, sqrt, sqrtf, hsqrt)
+// MATH_FUNC(sin, sin, sinf, hsin)
+// MATH_FUNC(cos, cos, cosf, hcos)
 //
-//template<typename T>
+// template<typename T>
 //__device__ void sincos(T val, T *sptr, T *cptr);
 //
-//template<>
+// template<>
 //__device__ void sincos(double val, double *sptr, double *cptr) {
 //    ::sincos(val, sptr, cptr);
 //}
 //
-//template<>
+// template<>
 //__device__ void sincos(float val, float *sptr, float *cptr) {
 //    sincosf(val, sptr, cptr);
 //}
 //
-//template<>
+// template<>
 //__device__ void sincos(__half val, __half *sptr, __half *cptr) {
 //#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
 //    *sptr = sin(val);
@@ -192,7 +191,7 @@ void sincospi(T val, T *sptr, T *cptr) {
     *cptr = sycl::cospi(val);
 }
 
-//template<>
+// template<>
 //__device__ void sincospi(__half val, __half *sptr, __half *cptr) {
 //    // CUDA cannot make __half into a constexpr as of CUDA 11 so we are
 //    // converting this offline
@@ -217,14 +216,13 @@ constexpr T neg_two() {
     return -2.0;
 }
 //
-//template<typename T>
-//constexpr __device__ T two_pi() {
+// template<typename T>
+// constexpr __device__ T two_pi() {
 //    return 2.0 * PI_VAL;
 //};
 //
 template<typename Tc>
-static void boxMullerTransform(cfloat *const cOut,
-                               const Tc &r1, const Tc &r2) {
+static void boxMullerTransform(cfloat *const cOut, const Tc &r1, const Tc &r2) {
     /*
      * The log of a real value x where 0 < x < 1 is negative.
      */
@@ -240,8 +238,8 @@ static void boxMullerTransform(cfloat *const cOut,
 }
 
 template<typename Tc>
-static void boxMullerTransform(cdouble *const cOut,
-                               const Tc &r1, const Tc &r2) {
+static void boxMullerTransform(cdouble *const cOut, const Tc &r1,
+                               const Tc &r2) {
     /*
      * The log of a real value x where 0 < x < 1 is negative.
      */
@@ -257,8 +255,8 @@ static void boxMullerTransform(cdouble *const cOut,
 }
 
 template<typename Td, typename Tc>
-static void boxMullerTransform(Td *const out1, Td *const out2,
-                               const Tc &r1, const Tc &r2) {
+static void boxMullerTransform(Td *const out1, Td *const out2, const Tc &r1,
+                               const Tc &r2) {
     /*
      * The log of a real value x where 0 < x < 1 is negative.
      */
@@ -272,7 +270,7 @@ static void boxMullerTransform(Td *const out1, Td *const out2,
     *out1 = static_cast<Td>(r * s);
     *out2 = static_cast<Td>(r * c);
 }
-//template<>
+// template<>
 //__device__ void boxMullerTransform<common::half, __half>(
 //    common::half *const out1, common::half *const out2, const __half &r1,
 //    const __half &r2) {
@@ -286,8 +284,8 @@ static void boxMullerTransform(Td *const out1, Td *const out2,
 
 // Writes without boundary checking
 static void writeOut128Bytes(uchar *out, const uint &index, const uint groupSz,
-                             const uint &r1, const uint &r2,
-                             const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     out[index]                = r1;
     out[index + groupSz]      = r1 >> 8;
     out[index + 2 * groupSz]  = r1 >> 16;
@@ -307,8 +305,8 @@ static void writeOut128Bytes(uchar *out, const uint &index, const uint groupSz,
 }
 
 static void writeOut128Bytes(char *out, const uint &index, const uint groupSz,
-                             const uint &r1, const uint &r2,
-                             const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     out[index]                = (r1)&0x1;
     out[index + groupSz]      = (r1 >> 1) & 0x1;
     out[index + 2 * groupSz]  = (r1 >> 2) & 0x1;
@@ -328,8 +326,8 @@ static void writeOut128Bytes(char *out, const uint &index, const uint groupSz,
 }
 
 static void writeOut128Bytes(short *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     out[index]               = r1;
     out[index + groupSz]     = r1 >> 16;
     out[index + 2 * groupSz] = r2;
@@ -341,14 +339,14 @@ static void writeOut128Bytes(short *out, const uint &index, const uint groupSz,
 }
 
 static void writeOut128Bytes(ushort *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     writeOut128Bytes((short *)(out), index, groupSz, r1, r2, r3, r4);
 }
 
 static void writeOut128Bytes(int *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     out[index]               = r1;
     out[index + groupSz]     = r2;
     out[index + 2 * groupSz] = r3;
@@ -356,14 +354,14 @@ static void writeOut128Bytes(int *out, const uint &index, const uint groupSz,
 }
 
 static void writeOut128Bytes(uint *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     writeOut128Bytes((int *)(out), index, groupSz, r1, r2, r3, r4);
 }
 
 static void writeOut128Bytes(intl *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     intl c1              = r2;
     c1                   = (c1 << 32) | r1;
     intl c2              = r4;
@@ -373,14 +371,14 @@ static void writeOut128Bytes(intl *out, const uint &index, const uint groupSz,
 }
 
 static void writeOut128Bytes(uintl *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     writeOut128Bytes((intl *)(out), index, groupSz, r1, r2, r3, r4);
 }
 
 static void writeOut128Bytes(float *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     out[index]               = 1.f - getFloat01(r1);
     out[index + groupSz]     = 1.f - getFloat01(r2);
     out[index + 2 * groupSz] = 1.f - getFloat01(r3);
@@ -388,131 +386,115 @@ static void writeOut128Bytes(float *out, const uint &index, const uint groupSz,
 }
 
 static void writeOut128Bytes(cfloat *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
-    out[index] = {1.f - getFloat01(r1), 1.f - getFloat01(r2)};
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
+    out[index]           = {1.f - getFloat01(r1), 1.f - getFloat01(r2)};
     out[index + groupSz] = {1.f - getFloat01(r3), 1.f - getFloat01(r4)};
 }
 
 static void writeOut128Bytes(double *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
     out[index]           = 1.0 - getDouble01(r1, r2);
     out[index + groupSz] = 1.0 - getDouble01(r3, r4);
 }
 
-static void writeOut128Bytes(cdouble *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
+static void writeOut128Bytes(cdouble *out, const uint &index,
+                             const uint groupSz, const uint &r1, const uint &r2,
+                             const uint &r3, const uint &r4) {
     out[index] = {1.0 - getDouble01(r1, r2), 1.0 - getDouble01(r3, r4)};
 }
 
-static void writeOut128Bytes(common::half *out, const uint &index, const uint groupSz,
-                                        const uint &r1, const uint &r2,
-                                        const uint &r3, const uint &r4) {
-   //out[index]               = oneMinusGetHalf01(r1);
-   //out[index + groupSz]     = oneMinusGetHalf01(r1 >> 16);
-   //out[index + 2 * groupSz] = oneMinusGetHalf01(r2);
-   //out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16);
-   //out[index + 4 * groupSz] = oneMinusGetHalf01(r3);
-   //out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16);
-   //out[index + 6 * groupSz] = oneMinusGetHalf01(r4);
-   //out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16);
+static void writeOut128Bytes(common::half *out, const uint &index,
+                             const uint groupSz, const uint &r1, const uint &r2,
+                             const uint &r3, const uint &r4) {
+    // out[index]               = oneMinusGetHalf01(r1);
+    // out[index + groupSz]     = oneMinusGetHalf01(r1 >> 16);
+    // out[index + 2 * groupSz] = oneMinusGetHalf01(r2);
+    // out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16);
+    // out[index + 4 * groupSz] = oneMinusGetHalf01(r3);
+    // out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16);
+    // out[index + 6 * groupSz] = oneMinusGetHalf01(r4);
+    // out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16);
 }
 
 // Normalized writes without boundary checking
 
-static void boxMullerWriteOut128Bytes(float *out, const uint &index, const uint groupSz,
-                                                 const uint &r1, const uint &r2,
-                                                 const uint &r3,
-                                                 const uint &r4) {
+static void boxMullerWriteOut128Bytes(float *out, const uint &index,
+                                      const uint groupSz, const uint &r1,
+                                      const uint &r2, const uint &r3,
+                                      const uint &r4) {
     boxMullerTransform(&out[index], &out[index + groupSz],
                        getFloatNegative11(r1), getFloat01(r2));
-    boxMullerTransform(&out[index + 2 * groupSz],
-                       &out[index + 3 * groupSz],
-                       getFloatNegative11(r3),
-                       getFloat01(r4));
+    boxMullerTransform(&out[index + 2 * groupSz], &out[index + 3 * groupSz],
+                       getFloatNegative11(r3), getFloat01(r4));
 }
 
-static void boxMullerWriteOut128Bytes(cfloat *out, const uint &index, const uint groupSz,
-                                                 const uint &r1, const uint &r2,
-                                                 const uint &r3,
-                                                 const uint &r4) {
+static void boxMullerWriteOut128Bytes(cfloat *out, const uint &index,
+                                      const uint groupSz, const uint &r1,
+                                      const uint &r2, const uint &r3,
+                                      const uint &r4) {
     boxMullerTransform(&out[index], getFloatNegative11(r1), getFloat01(r2));
-    boxMullerTransform(&out[index + groupSz], getFloatNegative11(r3), getFloat01(r4));
+    boxMullerTransform(&out[index + groupSz], getFloatNegative11(r3),
+                       getFloat01(r4));
 }
 
-static void boxMullerWriteOut128Bytes(double *out, const uint &index, const uint groupSz,
-                                                 const uint &r1, const uint &r2,
-                                                 const uint &r3,
-                                                 const uint &r4) {
+static void boxMullerWriteOut128Bytes(double *out, const uint &index,
+                                      const uint groupSz, const uint &r1,
+                                      const uint &r2, const uint &r3,
+                                      const uint &r4) {
     boxMullerTransform(&out[index], &out[index + groupSz],
                        getDoubleNegative11(r1, r2), getDouble01(r3, r4));
 }
 
-static void boxMullerWriteOut128Bytes(cdouble *out,
-                                                 const uint &index, const uint groupSz,
-                                                 const uint &r1, const uint &r2,
-                                                 const uint &r3,
-                                                 const uint &r4) {
-    boxMullerTransform(&out[index], getDoubleNegative11(r1, r2), getDouble01(r3, r4));
-}
-
-static void boxMullerWriteOut128Bytes(common::half *out,
-                                                 const uint &index, const uint groupSz,
-                                                 const uint &r1, const uint &r2,
-                                                 const uint &r3,
-                                                 const uint &r4) {
-//   boxMullerTransform(&out[index], &out[index + groupSz],
-//                      getHalfNegative11(r1), getHalf01(r1 >> 16));
-//   boxMullerTransform(&out[index + 2 * groupSz],
-//                      &out[index + 3 * groupSz], getHalfNegative11(r2),
-//                      getHalf01(r2 >> 16));
-//   boxMullerTransform(&out[index + 4 * groupSz],
-//                      &out[index + 5 * groupSz], getHalfNegative11(r3),
-//                      getHalf01(r3 >> 16));
-//   boxMullerTransform(&out[index + 6 * groupSz],
-//                      &out[index + 7 * groupSz], getHalfNegative11(r4),
-//                      getHalf01(r4 >> 16));
+static void boxMullerWriteOut128Bytes(cdouble *out, const uint &index,
+                                      const uint groupSz, const uint &r1,
+                                      const uint &r2, const uint &r3,
+                                      const uint &r4) {
+    boxMullerTransform(&out[index], getDoubleNegative11(r1, r2),
+                       getDouble01(r3, r4));
+}
+
+static void boxMullerWriteOut128Bytes(common::half *out, const uint &index,
+                                      const uint groupSz, const uint &r1,
+                                      const uint &r2, const uint &r3,
+                                      const uint &r4) {
+    //   boxMullerTransform(&out[index], &out[index + groupSz],
+    //                      getHalfNegative11(r1), getHalf01(r1 >> 16));
+    //   boxMullerTransform(&out[index + 2 * groupSz],
+    //                      &out[index + 3 * groupSz], getHalfNegative11(r2),
+    //                      getHalf01(r2 >> 16));
+    //   boxMullerTransform(&out[index + 4 * groupSz],
+    //                      &out[index + 5 * groupSz], getHalfNegative11(r3),
+    //                      getHalf01(r3 >> 16));
+    //   boxMullerTransform(&out[index + 6 * groupSz],
+    //                      &out[index + 7 * groupSz], getHalfNegative11(r4),
+    //                      getHalf01(r4 >> 16));
 }
 
 // Writes with boundary checking
 
-static void partialWriteOut128Bytes(uchar *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(uchar *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     if (index < elements) { out[index] = r1; }
     if (index + groupSz < elements) { out[index + groupSz] = r1 >> 8; }
-    if (index + 2 * groupSz < elements) {
-        out[index + 2 * groupSz] = r1 >> 16;
-    }
-    if (index + 3 * groupSz < elements) {
-        out[index + 3 * groupSz] = r1 >> 24;
-    }
+    if (index + 2 * groupSz < elements) { out[index + 2 * groupSz] = r1 >> 16; }
+    if (index + 3 * groupSz < elements) { out[index + 3 * groupSz] = r1 >> 24; }
     if (index + 4 * groupSz < elements) { out[index + 4 * groupSz] = r2; }
-    if (index + 5 * groupSz < elements) {
-        out[index + 5 * groupSz] = r2 >> 8;
-    }
-    if (index + 6 * groupSz < elements) {
-        out[index + 6 * groupSz] = r2 >> 16;
-    }
-    if (index + 7 * groupSz < elements) {
-        out[index + 7 * groupSz] = r2 >> 24;
-    }
+    if (index + 5 * groupSz < elements) { out[index + 5 * groupSz] = r2 >> 8; }
+    if (index + 6 * groupSz < elements) { out[index + 6 * groupSz] = r2 >> 16; }
+    if (index + 7 * groupSz < elements) { out[index + 7 * groupSz] = r2 >> 24; }
     if (index + 8 * groupSz < elements) { out[index + 8 * groupSz] = r3; }
-    if (index + 9 * groupSz < elements) {
-        out[index + 9 * groupSz] = r3 >> 8;
-    }
+    if (index + 9 * groupSz < elements) { out[index + 9 * groupSz] = r3 >> 8; }
     if (index + 10 * groupSz < elements) {
         out[index + 10 * groupSz] = r3 >> 16;
     }
     if (index + 11 * groupSz < elements) {
         out[index + 11 * groupSz] = r3 >> 24;
     }
-    if (index + 12 * groupSz < elements) {
-        out[index + 12 * groupSz] = r4;
-    }
+    if (index + 12 * groupSz < elements) { out[index + 12 * groupSz] = r4; }
     if (index + 13 * groupSz < elements) {
         out[index + 13 * groupSz] = r4 >> 8;
     }
@@ -524,23 +506,19 @@ static void partialWriteOut128Bytes(uchar *out, const uint &index, const uint gr
     }
 }
 
-static void partialWriteOut128Bytes(char *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(char *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     if (index < elements) { out[index] = (r1)&0x1; }
-    if (index + groupSz < elements) {
-        out[index + groupSz] = (r1 >> 1) & 0x1;
-    }
+    if (index + groupSz < elements) { out[index + groupSz] = (r1 >> 1) & 0x1; }
     if (index + 2 * groupSz < elements) {
         out[index + 2 * groupSz] = (r1 >> 2) & 0x1;
     }
     if (index + 3 * groupSz < elements) {
         out[index + 3 * groupSz] = (r1 >> 3) & 0x1;
     }
-    if (index + 4 * groupSz < elements) {
-        out[index + 4 * groupSz] = (r2)&0x1;
-    }
+    if (index + 4 * groupSz < elements) { out[index + 4 * groupSz] = (r2)&0x1; }
     if (index + 5 * groupSz < elements) {
         out[index + 5 * groupSz] = (r2 >> 1) & 0x1;
     }
@@ -550,9 +528,7 @@ static void partialWriteOut128Bytes(char *out, const uint &index, const uint gro
     if (index + 7 * groupSz < elements) {
         out[index + 7 * groupSz] = (r2 >> 3) & 0x1;
     }
-    if (index + 8 * groupSz < elements) {
-        out[index + 8 * groupSz] = (r3)&0x1;
-    }
+    if (index + 8 * groupSz < elements) { out[index + 8 * groupSz] = (r3)&0x1; }
     if (index + 9 * groupSz < elements) {
         out[index + 9 * groupSz] = (r3 >> 1) & 0x1;
     }
@@ -576,54 +552,50 @@ static void partialWriteOut128Bytes(char *out, const uint &index, const uint gro
     }
 }
 
-static void partialWriteOut128Bytes(short *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(short *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     if (index < elements) { out[index] = r1; }
     if (index + groupSz < elements) { out[index + groupSz] = r1 >> 16; }
     if (index + 2 * groupSz < elements) { out[index + 2 * groupSz] = r2; }
-    if (index + 3 * groupSz < elements) {
-        out[index + 3 * groupSz] = r2 >> 16;
-    }
+    if (index + 3 * groupSz < elements) { out[index + 3 * groupSz] = r2 >> 16; }
     if (index + 4 * groupSz < elements) { out[index + 4 * groupSz] = r3; }
-    if (index + 5 * groupSz < elements) {
-        out[index + 5 * groupSz] = r3 >> 16;
-    }
+    if (index + 5 * groupSz < elements) { out[index + 5 * groupSz] = r3 >> 16; }
     if (index + 6 * groupSz < elements) { out[index + 6 * groupSz] = r4; }
-    if (index + 7 * groupSz < elements) {
-        out[index + 7 * groupSz] = r4 >> 16;
-    }
+    if (index + 7 * groupSz < elements) { out[index + 7 * groupSz] = r4 >> 16; }
 }
 
-static void partialWriteOut128Bytes(ushort *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
-    partialWriteOut128Bytes((short *)(out), index, groupSz, r1, r2, r3, r4, elements);
+static void partialWriteOut128Bytes(ushort *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
+    partialWriteOut128Bytes((short *)(out), index, groupSz, r1, r2, r3, r4,
+                            elements);
 }
 
-static void partialWriteOut128Bytes(int *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(int *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     if (index < elements) { out[index] = r1; }
     if (index + groupSz < elements) { out[index + groupSz] = r2; }
     if (index + 2 * groupSz < elements) { out[index + 2 * groupSz] = r3; }
     if (index + 3 * groupSz < elements) { out[index + 3 * groupSz] = r4; }
 }
 
-static void partialWriteOut128Bytes(uint *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
-    partialWriteOut128Bytes((int *)(out), index, groupSz, r1, r2, r3, r4, elements);
+static void partialWriteOut128Bytes(uint *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
+    partialWriteOut128Bytes((int *)(out), index, groupSz, r1, r2, r3, r4,
+                            elements);
 }
 
-static void partialWriteOut128Bytes(intl *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(intl *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     intl c1 = r2;
     c1      = (c1 << 32) | r1;
     intl c2 = r4;
@@ -632,17 +604,18 @@ static void partialWriteOut128Bytes(intl *out, const uint &index, const uint gro
     if (index + groupSz < elements) { out[index + groupSz] = c2; }
 }
 
-static void partialWriteOut128Bytes(uintl *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
-    partialWriteOut128Bytes((intl *)(out), index, groupSz, r1, r2, r3, r4, elements);
+static void partialWriteOut128Bytes(uintl *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
+    partialWriteOut128Bytes((intl *)(out), index, groupSz, r1, r2, r3, r4,
+                            elements);
 }
 
-static void partialWriteOut128Bytes(float *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(float *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     if (index < elements) { out[index] = 1.f - getFloat01(r1); }
     if (index + groupSz < elements) {
         out[index + groupSz] = 1.f - getFloat01(r2);
@@ -655,10 +628,10 @@ static void partialWriteOut128Bytes(float *out, const uint &index, const uint gr
     }
 }
 
-static void partialWriteOut128Bytes(cfloat *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(cfloat *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     if (index < elements) {
         out[index] = {1.f - getFloat01(r1), 1.f - getFloat01(r2)};
     }
@@ -667,29 +640,31 @@ static void partialWriteOut128Bytes(cfloat *out, const uint &index, const uint g
     }
 }
 
-static void partialWriteOut128Bytes(double *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(double *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     if (index < elements) { out[index] = 1.0 - getDouble01(r1, r2); }
     if (index + groupSz < elements) {
         out[index + groupSz] = 1.0 - getDouble01(r3, r4);
     }
 }
 
-static void partialWriteOut128Bytes(cdouble *out, const uint &index, const uint groupSz,
-                                               const uint &r1, const uint &r2,
-                                               const uint &r3, const uint &r4,
-                                               const uint &elements) {
+static void partialWriteOut128Bytes(cdouble *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
     if (index < elements) {
         out[index] = {1.0 - getDouble01(r1, r2), 1.0 - getDouble01(r3, r4)};
     }
 }
 
 // Normalized writes with boundary checking
-static void partialBoxMullerWriteOut128Bytes(
-    float *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2,
-    const uint &r3, const uint &r4, const uint &elements) {
+static void partialBoxMullerWriteOut128Bytes(float *out, const uint &index,
+                                             const uint groupSz, const uint &r1,
+                                             const uint &r2, const uint &r3,
+                                             const uint &r4,
+                                             const uint &elements) {
     float n1, n2, n3, n4;
     boxMullerTransform(&n1, &n2, getFloatNegative11(r1), getFloat01(r2));
     boxMullerTransform(&n3, &n4, getFloatNegative11(r3), getFloat01(r4));
@@ -699,23 +674,23 @@ static void partialBoxMullerWriteOut128Bytes(
     if (index + 3 * groupSz < elements) { out[index + 3 * groupSz] = n4; }
 }
 
-static void partialBoxMullerWriteOut128Bytes(
-    cfloat *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2,
-    const uint &r3, const uint &r4, const uint &elements) {
+static void partialBoxMullerWriteOut128Bytes(cfloat *out, const uint &index,
+                                             const uint groupSz, const uint &r1,
+                                             const uint &r2, const uint &r3,
+                                             const uint &r4,
+                                             const uint &elements) {
     float n1, n2, n3, n4;
     boxMullerTransform(&n1, &n2, getFloatNegative11(r1), getFloat01(r2));
     boxMullerTransform(&n3, &n4, getFloatNegative11(r3), getFloat01(r4));
-    if (index < elements) {
-        out[index] = {n1, n2};
-    }
-    if (index + groupSz < elements) {
-        out[index + groupSz] = {n3, n4};
-    }
+    if (index < elements) { out[index] = {n1, n2}; }
+    if (index + groupSz < elements) { out[index + groupSz] = {n3, n4}; }
 }
 
-static void partialBoxMullerWriteOut128Bytes(
-    double *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2,
-    const uint &r3, const uint &r4, const uint &elements) {
+static void partialBoxMullerWriteOut128Bytes(double *out, const uint &index,
+                                             const uint groupSz, const uint &r1,
+                                             const uint &r2, const uint &r3,
+                                             const uint &r4,
+                                             const uint &elements) {
     double n1, n2;
     boxMullerTransform(&n1, &n2, getDoubleNegative11(r1, r2),
                        getDouble01(r3, r4));
@@ -723,82 +698,79 @@ static void partialBoxMullerWriteOut128Bytes(
     if (index + groupSz < elements) { out[index + groupSz] = n2; }
 }
 
-static void partialBoxMullerWriteOut128Bytes(
-    cdouble *out, const uint &index, const uint groupSz, const uint &r1, const uint &r2,
-    const uint &r3, const uint &r4, const uint &elements) {
+static void partialBoxMullerWriteOut128Bytes(cdouble *out, const uint &index,
+                                             const uint groupSz, const uint &r1,
+                                             const uint &r2, const uint &r3,
+                                             const uint &r4,
+                                             const uint &elements) {
     double n1, n2;
     boxMullerTransform(&n1, &n2, getDoubleNegative11(r1, r2),
                        getDouble01(r3, r4));
-    if (index < elements) {
-        out[index] = {n1, n2};
-    }
-}
-
-static void partialWriteOut128Bytes(common::half *out, 
-                                             const uint &index, const uint groupSz,
-                                             const uint &r1, const uint &r2,
-                                             const uint &r3, const uint &r4,
-                                             const uint &elements) {
-//  if (index < elements) { out[index] = oneMinusGetHalf01(r1); }
-//  if (index + groupSz < elements) {
-//      out[index + groupSz] = oneMinusGetHalf01(r1 >> 16);
-//  }
-//  if (index + 2 * groupSz < elements) {
-//      out[index + 2 * groupSz] = oneMinusGetHalf01(r2);
-//  }
-//  if (index + 3 * groupSz < elements) {
-//      out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16);
-//  }
-//  if (index + 4 * groupSz < elements) {
-//      out[index + 4 * groupSz] = oneMinusGetHalf01(r3);
-//  }
-//  if (index + 5 * groupSz < elements) {
-//      out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16);
-//  }
-//  if (index + 6 * groupSz < elements) {
-//      out[index + 6 * groupSz] = oneMinusGetHalf01(r4);
-//  }
-//  if (index + 7 * groupSz < elements) {
-//      out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16);
-//  }
+    if (index < elements) { out[index] = {n1, n2}; }
+}
+
+static void partialWriteOut128Bytes(common::half *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
+    //  if (index < elements) { out[index] = oneMinusGetHalf01(r1); }
+    //  if (index + groupSz < elements) {
+    //      out[index + groupSz] = oneMinusGetHalf01(r1 >> 16);
+    //  }
+    //  if (index + 2 * groupSz < elements) {
+    //      out[index + 2 * groupSz] = oneMinusGetHalf01(r2);
+    //  }
+    //  if (index + 3 * groupSz < elements) {
+    //      out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16);
+    //  }
+    //  if (index + 4 * groupSz < elements) {
+    //      out[index + 4 * groupSz] = oneMinusGetHalf01(r3);
+    //  }
+    //  if (index + 5 * groupSz < elements) {
+    //      out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16);
+    //  }
+    //  if (index + 6 * groupSz < elements) {
+    //      out[index + 6 * groupSz] = oneMinusGetHalf01(r4);
+    //  }
+    //  if (index + 7 * groupSz < elements) {
+    //      out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16);
+    //  }
 }
 
-
 // Normalized writes with boundary checking
 static void partialBoxMullerWriteOut128Bytes(
-    common::half *out, const uint &index, const uint groupSz,
-    const uint &r1, const uint &r2,
-    const uint &r3, const uint &r4, const uint &elements) {
-//    common::half n[8];
-//    boxMullerTransform(n + 0, n + 1, getHalfNegative11(r1),
-//                       getHalf01(r1 >> 16));
-//    boxMullerTransform(n + 2, n + 3, getHalfNegative11(r2),
-//                       getHalf01(r2 >> 16));
-//    boxMullerTransform(n + 4, n + 5, getHalfNegative11(r3),
-//                       getHalf01(r3 >> 16));
-//    boxMullerTransform(n + 6, n + 7, getHalfNegative11(r4),
-//                       getHalf01(r4 >> 16));
-//    if (index < elements) { out[index] = n[0]; }
-//    if (index + groupSz < elements) { out[index + groupSz] = n[1]; }
-//    if (index + 2 * groupSz < elements) {
-//        out[index + 2 * groupSz] = n[2];
-//    }
-//    if (index + 3 * groupSz < elements) {
-//        out[index + 3 * groupSz] = n[3];
-//    }
-//    if (index + 4 * groupSz < elements) {
-//        out[index + 4 * groupSz] = n[4];
-//    }
-//    if (index + 5 * groupSz < elements) {
-//        out[index + 5 * groupSz] = n[5];
-//    }
-//    if (index + 6 * groupSz < elements) {
-//        out[index + 6 * groupSz] = n[6];
-//    }
-//    if (index + 7 * groupSz < elements) {
-//        out[index + 7 * groupSz] = n[7];
-//    }
-}
-
-} // namespace kernel
-} // namespace oneapi
+    common::half *out, const uint &index, const uint groupSz, const uint &r1,
+    const uint &r2, const uint &r3, const uint &r4, const uint &elements) {
+    //    common::half n[8];
+    //    boxMullerTransform(n + 0, n + 1, getHalfNegative11(r1),
+    //                       getHalf01(r1 >> 16));
+    //    boxMullerTransform(n + 2, n + 3, getHalfNegative11(r2),
+    //                       getHalf01(r2 >> 16));
+    //    boxMullerTransform(n + 4, n + 5, getHalfNegative11(r3),
+    //                       getHalf01(r3 >> 16));
+    //    boxMullerTransform(n + 6, n + 7, getHalfNegative11(r4),
+    //                       getHalf01(r4 >> 16));
+    //    if (index < elements) { out[index] = n[0]; }
+    //    if (index + groupSz < elements) { out[index + groupSz] = n[1]; }
+    //    if (index + 2 * groupSz < elements) {
+    //        out[index + 2 * groupSz] = n[2];
+    //    }
+    //    if (index + 3 * groupSz < elements) {
+    //        out[index + 3 * groupSz] = n[3];
+    //    }
+    //    if (index + 4 * groupSz < elements) {
+    //        out[index + 4 * groupSz] = n[4];
+    //    }
+    //    if (index + 5 * groupSz < elements) {
+    //        out[index + 5 * groupSz] = n[5];
+    //    }
+    //    if (index + 6 * groupSz < elements) {
+    //        out[index + 6 * groupSz] = n[6];
+    //    }
+    //    if (index + 7 * groupSz < elements) {
+    //        out[index + 7 * groupSz] = n[7];
+    //    }
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
index 3a0c447035..d3106c5e7b 100644
--- a/src/backend/oneapi/kernel/range.hpp
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -26,17 +26,21 @@ namespace kernel {
 
 template<typename T>
 class rangeOp {
-public:
+   public:
     rangeOp(sycl::accessor<T> out, KParam oinfo, const int dim,
-               const int blocksPerMatX, const int blocksPerMatY, 
-               sycl::stream debug) :
-        out_(out), oinfo_(oinfo), dim_(dim),
-        blocksPerMatX_(blocksPerMatX), blocksPerMatY_(blocksPerMatY),
-        debug_(debug) {}
-
-    void operator() (sycl::nd_item<2> it) const {
-        //printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
-        //debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) << "]" << sycl::stream_manipulator::endl;
+            const int blocksPerMatX, const int blocksPerMatY,
+            sycl::stream debug)
+        : out_(out)
+        , oinfo_(oinfo)
+        , dim_(dim)
+        , blocksPerMatX_(blocksPerMatX)
+        , blocksPerMatY_(blocksPerMatY)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        // printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
+        // debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) <<
+        // "]" << sycl::stream_manipulator::endl;
 
         const int mul0 = (dim_ == 0);
         const int mul1 = (dim_ == 1);
@@ -44,8 +48,8 @@ class rangeOp {
         const int mul3 = (dim_ == 3);
 
         sycl::group g = it.get_group();
-        const int oz = g.get_group_id(0) / blocksPerMatX_;
-        const int ow = g.get_group_id(1) / blocksPerMatY_;
+        const int oz  = g.get_group_id(0) / blocksPerMatX_;
+        const int ow  = g.get_group_id(1) / blocksPerMatY_;
 
         const int blockIdx_x = g.get_group_id(0) - oz * blocksPerMatX_;
         const int blockIdx_y = g.get_group_id(1) - ow * blocksPerMatY_;
@@ -53,8 +57,8 @@ class rangeOp {
         const int xx = it.get_local_id(0) + blockIdx_x * it.get_local_range(0);
         const int yy = it.get_local_id(1) + blockIdx_y * it.get_local_range(1);
 
-        if (xx >= oinfo_.dims[0] || yy >= oinfo_.dims[1] || oz >= oinfo_.dims[2] ||
-            ow >= oinfo_.dims[3])
+        if (xx >= oinfo_.dims[0] || yy >= oinfo_.dims[1] ||
+            oz >= oinfo_.dims[2] || ow >= oinfo_.dims[3])
             return;
 
         const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
@@ -66,7 +70,7 @@ class rangeOp {
 
         T* optr = out_.get_pointer();
         for (int oy = yy; oy < oinfo_.dims[1]; oy += incy) {
-           T valYZW = valZW + (mul1 * oy);
+            T valYZW = valZW + (mul1 * oy);
             int oyzw = ozw + oy * oinfo_.strides[1];
             for (int ox = xx; ox < oinfo_.dims[0]; ox += incx) {
                 int oidx = oyzw + ox;
@@ -77,7 +81,7 @@ class rangeOp {
         }
     }
 
-protected:
+   protected:
     sycl::accessor<T> out_;
     KParam oinfo_;
     int dim_;
@@ -85,7 +89,6 @@ class rangeOp {
     sycl::stream debug_;
 };
 
-
 template<typename T>
 void range(Param<T> out, const int dim) {
     constexpr int RANGE_TX    = 32;
@@ -101,13 +104,14 @@ void range(Param<T> out, const int dim) {
                           local[1] * blocksPerMatY * out.info.dims[3]);
     sycl::nd_range<2> ndrange(global, local);
 
-    getQueue().submit([=] (sycl::handler &h) {
+    getQueue().submit([=](sycl::handler& h) {
         auto out_acc = out.data->get_access(h);
 
         sycl::stream debug_stream(2048, 128, h);
 
-        h.parallel_for(ndrange, rangeOp<T>(out_acc, out.info,
-            dim, blocksPerMatX, blocksPerMatY, debug_stream));
+        h.parallel_for(ndrange,
+                       rangeOp<T>(out_acc, out.info, dim, blocksPerMatX,
+                                  blocksPerMatY, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index ef87bc77b2..b2bc48a407 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -11,8 +11,8 @@
 
 #include <Param.hpp>
 #include <common/dispatch.hpp>
-#include <err_oneapi.hpp>
 #include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
 #include <traits.hpp>
 
 #include <string>
@@ -41,92 +41,104 @@ cdouble getConjugate(const cdouble &in) {
     return std::conj(in);
 }
 
-template <typename T, int dimensions>
+template<typename T, int dimensions>
 using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write, sycl::access::target::local>;
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
 
-template <typename T>
+template<typename T>
 class transposeKernel {
-public:
-  transposeKernel(sycl::accessor<T> oData, const KParam out,
-                  const sycl::accessor<T> iData, const KParam in,
-                  const int blocksPerMatX, const int blocksPerMatY,
-                  const bool conjugate, const bool IS32MULTIPLE,
-                  local_accessor<T, 1> shrdMem,
-                  sycl::stream debugStream) :
-    oData_(oData), out_(out), iData_(iData), in_(in), blocksPerMatX_(blocksPerMatX),
-    blocksPerMatY_(blocksPerMatY), conjugate_(conjugate), IS32MULTIPLE_(IS32MULTIPLE), shrdMem_(shrdMem), debugStream_(debugStream) {}
-  void operator() (sycl::nd_item<2> it) const {
-      const int shrdStride = TILE_DIM + 1;
-
-      const int oDim0 = out_.dims[0];
-      const int oDim1 = out_.dims[1];
-      const int iDim0 = in_.dims[0];
-      const int iDim1 = in_.dims[1];
-
-      // calculate strides
-      const int oStride1 = out_.strides[1];
-      const int iStride1 = in_.strides[1];
-
-      const int lx = it.get_local_id(0);
-      const int ly = it.get_local_id(1);
-
-      // batch based block Id
-      sycl::group g = it.get_group();
-      const int batchId_x  = g.get_group_id(0) / blocksPerMatX_;
-      const int blockIdx_x = (g.get_group_id(0) - batchId_x * blocksPerMatX_);
-
-      const int batchId_y  = g.get_group_id(1) / blocksPerMatY_;
-      const int blockIdx_y = (g.get_group_id(1) - batchId_y * blocksPerMatY_);
-
-      const int x0 = TILE_DIM * blockIdx_x;
-      const int y0 = TILE_DIM * blockIdx_y;
-
-      // calculate global in_dices
-      int gx = lx + x0;
-      int gy = ly + y0;
-
-      // offset in_ and out_ based on batch id
-      // also add the subBuffer offsets
-      T *iDataPtr = iData_.get_pointer(), *oDataPtr = oData_.get_pointer();
-      iDataPtr += batchId_x * in_.strides[2] + batchId_y * in_.strides[3] + in_.offset;
-      oDataPtr +=
-        batchId_x * out_.strides[2] + batchId_y * out_.strides[3] + out_.offset;
-
-      for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
-        int gy_ = gy + repeat;
-        if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
-          shrdMem_[(ly + repeat) * shrdStride + lx] =
-            iDataPtr[gy_ * iStride1 + gx];
-      }
-      it.barrier();
-
-      gx = lx + y0;
-      gy = ly + x0;
-
-      for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
-        int gy_ = gy + repeat;
-        if (IS32MULTIPLE_ || (gx < oDim0 && gy_ < oDim1)) {
-          const T val = shrdMem_[lx * shrdStride + ly + repeat];
-          oDataPtr[gy_ * oStride1 + gx] = conjugate_ ? getConjugate(val) : val;
+   public:
+    transposeKernel(sycl::accessor<T> oData, const KParam out,
+                    const sycl::accessor<T> iData, const KParam in,
+                    const int blocksPerMatX, const int blocksPerMatY,
+                    const bool conjugate, const bool IS32MULTIPLE,
+                    local_accessor<T, 1> shrdMem, sycl::stream debugStream)
+        : oData_(oData)
+        , out_(out)
+        , iData_(iData)
+        , in_(in)
+        , blocksPerMatX_(blocksPerMatX)
+        , blocksPerMatY_(blocksPerMatY)
+        , conjugate_(conjugate)
+        , IS32MULTIPLE_(IS32MULTIPLE)
+        , shrdMem_(shrdMem)
+        , debugStream_(debugStream) {}
+    void operator()(sycl::nd_item<2> it) const {
+        const int shrdStride = TILE_DIM + 1;
+
+        const int oDim0 = out_.dims[0];
+        const int oDim1 = out_.dims[1];
+        const int iDim0 = in_.dims[0];
+        const int iDim1 = in_.dims[1];
+
+        // calculate strides
+        const int oStride1 = out_.strides[1];
+        const int iStride1 = in_.strides[1];
+
+        const int lx = it.get_local_id(0);
+        const int ly = it.get_local_id(1);
+
+        // batch based block Id
+        sycl::group g        = it.get_group();
+        const int batchId_x  = g.get_group_id(0) / blocksPerMatX_;
+        const int blockIdx_x = (g.get_group_id(0) - batchId_x * blocksPerMatX_);
+
+        const int batchId_y  = g.get_group_id(1) / blocksPerMatY_;
+        const int blockIdx_y = (g.get_group_id(1) - batchId_y * blocksPerMatY_);
+
+        const int x0 = TILE_DIM * blockIdx_x;
+        const int y0 = TILE_DIM * blockIdx_y;
+
+        // calculate global in_dices
+        int gx = lx + x0;
+        int gy = ly + y0;
+
+        // offset in_ and out_ based on batch id
+        // also add the subBuffer offsets
+        T *iDataPtr = iData_.get_pointer(), *oDataPtr = oData_.get_pointer();
+        iDataPtr += batchId_x * in_.strides[2] + batchId_y * in_.strides[3] +
+                    in_.offset;
+        oDataPtr += batchId_x * out_.strides[2] + batchId_y * out_.strides[3] +
+                    out_.offset;
+
+        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+            int gy_ = gy + repeat;
+            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+                shrdMem_[(ly + repeat) * shrdStride + lx] =
+                    iDataPtr[gy_ * iStride1 + gx];
+        }
+        it.barrier();
+
+        gx = lx + y0;
+        gy = ly + x0;
+
+        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+            int gy_ = gy + repeat;
+            if (IS32MULTIPLE_ || (gx < oDim0 && gy_ < oDim1)) {
+                const T val = shrdMem_[lx * shrdStride + ly + repeat];
+                oDataPtr[gy_ * oStride1 + gx] =
+                    conjugate_ ? getConjugate(val) : val;
+            }
         }
-      }
-  }
-private:
-  sycl::accessor<T> oData_;
-  KParam out_;
-  sycl::accessor<T> iData_;
-  KParam in_;
-  int blocksPerMatX_;
-  int blocksPerMatY_;
-  sycl::stream debugStream_;
-  bool conjugate_;
-  bool IS32MULTIPLE_;
-  local_accessor<T, 1> shrdMem_;
+    }
+
+   private:
+    sycl::accessor<T> oData_;
+    KParam out_;
+    sycl::accessor<T> iData_;
+    KParam in_;
+    int blocksPerMatX_;
+    int blocksPerMatY_;
+    sycl::stream debugStream_;
+    bool conjugate_;
+    bool IS32MULTIPLE_;
+    local_accessor<T, 1> shrdMem_;
 };
 
 template<typename T>
-void transpose(Param<T> out, const Param<T> in, const bool conjugate, const bool IS32MULTIPLE) {
+void transpose(Param<T> out, const Param<T> in, const bool conjugate,
+               const bool IS32MULTIPLE) {
     auto local = sycl::range{THREADS_X, THREADS_Y};
 
     const int blk_x = divup(in.info.dims[0], TILE_DIM);
@@ -142,12 +154,10 @@ void transpose(Param<T> out, const Param<T> in, const bool conjugate, const bool
 
         auto shrdMem = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
 
-        h.parallel_for(sycl::nd_range{global, local},
-                       transposeKernel<T>(q, out.info,
-                                          r, in.info,
-                                          blk_x, blk_y,
-                                          conjugate, IS32MULTIPLE,
-                                          shrdMem, debugStream));
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            transposeKernel<T>(q, out.info, r, in.info, blk_x, blk_y, conjugate,
+                               IS32MULTIPLE, shrdMem, debugStream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/transpose_inplace.hpp b/src/backend/oneapi/kernel/transpose_inplace.hpp
old mode 100755
new mode 100644
index c5230f364e..81313b50da
--- a/src/backend/oneapi/kernel/transpose_inplace.hpp
+++ b/src/backend/oneapi/kernel/transpose_inplace.hpp
@@ -12,14 +12,13 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
-#include <err_oneapi.hpp>
 #include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
 #include <traits.hpp>
 
 #include <string>
 #include <vector>
 
-
 namespace oneapi {
 namespace kernel {
 
@@ -45,121 +44,132 @@ constexpr int TILE_DIM  = 16;
 constexpr int THREADS_X = TILE_DIM;
 constexpr int THREADS_Y = 256 / TILE_DIM;
 
-template <typename T, int dimensions>
+template<typename T, int dimensions>
 using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write, sycl::access::target::local>;
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
 
-template <typename T>
+template<typename T>
 class transposeInPlaceKernel {
-public:
-  transposeInPlaceKernel(const sycl::accessor<T> iData, const KParam in,
-                         const int blocksPerMatX, const int blocksPerMatY,
-                         const bool conjugate, const bool IS32MULTIPLE,
-                         local_accessor<T, 1> shrdMem_s, local_accessor<T, 1> shrdMem_d,
-                         sycl::stream debugStream) :
-    iData_(iData), in_(in), blocksPerMatX_(blocksPerMatX),
-    blocksPerMatY_(blocksPerMatY), conjugate_(conjugate), IS32MULTIPLE_(IS32MULTIPLE), shrdMem_s_(shrdMem_s), shrdMem_d_(shrdMem_d), debugStream_(debugStream) {}
-  void operator() (sycl::nd_item<2> it) const {
-    const int shrdStride = TILE_DIM + 1;
-
-    // create variables to hold output dimensions
-    const int iDim0 = in_.dims[0];
-    const int iDim1 = in_.dims[1];
-
-    // calculate strides
-    const int iStride1 = in_.strides[1];
-
-    const int lx = it.get_local_id(0);
-    const int ly = it.get_local_id(1);
-
-    // batch based block Id
-    sycl::group g = it.get_group();
-    const int batchId_x  = g.get_group_id(0) / blocksPerMatX_;
-    const int blockIdx_x = (g.get_group_id(0) - batchId_x * blocksPerMatX_);
-
-    const int batchId_y  = g.get_group_id(1) / blocksPerMatY_;
-    const int blockIdx_y = (g.get_group_id(1) - batchId_y * blocksPerMatY_);
-
-    const int x0 = TILE_DIM * blockIdx_x;
-    const int y0 = TILE_DIM * blockIdx_y;
-
-    T *iDataPtr = iData_.get_pointer();
-    iDataPtr += batchId_x * in_.strides[2] + batchId_y * in_.strides[3] + in_.offset;
-
-    if (blockIdx_y > blockIdx_x) {
-        // calculate global indices
-        int gx = lx + x0;
-        int gy = ly + y0;
-        int dx = lx + y0;
-        int dy = ly + x0;
-
-        // Copy to shared memory
-        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
-            int gy_ = gy + repeat;
-            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
-              shrdMem_s_[(ly + repeat) * shrdStride + lx] =
-                iDataPtr[gy_ * iStride1 + gx];
-
-            int dy_ = dy + repeat;
-            if (IS32MULTIPLE_ || (dx < iDim0 && dy_ < iDim1))
-                shrdMem_d_[(ly + repeat) * shrdStride + lx] =
-                    iDataPtr[dy_ * iStride1 + dx];
-        }
-
-        it.barrier();
-
-        // Copy from shared memory to global memory
-        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
-            int dy_ = dy + repeat;
-            if (IS32MULTIPLE_ || (dx < iDim0 && dy_ < iDim1))
-                iDataPtr[dy_ * iStride1 + dx] =
-                    doOp(shrdMem_s_[(ly + repeat) + (shrdStride * lx)]);
-
-            int gy_ = gy + repeat;
-            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
-                iDataPtr[gy_ * iStride1 + gx] =
-                    doOp(shrdMem_d_[(ly + repeat) + (shrdStride * lx)]);
-        }
-
-    } else if (blockIdx_y == blockIdx_x) {
-        // calculate global indices
-        int gx = lx + x0;
-        int gy = ly + y0;
-
-        // Copy to shared memory
-        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
-            int gy_ = gy + repeat;
-            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
-                shrdMem_s_[(ly + repeat) * shrdStride + lx] =
-                    iDataPtr[gy_ * iStride1 + gx];
-        }
-
-        it.barrier();
-
-        // Copy from shared memory to global memory
-        for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
-            int gy_ = gy + repeat;
-            if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
-                iDataPtr[gy_ * iStride1 + gx] =
-                    doOp(shrdMem_s_[(ly + repeat) + (shrdStride * lx)]);
+   public:
+    transposeInPlaceKernel(const sycl::accessor<T> iData, const KParam in,
+                           const int blocksPerMatX, const int blocksPerMatY,
+                           const bool conjugate, const bool IS32MULTIPLE,
+                           local_accessor<T, 1> shrdMem_s,
+                           local_accessor<T, 1> shrdMem_d,
+                           sycl::stream debugStream)
+        : iData_(iData)
+        , in_(in)
+        , blocksPerMatX_(blocksPerMatX)
+        , blocksPerMatY_(blocksPerMatY)
+        , conjugate_(conjugate)
+        , IS32MULTIPLE_(IS32MULTIPLE)
+        , shrdMem_s_(shrdMem_s)
+        , shrdMem_d_(shrdMem_d)
+        , debugStream_(debugStream) {}
+    void operator()(sycl::nd_item<2> it) const {
+        const int shrdStride = TILE_DIM + 1;
+
+        // create variables to hold output dimensions
+        const int iDim0 = in_.dims[0];
+        const int iDim1 = in_.dims[1];
+
+        // calculate strides
+        const int iStride1 = in_.strides[1];
+
+        const int lx = it.get_local_id(0);
+        const int ly = it.get_local_id(1);
+
+        // batch based block Id
+        sycl::group g        = it.get_group();
+        const int batchId_x  = g.get_group_id(0) / blocksPerMatX_;
+        const int blockIdx_x = (g.get_group_id(0) - batchId_x * blocksPerMatX_);
+
+        const int batchId_y  = g.get_group_id(1) / blocksPerMatY_;
+        const int blockIdx_y = (g.get_group_id(1) - batchId_y * blocksPerMatY_);
+
+        const int x0 = TILE_DIM * blockIdx_x;
+        const int y0 = TILE_DIM * blockIdx_y;
+
+        T *iDataPtr = iData_.get_pointer();
+        iDataPtr += batchId_x * in_.strides[2] + batchId_y * in_.strides[3] +
+                    in_.offset;
+
+        if (blockIdx_y > blockIdx_x) {
+            // calculate global indices
+            int gx = lx + x0;
+            int gy = ly + y0;
+            int dx = lx + y0;
+            int dy = ly + x0;
+
+            // Copy to shared memory
+            for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+                int gy_ = gy + repeat;
+                if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+                    shrdMem_s_[(ly + repeat) * shrdStride + lx] =
+                        iDataPtr[gy_ * iStride1 + gx];
+
+                int dy_ = dy + repeat;
+                if (IS32MULTIPLE_ || (dx < iDim0 && dy_ < iDim1))
+                    shrdMem_d_[(ly + repeat) * shrdStride + lx] =
+                        iDataPtr[dy_ * iStride1 + dx];
+            }
+
+            it.barrier();
+
+            // Copy from shared memory to global memory
+            for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+                int dy_ = dy + repeat;
+                if (IS32MULTIPLE_ || (dx < iDim0 && dy_ < iDim1))
+                    iDataPtr[dy_ * iStride1 + dx] =
+                        doOp(shrdMem_s_[(ly + repeat) + (shrdStride * lx)]);
+
+                int gy_ = gy + repeat;
+                if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+                    iDataPtr[gy_ * iStride1 + gx] =
+                        doOp(shrdMem_d_[(ly + repeat) + (shrdStride * lx)]);
+            }
+
+        } else if (blockIdx_y == blockIdx_x) {
+            // calculate global indices
+            int gx = lx + x0;
+            int gy = ly + y0;
+
+            // Copy to shared memory
+            for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+                int gy_ = gy + repeat;
+                if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+                    shrdMem_s_[(ly + repeat) * shrdStride + lx] =
+                        iDataPtr[gy_ * iStride1 + gx];
+            }
+
+            it.barrier();
+
+            // Copy from shared memory to global memory
+            for (int repeat = 0; repeat < TILE_DIM; repeat += THREADS_Y) {
+                int gy_ = gy + repeat;
+                if (IS32MULTIPLE_ || (gx < iDim0 && gy_ < iDim1))
+                    iDataPtr[gy_ * iStride1 + gx] =
+                        doOp(shrdMem_s_[(ly + repeat) + (shrdStride * lx)]);
+            }
         }
     }
-  }
-private:
-  sycl::accessor<T> iData_;
-  KParam in_;
-  int blocksPerMatX_;
-  int blocksPerMatY_;
-  sycl::stream debugStream_;
-  bool conjugate_;
-  bool IS32MULTIPLE_;
-  local_accessor<T, 1> shrdMem_s_;
-  local_accessor<T, 1> shrdMem_d_;
+
+   private:
+    sycl::accessor<T> iData_;
+    KParam in_;
+    int blocksPerMatX_;
+    int blocksPerMatY_;
+    sycl::stream debugStream_;
+    bool conjugate_;
+    bool IS32MULTIPLE_;
+    local_accessor<T, 1> shrdMem_s_;
+    local_accessor<T, 1> shrdMem_d_;
 };
 
 template<typename T>
-void transpose_inplace(Param<T> in, const bool conjugate, const bool IS32MULTIPLE)
-{
+void transpose_inplace(Param<T> in, const bool conjugate,
+                       const bool IS32MULTIPLE) {
     auto local = sycl::range{THREADS_X, THREADS_Y};
 
     int blk_x = divup(in.info.dims[0], TILE_DIM);
@@ -176,11 +186,9 @@ void transpose_inplace(Param<T> in, const bool conjugate, const bool IS32MULTIPL
         auto shrdMem_d = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
 
         h.parallel_for(sycl::nd_range{global, local},
-                       transposeInPlaceKernel<T>(r, in.info,
-                                                 blk_x, blk_y,
-                                                 conjugate, IS32MULTIPLE,
-                                                 shrdMem_s, shrdMem_d,
-                                                 debugStream));
+                       transposeInPlaceKernel<T>(
+                           r, in.info, blk_x, blk_y, conjugate, IS32MULTIPLE,
+                           shrdMem_s, shrdMem_d, debugStream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/triangle.hpp b/src/backend/oneapi/kernel/triangle.hpp
index 4f71ce1243..cf9c3e22a3 100644
--- a/src/backend/oneapi/kernel/triangle.hpp
+++ b/src/backend/oneapi/kernel/triangle.hpp
@@ -11,8 +11,8 @@
 
 #include <Param.hpp>
 #include <common/dispatch.hpp>
-#include <err_oneapi.hpp>
 #include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
 #include <traits.hpp>
 
 #include <string>
@@ -21,67 +21,77 @@
 namespace oneapi {
 namespace kernel {
 
-template <typename T, int dimensions>
+template<typename T, int dimensions>
 using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write, sycl::access::target::local>;
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
 
-template <typename T>
+template<typename T>
 class triangleKernel {
-public:
-  triangleKernel(sycl::accessor<T> rAcc, KParam rinfo, sycl::accessor<T> iAcc,
-                 KParam iinfo, const int groups_x, const int groups_y,
-                 const bool is_upper, const bool is_unit_diag) :
-    rAcc_(rAcc), rinfo_(rinfo), iAcc_(iAcc), iinfo_(iinfo), groups_x_(groups_x), groups_y_(groups_y), is_upper_(is_upper), is_unit_diag_(is_unit_diag) {}
-  void operator() (sycl::nd_item<2> it) const {
-    sycl::group g = it.get_group();
-    const int oz = g.get_group_id(0) / groups_x_;
-    const int ow = g.get_group_id(1) / groups_y_;
-
-    const int groupId_0 = g.get_group_id(0) - oz * groups_x_;
-    const int groupId_1 = g.get_group_id(1) - ow * groups_y_;
-
-    const int xx = it.get_local_id(0) + groupId_0 * it.get_local_range(0);
-    const int yy = it.get_local_id(1) + groupId_1 * it.get_local_range(1);
-
-    const int incy = groups_y_ * it.get_local_range(1);
-    const int incx = groups_x_ * it.get_local_range(0);
-
-    T *d_r = rAcc_.get_pointer();
-    const T *d_i = iAcc_.get_pointer() + iinfo_.offset;
-
-    if (oz < rinfo_.dims[2] && ow < rinfo_.dims[3]) {
-        d_i = d_i + oz * iinfo_.strides[2] + ow * iinfo_.strides[3];
-        d_r = d_r + oz * rinfo_.strides[2] + ow * rinfo_.strides[3];
-
-        for (int oy = yy; oy < rinfo_.dims[1]; oy += incy) {
-            const T *Yd_i = d_i + oy * iinfo_.strides[1];
-            T *Yd_r       = d_r + oy * rinfo_.strides[1];
-
-            for (int ox = xx; ox < rinfo_.dims[0]; ox += incx) {
-                bool cond         = is_upper_ ? (oy >= ox) : (oy <= ox);
-                bool do_unit_diag = is_unit_diag_ && (oy == ox);
-                if (cond) {
-                    Yd_r[ox] = do_unit_diag ? (T)(1) : Yd_i[ox];
-                } else {
-                    Yd_r[ox] = (T)(0);
+   public:
+    triangleKernel(sycl::accessor<T> rAcc, KParam rinfo, sycl::accessor<T> iAcc,
+                   KParam iinfo, const int groups_x, const int groups_y,
+                   const bool is_upper, const bool is_unit_diag)
+        : rAcc_(rAcc)
+        , rinfo_(rinfo)
+        , iAcc_(iAcc)
+        , iinfo_(iinfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , is_upper_(is_upper)
+        , is_unit_diag_(is_unit_diag) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const int oz  = g.get_group_id(0) / groups_x_;
+        const int ow  = g.get_group_id(1) / groups_y_;
+
+        const int groupId_0 = g.get_group_id(0) - oz * groups_x_;
+        const int groupId_1 = g.get_group_id(1) - ow * groups_y_;
+
+        const int xx = it.get_local_id(0) + groupId_0 * it.get_local_range(0);
+        const int yy = it.get_local_id(1) + groupId_1 * it.get_local_range(1);
+
+        const int incy = groups_y_ * it.get_local_range(1);
+        const int incx = groups_x_ * it.get_local_range(0);
+
+        T *d_r       = rAcc_.get_pointer();
+        const T *d_i = iAcc_.get_pointer() + iinfo_.offset;
+
+        if (oz < rinfo_.dims[2] && ow < rinfo_.dims[3]) {
+            d_i = d_i + oz * iinfo_.strides[2] + ow * iinfo_.strides[3];
+            d_r = d_r + oz * rinfo_.strides[2] + ow * rinfo_.strides[3];
+
+            for (int oy = yy; oy < rinfo_.dims[1]; oy += incy) {
+                const T *Yd_i = d_i + oy * iinfo_.strides[1];
+                T *Yd_r       = d_r + oy * rinfo_.strides[1];
+
+                for (int ox = xx; ox < rinfo_.dims[0]; ox += incx) {
+                    bool cond         = is_upper_ ? (oy >= ox) : (oy <= ox);
+                    bool do_unit_diag = is_unit_diag_ && (oy == ox);
+                    if (cond) {
+                        Yd_r[ox] = do_unit_diag ? (T)(1) : Yd_i[ox];
+                    } else {
+                        Yd_r[ox] = (T)(0);
+                    }
                 }
             }
         }
     }
-  }
-private:
-  sycl::accessor<T> rAcc_;
-  KParam rinfo_;
-  sycl::accessor<T> iAcc_;
-  KParam iinfo_;
-  const int groups_x_;
-  const int groups_y_;
-  const bool is_upper_;
-  const bool is_unit_diag_;
+
+   private:
+    sycl::accessor<T> rAcc_;
+    KParam rinfo_;
+    sycl::accessor<T> iAcc_;
+    KParam iinfo_;
+    const int groups_x_;
+    const int groups_y_;
+    const bool is_upper_;
+    const bool is_unit_diag_;
 };
 
 template<typename T>
-void triangle(Param<T> out, const Param<T> in, bool is_upper, bool is_unit_diag) {
+void triangle(Param<T> out, const Param<T> in, bool is_upper,
+              bool is_unit_diag) {
     constexpr unsigned TX    = 32;
     constexpr unsigned TY    = 8;
     constexpr unsigned TILEX = 128;
@@ -100,9 +110,10 @@ void triangle(Param<T> out, const Param<T> in, bool is_upper, bool is_unit_diag)
         auto rAcc = out.data->get_access(h);
         sycl::stream debugStream(128, 128, h);
 
-        h.parallel_for(sycl::nd_range{global, local},
-                       triangleKernel<T>(rAcc, out.info, iAcc, in.info, groups_x, groups_y,
-                                         is_upper, is_unit_diag));
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            triangleKernel<T>(rAcc, out.info, iAcc, in.info, groups_x, groups_y,
+                              is_upper, is_unit_diag));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/lu.cpp b/src/backend/oneapi/lu.cpp
index 849fea1426..170efca58c 100644
--- a/src/backend/oneapi/lu.cpp
+++ b/src/backend/oneapi/lu.cpp
@@ -59,13 +59,15 @@ template<typename T>
 void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
         const Array<T> &in) {
     ONEAPI_NOT_SUPPORTED("");
-    AF_ERROR("Linear Algebra is disabled on OneAPI backend", AF_ERR_NOT_CONFIGURED);
+    AF_ERROR("Linear Algebra is disabled on OneAPI backend",
+             AF_ERR_NOT_CONFIGURED);
 }
 
 template<typename T>
 Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
     ONEAPI_NOT_SUPPORTED("");
-    AF_ERROR("Linear Algebra is disabled on OneAPI backend", AF_ERR_NOT_CONFIGURED);
+    AF_ERROR("Linear Algebra is disabled on OneAPI backend",
+             AF_ERR_NOT_CONFIGURED);
 }
 
 bool isLAPACKAvailable() { return false; }
diff --git a/src/backend/oneapi/math.cpp b/src/backend/oneapi/math.cpp
index a3b9d07e7a..e9c1666960 100644
--- a/src/backend/oneapi/math.cpp
+++ b/src/backend/oneapi/math.cpp
@@ -12,42 +12,42 @@
 
 namespace oneapi {
 cfloat operator+(cfloat lhs, cfloat rhs) {
-    //cfloat res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
+    // cfloat res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
     cfloat res;
     return res;
 }
 
 cdouble operator+(cdouble lhs, cdouble rhs) {
-    //cdouble res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
+    // cdouble res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
     cdouble res;
     return res;
 }
 
 cfloat operator*(cfloat lhs, cfloat rhs) {
     cfloat out;
-    //out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1];
-    //out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0];
+    // out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1];
+    // out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0];
     return out;
 }
 
 cdouble operator*(cdouble lhs, cdouble rhs) {
     cdouble out;
-    //out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1];
-    //out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0];
+    // out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1];
+    // out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0];
     return out;
 }
 
 cfloat division(cfloat lhs, double rhs) {
     cfloat retVal;
-    //retVal.s[0] = real(lhs) / rhs;
-    //retVal.s[1] = imag(lhs) / rhs;
+    // retVal.s[0] = real(lhs) / rhs;
+    // retVal.s[1] = imag(lhs) / rhs;
     return retVal;
 }
 
 cdouble division(cdouble lhs, double rhs) {
     cdouble retVal;
-    //retVal.s[0] = real(lhs) / rhs;
-    //retVal.s[1] = imag(lhs) / rhs;
+    // retVal.s[0] = real(lhs) / rhs;
+    // retVal.s[1] = imag(lhs) / rhs;
     return retVal;
 }
 }  // namespace oneapi
diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp
index 2b4182d811..584efa1d14 100644
--- a/src/backend/oneapi/math.hpp
+++ b/src/backend/oneapi/math.hpp
@@ -146,7 +146,6 @@ inline common::half operator+(common::half lhs, common::half rhs) noexcept {
 }
 }  // namespace oneapi
 
-
 #if defined(__GNUC__) || defined(__GNUG__)
 /* GCC/G++, Clang/LLVM, Intel ICC */
 #pragma GCC diagnostic pop
diff --git a/src/backend/oneapi/mean.cpp b/src/backend/oneapi/mean.cpp
index 2fb632eb75..41d72a547e 100644
--- a/src/backend/oneapi/mean.cpp
+++ b/src/backend/oneapi/mean.cpp
@@ -7,8 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <mean.hpp>
 #include <err_oneapi.hpp>
+#include <mean.hpp>
 
 #include <common/half.hpp>
 // #include <kernel/mean.hpp>
@@ -21,7 +21,6 @@ using std::swap;
 namespace oneapi {
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in) {
-
     ONEAPI_NOT_SUPPORTED("mean Not supported");
 
     return To(0);
@@ -30,7 +29,6 @@ To mean(const Array<Ti>& in) {
 
 template<typename T, typename Tw>
 T mean(const Array<T>& in, const Array<Tw>& wts) {
-
     ONEAPI_NOT_SUPPORTED("mean Not supported");
 
     return T(0);
@@ -39,7 +37,6 @@ T mean(const Array<T>& in, const Array<Tw>& wts) {
 
 template<typename Ti, typename Tw, typename To>
 Array<To> mean(const Array<Ti>& in, const int dim) {
-
     ONEAPI_NOT_SUPPORTED("mean Not supported");
 
     dim4 odims    = in.dims();
@@ -51,7 +48,6 @@ Array<To> mean(const Array<Ti>& in, const int dim) {
 
 template<typename T, typename Tw>
 Array<T> mean(const Array<T>& in, const Array<Tw>& wts, const int dim) {
-
     ONEAPI_NOT_SUPPORTED("mean Not supported");
 
     dim4 odims   = in.dims();
diff --git a/src/backend/oneapi/meanshift.cpp b/src/backend/oneapi/meanshift.cpp
index 61823f1467..fa352ed5c1 100644
--- a/src/backend/oneapi/meanshift.cpp
+++ b/src/backend/oneapi/meanshift.cpp
@@ -20,12 +20,12 @@ template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
                    const float &chromaticSigma, const unsigned &numIterations,
                    const bool &isColor) {
-
     ONEAPI_NOT_SUPPORTED("meanshift Not supported");
 
     const dim4 &dims = in.dims();
     Array<T> out     = createEmptyArray<T>(dims);
-    // kernel::meanshift<T>(out, in, spatialSigma, chromaticSigma, numIterations,
+    // kernel::meanshift<T>(out, in, spatialSigma, chromaticSigma,
+    // numIterations,
     //                      isColor);
     return out;
 }
diff --git a/src/backend/oneapi/medfilt.cpp b/src/backend/oneapi/medfilt.cpp
index 526f505244..1729573628 100644
--- a/src/backend/oneapi/medfilt.cpp
+++ b/src/backend/oneapi/medfilt.cpp
@@ -20,7 +20,6 @@ namespace oneapi {
 template<typename T>
 Array<T> medfilt1(const Array<T> &in, const int w_wid,
                   const af::borderType pad) {
-
     ONEAPI_NOT_SUPPORTED("medfilt1 Not supported");
 
     // ARG_ASSERT(2, (w_wid <= kernel::MAX_MEDFILTER1_LEN));
@@ -38,7 +37,6 @@ Array<T> medfilt1(const Array<T> &in, const int w_wid,
 template<typename T>
 Array<T> medfilt2(const Array<T> &in, const int w_len, const int w_wid,
                   const af::borderType pad) {
-
     ONEAPI_NOT_SUPPORTED("medfilt2 Not supported");
 
     // ARG_ASSERT(2, (w_len % 2 != 0));
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index add529c8cc..314e1fd0a8 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -47,7 +47,8 @@ void signalMemoryCleanup() { memoryManager().signalMemoryCleanup(); }
 
 void shutdownMemoryManager() { memoryManager().shutdown(); }
 
-void shutdownPinnedMemoryManager() { /*pinnedMemoryManager().shutdown();*/ }
+void shutdownPinnedMemoryManager() { /*pinnedMemoryManager().shutdown();*/
+}
 
 void printMemInfo(const char *msg, const int device) {
     memoryManager().printInfo(msg, device);
@@ -55,10 +56,11 @@ void printMemInfo(const char *msg, const int device) {
 
 template<typename T>
 // unique_ptr<cl::Buffer, function<void(cl::Buffer *)>> memAlloc(
-//unique_ptr<int, function<void(int *)>> memAlloc(
-std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>> memAlloc(
-    const size_t &elements) {
-    return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>(new sycl::buffer<T>(sycl::range(elements)), bufferFree<T>);
+// unique_ptr<int, function<void(int *)>> memAlloc(
+std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>>
+memAlloc(const size_t &elements) {
+    return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>(
+        new sycl::buffer<T>(sycl::range(elements)), bufferFree<T>);
     // // TODO: make memAlloc aware of array shapes
     // if (elements) {
     //     dim4 dims(elements);
@@ -74,7 +76,6 @@ std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>> memAllo
 }
 
 void *memAllocUser(const size_t &bytes) {
-
     ONEAPI_NOT_SUPPORTED("memAllocUser Not supported");
     return nullptr;
 
@@ -86,7 +87,6 @@ void *memAllocUser(const size_t &bytes) {
 
 template<typename T>
 void memFree(T *ptr) {
-
     ONEAPI_NOT_SUPPORTED("memFree Not supported");
 
     // cl::Buffer *buf = reinterpret_cast<cl::Buffer *>(ptr);
@@ -96,7 +96,6 @@ void memFree(T *ptr) {
 }
 
 void memFreeUser(void *ptr) {
-
     ONEAPI_NOT_SUPPORTED("memFreeUser Not supported");
 
     // cl::Buffer *buf = static_cast<cl::Buffer *>(ptr);
@@ -107,7 +106,6 @@ void memFreeUser(void *ptr) {
 
 template<typename T>
 sycl::buffer<T> *bufferAlloc(const size_t &bytes) {
-
     ONEAPI_NOT_SUPPORTED("bufferAlloc Not supported");
     return nullptr;
 
@@ -124,9 +122,7 @@ sycl::buffer<T> *bufferAlloc(const size_t &bytes) {
 
 template<typename T>
 void bufferFree(sycl::buffer<T> *buf) {
-    if(buf) {
-        delete buf;
-    }
+    if (buf) { delete buf; }
     // if (buf) {
     //     cl_mem mem = (*buf)();
     //     delete buf;
@@ -136,7 +132,6 @@ void bufferFree(sycl::buffer<T> *buf) {
 
 template<typename T>
 void memLock(const sycl::buffer<T> *ptr) {
-
     ONEAPI_NOT_SUPPORTED("memLock Not supported");
 
     // cl_mem mem = static_cast<cl_mem>((*ptr)());
@@ -145,7 +140,6 @@ void memLock(const sycl::buffer<T> *ptr) {
 
 template<typename T>
 void memUnlock(const sycl::buffer<T> *ptr) {
-
     ONEAPI_NOT_SUPPORTED("memUnlock Not supported");
 
     // cl_mem mem = static_cast<cl_mem>((*ptr)());
@@ -164,7 +158,6 @@ void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
 
 template<typename T>
 T *pinnedAlloc(const size_t &elements) {
-
     ONEAPI_NOT_SUPPORTED("pinnedAlloc Not supported");
 
     // // TODO: make pinnedAlloc aware of array shapes
@@ -175,18 +168,19 @@ T *pinnedAlloc(const size_t &elements) {
 
 template<typename T>
 void pinnedFree(T *ptr) {
-    //pinnedMemoryManager().unlock(static_cast<void *>(ptr), false);
+    // pinnedMemoryManager().unlock(static_cast<void *>(ptr), false);
 }
 
-//template unique_ptr<int, function<void(int *)>> memAlloc<T>(
-#define INSTANTIATE(T)                                                                          \
-    template std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>> memAlloc( \
-        const size_t &elements);                                                                \
-    template void memFree(T *ptr);                                                              \
-    template T *pinnedAlloc(const size_t &elements);                                            \
-    template void pinnedFree(T *ptr);                                                           \
-    template void bufferFree(sycl::buffer<T> *buf);                                             \
-    template void memLock(const sycl::buffer<T> *buf);                                          \
+// template unique_ptr<int, function<void(int *)>> memAlloc<T>(
+#define INSTANTIATE(T)                                               \
+    template std::unique_ptr<sycl::buffer<T>,                        \
+                             std::function<void(sycl::buffer<T> *)>> \
+    memAlloc(const size_t &elements);                                \
+    template void memFree(T *ptr);                                   \
+    template T *pinnedAlloc(const size_t &elements);                 \
+    template void pinnedFree(T *ptr);                                \
+    template void bufferFree(sycl::buffer<T> *buf);                  \
+    template void memLock(const sycl::buffer<T> *buf);               \
     template void memUnlock(const sycl::buffer<T> *buf);
 
 INSTANTIATE(float)
@@ -206,7 +200,6 @@ INSTANTIATE(common::half)
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
 void Allocator::shutdown() {
-
     ONEAPI_NOT_SUPPORTED("Allocator::shutdown Not supported");
 
     // for (int n = 0; n < opencl::getDeviceCount(); n++) {
@@ -220,7 +213,6 @@ void Allocator::shutdown() {
 }
 
 int Allocator::getActiveDeviceId() {
-
     ONEAPI_NOT_SUPPORTED("Allocator::getActiveDeviceId Not supported");
 
     return 0;
@@ -228,7 +220,6 @@ int Allocator::getActiveDeviceId() {
 }
 
 size_t Allocator::getMaxMemorySize(int id) {
-
     ONEAPI_NOT_SUPPORTED("Allocator::getMaxMemorySize Not supported");
 
     return 0;
@@ -236,7 +227,6 @@ size_t Allocator::getMaxMemorySize(int id) {
 }
 
 void *Allocator::nativeAlloc(const size_t bytes) {
-
     ONEAPI_NOT_SUPPORTED("Allocator::nativeAlloc Not supported");
     return nullptr;
 
@@ -256,7 +246,6 @@ void *Allocator::nativeAlloc(const size_t bytes) {
 }
 
 void Allocator::nativeFree(void *ptr) {
-
     ONEAPI_NOT_SUPPORTED("Allocator::nativeFree Not supported");
 
     // cl_mem buffer = static_cast<cl_mem>(ptr);
@@ -272,22 +261,20 @@ AllocatorPinned::AllocatorPinned() : pinnedMaps(oneapi::getDeviceCount()) {
 }
 
 void AllocatorPinned::shutdown() {
-
     ONEAPI_NOT_SUPPORTED("AllocatorPinned::shutdown Not supported");
 
-//     for (int n = 0; n < opencl::getDeviceCount(); n++) {
-//         opencl::setDevice(n);
-//         shutdownPinnedMemoryManager();
-//         auto currIterator = pinnedMaps[n].begin();
-//         auto endIterator  = pinnedMaps[n].end();
-//         while (currIterator != endIterator) {
-//             pinnedMaps[n].erase(currIterator++);
-//         }
-//     }
+    //     for (int n = 0; n < opencl::getDeviceCount(); n++) {
+    //         opencl::setDevice(n);
+    //         shutdownPinnedMemoryManager();
+    //         auto currIterator = pinnedMaps[n].begin();
+    //         auto endIterator  = pinnedMaps[n].end();
+    //         while (currIterator != endIterator) {
+    //             pinnedMaps[n].erase(currIterator++);
+    //         }
+    //     }
 }
 
 int AllocatorPinned::getActiveDeviceId() {
-
     ONEAPI_NOT_SUPPORTED("AllocatorPinned::getActiveDeviceId Not supported");
     return 0;
 
@@ -295,38 +282,36 @@ int AllocatorPinned::getActiveDeviceId() {
 }
 
 size_t AllocatorPinned::getMaxMemorySize(int id) {
-
     ONEAPI_NOT_SUPPORTED("AllocatorPinned::getMaxMemorySize Not supported");
     return 0;
     // return opencl::getDeviceMemorySize(id);
 }
 
 void *AllocatorPinned::nativeAlloc(const size_t bytes) {
-
     ONEAPI_NOT_SUPPORTED("AllocatorPinned::nativeAlloc Not supported");
     return nullptr;
-//     void *ptr = NULL;
-
-//     cl_int err = CL_SUCCESS;
-//     auto buf   = clCreateBuffer(getContext()(), CL_MEM_ALLOC_HOST_PTR, bytes,
-//                               nullptr, &err);
-//     if (err != CL_SUCCESS) {
-//         AF_ERROR("Failed to allocate pinned memory.", AF_ERR_NO_MEM);
-//     }
-
-//     ptr = clEnqueueMapBuffer(getQueue()(), buf, CL_TRUE,
-//                              CL_MAP_READ | CL_MAP_WRITE, 0, bytes, 0, nullptr,
-//                              nullptr, &err);
-//     if (err != CL_SUCCESS) {
-//         AF_ERROR("Failed to map pinned memory", AF_ERR_RUNTIME);
-//     }
-//     AF_TRACE("Pinned::nativeAlloc: {:>7} {}", bytesToString(bytes), ptr);
-//     pinnedMaps[opencl::getActiveDeviceId()].emplace(ptr, new cl::Buffer(buf));
-//     return ptr;
+    //     void *ptr = NULL;
+
+    //     cl_int err = CL_SUCCESS;
+    //     auto buf   = clCreateBuffer(getContext()(), CL_MEM_ALLOC_HOST_PTR,
+    //     bytes,
+    //                               nullptr, &err);
+    //     if (err != CL_SUCCESS) {
+    //         AF_ERROR("Failed to allocate pinned memory.", AF_ERR_NO_MEM);
+    //     }
+
+    //     ptr = clEnqueueMapBuffer(getQueue()(), buf, CL_TRUE,
+    //                              CL_MAP_READ | CL_MAP_WRITE, 0, bytes, 0,
+    //                              nullptr, nullptr, &err);
+    //     if (err != CL_SUCCESS) {
+    //         AF_ERROR("Failed to map pinned memory", AF_ERR_RUNTIME);
+    //     }
+    //     AF_TRACE("Pinned::nativeAlloc: {:>7} {}", bytesToString(bytes), ptr);
+    //     pinnedMaps[opencl::getActiveDeviceId()].emplace(ptr, new
+    //     cl::Buffer(buf)); return ptr;
 }
 
 void AllocatorPinned::nativeFree(void *ptr) {
-
     ONEAPI_NOT_SUPPORTED("AllocatorPinned::nativeFree Not supported");
 
     // AF_TRACE("Pinned::nativeFree:          {}", ptr);
diff --git a/src/backend/oneapi/memory.hpp b/src/backend/oneapi/memory.hpp
index bb0e9f181e..2ed71fdd19 100644
--- a/src/backend/oneapi/memory.hpp
+++ b/src/backend/oneapi/memory.hpp
@@ -16,7 +16,6 @@
 #include <memory>
 #include <vector>
 
-
 namespace oneapi {
 template<typename T>
 sycl::buffer<T> *bufferAlloc(const size_t &bytes);
@@ -26,7 +25,7 @@ void bufferFree(sycl::buffer<T> *buf);
 
 template<typename T>
 using bufptr =
-  std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>>;
+    std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>>;
 
 template<typename T>
 bufptr<T> memAlloc(const size_t &elements);
@@ -67,7 +66,7 @@ void setMemStepSize(size_t step_bytes);
 size_t getMemStepSize(void);
 
 class Allocator final : public common::memory::AllocatorInterface {
-    public:
+   public:
     Allocator();
     ~Allocator() = default;
     void shutdown() override;
diff --git a/src/backend/oneapi/moments.cpp b/src/backend/oneapi/moments.cpp
index aa595c9269..119e01cbc9 100644
--- a/src/backend/oneapi/moments.cpp
+++ b/src/backend/oneapi/moments.cpp
@@ -22,7 +22,6 @@ static inline unsigned bitCount(unsigned v) {
 
 template<typename T>
 Array<float> moments(const Array<T> &in, const af_moment_type moment) {
-
     ONEAPI_NOT_SUPPORTED("moments Not supported");
 
     in.eval();
diff --git a/src/backend/oneapi/morph.cpp b/src/backend/oneapi/morph.cpp
index de38b446ac..adef3be8d6 100644
--- a/src/backend/oneapi/morph.cpp
+++ b/src/backend/oneapi/morph.cpp
@@ -21,7 +21,6 @@ namespace oneapi {
 
 template<typename T>
 Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation) {
-
     ONEAPI_NOT_SUPPORTED("morph Not supported");
 
     // const dim4 mdims = mask.dims();
@@ -39,7 +38,6 @@ Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation) {
 
 template<typename T>
 Array<T> morph3d(const Array<T> &in, const Array<T> &mask, bool isDilation) {
-
     ONEAPI_NOT_SUPPORTED("morph3d Not supported");
 
     // const dim4 mdims = mask.dims();
diff --git a/src/backend/oneapi/nearest_neighbour.cpp b/src/backend/oneapi/nearest_neighbour.cpp
index e4705f1126..30bc6d90d3 100644
--- a/src/backend/oneapi/nearest_neighbour.cpp
+++ b/src/backend/oneapi/nearest_neighbour.cpp
@@ -24,7 +24,6 @@ template<typename T, typename To, af_match_type dist_type>
 void nearest_neighbour_(Array<uint>& idx, Array<To>& dist,
                         const Array<T>& query, const Array<T>& train,
                         const uint dist_dim, const uint n_dist) {
-
     ONEAPI_NOT_SUPPORTED("nearest_neighbour_ Not supported");
 
     uint sample_dim   = (dist_dim == 0) ? 1 : 0;
diff --git a/src/backend/oneapi/orb.cpp b/src/backend/oneapi/orb.cpp
index db7bd31207..aaca439632 100644
--- a/src/backend/oneapi/orb.cpp
+++ b/src/backend/oneapi/orb.cpp
@@ -25,7 +25,6 @@ unsigned orb(Array<float> &x_out, Array<float> &y_out, Array<float> &score_out,
              Array<uint> &desc_out, const Array<T> &image, const float fast_thr,
              const unsigned max_feat, const float scl_fctr,
              const unsigned levels, const bool blur_img) {
-
     ONEAPI_NOT_SUPPORTED("orb Not supported");
     return 0;
 
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index c466ff60af..b65ad6698d 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -7,19 +7,19 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <common/graphics_common.hpp>
 #include <GraphicsResourceManager.hpp>
 #include <blas.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
+#include <common/graphics_common.hpp>
 #include <common/host_memory.hpp>
 #include <common/util.hpp>
 #include <device_manager.hpp>
 #include <err_oneapi.hpp>
 #include <errorcodes.hpp>
+#include <memory.hpp>
 #include <version.hpp>
 #include <af/version.h>
-#include <memory.hpp>
 
 #ifdef OS_MAC
 #include <OpenGL/CGLCurrent.h>
@@ -36,10 +36,10 @@
 #include <utility>
 #include <vector>
 
-using sycl::queue;
 using sycl::context;
 using sycl::device;
 using sycl::platform;
+using sycl::queue;
 
 using std::begin;
 using std::call_once;
@@ -90,7 +90,7 @@ bool verify_present(const string& pname, const string ref) {
     return iter != end(pname);
 }
 
-//TODO: update to new platforms?
+// TODO: update to new platforms?
 static string platformMap(string& platStr) {
     using strmap_t                = map<string, string>;
     static const strmap_t platMap = {
@@ -141,7 +141,7 @@ string getDeviceInfo() noexcept {
         common::lock_guard_t lock(devMngr.deviceMutex);
         unsigned nDevices = 0;
         for (auto& device : devMngr.mDevices) {
-            //const Platform platform(device->getInfo<CL_DEVICE_PLATFORM>());
+            // const Platform platform(device->getInfo<CL_DEVICE_PLATFORM>());
 
             string dstr = device->get_info<sycl::info::device::name>();
             bool show_braces =
@@ -150,21 +150,22 @@ string getDeviceInfo() noexcept {
             string id = (show_braces ? string("[") : "-") +
                         to_string(nDevices) + (show_braces ? string("]") : "-");
 
-            size_t msize = device->get_info<sycl::info::device::global_mem_size>();
+            size_t msize =
+                device->get_info<sycl::info::device::global_mem_size>();
             info << id << " " << getPlatformName(*device) << ": " << ltrim(dstr)
                  << ", " << msize / 1048576 << " MB";
 #ifndef NDEBUG
             info << " -- ";
             string devVersion = device->get_info<sycl::info::device::version>();
-            string driVersion = device->get_info<sycl::info::device::driver_version>();
+            string driVersion =
+                device->get_info<sycl::info::device::driver_version>();
             info << devVersion;
             info << " -- Device driver " << driVersion;
-            info
-                << " -- FP64 Support: "
-                << (device->get_info<sycl::info::device::preferred_vector_width_double>() >
-                            0
-                        ? "True"
-                        : "False");
+            info << " -- FP64 Support: "
+                 << (device->get_info<sycl::info::device::
+                                          preferred_vector_width_double>() > 0
+                         ? "True"
+                         : "False");
             info << " -- Unified Memory ("
                  << (isHostUnifiedMemory(*device) ? "True" : "False") << ")";
 #endif
@@ -182,8 +183,9 @@ string getDeviceInfo() noexcept {
 }
 
 string getPlatformName(const sycl::device& device) {
-    std::string platStr = device.get_platform().get_info<sycl::info::platform::name>();
-    //return platformMap(platStr);
+    std::string platStr =
+        device.get_platform().get_info<sycl::info::platform::name>();
+    // return platformMap(platStr);
     return platStr;
 }
 
@@ -309,7 +311,8 @@ size_t getHostMemorySize() { return common::getHostMemorySize(); }
 
 sycl::info::device_type getDeviceType() {
     const sycl::device& device = getDevice();
-    sycl::info::device_type type = device.get_info<sycl::info::device::device_type>();
+    sycl::info::device_type type =
+        device.get_info<sycl::info::device::device_type>();
     return type;
 }
 
@@ -409,20 +412,20 @@ void addDeviceContext(sycl::device dev, sycl::context ctx, sycl::queue que) {
         auto tDevice  = make_unique<sycl::device>(dev);
         auto tContext = make_unique<sycl::context>(ctx);
         // queue atleast has implicit context and device if created
-        auto tQueue   = make_unique<sycl::queue>(que);
+        auto tQueue = make_unique<sycl::queue>(que);
 
         devMngr.mPlatforms.push_back(getPlatformEnum(*tDevice));
         // FIXME: add OpenGL Interop for user provided contexts later
         devMngr.mIsGLSharingOn.push_back(false);
-        devMngr.mDeviceTypes.push_back(
-            static_cast<int>(tDevice->get_info<sycl::info::device::device_type>()));
+        devMngr.mDeviceTypes.push_back(static_cast<int>(
+            tDevice->get_info<sycl::info::device::device_type>()));
 
         devMngr.mDevices.push_back(move(tDevice));
         devMngr.mContexts.push_back(move(tContext));
         devMngr.mQueues.push_back(move(tQueue));
         nDevices = static_cast<int>(devMngr.mDevices.size()) - 1;
 
-        //TODO: cache?
+        // TODO: cache?
     }
 
     // Last/newly added device needs memory management
@@ -437,8 +440,7 @@ void setDeviceContext(sycl::device dev, sycl::context ctx) {
 
     const int dCount = static_cast<int>(devMngr.mDevices.size());
     for (int i = 0; i < dCount; ++i) {
-        if (*devMngr.mDevices[i] == dev &&
-            *devMngr.mContexts[i] == ctx) {
+        if (*devMngr.mDevices[i] == dev && *devMngr.mContexts[i] == ctx) {
             setActiveContext(i);
             return;
         }
@@ -459,8 +461,7 @@ void removeDeviceContext(sycl::device dev, sycl::context ctx) {
 
         const int dCount = static_cast<int>(devMngr.mDevices.size());
         for (int i = 0; i < dCount; ++i) {
-            if (*devMngr.mDevices[i] == dev &&
-                *devMngr.mContexts[i] == ctx) {
+            if (*devMngr.mDevices[i] == dev && *devMngr.mContexts[i] == ctx) {
                 deleteIdx = i;
                 break;
             }
@@ -608,7 +609,8 @@ GraphicsResourceManager& interopManager() {
 }  // namespace oneapi
 
 /*
-//TODO: select which external api functions to expose and add to header+implement
+//TODO: select which external api functions to expose and add to
+header+implement
 
 using namespace oneapi;
 
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index d82868454e..46d24393f3 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -62,7 +62,7 @@ size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();
 
-//sycl::device::is_cpu,is_gpu,is_accelerator
+// sycl::device::is_cpu,is_gpu,is_accelerator
 sycl::info::device_type getDeviceType();
 
 bool isHostUnifiedMemory(const sycl::device& device);
@@ -114,7 +114,7 @@ graphics::ForgeManager& forgeManager();
 
 GraphicsResourceManager& interopManager();
 
-//afcl::platform getPlatformEnum(cl::Device dev);
+// afcl::platform getPlatformEnum(cl::Device dev);
 
 void setActiveContext(int device);
 
diff --git a/src/backend/oneapi/plot.cpp b/src/backend/oneapi/plot.cpp
index 544cc61568..6abf9896a3 100644
--- a/src/backend/oneapi/plot.cpp
+++ b/src/backend/oneapi/plot.cpp
@@ -36,12 +36,14 @@ void copy_plot(const Array<T> &P, fg_plot plot) {
     //     glFinish();
 
     //     // Use of events:
-    //     // https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReleaseGLObjects.html
+    //     //
+    //     https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReleaseGLObjects.html
     //     cl::Event event;
 
     //     getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
     //     event.wait();
-    //     getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes, NULL,
+    //     getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes,
+    //     NULL,
     //                                  &event);
     //     getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
     //     event.wait();
@@ -56,7 +58,8 @@ void copy_plot(const Array<T> &P, fg_plot plot) {
     //     CheckGL("Begin OpenCL fallback-resource copy");
     //     glBindBuffer(GL_ARRAY_BUFFER, buffer);
     //     auto *ptr =
-    //         static_cast<GLubyte *>(glMapBuffer(GL_ARRAY_BUFFER, GL_WRITE_ONLY));
+    //         static_cast<GLubyte *>(glMapBuffer(GL_ARRAY_BUFFER,
+    //         GL_WRITE_ONLY));
     //     if (ptr) {
     //         getQueue().enqueueReadBuffer(*P.get(), CL_TRUE, 0, bytes, ptr);
     //         glUnmapBuffer(GL_ARRAY_BUFFER);
diff --git a/src/backend/oneapi/random_engine.cpp b/src/backend/oneapi/random_engine.cpp
index 5f8231706e..cff66a7170 100644
--- a/src/backend/oneapi/random_engine.cpp
+++ b/src/backend/oneapi/random_engine.cpp
@@ -8,25 +8,24 @@
  ********************************************************/
 
 #include <Array.hpp>
-#include <err_oneapi.hpp>
 #include <common/half.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/random_engine.hpp>
 #include <af/defines.h>
 #include <af/dim4.hpp>
-#include <kernel/random_engine.hpp>
 
 using common::half;
 
 namespace oneapi {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl) {
-     kernel::initMersenneState(state, tbl, seed);
+    kernel::initMersenneState(state, tbl, seed);
 }
 
 template<typename T>
 Array<T> uniformDistribution(const af::dim4 &dims,
                              const af_random_engine_type type,
                              const uintl &seed, uintl &counter) {
-
     Array<T> out = createEmptyArray<T>(dims);
     kernel::uniformDistributionCBRNG<T>(out, out.elements(), type, seed,
                                         counter);
@@ -49,9 +48,8 @@ Array<T> uniformDistribution(const af::dim4 &dims, Array<uint> pos,
                              Array<uint> recursion_table,
                              Array<uint> temper_table, Array<uint> state) {
     Array<T> out = createEmptyArray<T>(dims);
-    kernel::uniformDistributionMT<T>(
-        out, out.elements(), state, pos, sh1,
-        sh2, mask, recursion_table, temper_table);
+    kernel::uniformDistributionMT<T>(out, out.elements(), state, pos, sh1, sh2,
+                                     mask, recursion_table, temper_table);
     return out;
 }
 
@@ -61,9 +59,8 @@ Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
                             Array<uint> recursion_table,
                             Array<uint> temper_table, Array<uint> state) {
     Array<T> out = createEmptyArray<T>(dims);
-    kernel::normalDistributionMT<T>(
-        out, out.elements(), state, pos, sh1,
-        sh2, mask, recursion_table, temper_table);
+    kernel::normalDistributionMT<T>(out, out.elements(), state, pos, sh1, sh2,
+                                    mask, recursion_table, temper_table);
     return out;
 }
 
diff --git a/src/backend/oneapi/range.cpp b/src/backend/oneapi/range.cpp
index 015ae955db..e5498d12d8 100644
--- a/src/backend/oneapi/range.cpp
+++ b/src/backend/oneapi/range.cpp
@@ -6,8 +6,8 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
-#include <range.hpp>
 #include <kernel/range.hpp>
+#include <range.hpp>
 
 #include <Array.hpp>
 #include <common/half.hpp>
diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp
index d2763c92ac..0300fa99b0 100644
--- a/src/backend/oneapi/reduce_impl.hpp
+++ b/src/backend/oneapi/reduce_impl.hpp
@@ -24,7 +24,6 @@ Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan,
     ONEAPI_NOT_SUPPORTED("");
     Array<To> out = createEmptyArray<To>(1);
     return out;
-
 }
 
 template<af_op_t op, typename Ti, typename Tk, typename To>
diff --git a/src/backend/oneapi/regions.cpp b/src/backend/oneapi/regions.cpp
index cc74fb9543..73ebccc46e 100644
--- a/src/backend/oneapi/regions.cpp
+++ b/src/backend/oneapi/regions.cpp
@@ -19,7 +19,6 @@ namespace oneapi {
 
 template<typename T>
 Array<T> regions(const Array<char> &in, af_connectivity connectivity) {
-
     ONEAPI_NOT_SUPPORTED("regions Not supported");
 
     const af::dim4 &dims = in.dims();
diff --git a/src/backend/oneapi/reorder.cpp b/src/backend/oneapi/reorder.cpp
index 7cced14197..fe5bf98854 100644
--- a/src/backend/oneapi/reorder.cpp
+++ b/src/backend/oneapi/reorder.cpp
@@ -19,7 +19,6 @@ using common::half;
 namespace oneapi {
 template<typename T>
 Array<T> reorder(const Array<T> &in, const af::dim4 &rdims) {
-
     ONEAPI_NOT_SUPPORTED("reorder Not supported");
 
     const af::dim4 &iDims = in.dims();
diff --git a/src/backend/oneapi/reshape.cpp b/src/backend/oneapi/reshape.cpp
index 9331038986..87a7e7d28e 100644
--- a/src/backend/oneapi/reshape.cpp
+++ b/src/backend/oneapi/reshape.cpp
@@ -21,7 +21,6 @@ namespace oneapi {
 template<typename inType, typename outType>
 Array<outType> reshape(const Array<inType> &in, const dim4 &outDims,
                        outType defaultValue, double scale) {
-
     ONEAPI_NOT_SUPPORTED("reshape Not supported");
 
     Array<outType> out = createEmptyArray<outType>(outDims);
diff --git a/src/backend/oneapi/rotate.cpp b/src/backend/oneapi/rotate.cpp
index fc49dd6baa..37f8abbe00 100644
--- a/src/backend/oneapi/rotate.cpp
+++ b/src/backend/oneapi/rotate.cpp
@@ -7,8 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <rotate.hpp>
 #include <err_oneapi.hpp>
+#include <rotate.hpp>
 
 // #include <kernel/rotate.hpp>
 
@@ -16,7 +16,6 @@ namespace oneapi {
 template<typename T>
 Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
                 const af_interp_type method) {
-
     ONEAPI_NOT_SUPPORTED("rotate Not supported");
 
     Array<T> out = createEmptyArray<T>(odims);
diff --git a/src/backend/oneapi/scan.cpp b/src/backend/oneapi/scan.cpp
index 572746035c..c71564cc65 100644
--- a/src/backend/oneapi/scan.cpp
+++ b/src/backend/oneapi/scan.cpp
@@ -7,8 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <scan.hpp>
 #include <err_oneapi.hpp>
+#include <scan.hpp>
 
 // #include <kernel/scan_dim.hpp>
 // #include <kernel/scan_first.hpp>
@@ -16,7 +16,6 @@
 namespace oneapi {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusiveScan) {
-
     ONEAPI_NOT_SUPPORTED("scan Not supported");
 
     Array<To> out = createEmptyArray<To>(in.dims());
diff --git a/src/backend/oneapi/scan_by_key.cpp b/src/backend/oneapi/scan_by_key.cpp
index 08a4969905..555817819c 100644
--- a/src/backend/oneapi/scan_by_key.cpp
+++ b/src/backend/oneapi/scan_by_key.cpp
@@ -20,7 +20,6 @@ namespace oneapi {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
                bool inclusive_scan) {
-
     ONEAPI_NOT_SUPPORTED("scan Not supported");
 
     Array<To> out = createEmptyArray<To>(in.dims());
@@ -30,9 +29,11 @@ Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
     // Param In  = in;
 
     // if (dim == 0) {
-    //     // kernel::scanFirstByKey<Ti, Tk, To, op>(Out, In, Key, inclusive_scan);
+    //     // kernel::scanFirstByKey<Ti, Tk, To, op>(Out, In, Key,
+    //     inclusive_scan);
     // } else {
-    //     // kernel::scanDimByKey<Ti, Tk, To, op>(Out, In, Key, dim, inclusive_scan);
+    //     // kernel::scanDimByKey<Ti, Tk, To, op>(Out, In, Key, dim,
+    //     inclusive_scan);
     // }
     return out;
 }
diff --git a/src/backend/oneapi/select.cpp b/src/backend/oneapi/select.cpp
index f15e2ab61c..beea59a771 100644
--- a/src/backend/oneapi/select.cpp
+++ b/src/backend/oneapi/select.cpp
@@ -30,7 +30,6 @@ namespace oneapi {
 template<typename T>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
                           const Array<T> &b, const dim4 &odims) {
-
     ONEAPI_NOT_SUPPORTED("createSelectNode Not supported");
 
     auto cond_node   = cond.getNode();
@@ -41,9 +40,9 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
     auto cond_height = cond_node->getHeight();
     const int height = max(max(a_height, b_height), cond_height) + 1;
 
-    auto node = make_shared<NaryNode>(
-        NaryNode(static_cast<af::dtype>(af::dtype_traits<T>::af_type), "__select",
-                 3, {{cond_node, a_node, b_node}}, af_select_t, height));
+    auto node = make_shared<NaryNode>(NaryNode(
+        static_cast<af::dtype>(af::dtype_traits<T>::af_type), "__select", 3,
+        {{cond_node, a_node, b_node}}, af_select_t, height));
     std::array<common::Node *, 1> nodes{node.get()};
     if (detail::passesJitHeuristics<T>(nodes) != kJITHeuristics::Pass) {
         if (a_height > max(b_height, cond_height)) {
@@ -61,7 +60,6 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
 template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
                           const T &b_val, const dim4 &odims) {
-
     ONEAPI_NOT_SUPPORTED("createSelectNode Not supported");
 
     auto cond_node   = cond.getNode();
diff --git a/src/backend/oneapi/set.cpp b/src/backend/oneapi/set.cpp
index 2001729eca..01fa0a6bcf 100644
--- a/src/backend/oneapi/set.cpp
+++ b/src/backend/oneapi/set.cpp
@@ -8,12 +8,12 @@
  ********************************************************/
 
 #include <Array.hpp>
+#include <common/deprecated.hpp>
 #include <copy.hpp>
 #include <err_oneapi.hpp>
 #include <set.hpp>
 #include <sort.hpp>
 #include <af/dim4.hpp>
-#include <common/deprecated.hpp>
 
 namespace oneapi {
 using af::dim4;
@@ -29,7 +29,6 @@ using type_t =
 
 template<typename T>
 Array<T> setUnique(const Array<T> &in, const bool is_sorted) {
-
     ONEAPI_NOT_SUPPORTED("setUnique Not supported");
     return createEmptyArray<T>(dim4(1, 1, 1, 1));
 
@@ -50,13 +49,13 @@ Array<T> setUnique(const Array<T> &in, const bool is_sorted) {
     //     out.resetDims(dim4(std::distance(begin, end), 1, 1, 1));
 
     //     return out;
-    // } catch (const std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+    // } catch (const std::exception &ex) { AF_ERROR(ex.what(),
+    // AF_ERR_INTERNAL); }
 }
 
 template<typename T>
 Array<T> setUnion(const Array<T> &first, const Array<T> &second,
                   const bool is_unique) {
-
     ONEAPI_NOT_SUPPORTED("setUnion Not supported");
     return createEmptyArray<T>(dim4(1, 1, 1, 1));
 
@@ -87,18 +86,19 @@ Array<T> setUnion(const Array<T> &first, const Array<T> &second,
     //     compute::buffer_iterator<type_t<T>> out_begin(out_data, 0);
 
     //     compute::buffer_iterator<type_t<T>> out_end = compute::set_union(
-    //         first_begin, first_end, second_begin, second_end, out_begin, queue);
+    //         first_begin, first_end, second_begin, second_end, out_begin,
+    //         queue);
 
     //     out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
     //     return out;
 
-    // } catch (const std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+    // } catch (const std::exception &ex) { AF_ERROR(ex.what(),
+    // AF_ERR_INTERNAL); }
 }
 
 template<typename T>
 Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
                       const bool is_unique) {
-
     ONEAPI_NOT_SUPPORTED("setIntersect Not supported");
     return createEmptyArray<T>(dim4(1, 1, 1, 1));
 
@@ -129,12 +129,15 @@ Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
     //         second_data, unique_second.elements());
     //     compute::buffer_iterator<type_t<T>> out_begin(out_data, 0);
 
-    //     compute::buffer_iterator<type_t<T>> out_end = compute::set_intersection(
-    //         first_begin, first_end, second_begin, second_end, out_begin, queue);
+    //     compute::buffer_iterator<type_t<T>> out_end =
+    //     compute::set_intersection(
+    //         first_begin, first_end, second_begin, second_end, out_begin,
+    //         queue);
 
     //     out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
     //     return out;
-    // } catch (const std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+    // } catch (const std::exception &ex) { AF_ERROR(ex.what(),
+    // AF_ERR_INTERNAL); }
 }
 
 #define INSTANTIATE(T)                                                        \
diff --git a/src/backend/oneapi/shift.cpp b/src/backend/oneapi/shift.cpp
index b3941f1960..e4ada40a5c 100644
--- a/src/backend/oneapi/shift.cpp
+++ b/src/backend/oneapi/shift.cpp
@@ -70,4 +70,4 @@ INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
-}  // namespace opencl
+}  // namespace oneapi
diff --git a/src/backend/oneapi/sift.cpp b/src/backend/oneapi/sift.cpp
index af2f7bf10d..9197c23d14 100644
--- a/src/backend/oneapi/sift.cpp
+++ b/src/backend/oneapi/sift.cpp
@@ -26,7 +26,6 @@ unsigned sift(Array<float>& x_out, Array<float>& y_out, Array<float>& score_out,
               const float edge_thr, const float init_sigma,
               const bool double_input, const float img_scale,
               const float feature_ratio, const bool compute_GLOH) {
-
     ONEAPI_NOT_SUPPORTED("sift Not supported");
     return 0;
 
diff --git a/src/backend/oneapi/sobel.cpp b/src/backend/oneapi/sobel.cpp
index f76b8685db..7d722e7f4d 100644
--- a/src/backend/oneapi/sobel.cpp
+++ b/src/backend/oneapi/sobel.cpp
@@ -20,7 +20,6 @@ namespace oneapi {
 template<typename Ti, typename To>
 std::pair<Array<To>, Array<To>> sobelDerivatives(const Array<Ti> &img,
                                                  const unsigned &ker_size) {
-
     ONEAPI_NOT_SUPPORTED("sobelDerivatives Not supported");
 
     Array<To> dx = createEmptyArray<To>(img.dims());
diff --git a/src/backend/oneapi/solve.cpp b/src/backend/oneapi/solve.cpp
index b38461d0f1..ee662de210 100644
--- a/src/backend/oneapi/solve.cpp
+++ b/src/backend/oneapi/solve.cpp
@@ -37,7 +37,6 @@ namespace oneapi {
 template<typename T>
 Array<T> solveLU(const Array<T> &A, const Array<int> &pivot, const Array<T> &b,
                  const af_mat_prop options) {
-
     ONEAPI_NOT_SUPPORTED("solveLU Not supported");
 
     if (OpenCLCPUOffload()) { return cpu::solveLU(A, pivot, b, options); }
@@ -62,7 +61,6 @@ Array<T> solveLU(const Array<T> &A, const Array<int> &pivot, const Array<T> &b,
 
 template<typename T>
 Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
-
     ONEAPI_NOT_SUPPORTED("generalSolve Not supported");
 
     // dim4 aDims = a.dims();
@@ -102,7 +100,6 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
 
 template<typename T>
 Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
-
     ONEAPI_NOT_SUPPORTED("leastSquares Not supported");
 
     int M  = a.dims()[0];
diff --git a/src/backend/oneapi/sort.cpp b/src/backend/oneapi/sort.cpp
index f9c13b7429..b5e0eb73fd 100644
--- a/src/backend/oneapi/sort.cpp
+++ b/src/backend/oneapi/sort.cpp
@@ -19,7 +19,6 @@
 namespace oneapi {
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending) {
-
     ONEAPI_NOT_SUPPORTED("sort Not supported");
 
     try {
diff --git a/src/backend/oneapi/sort_index.cpp b/src/backend/oneapi/sort_index.cpp
index ebf5ce65f7..6600db9f7c 100644
--- a/src/backend/oneapi/sort_index.cpp
+++ b/src/backend/oneapi/sort_index.cpp
@@ -24,7 +24,6 @@ namespace oneapi {
 template<typename T>
 void sort_index(Array<T> &okey, Array<uint> &oval, const Array<T> &in,
                 const uint dim, bool isAscending) {
-
     ONEAPI_NOT_SUPPORTED("sort_index Not supported");
 
     try {
@@ -34,12 +33,10 @@ void sort_index(Array<T> &okey, Array<uint> &oval, const Array<T> &in,
         oval.eval();
 
         // switch (dim) {
-        //     case 0: kernel::sort0ByKey<T, uint>(okey, oval, isAscending); break;
-        //     case 1:
-        //     case 2:
-        //     case 3:
-        //         kernel::sortByKeyBatched<T, uint>(okey, oval, dim, isAscending);
-        //         break;
+        //     case 0: kernel::sort0ByKey<T, uint>(okey, oval, isAscending);
+        //     break; case 1: case 2: case 3:
+        //         kernel::sortByKeyBatched<T, uint>(okey, oval, dim,
+        //         isAscending); break;
         //     default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
         // }
 
diff --git a/src/backend/oneapi/sparse.cpp b/src/backend/oneapi/sparse.cpp
index 70de66f6ee..ba776efb18 100644
--- a/src/backend/oneapi/sparse.cpp
+++ b/src/backend/oneapi/sparse.cpp
@@ -115,7 +115,7 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in_) {
     const Array<int> &colIdx = in_.getColIdx();
 
     if (stype == AF_STORAGE_CSR) {
-      // kernel::csr2dense<T>(dense_, values, rowIdx, colIdx);
+        // kernel::csr2dense<T>(dense_, values, rowIdx, colIdx);
     } else {
         AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
                  AF_ERR_NOT_SUPPORTED);
@@ -144,7 +144,8 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
         const Array<int> &irowIdx = in.getRowIdx();
         const Array<int> &icolIdx = in.getColIdx();
 
-        // kernel::csr2coo<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, icolIdx,
+        // kernel::csr2coo<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx,
+        // icolIdx,
         //                    index);
 
     } else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) {
@@ -161,7 +162,8 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
         Array<int> rowCopy = copyArray<int>(irowIdx);
         rowCopy.eval();
 
-        // kernel::coo2csr<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, icolIdx,
+        // kernel::coo2csr<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx,
+        // icolIdx,
         //                    index, rowCopy, in.dims()[0]);
 
     } else {
diff --git a/src/backend/oneapi/sparse_arith.cpp b/src/backend/oneapi/sparse_arith.cpp
index 40e9e24ff4..2bb14c2b1d 100644
--- a/src/backend/oneapi/sparse_arith.cpp
+++ b/src/backend/oneapi/sparse_arith.cpp
@@ -8,8 +8,8 @@
  ********************************************************/
 
 // #include <kernel/sparse_arith.hpp>
-#include <sparse.hpp>
 #include <err_oneapi.hpp>
+#include <sparse.hpp>
 
 #include <stdexcept>
 #include <string>
@@ -141,9 +141,9 @@ SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
     auto outValues = createEmptyArray<T>(dim4(nnzC));
 
     // kernel::ssArithCSR<T, op>(outValues, outColIdx, outRowIdx, M, N, nnzA,
-    //                           lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(),
-    //                           nnzB, rhs.getValues(), rhs.getRowIdx(),
-    //                           rhs.getColIdx());
+    //                           lhs.getValues(), lhs.getRowIdx(),
+    //                           lhs.getColIdx(), nnzB, rhs.getValues(),
+    //                           rhs.getRowIdx(), rhs.getColIdx());
 
     SparseArray<T> retVal = createArrayDataSparseArray(
         ldims, outValues, outRowIdx, outColIdx, sfmt);
diff --git a/src/backend/oneapi/sparse_blas.cpp b/src/backend/oneapi/sparse_blas.cpp
index bc06759dde..a7c04abc40 100644
--- a/src/backend/oneapi/sparse_blas.cpp
+++ b/src/backend/oneapi/sparse_blas.cpp
@@ -37,7 +37,7 @@ using namespace common;
 template<typename T>
 Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhsIn,
                 af_mat_prop optLhs, af_mat_prop optRhs) {
-  ONEAPI_NOT_SUPPORTED("sparse matmul Not supported");
+    ONEAPI_NOT_SUPPORTED("sparse matmul Not supported");
 #if defined(WITH_LINEAR_ALGEBRA)
     if (OpenCLCPUOffload(
             false)) {  // Do not force offload gemm on OSX Intel devices
diff --git a/src/backend/oneapi/surface.cpp b/src/backend/oneapi/surface.cpp
index 7efebfc43c..38ad3388f5 100644
--- a/src/backend/oneapi/surface.cpp
+++ b/src/backend/oneapi/surface.cpp
@@ -37,12 +37,14 @@ void copy_surface(const Array<T> &P, fg_surface surface) {
     //     glFinish();
 
     //     // Use of events:
-    //     // https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReleaseGLObjects.html
+    //     //
+    //     https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/clEnqueueReleaseGLObjects.html
     //     cl::Event event;
 
     //     getQueue().enqueueAcquireGLObjects(&shared_objects, NULL, &event);
     //     event.wait();
-    //     getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes, NULL,
+    //     getQueue().enqueueCopyBuffer(*d_P, *(res[0].get()), 0, 0, bytes,
+    //     NULL,
     //                                  &event);
     //     getQueue().enqueueReleaseGLObjects(&shared_objects, NULL, &event);
     //     event.wait();
@@ -57,7 +59,8 @@ void copy_surface(const Array<T> &P, fg_surface surface) {
     //     CheckGL("Begin OpenCL fallback-resource copy");
     //     glBindBuffer(GL_ARRAY_BUFFER, buffer);
     //     auto *ptr =
-    //         static_cast<GLubyte *>(glMapBuffer(GL_ARRAY_BUFFER, GL_WRITE_ONLY));
+    //         static_cast<GLubyte *>(glMapBuffer(GL_ARRAY_BUFFER,
+    //         GL_WRITE_ONLY));
     //     if (ptr) {
     //         getQueue().enqueueReadBuffer(*P.get(), CL_TRUE, 0, bytes, ptr);
     //         glUnmapBuffer(GL_ARRAY_BUFFER);
diff --git a/src/backend/oneapi/susan.cpp b/src/backend/oneapi/susan.cpp
index e6fe536918..94173b3e4c 100644
--- a/src/backend/oneapi/susan.cpp
+++ b/src/backend/oneapi/susan.cpp
@@ -36,7 +36,8 @@ unsigned susan(Array<float> &x_out, Array<float> &y_out, Array<float> &resp_out,
     ONEAPI_NOT_SUPPORTED("");
     return 0;
 
-    // kernel::susan<T>(resp.get(), in.get(), in.getOffset(), idims[0], idims[1],
+    // kernel::susan<T>(resp.get(), in.get(), in.getOffset(), idims[0],
+    // idims[1],
     //                  diff_thr, geom_thr, edge, radius);
 
     // unsigned corners_found = kernel::nonMaximal<T>(
@@ -72,4 +73,4 @@ INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 
-}  // namespace oneap
+}  // namespace oneapi
diff --git a/src/backend/oneapi/svd.cpp b/src/backend/oneapi/svd.cpp
index 8fef95ba6c..8a886983f9 100644
--- a/src/backend/oneapi/svd.cpp
+++ b/src/backend/oneapi/svd.cpp
@@ -137,8 +137,8 @@ void svd(Array<T> &arrU, Array<Tr> &arrS, Array<T> &arrVT, Array<T> &arrA,
 
     if (want_vectors) {
         mappedU  = static_cast<T *>(getQueue().enqueueMapBuffer(
-            *arrU.get(), CL_FALSE, CL_MAP_WRITE, sizeof(T) * arrU.getOffset(),
-            sizeof(T) * arrU.elements()));
+             *arrU.get(), CL_FALSE, CL_MAP_WRITE, sizeof(T) * arrU.getOffset(),
+             sizeof(T) * arrU.elements()));
         mappedVT = static_cast<T *>(getQueue().enqueueMapBuffer(
             *arrVT.get(), CL_TRUE, CL_MAP_WRITE, sizeof(T) * arrVT.getOffset(),
             sizeof(T) * arrVT.elements()));
@@ -234,7 +234,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
-}  // namespace opencl
+}  // namespace oneapi
 
 #else  // WITH_LINEAR_ALGEBRA
 
diff --git a/src/backend/oneapi/tile.cpp b/src/backend/oneapi/tile.cpp
index 5aac53265b..384c0f0710 100644
--- a/src/backend/oneapi/tile.cpp
+++ b/src/backend/oneapi/tile.cpp
@@ -7,8 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 //#include <kernel/tile.hpp>
-#include <tile.hpp>
 #include <err_oneapi.hpp>
+#include <tile.hpp>
 
 #include <Array.hpp>
 #include <common/half.hpp>
diff --git a/src/backend/oneapi/topk.cpp b/src/backend/oneapi/topk.cpp
index 06d4218221..8d963ac4c2 100644
--- a/src/backend/oneapi/topk.cpp
+++ b/src/backend/oneapi/topk.cpp
@@ -48,7 +48,6 @@ vector<af_index_t> indexForTopK(const int k) {
 template<typename T>
 void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
           const int k, const int dim, const af::topkFunction order) {
-
     ONEAPI_NOT_SUPPORTED("topk Not supported");
 
     // if (getDeviceType() == CL_DEVICE_TYPE_CPU) {
@@ -57,7 +56,8 @@ void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
 
     //     // TODO(umar): implement this in the kernel namespace
 
-    //     // The out_dims is of size k along the dimension of the topk operation
+    //     // The out_dims is of size k along the dimension of the topk
+    //     operation
     //     // and the same as the input dimension otherwise.
     //     dim4 out_dims(1);
     //     int ndims = in.dims().ndims();
@@ -84,7 +84,8 @@ void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
     //         *ibuf, CL_FALSE, CL_MAP_READ | CL_MAP_WRITE, 0, k * sizeof(uint),
     //         nullptr, &ev_ind));
     //     T* vptr    = static_cast<T*>(getQueue().enqueueMapBuffer(
-    //         *vbuf, CL_FALSE, CL_MAP_WRITE, 0, k * sizeof(T), nullptr, &ev_val));
+    //         *vbuf, CL_FALSE, CL_MAP_WRITE, 0, k * sizeof(T), nullptr,
+    //         &ev_val));
 
     //     vector<uint> idx(in.elements());
 
diff --git a/src/backend/oneapi/transform.cpp b/src/backend/oneapi/transform.cpp
index 79cb584264..732ba39cc0 100644
--- a/src/backend/oneapi/transform.cpp
+++ b/src/backend/oneapi/transform.cpp
@@ -22,15 +22,18 @@ void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            // kernel::transform<T>(out, in, tf, inverse, perspective, method, 1);
+            // kernel::transform<T>(out, in, tf, inverse, perspective, method,
+            // 1);
             break;
         case AF_INTERP_BILINEAR:
         case AF_INTERP_BILINEAR_COSINE:
-            // kernel::transform<T>(out, in, tf, inverse, perspective, method, 2);
+            // kernel::transform<T>(out, in, tf, inverse, perspective, method,
+            // 2);
             break;
         case AF_INTERP_BICUBIC:
         case AF_INTERP_BICUBIC_SPLINE:
-            // kernel::transform<T>(out, in, tf, inverse, perspective, method, 3);
+            // kernel::transform<T>(out, in, tf, inverse, perspective, method,
+            // 3);
             break;
         default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG);
     }
diff --git a/src/backend/oneapi/transpose.cpp b/src/backend/oneapi/transpose.cpp
index 0985bc48fa..cef137b561 100644
--- a/src/backend/oneapi/transpose.cpp
+++ b/src/backend/oneapi/transpose.cpp
@@ -6,9 +6,9 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
+#include <err_oneapi.hpp>
 #include <kernel/transpose.hpp>
 #include <transpose.hpp>
-#include <err_oneapi.hpp>
 
 #include <Array.hpp>
 #include <common/half.hpp>
diff --git a/src/backend/oneapi/triangle.cpp b/src/backend/oneapi/triangle.cpp
index f514b8d64b..afe0c27b7f 100644
--- a/src/backend/oneapi/triangle.cpp
+++ b/src/backend/oneapi/triangle.cpp
@@ -7,8 +7,9 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 #include <kernel/triangle.hpp>
-#include <triangle.hpp>
+
 #include <err_oneapi.hpp>
+#include <triangle.hpp>
 
 #include <Array.hpp>
 #include <common/half.hpp>
@@ -52,4 +53,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 
-}  // namespace opencl
+}  // namespace oneapi
diff --git a/src/backend/oneapi/unwrap.cpp b/src/backend/oneapi/unwrap.cpp
index 200da9d307..cbb2910ef7 100644
--- a/src/backend/oneapi/unwrap.cpp
+++ b/src/backend/oneapi/unwrap.cpp
@@ -60,4 +60,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 #undef INSTANTIATE
 
-}  // namespace opencl
+}  // namespace oneapi
diff --git a/src/backend/oneapi/vector_field.cpp b/src/backend/oneapi/vector_field.cpp
index 40c7be146d..d42c86c270 100644
--- a/src/backend/oneapi/vector_field.cpp
+++ b/src/backend/oneapi/vector_field.cpp
@@ -18,8 +18,7 @@ namespace oneapi {
 
 template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
-                       fg_vector_field vfield) {
-}
+                       fg_vector_field vfield) {}
 
 #define INSTANTIATE(T)                                                     \
     template void copy_vector_field<T>(const Array<T> &, const Array<T> &, \
diff --git a/src/backend/oneapi/where.cpp b/src/backend/oneapi/where.cpp
index 4dc3e42565..df9267df72 100644
--- a/src/backend/oneapi/where.cpp
+++ b/src/backend/oneapi/where.cpp
@@ -18,11 +18,11 @@ namespace oneapi {
 
 template<typename T>
 Array<uint> where(const Array<T> &in) {
-    //Param<uint> Out;
+    // Param<uint> Out;
     // Param<T> In = in;
     ONEAPI_NOT_SUPPORTED("where Not supported");
     // kernel::where<T>(Out, In);
-    //return createParamArray<uint>(Out, true);
+    // return createParamArray<uint>(Out, true);
     return createEmptyArray<uint>(af::dim4(1));
 }
 
@@ -41,4 +41,4 @@ INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 
-}  // namespace opencl
+}  // namespace oneapi
diff --git a/src/backend/oneapi/wrap.cpp b/src/backend/oneapi/wrap.cpp
index 5dd0d7d78f..e3a9b2fc1f 100644
--- a/src/backend/oneapi/wrap.cpp
+++ b/src/backend/oneapi/wrap.cpp
@@ -24,8 +24,8 @@ template<typename T>
 void wrap(Array<T> &out, const Array<T> &in, const dim_t wx, const dim_t wy,
           const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
           const bool is_column) {
-  ONEAPI_NOT_SUPPORTED("wrap Not supported");
-  // kernel::wrap<T>(out, in, wx, wy, sx, sy, px, py, is_column);
+    ONEAPI_NOT_SUPPORTED("wrap Not supported");
+    // kernel::wrap<T>(out, in, wx, wy, sx, sy, px, py, is_column);
 }
 
 #define INSTANTIATE(T)                                                        \
@@ -57,7 +57,8 @@ Array<T> wrap_dilated(const Array<T> &in, const dim_t ox, const dim_t oy,
     af::dim4 odims(ox, oy, idims[2], idims[3]);
     Array<T> out = createValueArray<T>(odims, scalar<T>(0));
 
-    // kernel::wrap_dilated<T>(out, in, wx, wy, sx, sy, px, py, dx, dy, is_column);
+    // kernel::wrap_dilated<T>(out, in, wx, wy, sx, sy, px, py, dx, dy,
+    // is_column);
     ONEAPI_NOT_SUPPORTED("wrap_dilated Not supported");
     return out;
 }
@@ -73,4 +74,4 @@ INSTANTIATE(double)
 INSTANTIATE(half)
 #undef INSTANTIATE
 
-}  // namespace opencl
+}  // namespace oneapi

From 87cfde362389c515815f380501aac36a3c867ddc Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Wed, 5 Oct 2022 16:39:41 -0400
Subject: [PATCH 107/473] diagonal port to oneapi. its tests pass except
 *LargeDim* and *GFOR*

---
 src/backend/oneapi/CMakeLists.txt      |   1 +
 src/backend/oneapi/diagonal.cpp        |   9 +-
 src/backend/oneapi/kernel/diagonal.hpp | 163 +++++++++++++++++++++++++
 3 files changed, 170 insertions(+), 3 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/diagonal.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 4559aa9292..10dec177d4 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -206,6 +206,7 @@ target_sources(afoneapi
   PRIVATE
     kernel/KParam.hpp
     kernel/assign.hpp
+    kernel/diagonal.hpp
     kernel/iota.hpp
     kernel/memcopy.hpp
     kernel/random_engine.hpp
diff --git a/src/backend/oneapi/diagonal.cpp b/src/backend/oneapi/diagonal.cpp
index f22b2440c2..b9d443c662 100644
--- a/src/backend/oneapi/diagonal.cpp
+++ b/src/backend/oneapi/diagonal.cpp
@@ -11,7 +11,7 @@
 #include <common/half.hpp>
 #include <diagonal.hpp>
 #include <err_oneapi.hpp>
-//#include <kernel/diagonal.hpp>
+#include <kernel/diagonal.hpp>
 #include <math.hpp>
 #include <af/dim4.hpp>
 
@@ -20,20 +20,23 @@ using common::half;
 namespace oneapi {
 template<typename T>
 Array<T> diagCreate(const Array<T> &in, const int num) {
-    ONEAPI_NOT_SUPPORTED("");
     int size     = in.dims()[0] + std::abs(num);
     int batch    = in.dims()[1];
     Array<T> out = createEmptyArray<T>(dim4(size, size, batch));
+
+    kernel::diagCreate<T>(out, in, num);
+
     return out;
 }
 
 template<typename T>
 Array<T> diagExtract(const Array<T> &in, const int num) {
-    ONEAPI_NOT_SUPPORTED("");
     const dim_t *idims = in.dims().get();
     dim_t size         = std::min(idims[0], idims[1]) - std::abs(num);
     Array<T> out       = createEmptyArray<T>(dim4(size, 1, idims[2], idims[3]));
 
+    kernel::diagExtract<T>(out, in, num);
+
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/diagonal.hpp b/src/backend/oneapi/kernel/diagonal.hpp
new file mode 100644
index 0000000000..4668fee5bd
--- /dev/null
+++ b/src/backend/oneapi/kernel/diagonal.hpp
@@ -0,0 +1,163 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename T>
+class diagCreateKernel {
+   public:
+    diagCreateKernel(sycl::accessor<T> oData, KParam oInfo,
+                     sycl::accessor<T> iData, KParam iInfo, int num,
+                     int groups_x)
+        : oData_(oData)
+        , oInfo_(oInfo)
+        , iData_(iData)
+        , iInfo_(iInfo)
+        , num_(num)
+        , groups_x_(groups_x) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g      = it.get_group();
+        unsigned idz       = g.get_group_id(0) / groups_x_;
+        unsigned groupId_x = g.get_group_id(0) - idz * groups_x_;
+
+        unsigned idx = it.get_local_id(0) + groupId_x * g.get_local_range(0);
+        unsigned idy = it.get_global_id(1);
+
+        if (idx >= oInfo_.dims[0] || idy >= oInfo_.dims[1] ||
+            idz >= oInfo_.dims[2])
+            return;
+
+        T *optr = oData_.get_pointer();
+        optr += idz * oInfo_.strides[2] + idy * oInfo_.strides[1] + idx;
+
+        const T *iptr = iData_.get_pointer();
+        iptr +=
+            idz * iInfo_.strides[1] + ((num_ > 0) ? idx : idy) + iInfo_.offset;
+
+        T val = (idx == (idy - num_)) ? *iptr : (T)(0);
+        *optr = val;
+    }
+
+   private:
+    sycl::accessor<T> oData_;
+    KParam oInfo_;
+    sycl::accessor<T> iData_;
+    KParam iInfo_;
+    int num_;
+    int groups_x_;
+};
+
+template<typename T>
+static void diagCreate(Param<T> out, Param<T> in, int num) {
+    auto local   = sycl::range{32, 8};
+    int groups_x = divup(out.info.dims[0], local[0]);
+    int groups_y = divup(out.info.dims[1], local[1]);
+    auto global  = sycl::range{groups_x * local[0] * out.info.dims[2],
+                              groups_y * local[1]};
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto oData = out.data->get_access(h);
+        auto iData = in.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       diagCreateKernel<T>(oData, out.info, iData, in.info, num,
+                                           groups_x));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+class diagExtractKernel {
+   public:
+    diagExtractKernel(sycl::accessor<T> oData, KParam oInfo,
+                      sycl::accessor<T> iData, KParam iInfo, int num,
+                      int groups_z)
+        : oData_(oData)
+        , oInfo_(oInfo)
+        , iData_(iData)
+        , iInfo_(iInfo)
+        , num_(num)
+        , groups_z_(groups_z) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        unsigned idw  = g.get_group_id(1) / groups_z_;
+        unsigned idz  = g.get_group_id(1) - idw * groups_z_;
+
+        unsigned idx = it.get_global_id(0);
+
+        if (idx >= oInfo_.dims[0] || idz >= oInfo_.dims[2] ||
+            idw >= oInfo_.dims[3])
+            return;
+
+        T *optr = oData_.get_pointer();
+        optr += idz * oInfo_.strides[2] + idw * oInfo_.strides[3] + idx;
+
+        if (idx >= iInfo_.dims[0] || idx >= iInfo_.dims[1]) {
+            *optr = (T)(0);
+            return;
+        }
+
+        int i_off = (num_ > 0) ? (num_ * iInfo_.strides[1] + idx)
+                               : (idx - num_) + iInfo_.offset;
+
+        const T *iptr = iData_.get_pointer();
+        iptr += idz * iInfo_.strides[2] + idw * iInfo_.strides[3] + i_off;
+
+        *optr = iptr[idx * iInfo_.strides[1]];
+    }
+
+   private:
+    sycl::accessor<T> oData_;
+    KParam oInfo_;
+    sycl::accessor<T> iData_;
+    KParam iInfo_;
+    int num_;
+    int groups_z_;
+};
+
+template<typename T>
+static void diagExtract(Param<T> out, Param<T> in, int num) {
+    auto local   = sycl::range{256, 1};
+    int groups_x = divup(out.info.dims[0], local[0]);
+    int groups_z = out.info.dims[2];
+    auto global  = sycl::range{groups_x * local[0],
+                              groups_z * local[1] * out.info.dims[3]};
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto oData = out.data->get_access(h);
+        auto iData = in.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       diagExtractKernel<T>(oData, out.info, iData, in.info,
+                                            num, groups_z));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi

From eb31c7f2c9016c8ad6c32159ebb5588cf73c2fde Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Fri, 7 Oct 2022 12:49:24 -0400
Subject: [PATCH 108/473] diff port to oneapi. its tests pass except *LargeDim*
 and *GFOR* (#3304)

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Co-authored-by: Umar Arshad <umar@arrayfire.com>
---
 src/backend/oneapi/CMakeLists.txt  |   1 +
 src/backend/oneapi/diff.cpp        |   7 +-
 src/backend/oneapi/kernel/diff.hpp | 124 +++++++++++++++++++++++++++++
 3 files changed, 127 insertions(+), 5 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/diff.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 10dec177d4..67f9ec8b23 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -207,6 +207,7 @@ target_sources(afoneapi
     kernel/KParam.hpp
     kernel/assign.hpp
     kernel/diagonal.hpp
+    kernel/diff.hpp
     kernel/iota.hpp
     kernel/memcopy.hpp
     kernel/random_engine.hpp
diff --git a/src/backend/oneapi/diff.cpp b/src/backend/oneapi/diff.cpp
index 71e331a122..ad9da16697 100644
--- a/src/backend/oneapi/diff.cpp
+++ b/src/backend/oneapi/diff.cpp
@@ -9,8 +9,7 @@
 
 #include <Array.hpp>
 #include <diff.hpp>
-//#include <kernel/diff.hpp>
-#include <err_oneapi.hpp>
+#include <kernel/diff.hpp>
 #include <af/dim4.hpp>
 #include <stdexcept>
 
@@ -18,7 +17,6 @@ namespace oneapi {
 
 template<typename T>
 Array<T> diff(const Array<T> &in, const int dim, const bool isDiff2) {
-    ONEAPI_NOT_SUPPORTED("");
     const af::dim4 &iDims = in.dims();
     af::dim4 oDims        = iDims;
     oDims[dim] -= (isDiff2 + 1);
@@ -27,18 +25,17 @@ Array<T> diff(const Array<T> &in, const int dim, const bool isDiff2) {
         throw std::runtime_error("Elements are 0");
     }
     Array<T> out = createEmptyArray<T>(oDims);
+    kernel::diff<T>(out, in, in.ndims(), dim, isDiff2);
     return out;
 }
 
 template<typename T>
 Array<T> diff1(const Array<T> &in, const int dim) {
-    ONEAPI_NOT_SUPPORTED("");
     return diff<T>(in, dim, false);
 }
 
 template<typename T>
 Array<T> diff2(const Array<T> &in, const int dim) {
-    ONEAPI_NOT_SUPPORTED("");
     return diff<T>(in, dim, true);
 }
 
diff --git a/src/backend/oneapi/kernel/diff.hpp b/src/backend/oneapi/kernel/diff.hpp
new file mode 100644
index 0000000000..d624cd5283
--- /dev/null
+++ b/src/backend/oneapi/kernel/diff.hpp
@@ -0,0 +1,124 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename T>
+class diffKernel {
+   public:
+    diffKernel(sycl::accessor<T> outAcc, const sycl::accessor<T> inAcc,
+               const KParam op, const KParam ip, const int oElem,
+               const int blocksPerMatX, const int blocksPerMatY,
+               const bool isDiff2, const unsigned DIM)
+        : outAcc_(outAcc)
+        , inAcc_(inAcc)
+        , op_(op)
+        , ip_(ip)
+        , oElem_(oElem)
+        , blocksPerMatX_(blocksPerMatX)
+        , blocksPerMatY_(blocksPerMatY)
+        , isDiff2_(isDiff2)
+        , DIM_(DIM) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const int idz = g.get_group_id(0) / blocksPerMatX_;
+        const int idw = g.get_group_id(1) / blocksPerMatY_;
+
+        const int blockIdx_x = g.get_group_id(0) - idz * blocksPerMatX_;
+        const int blockIdx_y = g.get_group_id(1) - idw * blocksPerMatY_;
+
+        const int idx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+        const int idy = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+        if (idx >= op_.dims[0] || idy >= op_.dims[1] || idz >= op_.dims[2] ||
+            idw >= op_.dims[3])
+            return;
+
+        int iMem0 = idw * ip_.strides[3] + idz * ip_.strides[2] +
+                    idy * ip_.strides[1] + idx;
+        int iMem1 = iMem0 + ip_.strides[DIM_];
+        int iMem2 = iMem1 + ip_.strides[DIM_];
+
+        int oMem = idw * op_.strides[3] + idz * op_.strides[2] +
+                   idy * op_.strides[1] + idx;
+
+        iMem2 *= isDiff2_;
+
+        T *out      = outAcc_.get_pointer();
+        const T *in = inAcc_.get_pointer() + ip_.offset;
+        if (isDiff2_ == 0) {
+            out[oMem] = in[iMem1] - in[iMem0];
+        } else {
+            out[oMem] = in[iMem2] - in[iMem1] - in[iMem1] + in[iMem0];
+        }
+
+        // diff_this(out, in + ip.offset, oMem, iMem0, iMem1, iMem2);
+    }
+
+   private:
+    sycl::accessor<T> outAcc_;
+    const sycl::accessor<T> inAcc_;
+    const KParam op_;
+    const KParam ip_;
+    const int oElem_;
+    const int blocksPerMatX_;
+    const int blocksPerMatY_;
+    const bool isDiff2_;
+    const unsigned DIM_;
+};
+
+template<typename T>
+void diff(Param<T> out, const Param<T> in, const unsigned indims,
+          const unsigned dim, const bool isDiff2) {
+    constexpr int TX = 16;
+    constexpr int TY = 16;
+
+    auto local = sycl::range{TX, TY};
+    if (dim == 0 && indims == 1) { local = sycl::range{TX * TY, 1}; }
+
+    int blocksPerMatX = divup(out.info.dims[0], local[0]);
+    int blocksPerMatY = divup(out.info.dims[1], local[1]);
+    auto global       = sycl::range{local[0] * blocksPerMatX * out.info.dims[2],
+                              local[1] * blocksPerMatY * out.info.dims[3]};
+
+    const int oElem = out.info.dims[0] * out.info.dims[1] * out.info.dims[2] *
+                      out.info.dims[3];
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto inAcc  = in.data->get_access(h);
+        auto outAcc = out.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            diffKernel<T>(outAcc, inAcc, out.info, in.info, oElem,
+                          blocksPerMatX, blocksPerMatY, isDiff2, dim));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi

From 9b4642e19dde97e79d59e3702a09e29013d49d98 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 7 Oct 2022 15:13:21 -0400
Subject: [PATCH 109/473] Fix several warnings in oneAPI backend

---
 extern/half/include/half.hpp                  |  14 ++
 src/backend/common/Logger.hpp                 |   1 +
 src/backend/oneapi/Array.cpp                  |   8 +-
 src/backend/oneapi/Module.hpp                 |   6 +-
 src/backend/oneapi/iota.cpp                   |   7 +-
 src/backend/oneapi/jit/kernel_generators.hpp  |   5 +-
 src/backend/oneapi/kernel/assign.hpp          |   2 +-
 src/backend/oneapi/kernel/memcopy.hpp         |  23 +-
 .../oneapi/kernel/random_engine_write.hpp     |   8 +-
 src/backend/oneapi/kernel/transpose.hpp       |   2 +-
 .../oneapi/kernel/transpose_inplace.hpp       |   8 +-
 src/backend/oneapi/platform.cpp               |   5 +-
 src/backend/oneapi/sparse.cpp                 | 208 +++++++++---------
 src/backend/oneapi/sparse_arith.cpp           | 140 ++++++------
 src/backend/oneapi/sparse_blas.cpp            |  95 ++++----
 test/CMakeLists.txt                           |  26 ++-
 test/arrayfire_test.cpp                       |   1 +
 17 files changed, 287 insertions(+), 272 deletions(-)

diff --git a/extern/half/include/half.hpp b/extern/half/include/half.hpp
index ab70791db9..e8dfc1995a 100644
--- a/extern/half/include/half.hpp
+++ b/extern/half/include/half.hpp
@@ -403,7 +403,14 @@ namespace half_float
 		template<typename T> bool builtin_isinf(T arg)
 		{
 		#if HALF_ENABLE_CPP11_CMATH
+#ifdef __clang__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wtautological-constant-compare"
+#endif
 			return std::isinf(arg);
+#ifdef __clang__
+#pragma GCC diagnostic pop
+#endif
 		#elif defined(_MSC_VER)
 			return !::_finite(static_cast<double>(arg)) && !::_isnan(static_cast<double>(arg));
 		#else
@@ -419,7 +426,14 @@ namespace half_float
 		template<typename T> bool builtin_isnan(T arg)
 		{
 		#if HALF_ENABLE_CPP11_CMATH
+#ifdef __clang__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wtautological-constant-compare"
+#endif
 			return std::isnan(arg);
+#ifdef __clang__
+#pragma GCC diagnostic pop
+#endif
 		#elif defined(_MSC_VER)
 			return ::_isnan(static_cast<double>(arg)) != 0;
 		#else
diff --git a/src/backend/common/Logger.hpp b/src/backend/common/Logger.hpp
index 5241dc9126..4b7b4d419e 100644
--- a/src/backend/common/Logger.hpp
+++ b/src/backend/common/Logger.hpp
@@ -17,6 +17,7 @@
 /* Clang/LLVM */
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wignored-attributes"
+#pragma clang diagnostic ignored "-Wtautological-constant-compare"
 #elif defined(__ICC) || defined(__INTEL_COMPILER)
 /* Intel ICC/ICPC */
 // Fix the warning code here, if any
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index db4bce10e3..bd5676fd01 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -313,12 +313,12 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
             return kJITHeuristics::TreeHeight;
         }
     }
+    ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
 
-    bool isBufferLimit = getMemoryPressure() >= getMemoryPressureThreshold();
-    auto platform      = getActivePlatform();
+    // bool isBufferLimit = getMemoryPressure() >= getMemoryPressureThreshold();
+    // auto platform      = getActivePlatform();
 
     // The Apple platform can have the nvidia card or the AMD card
-    ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
     // bool isIntel = platform == AFCL_PLATFORM_INTEL;
 
     // /// Intels param_size limit is much smaller than the other platforms
@@ -502,8 +502,6 @@ void writeDeviceDataArray(Array<T> &arr, const void *const data,
                           const size_t bytes) {
     if (!arr.isOwner()) { arr = copyArray<T>(arr); }
 
-    buffer<T> &buf = *arr.get();
-
     // clRetainMemObject(
     //    reinterpret_cast<buffer<T> *>(const_cast<void *>(data)));
     // buffer<T> data_buf =
diff --git a/src/backend/oneapi/Module.hpp b/src/backend/oneapi/Module.hpp
index 0aa1cc790d..5637fa5d06 100644
--- a/src/backend/oneapi/Module.hpp
+++ b/src/backend/oneapi/Module.hpp
@@ -17,9 +17,9 @@ namespace oneapi {
 /// oneapi backend wrapper for cl::Program object
 class Module
     : public common::ModuleInterface<
-          sycl::kernel_bundle<sycl::bundle_state::executable>> {
+          sycl::kernel_bundle<sycl::bundle_state::executable>*> {
    public:
-    using ModuleType = sycl::kernel_bundle<sycl::bundle_state::executable>;
+    using ModuleType = sycl::kernel_bundle<sycl::bundle_state::executable>*;
     using BaseClass  = common::ModuleInterface<ModuleType>;
 
     /// \brief Create an uninitialized Module
@@ -29,7 +29,7 @@ class Module
     Module(ModuleType mod) : BaseClass(mod) {}
 
     /// \brief Unload module
-    operator bool() const final { return get().empty(); }
+    operator bool() const final { return get()->empty(); }
 
     /// Unload the module
     void unload() final {
diff --git a/src/backend/oneapi/iota.cpp b/src/backend/oneapi/iota.cpp
index bb6380993b..18077e5199 100644
--- a/src/backend/oneapi/iota.cpp
+++ b/src/backend/oneapi/iota.cpp
@@ -31,10 +31,10 @@ Array<T> iota(const dim4 &dims, const dim4 &tile_dims) {
 template<>
 Array<half> iota(const dim4 &dims, const dim4 &tile_dims) {
     ONEAPI_NOT_SUPPORTED("");
-    dim4 outdims = dims * tile_dims;
+    // dim4 outdims = dims * tile_dims;
 
-    Array<half> out = createEmptyArray<half>(outdims);
-    return out;
+    // Array<half> out = createEmptyArray<half>(outdims);
+    // return out;
 }
 
 #define INSTANTIATE(T) \
@@ -49,5 +49,4 @@ INSTANTIATE(uintl)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
-INSTANTIATE(half)
 }  // namespace oneapi
diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
index a49b25de0c..202403f4cb 100644
--- a/src/backend/oneapi/jit/kernel_generators.hpp
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -18,8 +18,9 @@ namespace oneapi {
 namespace {
 
 /// Creates a string that will be used to declare the parameter of kernel
-void generateParamDeclaration(std::stringstream& kerStream, int id,
-                              bool is_linear, const std::string& m_type_str) {
+inline void generateParamDeclaration(std::stringstream& kerStream, int id,
+                                     bool is_linear,
+                                     const std::string& m_type_str) {
     if (is_linear) {
         kerStream << "__global " << m_type_str << " *in" << id
                   << ", dim_t iInfo" << id << "_offset, \n";
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 7a75735f50..d4cc7e2b6c 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -47,8 +47,8 @@ class assignKernel {
                  sycl::accessor<uint> ptr3, const int nBBS0, const int nBBS1,
                  sycl::stream debug)
         : out_(out)
-        , oInfo_(oInfo)
         , in_(in)
+        , oInfo_(oInfo)
         , iInfo_(iInfo)
         , p_(p)
         , ptr0_(ptr0)
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 3f3fdce1ae..701060820f 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -36,8 +36,8 @@ class memCopy {
             dims_t idims, dims_t istrides, int offset, int groups_0,
             int groups_1, sycl::stream debug)
         : out_(out)
-        , ostrides_(ostrides)
         , in_(in)
+        , ostrides_(ostrides)
         , idims_(idims)
         , istrides_(istrides)
         , offset_(offset)
@@ -46,9 +46,6 @@ class memCopy {
         , debug_(debug) {}
 
     void operator()(sycl::nd_item<2> it) const {
-        // printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
-        // debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) <<
-        // "]" << sycl::stream_manipulator::endl;
         const int lid0 = it.get_local_id(0);
         const int lid1 = it.get_local_id(1);
 
@@ -112,11 +109,6 @@ void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
                           groups_1 * idims[3] * local_size[1]);
     sycl::nd_range<2> ndrange(global, local);
 
-    printf("<%d, %d> <%d, %d>\n", ndrange.get_global_range().get(0),
-           ndrange.get_global_range().get(1), ndrange.get_local_range().get(0),
-           ndrange.get_local_range().get(1));
-    printf("<%d, %d> ", ndrange.get_group_range().get(0),
-           ndrange.get_group_range().get(1));
     getQueue().submit([=](sycl::handler &h) {
         auto out_acc = out->get_access(h);
         auto in_acc  = const_cast<sycl::buffer<T> *>(in)->get_access(h);
@@ -137,12 +129,13 @@ static T scale(T value, double factor) {
 
 template<>
 cfloat scale<cfloat>(cfloat value, double factor) {
-    return (cfloat)(value.real() * factor, value.imag() * factor);
+    return cfloat{static_cast<float>(value.real() * factor),
+                  static_cast<float>(value.imag() * factor)};
 }
 
 template<>
 cdouble scale<cdouble>(cdouble value, double factor) {
-    return (cdouble)(value.real() * factor, value.imag() * factor);
+    return cdouble{value.real() * factor, value.imag() * factor};
 }
 
 template<typename inType, typename outType>
@@ -213,8 +206,8 @@ class reshapeCopy {
                 float factor, dims_t trgt, int blk_x, int blk_y,
                 sycl::stream debug)
         : dst_(dst)
-        , oInfo_(oInfo)
         , src_(src)
+        , oInfo_(oInfo)
         , iInfo_(iInfo)
         , default_value_(default_value)
         , factor_(factor)
@@ -293,12 +286,6 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
                           blk_y * dst.info.dims[3] * DIM1);
 
     sycl::nd_range<2> ndrange(global, local);
-    printf("reshape wat?\n");
-    printf("<%d, %d> <%d, %d>\n", ndrange.get_global_range().get(0),
-           ndrange.get_global_range().get(1), ndrange.get_local_range().get(0),
-           ndrange.get_local_range().get(1));
-    printf("<%d, %d> ", ndrange.get_group_range().get(0),
-           ndrange.get_group_range().get(1));
 
     dims_t trgt_dims;
     if (same_dims) {
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index 3b2857b92f..09f7a9c6e5 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -102,8 +102,9 @@ static double getDouble01(uint num1, uint num2) {
     n1 <<= 32;
     uint64_t num = n1 | n2;
     constexpr double factor =
-        ((1.0) / (std::numeric_limits<unsigned long long>::max() +
-                  static_cast<double>(1.0)));
+        ((1.0) /
+         (static_cast<double>(std::numeric_limits<unsigned long long>::max()) +
+          static_cast<double>(1.0)));
     constexpr double half_factor((0.5) * factor);
 
     return sycl::fma(static_cast<double>(num), factor, half_factor);
@@ -111,7 +112,8 @@ static double getDouble01(uint num1, uint num2) {
 
 // Conversion to doubles adapted from Random123
 constexpr double signed_factor =
-    ((1.0l) / (std::numeric_limits<long long>::max() + (1.0l)));
+    ((1.0l) / (static_cast<long double>(std::numeric_limits<long long>::max()) +
+               (1.0l)));
 constexpr double half_factor = ((0.5) * signed_factor);
 
 // Generates rationals in (-1, 1]
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index b2bc48a407..8c7fef325f 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -130,10 +130,10 @@ class transposeKernel {
     KParam in_;
     int blocksPerMatX_;
     int blocksPerMatY_;
-    sycl::stream debugStream_;
     bool conjugate_;
     bool IS32MULTIPLE_;
     local_accessor<T, 1> shrdMem_;
+    sycl::stream debugStream_;
 };
 
 template<typename T>
diff --git a/src/backend/oneapi/kernel/transpose_inplace.hpp b/src/backend/oneapi/kernel/transpose_inplace.hpp
index 81313b50da..108b9596f9 100644
--- a/src/backend/oneapi/kernel/transpose_inplace.hpp
+++ b/src/backend/oneapi/kernel/transpose_inplace.hpp
@@ -23,18 +23,18 @@ namespace oneapi {
 namespace kernel {
 
 template<typename T>
-T static getConjugate(const T &in) {
+static T getConjugate(const T &in) {
     // For non-complex types return same
     return in;
 }
 
 template<>
-cfloat static getConjugate(const cfloat &in) {
+cfloat getConjugate(const cfloat &in) {
     return std::conj(in);
 }
 
 template<>
-cdouble static getConjugate(const cdouble &in) {
+cdouble getConjugate(const cdouble &in) {
     return std::conj(in);
 }
 
@@ -160,11 +160,11 @@ class transposeInPlaceKernel {
     KParam in_;
     int blocksPerMatX_;
     int blocksPerMatY_;
-    sycl::stream debugStream_;
     bool conjugate_;
     bool IS32MULTIPLE_;
     local_accessor<T, 1> shrdMem_s_;
     local_accessor<T, 1> shrdMem_d_;
+    sycl::stream debugStream_;
 };
 
 template<typename T>
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index b65ad6698d..f2128c5ac5 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -91,7 +91,7 @@ bool verify_present(const string& pname, const string ref) {
 }
 
 // TODO: update to new platforms?
-static string platformMap(string& platStr) {
+inline string platformMap(string& platStr) {
     using strmap_t                = map<string, string>;
     static const strmap_t platMap = {
         make_pair("NVIDIA CUDA", "NVIDIA"),
@@ -134,7 +134,6 @@ string getDeviceInfo() noexcept {
     info << "ArrayFire v" << AF_VERSION << " (OpenCL, " << get_system()
          << ", build " << AF_REVISION << ")\n";
 
-    vector<sycl::device*> devices;
     try {
         DeviceManager& devMngr = DeviceManager::getInstance();
 
@@ -688,4 +687,4 @@ af_err afcl_delete_device_context(cl_device_id dev, cl_context ctx) {
     CATCHALL;
     return AF_SUCCESS;
 }
-*/
\ No newline at end of file
+*/
diff --git a/src/backend/oneapi/sparse.cpp b/src/backend/oneapi/sparse.cpp
index ba776efb18..18e1d48e81 100644
--- a/src/backend/oneapi/sparse.cpp
+++ b/src/backend/oneapi/sparse.cpp
@@ -35,46 +35,46 @@ using namespace common;
 template<typename T>
 SparseArray<T> sparseConvertDenseToCOO(const Array<T> &in) {
     ONEAPI_NOT_SUPPORTED("sparseConvertDenseToCOO Not supported");
-    in.eval();
+    // in.eval();
 
-    Array<uint> nonZeroIdx_ = where<T>(in);
-    Array<int> nonZeroIdx   = cast<int, uint>(nonZeroIdx_);
+    // Array<uint> nonZeroIdx_ = where<T>(in);
+    // Array<int> nonZeroIdx   = cast<int, uint>(nonZeroIdx_);
 
-    dim_t nNZ = nonZeroIdx.elements();
+    // dim_t nNZ = nonZeroIdx.elements();
 
-    Array<int> constDim = createValueArray<int>(dim4(nNZ), in.dims()[0]);
-    constDim.eval();
+    // Array<int> constDim = createValueArray<int>(dim4(nNZ), in.dims()[0]);
+    // constDim.eval();
 
-    Array<int> rowIdx =
-        arithOp<int, af_mod_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
-    Array<int> colIdx =
-        arithOp<int, af_div_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
+    // Array<int> rowIdx =
+    //     arithOp<int, af_mod_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
+    // Array<int> colIdx =
+    //     arithOp<int, af_div_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
 
-    Array<T> values = copyArray<T>(in);
-    values          = modDims(values, dim4(values.elements()));
-    values          = lookup<T, int>(values, nonZeroIdx, 0);
+    // Array<T> values = copyArray<T>(in);
+    // values          = modDims(values, dim4(values.elements()));
+    // values          = lookup<T, int>(values, nonZeroIdx, 0);
 
-    return createArrayDataSparseArray<T>(in.dims(), values, rowIdx, colIdx,
-                                         AF_STORAGE_COO);
+    // return createArrayDataSparseArray<T>(in.dims(), values, rowIdx, colIdx,
+    //                                      AF_STORAGE_COO);
 }
 
 template<typename T, af_storage stype>
 SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in_) {
     ONEAPI_NOT_SUPPORTED("sparseConvertDenseToStorage Not supported");
-    in_.eval();
-
-    uint nNZ = getScalar<uint>(reduce_all<af_notzero_t, T, uint>(in_));
-
-    SparseArray<T> sparse_ = createEmptySparseArray<T>(in_.dims(), nNZ, stype);
-    sparse_.eval();
-
-    Array<T> &values   = sparse_.getValues();
-    Array<int> &rowIdx = sparse_.getRowIdx();
-    Array<int> &colIdx = sparse_.getColIdx();
+    //     in_.eval();
+    //
+    //     uint nNZ = getScalar<uint>(reduce_all<af_notzero_t, T, uint>(in_));
+    //
+    //     SparseArray<T> sparse_ = createEmptySparseArray<T>(in_.dims(), nNZ,
+    //     stype); sparse_.eval();
+    //
+    //     Array<T> &values   = sparse_.getValues();
+    //     Array<int> &rowIdx = sparse_.getRowIdx();
+    //     Array<int> &colIdx = sparse_.getColIdx();
 
     // kernel::dense2csr<T>(values, rowIdx, colIdx, in_);
 
-    return sparse_;
+    // return sparse_;
 }
 
 // Partial template specialization of sparseConvertStorageToDense for COO
@@ -82,97 +82,97 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in_) {
 template<typename T>
 Array<T> sparseConvertCOOToDense(const SparseArray<T> &in) {
     ONEAPI_NOT_SUPPORTED("sparseConvertCOOToDense Not supported");
-    in.eval();
-
-    Array<T> dense = createValueArray<T>(in.dims(), scalar<T>(0));
-    dense.eval();
-
-    const Array<T> values   = in.getValues();
-    const Array<int> rowIdx = in.getRowIdx();
-    const Array<int> colIdx = in.getColIdx();
+    //    in.eval();
+    //
+    //    Array<T> dense = createValueArray<T>(in.dims(), scalar<T>(0));
+    //    dense.eval();
+    //
+    //    const Array<T> values   = in.getValues();
+    //    const Array<int> rowIdx = in.getRowIdx();
+    //    const Array<int> colIdx = in.getColIdx();
 
     // kernel::coo2dense<T>(dense, values, rowIdx, colIdx);
 
-    return dense;
+    // return dense;
 }
 
 template<typename T, af_storage stype>
 Array<T> sparseConvertStorageToDense(const SparseArray<T> &in_) {
     ONEAPI_NOT_SUPPORTED("sparseConvertStorageToDense Not supported");
-
-    if (stype != AF_STORAGE_CSR) {
-        AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
-                 AF_ERR_NOT_SUPPORTED);
-    }
-
-    in_.eval();
-
-    Array<T> dense_ = createValueArray<T>(in_.dims(), scalar<T>(0));
-    dense_.eval();
-
-    const Array<T> &values   = in_.getValues();
-    const Array<int> &rowIdx = in_.getRowIdx();
-    const Array<int> &colIdx = in_.getColIdx();
-
-    if (stype == AF_STORAGE_CSR) {
-        // kernel::csr2dense<T>(dense_, values, rowIdx, colIdx);
-    } else {
-        AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
-                 AF_ERR_NOT_SUPPORTED);
-    }
-
-    return dense_;
+    //
+    //    if (stype != AF_STORAGE_CSR) {
+    //        AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
+    //                 AF_ERR_NOT_SUPPORTED);
+    //    }
+    //
+    //    in_.eval();
+    //
+    //    Array<T> dense_ = createValueArray<T>(in_.dims(), scalar<T>(0));
+    //    dense_.eval();
+    //
+    //    const Array<T> &values   = in_.getValues();
+    //    const Array<int> &rowIdx = in_.getRowIdx();
+    //    const Array<int> &colIdx = in_.getColIdx();
+    //
+    //    if (stype == AF_STORAGE_CSR) {
+    //        // kernel::csr2dense<T>(dense_, values, rowIdx, colIdx);
+    //    } else {
+    //        AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
+    //                 AF_ERR_NOT_SUPPORTED);
+    //    }
+    //
+    //    return dense_;
 }
 
 template<typename T, af_storage dest, af_storage src>
 SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
     ONEAPI_NOT_SUPPORTED("sparseConvertStorageToStorage Not supported");
-    in.eval();
-
-    SparseArray<T> converted = createEmptySparseArray<T>(
-        in.dims(), static_cast<int>(in.getNNZ()), dest);
-    converted.eval();
-
-    if (src == AF_STORAGE_CSR && dest == AF_STORAGE_COO) {
-        Array<int> index = range<int>(in.getNNZ(), 0);
-        index.eval();
-
-        Array<T> &ovalues         = converted.getValues();
-        Array<int> &orowIdx       = converted.getRowIdx();
-        Array<int> &ocolIdx       = converted.getColIdx();
-        const Array<T> &ivalues   = in.getValues();
-        const Array<int> &irowIdx = in.getRowIdx();
-        const Array<int> &icolIdx = in.getColIdx();
-
-        // kernel::csr2coo<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx,
-        // icolIdx,
-        //                    index);
-
-    } else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) {
-        Array<int> index = range<int>(in.getNNZ(), 0);
-        index.eval();
-
-        Array<T> &ovalues         = converted.getValues();
-        Array<int> &orowIdx       = converted.getRowIdx();
-        Array<int> &ocolIdx       = converted.getColIdx();
-        const Array<T> &ivalues   = in.getValues();
-        const Array<int> &irowIdx = in.getRowIdx();
-        const Array<int> &icolIdx = in.getColIdx();
-
-        Array<int> rowCopy = copyArray<int>(irowIdx);
-        rowCopy.eval();
-
-        // kernel::coo2csr<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx,
-        // icolIdx,
-        //                    index, rowCopy, in.dims()[0]);
-
-    } else {
-        // Should never come here
-        AF_ERROR("OpenCL Backend invalid conversion combination",
-                 AF_ERR_NOT_SUPPORTED);
-    }
-
-    return converted;
+    // in.eval();
+
+    // SparseArray<T> converted = createEmptySparseArray<T>(
+    //     in.dims(), static_cast<int>(in.getNNZ()), dest);
+    // converted.eval();
+
+    // if (src == AF_STORAGE_CSR && dest == AF_STORAGE_COO) {
+    //     Array<int> index = range<int>(in.getNNZ(), 0);
+    //     index.eval();
+
+    //    Array<T> &ovalues         = converted.getValues();
+    //    Array<int> &orowIdx       = converted.getRowIdx();
+    //    Array<int> &ocolIdx       = converted.getColIdx();
+    //    const Array<T> &ivalues   = in.getValues();
+    //    const Array<int> &irowIdx = in.getRowIdx();
+    //    const Array<int> &icolIdx = in.getColIdx();
+
+    //    // kernel::csr2coo<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx,
+    //    // icolIdx,
+    //    //                    index);
+
+    //} else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) {
+    //    Array<int> index = range<int>(in.getNNZ(), 0);
+    //    index.eval();
+
+    //    Array<T> &ovalues         = converted.getValues();
+    //    Array<int> &orowIdx       = converted.getRowIdx();
+    //    Array<int> &ocolIdx       = converted.getColIdx();
+    //    const Array<T> &ivalues   = in.getValues();
+    //    const Array<int> &irowIdx = in.getRowIdx();
+    //    const Array<int> &icolIdx = in.getColIdx();
+
+    //    Array<int> rowCopy = copyArray<int>(irowIdx);
+    //    rowCopy.eval();
+
+    //    kernel::coo2csr<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx,
+    //    icolIdx,
+    //                       index, rowCopy, in.dims()[0]);
+
+    //} else {
+    //    // Should never come here
+    //    AF_ERROR("OpenCL Backend invalid conversion combination",
+    //             AF_ERR_NOT_SUPPORTED);
+    //}
+
+    // return converted;
 }
 
 #define INSTANTIATE_TO_STORAGE(T, S)                     \
diff --git a/src/backend/oneapi/sparse_arith.cpp b/src/backend/oneapi/sparse_arith.cpp
index 2bb14c2b1d..e39bed14e4 100644
--- a/src/backend/oneapi/sparse_arith.cpp
+++ b/src/backend/oneapi/sparse_arith.cpp
@@ -51,103 +51,103 @@ template<typename T, af_op_t op>
 Array<T> arithOpD(const SparseArray<T> &lhs, const Array<T> &rhs,
                   const bool reverse) {
     ONEAPI_NOT_SUPPORTED("arithOpD Not supported");
-    lhs.eval();
-    rhs.eval();
-
-    Array<T> out  = createEmptyArray<T>(dim4(0));
-    Array<T> zero = createValueArray<T>(rhs.dims(), scalar<T>(0));
-    switch (op) {
-        case af_add_t: out = copyArray<T>(rhs); break;
-        case af_sub_t:
-            out = reverse ? copyArray<T>(rhs)
-                          : arithOp<T, af_sub_t>(zero, rhs, rhs.dims());
-            break;
-        default: out = copyArray<T>(rhs);
-    }
-    out.eval();
-    switch (lhs.getStorage()) {
-        case AF_STORAGE_CSR:
-            // kernel::sparseArithOpCSR<T, op>(out, lhs.getValues(),
-            //                                 lhs.getRowIdx(), lhs.getColIdx(),
-            //                                 rhs, reverse);
-            break;
-        case AF_STORAGE_COO:
-            // kernel::sparseArithOpCOO<T, op>(out, lhs.getValues(),
-            //                                 lhs.getRowIdx(), lhs.getColIdx(),
-            //                                 rhs, reverse);
-            break;
-        default:
-            AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
-                     AF_ERR_NOT_SUPPORTED);
-    }
-
-    return out;
+    // lhs.eval();
+    // rhs.eval();
+
+    // Array<T> out  = createEmptyArray<T>(dim4(0));
+    // Array<T> zero = createValueArray<T>(rhs.dims(), scalar<T>(0));
+    // switch (op) {
+    //     case af_add_t: out = copyArray<T>(rhs); break;
+    //     case af_sub_t:
+    //         out = reverse ? copyArray<T>(rhs)
+    //                       : arithOp<T, af_sub_t>(zero, rhs, rhs.dims());
+    //         break;
+    //     default: out = copyArray<T>(rhs);
+    // }
+    // out.eval();
+    // switch (lhs.getStorage()) {
+    //     case AF_STORAGE_CSR:
+    //         kernel::sparseArithOpCSR<T, op>(out, lhs.getValues(),
+    //                                         lhs.getRowIdx(), lhs.getColIdx(),
+    //                                         rhs, reverse);
+    //         break;
+    //     case AF_STORAGE_COO:
+    //         kernel::sparseArithOpCOO<T, op>(out, lhs.getValues(),
+    //                                         lhs.getRowIdx(), lhs.getColIdx(),
+    //                                         rhs, reverse);
+    //         break;
+    //     default:
+    //         AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
+    //                  AF_ERR_NOT_SUPPORTED);
+    // }
+
+    // return out;
 }
 
 template<typename T, af_op_t op>
 SparseArray<T> arithOp(const SparseArray<T> &lhs, const Array<T> &rhs,
                        const bool reverse) {
     ONEAPI_NOT_SUPPORTED("arithOp Not supported");
-    lhs.eval();
-    rhs.eval();
-
-    SparseArray<T> out = createArrayDataSparseArray<T>(
-        lhs.dims(), lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(),
-        lhs.getStorage(), true);
-    out.eval();
-    switch (lhs.getStorage()) {
-        case AF_STORAGE_CSR:
-            // kernel::sparseArithOpCSR<T, op>(out.getValues(), out.getRowIdx(),
-            //                                 out.getColIdx(), rhs, reverse);
-            break;
-        case AF_STORAGE_COO:
-            // kernel::sparseArithOpCOO<T, op>(out.getValues(), out.getRowIdx(),
-            //                                 out.getColIdx(), rhs, reverse);
-            break;
-        default:
-            AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
-                     AF_ERR_NOT_SUPPORTED);
-    }
-
-    return out;
+    // lhs.eval();
+    // rhs.eval();
+
+    // SparseArray<T> out = createArrayDataSparseArray<T>(
+    //     lhs.dims(), lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(),
+    //     lhs.getStorage(), true);
+    // out.eval();
+    // switch (lhs.getStorage()) {
+    //     case AF_STORAGE_CSR:
+    //         kernel::sparseArithOpCSR<T, op>(out.getValues(), out.getRowIdx(),
+    //                                         out.getColIdx(), rhs, reverse);
+    //         break;
+    //     case AF_STORAGE_COO:
+    //         kernel::sparseArithOpCOO<T, op>(out.getValues(), out.getRowIdx(),
+    //                                         out.getColIdx(), rhs, reverse);
+    //         break;
+    //     default:
+    //         AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
+    //                  AF_ERR_NOT_SUPPORTED);
+    // }
+
+    // return out;
 }
 
 template<typename T, af_op_t op>
 SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
     ONEAPI_NOT_SUPPORTED("arithOp Not supported");
-    lhs.eval();
-    rhs.eval();
-    af::storage sfmt = lhs.getStorage();
+    // lhs.eval();
+    // rhs.eval();
+    // af::storage sfmt = lhs.getStorage();
 
-    const dim4 &ldims = lhs.dims();
+    // const dim4 &ldims = lhs.dims();
 
-    const uint M = ldims[0];
-    const uint N = ldims[1];
+    // const uint M = ldims[0];
+    // const uint N = ldims[1];
 
-    const dim_t nnzA = lhs.getNNZ();
-    const dim_t nnzB = rhs.getNNZ();
+    // const dim_t nnzA = lhs.getNNZ();
+    // const dim_t nnzB = rhs.getNNZ();
 
-    auto temp = createValueArray<int>(dim4(M + 1), scalar<int>(0));
-    temp.eval();
+    // auto temp = createValueArray<int>(dim4(M + 1), scalar<int>(0));
+    // temp.eval();
 
-    unsigned nnzC = 0;
+    // unsigned nnzC = 0;
     // kernel::csrCalcOutNNZ(temp, nnzC, M, N, nnzA, lhs.getRowIdx(),
     //                       lhs.getColIdx(), nnzB, rhs.getRowIdx(),
     //                       rhs.getColIdx());
 
-    auto outRowIdx = scan<af_add_t, int, int>(temp, 0);
+    // auto outRowIdx = scan<af_add_t, int, int>(temp, 0);
 
-    auto outColIdx = createEmptyArray<int>(dim4(nnzC));
-    auto outValues = createEmptyArray<T>(dim4(nnzC));
+    // auto outColIdx = createEmptyArray<int>(dim4(nnzC));
+    // auto outValues = createEmptyArray<T>(dim4(nnzC));
 
     // kernel::ssArithCSR<T, op>(outValues, outColIdx, outRowIdx, M, N, nnzA,
     //                           lhs.getValues(), lhs.getRowIdx(),
     //                           lhs.getColIdx(), nnzB, rhs.getValues(),
     //                           rhs.getRowIdx(), rhs.getColIdx());
 
-    SparseArray<T> retVal = createArrayDataSparseArray(
-        ldims, outValues, outRowIdx, outColIdx, sfmt);
-    return retVal;
+    // SparseArray<T> retVal = createArrayDataSparseArray(
+    //     ldims, outValues, outRowIdx, outColIdx, sfmt);
+    // return retVal;
 }
 
 #define INSTANTIATE(T)                                                         \
diff --git a/src/backend/oneapi/sparse_blas.cpp b/src/backend/oneapi/sparse_blas.cpp
index a7c04abc40..b9fcd6fb52 100644
--- a/src/backend/oneapi/sparse_blas.cpp
+++ b/src/backend/oneapi/sparse_blas.cpp
@@ -38,52 +38,55 @@ template<typename T>
 Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhsIn,
                 af_mat_prop optLhs, af_mat_prop optRhs) {
     ONEAPI_NOT_SUPPORTED("sparse matmul Not supported");
-#if defined(WITH_LINEAR_ALGEBRA)
-    if (OpenCLCPUOffload(
-            false)) {  // Do not force offload gemm on OSX Intel devices
-        return cpu::matmul(lhs, rhsIn, optLhs, optRhs);
-    }
-#endif
-
-    int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1;
-    // int lColDim = (optLhs == AF_MAT_NONE) ? 1 : 0;
-    static const int rColDim =
-        1;  // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0;
-
-    dim4 lDims = lhs.dims();
-    dim4 rDims = rhsIn.dims();
-    int M      = lDims[lRowDim];
-    int N      = rDims[rColDim];
-    // int K = lDims[lColDim];
-
-    const Array<T> rhs =
-        (N != 1 && optLhs == AF_MAT_NONE) ? transpose(rhsIn, false) : rhsIn;
-    Array<T> out = createEmptyArray<T>(af::dim4(M, N, 1, 1));
-
-    static const T alpha = scalar<T>(1.0);
-    static const T beta  = scalar<T>(0.0);
-
-    const Array<T>& values   = lhs.getValues();
-    const Array<int>& rowIdx = lhs.getRowIdx();
-    const Array<int>& colIdx = lhs.getColIdx();
-
-    if (optLhs == AF_MAT_NONE) {
-        // if (N == 1) {
-        //     kernel::csrmv(out, values, rowIdx, colIdx, rhs, alpha, beta);
-        // } else {
-        //     kernel::csrmm_nt(out, values, rowIdx, colIdx, rhs, alpha, beta);
-        // }
-    } else {
-        // // CSR transpose is a CSC matrix
-        // if (N == 1) {
-        //     kernel::cscmv(out, values, rowIdx, colIdx, rhs, alpha, beta,
-        //                   optLhs == AF_MAT_CTRANS);
-        // } else {
-        //     kernel::cscmm_nn(out, values, rowIdx, colIdx, rhs, alpha, beta,
-        //                      optLhs == AF_MAT_CTRANS);
-        // }
-    }
-    return out;
+    //#if defined(WITH_LINEAR_ALGEBRA)
+    //    if (OpenCLCPUOffload(
+    //            false)) {  // Do not force offload gemm on OSX Intel devices
+    //        return cpu::matmul(lhs, rhsIn, optLhs, optRhs);
+    //    }
+    //#endif
+    //
+    //    int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1;
+    //    // int lColDim = (optLhs == AF_MAT_NONE) ? 1 : 0;
+    //    static const int rColDim =
+    //        1;  // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0;
+    //
+    //    dim4 lDims = lhs.dims();
+    //    dim4 rDims = rhsIn.dims();
+    //    int M      = lDims[lRowDim];
+    //    int N      = rDims[rColDim];
+    //    // int K = lDims[lColDim];
+    //
+    //    const Array<T> rhs =
+    //        (N != 1 && optLhs == AF_MAT_NONE) ? transpose(rhsIn, false) :
+    //        rhsIn;
+    //    Array<T> out = createEmptyArray<T>(af::dim4(M, N, 1, 1));
+    //
+    //    static const T alpha = scalar<T>(1.0);
+    //    static const T beta  = scalar<T>(0.0);
+    //
+    //    const Array<T>& values   = lhs.getValues();
+    //    const Array<int>& rowIdx = lhs.getRowIdx();
+    //    const Array<int>& colIdx = lhs.getColIdx();
+    //
+    //    if (optLhs == AF_MAT_NONE) {
+    //        if (N == 1) {
+    //            kernel::csrmv(out, values, rowIdx, colIdx, rhs, alpha, beta);
+    //        } else {
+    //            kernel::csrmm_nt(out, values, rowIdx, colIdx, rhs, alpha,
+    //            beta);
+    //        }
+    //    } else {
+    //        // CSR transpose is a CSC matrix
+    //        if (N == 1) {
+    //            kernel::cscmv(out, values, rowIdx, colIdx, rhs, alpha, beta,
+    //                          optLhs == AF_MAT_CTRANS);
+    //        } else {
+    //            kernel::cscmm_nn(out, values, rowIdx, colIdx, rhs, alpha,
+    //            beta,
+    //                             optLhs == AF_MAT_CTRANS);
+    //        }
+    //    }
+    //    return out;
 }
 
 #define INSTANTIATE_SPARSE(T)                                            \
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index aa46bdaebb..7fcc708d32 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -107,6 +107,7 @@ if(AF_BUILD_UNIFIED)
   list(APPEND enabled_backends "unified")
 endif(AF_BUILD_UNIFIED)
 
+check_cxx_compiler_flag("-Wtautological-constant-compare" has_tautological_constant_compare_flag)
 
 add_library(arrayfire_test STATIC
   testHelpers.hpp
@@ -120,11 +121,15 @@ target_include_directories(arrayfire_test
     ${ArrayFire_SOURCE_DIR}/extern/half/include
   )
 
+# The tautological-constant-compare warning is always thrown for std::nan
+# and std::info calls. Its unnecessarily verbose.
+target_compile_options(arrayfire_test
+  PRIVATE
+    $<$<BOOL:${has_tautological_constant_compare_flag}>:-Wno-tautological-constant-compare>
+    $<$<CXX_COMPILER_ID:MSVC>: /bigobj
+                               /EHsc>
+  )
 if(WIN32)
-  target_compile_options(arrayfire_test
-    PRIVATE
-      /bigobj
-      /EHsc)
   target_compile_definitions(arrayfire_test
     PRIVATE
       WIN32_LEAN_AND_MEAN
@@ -185,6 +190,15 @@ function(make_test)
         arrayfire_test
       )
 
+    # The tautological-constant-compare warning is always thrown for std::nan
+    # and std::info calls. Its unnecessarily verbose.
+    target_compile_options(${target}
+      PRIVATE
+        $<$<BOOL:${has_tautological_constant_compare_flag}>:-Wno-tautological-constant-compare>
+        $<$<CXX_COMPILER_ID:MSVC>: /bigobj
+                                   /EHsc>
+      )
+
     if(${backend} STREQUAL "unified")
       target_link_libraries(${target}
         PRIVATE
@@ -221,10 +235,6 @@ function(make_test)
         )
     endif()
     if(WIN32)
-      target_compile_options(${target}
-        PRIVATE
-          /bigobj
-          /EHsc)
       target_compile_definitions(${target}
         PRIVATE
           WIN32_LEAN_AND_MEAN
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index 6a7f6e7000..fda3d887d6 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -56,6 +56,7 @@ std::ostream &operator<<(std::ostream &os, af::Backend bk) {
         case AF_BACKEND_CPU: os << "AF_BACKEND_CPU"; break;
         case AF_BACKEND_CUDA: os << "AF_BACKEND_CUDA"; break;
         case AF_BACKEND_OPENCL: os << "AF_BACKEND_OPENCL"; break;
+        case AF_BACKEND_ONEAPI: os << "AF_BACKEND_ONEAPI"; break;
         case AF_BACKEND_DEFAULT: os << "AF_BACKEND_DEFAULT"; break;
     }
     return os;

From 227b1b10b3a0144cd005574c2a40663b2689a847 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 7 Oct 2022 15:14:38 -0400
Subject: [PATCH 110/473] Add support for oneAPI to the unified backend

---
 src/api/cpp/array.cpp                       | 9 +++++++++
 src/api/unified/symbol_manager.cpp          | 8 ++++++--
 src/backend/common/DefaultMemoryManager.hpp | 8 ++++----
 src/backend/common/jit/BufferNodeBase.hpp   | 2 +-
 4 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/src/api/cpp/array.cpp b/src/api/cpp/array.cpp
index 5889c0d99c..832c2999e5 100644
--- a/src/api/cpp/array.cpp
+++ b/src/api/cpp/array.cpp
@@ -286,6 +286,15 @@ array::~array() {
                     release_func(get());
                     break;
                 }
+                case AF_BACKEND_ONEAPI: {
+                    static auto *oneapi_handle = unified::getActiveHandle();
+                    static auto release_func =
+                        reinterpret_cast<af_release_array_ptr>(
+                            common::getFunctionPointer(oneapi_handle,
+                                                       "af_release_array"));
+                    release_func(get());
+                    break;
+                }
                 case AF_BACKEND_DEFAULT:
                     assert(1 != 1 &&
                            "AF_BACKEND_DEFAULT cannot be set as a backend for "
diff --git a/src/api/unified/symbol_manager.cpp b/src/api/unified/symbol_manager.cpp
index 8e1f846c54..ca11238773 100644
--- a/src/api/unified/symbol_manager.cpp
+++ b/src/api/unified/symbol_manager.cpp
@@ -68,6 +68,9 @@ string getBkndLibName(const af_backend backend) {
         case AF_BACKEND_CPU:
             ret = string(LIB_AF_BKND_PREFIX) + "afcpu" + LIB_AF_BKND_SUFFIX;
             break;
+        case AF_BACKEND_ONEAPI:
+            ret = string(LIB_AF_BKND_PREFIX) + "afoneapi" + LIB_AF_BKND_SUFFIX;
+            break;
         default: assert(1 != 1 && "Invalid backend");
     }
     return ret;
@@ -78,6 +81,7 @@ string getBackendDirectoryName(const af_backend backend) {
         case AF_BACKEND_CUDA: ret = "cuda"; break;
         case AF_BACKEND_OPENCL: ret = "opencl"; break;
         case AF_BACKEND_CPU: ret = "cpu"; break;
+        case AF_BACKEND_ONEAPI: ret = "oneapi"; break;
         default: assert(1 != 1 && "Invalid backend");
     }
     return ret;
@@ -185,8 +189,8 @@ AFSymbolManager::AFSymbolManager()
     , backendsAvailable(0)
     , logger(loggerFactory("unified")) {
     // In order of priority.
-    static const af_backend order[] = {AF_BACKEND_CUDA, AF_BACKEND_OPENCL,
-                                       AF_BACKEND_CPU};
+    static const af_backend order[] = {AF_BACKEND_CUDA, AF_BACKEND_ONEAPI,
+                                       AF_BACKEND_OPENCL, AF_BACKEND_CPU};
 
     LibHandle handle    = nullptr;
     af::Backend backend = AF_BACKEND_DEFAULT;
diff --git a/src/backend/common/DefaultMemoryManager.hpp b/src/backend/common/DefaultMemoryManager.hpp
index 0881f318a1..83af36d390 100644
--- a/src/backend/common/DefaultMemoryManager.hpp
+++ b/src/backend/common/DefaultMemoryManager.hpp
@@ -121,11 +121,11 @@ class DefaultMemoryManager final : public common::memory::MemoryManagerBase {
     ~DefaultMemoryManager() = default;
 
    protected:
-    DefaultMemoryManager()                                  = delete;
-    DefaultMemoryManager(const DefaultMemoryManager &other) = delete;
-    DefaultMemoryManager(DefaultMemoryManager &&other)      = default;
+    DefaultMemoryManager()                                             = delete;
+    DefaultMemoryManager(const DefaultMemoryManager &other)            = delete;
+    DefaultMemoryManager(DefaultMemoryManager &&other)                 = delete;
     DefaultMemoryManager &operator=(const DefaultMemoryManager &other) = delete;
-    DefaultMemoryManager &operator=(DefaultMemoryManager &&other) = default;
+    DefaultMemoryManager &operator=(DefaultMemoryManager &&other)      = delete;
     common::mutex_t memory_mutex;
     // backend-specific
     std::vector<memory_info> memory;
diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index a7d6747036..9633b2a867 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -93,7 +93,7 @@ class BufferNodeBase : public common::Node {
 
     size_t getBytes() const final { return m_bytes; }
 
-    size_t getHash() const noexcept {
+    size_t getHash() const noexcept override {
         size_t out = 0;
         auto ptr   = m_data.get();
         std::memcpy(&out, &ptr, std::max(sizeof(Node *), sizeof(size_t)));

From a976c076ffb7ac984cbb34654dc8f3a12c5db367 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 12 Oct 2022 13:11:27 -0400
Subject: [PATCH 111/473] Use span instead of vector in getKernel APIs

---
 CMakeLists.txt                                |  1 +
 src/backend/common/TemplateArg.cpp            |  5 +++
 src/backend/common/TemplateArg.hpp            | 14 +++++++
 src/backend/common/TemplateTypename.hpp       | 16 ++++---
 src/backend/common/compile_module.hpp         |  7 ++--
 src/backend/common/kernel_cache.cpp           | 18 ++++----
 src/backend/common/kernel_cache.hpp           |  9 ++--
 src/backend/common/util.cpp                   |  6 ++-
 src/backend/common/util.hpp                   |  5 ++-
 src/backend/cuda/compile_module.cpp           |  7 ++--
 src/backend/cuda/jit.cpp                      |  3 +-
 .../cuda/kernel/anisotropic_diffusion.hpp     |  9 ++--
 src/backend/cuda/kernel/approx.hpp            | 14 +++----
 src/backend/cuda/kernel/assign.hpp            |  5 ++-
 src/backend/cuda/kernel/bilateral.hpp         |  6 +--
 src/backend/cuda/kernel/canny.hpp             | 28 +++++++------
 src/backend/cuda/kernel/convolve.hpp          | 40 ++++++++++--------
 src/backend/cuda/kernel/diagonal.hpp          | 10 +++--
 src/backend/cuda/kernel/diff.hpp              |  7 ++--
 src/backend/cuda/kernel/exampleFunction.hpp   |  8 ++--
 src/backend/cuda/kernel/fftconvolve.hpp       | 26 ++++++------
 src/backend/cuda/kernel/flood_fill.hpp        | 13 +++---
 src/backend/cuda/kernel/gradient.hpp          |  9 ++--
 src/backend/cuda/kernel/histogram.hpp         |  8 ++--
 src/backend/cuda/kernel/hsv_rgb.hpp           |  6 +--
 src/backend/cuda/kernel/identity.hpp          |  5 ++-
 src/backend/cuda/kernel/iir.hpp               |  7 ++--
 src/backend/cuda/kernel/index.hpp             |  4 +-
 src/backend/cuda/kernel/iota.hpp              |  4 +-
 src/backend/cuda/kernel/ireduce.hpp           | 18 ++++----
 src/backend/cuda/kernel/lookup.hpp            | 14 +++----
 src/backend/cuda/kernel/lu_split.hpp          |  8 ++--
 src/backend/cuda/kernel/match_template.hpp    |  6 +--
 src/backend/cuda/kernel/meanshift.hpp         | 10 ++---
 src/backend/cuda/kernel/medfilt.hpp           | 17 ++++----
 src/backend/cuda/kernel/memcopy.hpp           | 41 +++++++++---------
 src/backend/cuda/kernel/moments.hpp           |  5 ++-
 src/backend/cuda/kernel/morph.hpp             | 18 ++++----
 src/backend/cuda/kernel/pad_array_borders.hpp |  8 ++--
 src/backend/cuda/kernel/range.hpp             |  4 +-
 src/backend/cuda/kernel/reorder.hpp           |  5 ++-
 src/backend/cuda/kernel/resize.hpp            |  6 +--
 src/backend/cuda/kernel/rotate.hpp            |  6 +--
 src/backend/cuda/kernel/scan_dim.hpp          | 18 ++++----
 .../cuda/kernel/scan_dim_by_key_impl.hpp      | 20 ++++-----
 src/backend/cuda/kernel/scan_first.hpp        | 18 ++++----
 .../cuda/kernel/scan_first_by_key_impl.hpp    | 22 +++++-----
 src/backend/cuda/kernel/select.hpp            | 12 +++---
 src/backend/cuda/kernel/sobel.hpp             | 11 ++---
 src/backend/cuda/kernel/sparse.hpp            |  6 +--
 src/backend/cuda/kernel/sparse_arith.hpp      | 26 ++++++------
 src/backend/cuda/kernel/susan.hpp             |  9 ++--
 src/backend/cuda/kernel/tile.hpp              |  4 +-
 src/backend/cuda/kernel/transform.hpp         |  5 ++-
 src/backend/cuda/kernel/transpose.hpp         | 10 ++---
 src/backend/cuda/kernel/transpose_inplace.hpp | 10 ++---
 src/backend/cuda/kernel/triangle.hpp          |  8 ++--
 src/backend/cuda/kernel/unwrap.hpp            |  6 +--
 src/backend/cuda/kernel/where.hpp             |  4 +-
 src/backend/cuda/kernel/wrap.hpp              | 12 +++---
 src/backend/opencl/compile_module.cpp         | 12 +++---
 src/backend/opencl/jit.cpp                    |  3 +-
 .../opencl/kernel/anisotropic_diffusion.hpp   |  6 +--
 src/backend/opencl/kernel/approx.hpp          | 10 +++--
 src/backend/opencl/kernel/assign.hpp          | 11 +++--
 src/backend/opencl/kernel/bilateral.hpp       |  6 +--
 src/backend/opencl/kernel/canny.hpp           | 22 +++++-----
 .../opencl/kernel/convolve/conv2_impl.hpp     |  5 ++-
 .../opencl/kernel/convolve/conv_common.hpp    |  5 ++-
 .../opencl/kernel/convolve_separable.cpp      | 13 +++---
 src/backend/opencl/kernel/cscmm.hpp           |  9 ++--
 src/backend/opencl/kernel/cscmv.hpp           | 11 +++--
 src/backend/opencl/kernel/csrmm.hpp           |  9 ++--
 src/backend/opencl/kernel/csrmv.hpp           | 12 +++---
 src/backend/opencl/kernel/diagonal.hpp        | 22 +++++-----
 src/backend/opencl/kernel/diff.hpp            | 14 +++----
 src/backend/opencl/kernel/exampleFunction.hpp | 18 ++++----
 src/backend/opencl/kernel/fast.hpp            | 21 +++++-----
 src/backend/opencl/kernel/fftconvolve.hpp     | 33 +++++++--------
 src/backend/opencl/kernel/flood_fill.hpp      | 42 +++++++++----------
 src/backend/opencl/kernel/gradient.hpp        | 11 +++--
 src/backend/opencl/kernel/harris.hpp          | 18 ++++----
 src/backend/opencl/kernel/histogram.hpp       |  6 +--
 src/backend/opencl/kernel/homography.hpp      | 28 ++++++-------
 src/backend/opencl/kernel/hsv_rgb.hpp         |  9 ++--
 src/backend/opencl/kernel/identity.hpp        | 11 +++--
 src/backend/opencl/kernel/iir.hpp             | 13 +++---
 src/backend/opencl/kernel/index.hpp           | 10 ++---
 src/backend/opencl/kernel/iota.hpp            |  9 ++--
 src/backend/opencl/kernel/ireduce.hpp         | 26 ++++++------
 src/backend/opencl/kernel/laset.hpp           | 14 +++----
 src/backend/opencl/kernel/laset_band.hpp      |  6 +--
 src/backend/opencl/kernel/laswp.hpp           | 13 +++---
 src/backend/opencl/kernel/lookup.hpp          | 12 +++---
 src/backend/opencl/kernel/lu_split.hpp        | 14 +++----
 src/backend/opencl/kernel/match_template.hpp  | 11 +++--
 src/backend/opencl/kernel/mean.hpp            | 13 +++---
 src/backend/opencl/kernel/meanshift.hpp       | 11 +++--
 src/backend/opencl/kernel/medfilt.hpp         | 22 +++++-----
 src/backend/opencl/kernel/memcopy.hpp         | 18 ++++----
 src/backend/opencl/kernel/moments.hpp         | 11 +++--
 src/backend/opencl/kernel/morph.hpp           |  6 ++-
 .../opencl/kernel/nearest_neighbour.hpp       |  7 ++--
 src/backend/opencl/kernel/orb.hpp             | 12 ++++--
 .../opencl/kernel/pad_array_borders.hpp       |  5 ++-
 src/backend/opencl/kernel/random_engine.hpp   |  7 ++--
 src/backend/opencl/kernel/range.hpp           | 11 +++--
 src/backend/opencl/kernel/reduce.hpp          | 33 +++++++--------
 src/backend/opencl/kernel/reduce_by_key.hpp   | 42 ++++++++++---------
 src/backend/opencl/kernel/regions.hpp         |  9 ++--
 src/backend/opencl/kernel/reorder.hpp         | 11 +++--
 src/backend/opencl/kernel/resize.hpp          | 10 ++---
 src/backend/opencl/kernel/rotate.hpp          |  5 ++-
 .../opencl/kernel/scan_by_key/CMakeLists.txt  |  2 +
 src/backend/opencl/kernel/scan_dim.hpp        |  4 +-
 .../opencl/kernel/scan_dim_by_key_impl.hpp    |  3 +-
 src/backend/opencl/kernel/scan_first.hpp      |  4 +-
 .../opencl/kernel/scan_first_by_key_impl.hpp  |  3 +-
 src/backend/opencl/kernel/select.hpp          | 28 ++++++-------
 src/backend/opencl/kernel/sift.hpp            | 25 +++++------
 src/backend/opencl/kernel/sobel.hpp           |  4 +-
 src/backend/opencl/kernel/sparse.hpp          | 24 +++++------
 src/backend/opencl/kernel/sparse_arith.hpp    | 12 +++---
 src/backend/opencl/kernel/susan.hpp           |  8 ++--
 src/backend/opencl/kernel/swapdblk.hpp        |  4 +-
 src/backend/opencl/kernel/tile.hpp            |  3 +-
 src/backend/opencl/kernel/transform.hpp       |  6 +--
 src/backend/opencl/kernel/transpose.hpp       |  4 +-
 .../opencl/kernel/transpose_inplace.hpp       |  6 +--
 src/backend/opencl/kernel/triangle.hpp        |  4 +-
 src/backend/opencl/kernel/unwrap.hpp          |  4 +-
 src/backend/opencl/kernel/where.hpp           |  4 +-
 src/backend/opencl/kernel/wrap.hpp            |  9 ++--
 133 files changed, 805 insertions(+), 751 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 72b2ca4317..5689a8094b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -322,6 +322,7 @@ else()
   set_target_properties(bin2cpp
     PROPERTIES
       CXX_STANDARD 17)
+  target_link_libraries(bin2cpp PRIVATE nonstd::span-lite)
 
   # NOSPDLOG is used to remove the spdlog dependency from bin2cpp
   target_compile_definitions(bin2cpp PRIVATE NOSPDLOG)
diff --git a/src/backend/common/TemplateArg.cpp b/src/backend/common/TemplateArg.cpp
index 740138b337..8cff5c4e24 100644
--- a/src/backend/common/TemplateArg.cpp
+++ b/src/backend/common/TemplateArg.cpp
@@ -33,6 +33,11 @@ template string toString<float>(float);
 template string toString<double>(double);
 template string toString<long double>(long double);
 
+template<>
+string toString(TemplateArg arg) {
+    return arg._tparam;
+}
+
 template<>
 string toString(bool val) {
     return string(val ? "true" : "false");
diff --git a/src/backend/common/TemplateArg.hpp b/src/backend/common/TemplateArg.hpp
index d82d30e12a..a7dfbe4ceb 100644
--- a/src/backend/common/TemplateArg.hpp
+++ b/src/backend/common/TemplateArg.hpp
@@ -9,10 +9,15 @@
 
 #pragma once
 
+#include <array>
 #include <string>
 #include <utility>
 
 #include <optypes.hpp>
+#include <traits.hpp>
+
+template<typename T>
+class TemplateTypename;
 
 template<typename T>
 std::string toString(T value);
@@ -24,10 +29,19 @@ struct TemplateArg {
 
     TemplateArg(std::string str) : _tparam(std::move(str)) {}
 
+    template<typename T>
+    constexpr TemplateArg(TemplateTypename<T> arg) noexcept : _tparam(arg) {}
+
     template<typename T>
     constexpr TemplateArg(T value) noexcept : _tparam(toString(value)) {}
 };
 
+template<typename... Targs>
+std::array<TemplateArg, sizeof...(Targs)> TemplateArgs(Targs &&...args) {
+    return std::array<TemplateArg, sizeof...(Targs)>{
+        std::forward<Targs>(args)...};
+}
+
 #define DefineKey(arg) " -D " #arg
 #define DefineValue(arg) " -D " #arg "=" + toString(arg)
 #define DefineKeyValue(key, arg) " -D " #key "=" + toString(arg)
diff --git a/src/backend/common/TemplateTypename.hpp b/src/backend/common/TemplateTypename.hpp
index 6191348aae..682070510a 100644
--- a/src/backend/common/TemplateTypename.hpp
+++ b/src/backend/common/TemplateTypename.hpp
@@ -19,14 +19,18 @@ struct TemplateTypename {
     operator TemplateArg() const noexcept {
         return {std::string(dtype_traits<T>::getName())};
     }
+    operator std::string() const noexcept {
+        return {std::string(dtype_traits<T>::getName())};
+    }
 };
 
-#define SPECIALIZE(TYPE, NAME)                      \
-    template<>                                      \
-    struct TemplateTypename<TYPE> {                 \
-        operator TemplateArg() const noexcept {     \
-            return TemplateArg(std::string(#NAME)); \
-        }                                           \
+#define SPECIALIZE(TYPE, NAME)                                  \
+    template<>                                                  \
+    struct TemplateTypename<TYPE> {                             \
+        operator TemplateArg() const noexcept {                 \
+            return TemplateArg(std::string(#NAME));             \
+        }                                                       \
+        operator std::string() const noexcept { return #NAME; } \
     }
 
 SPECIALIZE(unsigned char, detail::uchar);
diff --git a/src/backend/common/compile_module.hpp b/src/backend/common/compile_module.hpp
index dc8a0b7dd0..c2abe76ecd 100644
--- a/src/backend/common/compile_module.hpp
+++ b/src/backend/common/compile_module.hpp
@@ -14,6 +14,7 @@
 #include <Module.hpp>
 #include <backend.hpp>
 
+#include <nonstd/span.hpp>
 #include <string>
 #include <vector>
 
@@ -43,9 +44,9 @@ namespace common {
 ///
 /// \returns Backend specific binary module that contains associated kernel
 detail::Module compileModule(const std::string& moduleKey,
-                             const std::vector<std::string>& sources,
-                             const std::vector<std::string>& options,
-                             const std::vector<std::string>& kInstances,
+                             nonstd::span<const std::string> sources,
+                             nonstd::span<const std::string> options,
+                             nonstd::span<const std::string> kInstances,
                              const bool isJIT);
 
 /// \brief Load module binary from disk cache
diff --git a/src/backend/common/kernel_cache.cpp b/src/backend/common/kernel_cache.cpp
index 869ea8d5e9..ff2b53c787 100644
--- a/src/backend/common/kernel_cache.cpp
+++ b/src/backend/common/kernel_cache.cpp
@@ -15,6 +15,7 @@
 #include <device_manager.hpp>
 #include <platform.hpp>
 
+#include <nonstd/span.hpp>
 #include <algorithm>
 #include <shared_mutex>
 #include <string>
@@ -24,11 +25,15 @@
 using detail::Kernel;
 using detail::Module;
 
+using nonstd::span;
+using std::array;
 using std::back_inserter;
+using std::shared_lock;
 using std::shared_timed_mutex;
 using std::string;
 using std::to_string;
 using std::transform;
+using std::unique_lock;
 using std::unordered_map;
 using std::vector;
 
@@ -48,17 +53,16 @@ ModuleMap& getCache(const int device) {
 }
 
 Module findModule(const int device, const size_t& key) {
-    std::shared_lock<shared_timed_mutex> readLock(getCacheMutex(device));
+    shared_lock<shared_timed_mutex> readLock(getCacheMutex(device));
     auto& cache = getCache(device);
     auto iter   = cache.find(key);
     if (iter != cache.end()) { return iter->second; }
     return Module{};
 }
 
-Kernel getKernel(const string& kernelName,
-                 const vector<common::Source>& sources,
-                 const vector<TemplateArg>& targs,
-                 const vector<string>& options, const bool sourceIsJIT) {
+Kernel getKernel(const string& kernelName, span<const common::Source> sources,
+                 span<const TemplateArg> targs, span<const string> options,
+                 const bool sourceIsJIT) {
     string tInstance = kernelName;
 
 #if defined(AF_CUDA)
@@ -116,10 +120,10 @@ Kernel getKernel(const string& kernelName,
                 sources_str.push_back({s.ptr, s.length});
             }
             currModule = compileModule(to_string(moduleKeyDisk), sources_str,
-                                       options, {tInstance}, sourceIsJIT);
+                                       options, array{tInstance}, sourceIsJIT);
         }
 
-        std::unique_lock<shared_timed_mutex> writeLock(getCacheMutex(device));
+        unique_lock<shared_timed_mutex> writeLock(getCacheMutex(device));
         auto& cache = getCache(device);
         auto iter   = cache.find(moduleKeyCache);
         if (iter == cache.end()) {
diff --git a/src/backend/common/kernel_cache.hpp b/src/backend/common/kernel_cache.hpp
index c63c4278a4..b021919a21 100644
--- a/src/backend/common/kernel_cache.hpp
+++ b/src/backend/common/kernel_cache.hpp
@@ -17,6 +17,7 @@
 #include <common/TemplateTypename.hpp>
 #include <common/util.hpp>
 
+#include <nonstd/span.hpp>
 #include <string>
 #include <vector>
 
@@ -46,7 +47,7 @@ namespace common {
 /// Example Usage: transpose
 ///
 /// \code
-/// auto transpose = getKernel("cuda::transpose", {transpase_cuh_src},
+/// auto transpose = getKernel("cuda::transpose", std::array{transpase_cuh_src},
 ///         {
 ///           TemplateTypename<T>(),
 ///           TemplateArg(conjugate),
@@ -70,9 +71,9 @@ namespace common {
 ///            the kernel compilation.
 ///
 detail::Kernel getKernel(const std::string& kernelName,
-                         const std::vector<common::Source>& sources,
-                         const std::vector<TemplateArg>& templateArgs,
-                         const std::vector<std::string>& options = {},
+                         nonstd::span<const common::Source> sources,
+                         nonstd::span<const TemplateArg> templateArgs,
+                         nonstd::span<const std::string> options = {},
                          const bool sourceIsJIT                  = false);
 
 /// \brief Lookup a Module that matches the given key
diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp
index a5af7f80e6..bac4cb573d 100644
--- a/src/backend/common/util.cpp
+++ b/src/backend/common/util.cpp
@@ -23,6 +23,7 @@
 #include <common/util.hpp>
 #include <af/defines.h>
 
+#include <nonstd/span.hpp>
 #include <sys/stat.h>
 #include <algorithm>
 #include <cstdio>
@@ -34,6 +35,7 @@
 #include <thread>
 #include <vector>
 
+using nonstd::span;
 using std::accumulate;
 using std::hash;
 using std::ofstream;
@@ -248,13 +250,13 @@ size_t deterministicHash(const string& data, const size_t prevHash) {
     return deterministicHash(data.data(), data.size(), prevHash);
 }
 
-size_t deterministicHash(const vector<string>& list, const size_t prevHash) {
+size_t deterministicHash(span<const string> list, const size_t prevHash) {
     size_t hash = prevHash;
     for (auto s : list) { hash = deterministicHash(s.data(), s.size(), hash); }
     return hash;
 }
 
-size_t deterministicHash(const vector<common::Source>& list) {
+size_t deterministicHash(span<const common::Source> list) {
     // Combine the different source codes, via their hashes
     size_t hash = FNV1A_BASE_OFFSET;
     for (auto s : list) {
diff --git a/src/backend/common/util.hpp b/src/backend/common/util.hpp
index c0f712ec0e..fb6c195af6 100644
--- a/src/backend/common/util.hpp
+++ b/src/backend/common/util.hpp
@@ -12,6 +12,7 @@
 
 #include <af/defines.h>
 
+#include <nonstd/span.hpp>
 #include <iosfwd>
 #include <string>
 #include <vector>
@@ -78,8 +79,8 @@ std::size_t deterministicHash(const std::string& data,
                               const std::size_t prevHash = FNV1A_BASE_OFFSET);
 
 // This concatenates strings in the vector and computes hash
-std::size_t deterministicHash(const std::vector<std::string>& list,
+std::size_t deterministicHash(nonstd::span<const std::string> list,
                               const std::size_t prevHash = FNV1A_BASE_OFFSET);
 
 // This concatenates hashes of multiple sources
-std::size_t deterministicHash(const std::vector<common::Source>& list);
+std::size_t deterministicHash(nonstd::span<const common::Source> list);
diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp
index cbc7d98517..ee10077477 100644
--- a/src/backend/cuda/compile_module.cpp
+++ b/src/backend/cuda/compile_module.cpp
@@ -64,6 +64,7 @@
 using namespace cuda;
 
 using detail::Module;
+using nonstd::span;
 using std::accumulate;
 using std::array;
 using std::back_insert_iterator;
@@ -140,9 +141,9 @@ string getKernelCacheFilename(const int device, const string &key) {
 
 namespace common {
 
-Module compileModule(const string &moduleKey, const vector<string> &sources,
-                     const vector<string> &opts,
-                     const vector<string> &kInstances, const bool sourceIsJIT) {
+Module compileModule(const string &moduleKey, span<const string> sources,
+                     span<const string> opts, span<const string> kInstances,
+                     const bool sourceIsJIT) {
     nvrtcProgram prog;
     if (sourceIsJIT) {
         constexpr const char *header_names[] = {
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 6904d0673d..37ff605cb4 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -318,7 +318,8 @@ static CUfunction getKernel(const vector<Node*>& output_nodes,
         const common::Source jit_src{jitKer.c_str(), jitKer.size(),
                                      deterministicHash(jitKer)};
 
-        return common::getKernel(funcName, {jit_src}, {}, {}, true).get();
+        return common::getKernel(funcName, std::array{jit_src}, {}, {}, true)
+            .get();
     }
     return common::getKernel(entry, funcName, true).get();
 }
diff --git a/src/backend/cuda/kernel/anisotropic_diffusion.hpp b/src/backend/cuda/kernel/anisotropic_diffusion.hpp
index 32e10b9942..1c247bb499 100644
--- a/src/backend/cuda/kernel/anisotropic_diffusion.hpp
+++ b/src/backend/cuda/kernel/anisotropic_diffusion.hpp
@@ -27,10 +27,11 @@ template<typename T>
 void anisotropicDiffusion(Param<T> inout, const float dt, const float mct,
                           const af::fluxFunction fftype, bool isMCDE) {
     auto diffUpdate = common::getKernel(
-        "cuda::diffUpdate", {anisotropic_diffusion_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(fftype), TemplateArg(isMCDE)},
-        {DefineValue(THREADS_X), DefineValue(THREADS_Y),
-         DefineValue(YDIM_LOAD)});
+        "cuda::diffUpdate", std::array{anisotropic_diffusion_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(fftype),
+                     TemplateArg(isMCDE)),
+        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y),
+                   DefineValue(YDIM_LOAD)});
 
     dim3 threads(THREADS_X, THREADS_Y, 1);
 
diff --git a/src/backend/cuda/kernel/approx.hpp b/src/backend/cuda/kernel/approx.hpp
index 47473a4f03..66dea16fe6 100644
--- a/src/backend/cuda/kernel/approx.hpp
+++ b/src/backend/cuda/kernel/approx.hpp
@@ -27,10 +27,10 @@ template<typename Ty, typename Tp>
 void approx1(Param<Ty> yo, CParam<Ty> yi, CParam<Tp> xo, const int xdim,
              const Tp &xi_beg, const Tp &xi_step, const float offGrid,
              const af::interpType method, const int order) {
-    auto approx1 =
-        common::getKernel("cuda::approx1", {approx1_cuh_src},
-                          {TemplateTypename<Ty>(), TemplateTypename<Tp>(),
-                           TemplateArg(xdim), TemplateArg(order)});
+    auto approx1 = common::getKernel(
+        "cuda::approx1", std::array{approx1_cuh_src},
+        TemplateArgs(TemplateTypename<Ty>(), TemplateTypename<Tp>(),
+                     TemplateArg(xdim), TemplateArg(order)));
 
     dim3 threads(THREADS, 1, 1);
     int blocksPerMat = divup(yo.dims[0], threads.x);
@@ -57,9 +57,9 @@ void approx2(Param<Ty> zo, CParam<Ty> zi, CParam<Tp> xo, const int xdim,
              const Tp &yi_beg, const Tp &yi_step, const float offGrid,
              const af::interpType method, const int order) {
     auto approx2 = common::getKernel(
-        "cuda::approx2", {approx2_cuh_src},
-        {TemplateTypename<Ty>(), TemplateTypename<Tp>(), TemplateArg(xdim),
-         TemplateArg(ydim), TemplateArg(order)});
+        "cuda::approx2", std::array{approx2_cuh_src},
+        TemplateArgs(TemplateTypename<Ty>(), TemplateTypename<Tp>(),
+                     TemplateArg(xdim), TemplateArg(ydim), TemplateArg(order)));
 
     dim3 threads(TX, TY, 1);
     int blocksPerMatX = divup(zo.dims[0], threads.x);
diff --git a/src/backend/cuda/kernel/assign.hpp b/src/backend/cuda/kernel/assign.hpp
index 9632892cc4..523dad2505 100644
--- a/src/backend/cuda/kernel/assign.hpp
+++ b/src/backend/cuda/kernel/assign.hpp
@@ -22,8 +22,9 @@ void assign(Param<T> out, CParam<T> in, const AssignKernelParam& p) {
     constexpr int THREADS_X = 32;
     constexpr int THREADS_Y = 8;
 
-    auto assignKer = common::getKernel("cuda::assign", {assign_cuh_src},
-                                       {TemplateTypename<T>()});
+    auto assignKer =
+        common::getKernel("cuda::assign", std::array{assign_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     const dim3 threads(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/bilateral.hpp b/src/backend/cuda/kernel/bilateral.hpp
index a7788a5deb..357b57a8bc 100644
--- a/src/backend/cuda/kernel/bilateral.hpp
+++ b/src/backend/cuda/kernel/bilateral.hpp
@@ -23,9 +23,9 @@ template<typename inType, typename outType>
 void bilateral(Param<outType> out, CParam<inType> in, float s_sigma,
                float c_sigma) {
     auto bilateral = common::getKernel(
-        "cuda::bilateral", {bilateral_cuh_src},
-        {TemplateTypename<inType>(), TemplateTypename<outType>()},
-        {DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        "cuda::bilateral", std::array{bilateral_cuh_src},
+        TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>()),
+        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
 
     dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/canny.hpp b/src/backend/cuda/kernel/canny.hpp
index 4dd6ce739c..cc63a029c4 100644
--- a/src/backend/cuda/kernel/canny.hpp
+++ b/src/backend/cuda/kernel/canny.hpp
@@ -27,9 +27,10 @@ template<typename T>
 void nonMaxSuppression(Param<T> output, CParam<T> magnitude, CParam<T> dx,
                        CParam<T> dy) {
     auto nonMaxSuppress = common::getKernel(
-        "cuda::nonMaxSuppression", {canny_cuh_src}, {TemplateTypename<T>()},
-        {DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
-         DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        "cuda::nonMaxSuppression", std::array{canny_cuh_src},
+        TemplateArgs(TemplateTypename<T>()),
+        std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
+                   DefineValue(THREADS_X), DefineValue(THREADS_Y)});
 
     dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
@@ -48,17 +49,20 @@ void nonMaxSuppression(Param<T> output, CParam<T> magnitude, CParam<T> dx,
 template<typename T>
 void edgeTrackingHysteresis(Param<T> output, CParam<T> strong, CParam<T> weak) {
     auto initEdgeOut = common::getKernel(
-        "cuda::initEdgeOut", {canny_cuh_src}, {TemplateTypename<T>()},
-        {DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
-         DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        "cuda::initEdgeOut", std::array{canny_cuh_src},
+        TemplateArgs(TemplateTypename<T>()),
+        std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
+                   DefineValue(THREADS_X), DefineValue(THREADS_Y)});
     auto edgeTrack = common::getKernel(
-        "cuda::edgeTrack", {canny_cuh_src}, {TemplateTypename<T>()},
-        {DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
-         DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        "cuda::edgeTrack", std::array{canny_cuh_src},
+        TemplateArgs(TemplateTypename<T>()),
+        std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
+                   DefineValue(THREADS_X), DefineValue(THREADS_Y)});
     auto suppressLeftOver = common::getKernel(
-        "cuda::suppressLeftOver", {canny_cuh_src}, {TemplateTypename<T>()},
-        {DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
-         DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        "cuda::suppressLeftOver", std::array{canny_cuh_src},
+        TemplateArgs(TemplateTypename<T>()),
+        std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
+                   DefineValue(THREADS_X), DefineValue(THREADS_Y)});
 
     dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/convolve.hpp b/src/backend/cuda/kernel/convolve.hpp
index 40485d0148..7b105ef842 100644
--- a/src/backend/cuda/kernel/convolve.hpp
+++ b/src/backend/cuda/kernel/convolve.hpp
@@ -101,9 +101,11 @@ template<typename T, typename aT>
 void convolve_1d(conv_kparam_t& p, Param<T> out, CParam<T> sig, CParam<aT> filt,
                  const bool expand) {
     auto convolve1 = common::getKernel(
-        "cuda::convolve1", {convolve1_cuh_src},
-        {TemplateTypename<T>(), TemplateTypename<aT>(), TemplateArg(expand)},
-        {DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS)});
+        "cuda::convolve1", std::array{convolve1_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
+                     TemplateArg(expand)),
+        std::array{DefineValue(MAX_CONV1_FILTER_LEN),
+                   DefineValue(CONV_THREADS)});
 
     prepareKernelArgs<T>(p, out.dims, filt.dims, 1);
 
@@ -156,11 +158,11 @@ void conv2Helper(const conv_kparam_t& p, Param<T> out, CParam<T> sig,
     }
 
     auto convolve2 = common::getKernel(
-        "cuda::convolve2", {convolve2_cuh_src},
-        {TemplateTypename<T>(), TemplateTypename<aT>(), TemplateArg(expand),
-         TemplateArg(f0), TemplateArg(f1)},
-        {DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
-         DefineValue(CONV2_THREADS_X), DefineValue(CONV2_THREADS_Y)});
+        "cuda::convolve2", std::array{convolve2_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
+                     TemplateArg(expand), TemplateArg(f0), TemplateArg(f1)),
+        std::array{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
+                   DefineValue(CONV2_THREADS_X), DefineValue(CONV2_THREADS_Y)});
 
     // FIXME: case where filter array is strided
     auto constMemPtr = convolve2.getDevPtr(conv_c_name);
@@ -201,11 +203,12 @@ template<typename T, typename aT>
 void convolve_3d(conv_kparam_t& p, Param<T> out, CParam<T> sig, CParam<aT> filt,
                  const bool expand) {
     auto convolve3 = common::getKernel(
-        "cuda::convolve3", {convolve3_cuh_src},
-        {TemplateTypename<T>(), TemplateTypename<aT>(), TemplateArg(expand)},
-        {DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
-         DefineValue(CONV3_CUBE_X), DefineValue(CONV3_CUBE_Y),
-         DefineValue(CONV3_CUBE_Z)});
+        "cuda::convolve3", std::array{convolve3_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
+                     TemplateArg(expand)),
+        std::array{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
+                   DefineValue(CONV3_CUBE_X), DefineValue(CONV3_CUBE_Y),
+                   DefineValue(CONV3_CUBE_Z)});
 
     prepareKernelArgs<T>(p, out.dims, filt.dims, 3);
 
@@ -305,11 +308,12 @@ void convolve2(Param<T> out, CParam<T> signal, CParam<aT> filter, int conv_dim,
     }
 
     auto convolve2_separable = common::getKernel(
-        "cuda::convolve2_separable", {convolve_separable_cuh_src},
-        {TemplateTypename<T>(), TemplateTypename<aT>(), TemplateArg(conv_dim),
-         TemplateArg(expand), TemplateArg(fLen)},
-        {DefineValue(MAX_SCONV_FILTER_LEN), DefineValue(SCONV_THREADS_X),
-         DefineValue(SCONV_THREADS_Y)});
+        "cuda::convolve2_separable", std::array{convolve_separable_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
+                     TemplateArg(conv_dim), TemplateArg(expand),
+                     TemplateArg(fLen)),
+        std::array{DefineValue(MAX_SCONV_FILTER_LEN),
+                   DefineValue(SCONV_THREADS_X), DefineValue(SCONV_THREADS_Y)});
 
     dim3 threads(SCONV_THREADS_X, SCONV_THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/diagonal.hpp b/src/backend/cuda/kernel/diagonal.hpp
index 93b974420e..87ba53965b 100644
--- a/src/backend/cuda/kernel/diagonal.hpp
+++ b/src/backend/cuda/kernel/diagonal.hpp
@@ -20,8 +20,9 @@ namespace kernel {
 
 template<typename T>
 void diagCreate(Param<T> out, CParam<T> in, int num) {
-    auto genDiagMat = common::getKernel(
-        "cuda::createDiagonalMat", {diagonal_cuh_src}, {TemplateTypename<T>()});
+    auto genDiagMat = common::getKernel("cuda::createDiagonalMat",
+                                        std::array{diagonal_cuh_src},
+                                        TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(32, 8);
     int blocks_x = divup(out.dims[0], threads.x);
@@ -45,8 +46,9 @@ void diagCreate(Param<T> out, CParam<T> in, int num) {
 
 template<typename T>
 void diagExtract(Param<T> out, CParam<T> in, int num) {
-    auto extractDiag = common::getKernel(
-        "cuda::extractDiagonal", {diagonal_cuh_src}, {TemplateTypename<T>()});
+    auto extractDiag =
+        common::getKernel("cuda::extractDiagonal", std::array{diagonal_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(256, 1);
     int blocks_x = divup(out.dims[0], threads.x);
diff --git a/src/backend/cuda/kernel/diff.hpp b/src/backend/cuda/kernel/diff.hpp
index 1d3d4c5278..fb157af798 100644
--- a/src/backend/cuda/kernel/diff.hpp
+++ b/src/backend/cuda/kernel/diff.hpp
@@ -24,9 +24,10 @@ void diff(Param<T> out, CParam<T> in, const int indims, const unsigned dim,
     constexpr unsigned TX = 16;
     constexpr unsigned TY = 16;
 
-    auto diff = common::getKernel(
-        "cuda::diff", {diff_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(dim), TemplateArg(isDiff2)});
+    auto diff =
+        common::getKernel("cuda::diff", std::array{diff_cuh_src},
+                          TemplateArgs(TemplateTypename<T>(), TemplateArg(dim),
+                                       TemplateArg(isDiff2)));
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/exampleFunction.hpp b/src/backend/cuda/kernel/exampleFunction.hpp
index 64229c88d7..019b8c9743 100644
--- a/src/backend/cuda/kernel/exampleFunction.hpp
+++ b/src/backend/cuda/kernel/exampleFunction.hpp
@@ -27,11 +27,9 @@ static const unsigned TY = 16;  // Kernel Launch Config Values
 
 template<typename T>  // CUDA kernel wrapper function
 void exampleFunc(Param<T> c, CParam<T> a, CParam<T> b, const af_someenum_t p) {
-    auto exampleFunc =
-        common::getKernel("cuda::exampleFunc", {exampleFunction_cuh_src},
-                          {
-                              TemplateTypename<T>(),
-                          });
+    auto exampleFunc = common::getKernel("cuda::exampleFunc",
+                                         std::array{exampleFunction_cuh_src},
+                                         TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(TX, TY, 1);  // set your cuda launch config for blocks
 
diff --git a/src/backend/cuda/kernel/fftconvolve.hpp b/src/backend/cuda/kernel/fftconvolve.hpp
index df6836c8af..6ca9569206 100644
--- a/src/backend/cuda/kernel/fftconvolve.hpp
+++ b/src/backend/cuda/kernel/fftconvolve.hpp
@@ -23,12 +23,12 @@ static const int THREADS = 256;
 template<typename convT, typename T>
 void packDataHelper(Param<convT> sig_packed, Param<convT> filter_packed,
                     CParam<T> sig, CParam<T> filter) {
-    auto packData =
-        common::getKernel("cuda::packData", {fftconvolve_cuh_src},
-                          {TemplateTypename<convT>(), TemplateTypename<T>()});
-    auto padArray =
-        common::getKernel("cuda::padArray", {fftconvolve_cuh_src},
-                          {TemplateTypename<convT>(), TemplateTypename<T>()});
+    auto packData = common::getKernel(
+        "cuda::packData", std::array{fftconvolve_cuh_src},
+        TemplateArgs(TemplateTypename<convT>(), TemplateTypename<T>()));
+    auto padArray = common::getKernel(
+        "cuda::padArray", std::array{fftconvolve_cuh_src},
+        TemplateArgs(TemplateTypename<convT>(), TemplateTypename<T>()));
 
     dim_t *sd = sig.dims;
 
@@ -67,9 +67,9 @@ void packDataHelper(Param<convT> sig_packed, Param<convT> filter_packed,
 template<typename T, typename convT>
 void complexMultiplyHelper(Param<convT> sig_packed, Param<convT> filter_packed,
                            AF_BATCH_KIND kind) {
-    auto cplxMul =
-        common::getKernel("cuda::complexMultiply", {fftconvolve_cuh_src},
-                          {TemplateTypename<convT>(), TemplateArg(kind)});
+    auto cplxMul = common::getKernel(
+        "cuda::complexMultiply", std::array{fftconvolve_cuh_src},
+        TemplateArgs(TemplateTypename<convT>(), TemplateArg(kind)));
 
     int sig_packed_elem    = 1;
     int filter_packed_elem = 1;
@@ -100,10 +100,10 @@ void reorderOutputHelper(Param<T> out, Param<convT> packed, CParam<T> sig,
                          CParam<T> filter, bool expand, int rank) {
     constexpr bool RoundResult = std::is_integral<T>::value;
 
-    auto reorderOut =
-        common::getKernel("cuda::reorderOutput", {fftconvolve_cuh_src},
-                          {TemplateTypename<T>(), TemplateTypename<convT>(),
-                           TemplateArg(expand), TemplateArg(RoundResult)});
+    auto reorderOut = common::getKernel(
+        "cuda::reorderOutput", std::array{fftconvolve_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateTypename<convT>(),
+                     TemplateArg(expand), TemplateArg(RoundResult)));
 
     dim_t *sd    = sig.dims;
     int fftScale = 1;
diff --git a/src/backend/cuda/kernel/flood_fill.hpp b/src/backend/cuda/kernel/flood_fill.hpp
index b6f9615a6c..ad6366a286 100644
--- a/src/backend/cuda/kernel/flood_fill.hpp
+++ b/src/backend/cuda/kernel/flood_fill.hpp
@@ -45,13 +45,16 @@ void floodFill(Param<T> out, CParam<T> image, CParam<uint> seedsx,
         CUDA_NOT_SUPPORTED(errMessage);
     }
 
-    auto initSeeds = common::getKernel("cuda::initSeeds", {flood_fill_cuh_src},
-                                       {TemplateTypename<T>()});
+    auto initSeeds =
+        common::getKernel("cuda::initSeeds", std::array{flood_fill_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
     auto floodStep = common::getKernel(
-        "cuda::floodStep", {flood_fill_cuh_src}, {TemplateTypename<T>()},
-        {DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        "cuda::floodStep", std::array{flood_fill_cuh_src},
+        TemplateArgs(TemplateTypename<T>()),
+        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
     auto finalizeOutput = common::getKernel(
-        "cuda::finalizeOutput", {flood_fill_cuh_src}, {TemplateTypename<T>()});
+        "cuda::finalizeOutput", std::array{flood_fill_cuh_src},
+        TemplateArgs(TemplateTypename<T>()));
 
     EnqueueArgs qArgs(dim3(divup(seedsx.elements(), THREADS)), dim3(THREADS),
                       getActiveStream());
diff --git a/src/backend/cuda/kernel/gradient.hpp b/src/backend/cuda/kernel/gradient.hpp
index f413faec2d..8f1306e2b0 100644
--- a/src/backend/cuda/kernel/gradient.hpp
+++ b/src/backend/cuda/kernel/gradient.hpp
@@ -15,6 +15,8 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/gradient_cuh.hpp>
 
+#include <array>
+
 namespace cuda {
 namespace kernel {
 
@@ -23,9 +25,10 @@ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in) {
     constexpr unsigned TX = 32;
     constexpr unsigned TY = 8;
 
-    auto gradient = common::getKernel("cuda::gradient", {gradient_cuh_src},
-                                      {TemplateTypename<T>()},
-                                      {DefineValue(TX), DefineValue(TY)});
+    auto gradient =
+        common::getKernel("cuda::gradient", std::array{gradient_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()),
+                          std::array{DefineValue(TX), DefineValue(TY)});
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/histogram.hpp b/src/backend/cuda/kernel/histogram.hpp
index bdf7d2283e..4e4fe8c901 100644
--- a/src/backend/cuda/kernel/histogram.hpp
+++ b/src/backend/cuda/kernel/histogram.hpp
@@ -23,10 +23,10 @@ constexpr int THRD_LOAD = 16;
 template<typename T>
 void histogram(Param<uint> out, CParam<T> in, int nbins, float minval,
                float maxval, bool isLinear) {
-    auto histogram =
-        common::getKernel("cuda::histogram", {histogram_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(isLinear)},
-                          {DefineValue(MAX_BINS), DefineValue(THRD_LOAD)});
+    auto histogram = common::getKernel(
+        "cuda::histogram", std::array{histogram_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(isLinear)),
+        std::array{DefineValue(MAX_BINS), DefineValue(THRD_LOAD)});
 
     dim3 threads(kernel::THREADS_X, 1);
 
diff --git a/src/backend/cuda/kernel/hsv_rgb.hpp b/src/backend/cuda/kernel/hsv_rgb.hpp
index ec3f0098eb..a10a6ade93 100644
--- a/src/backend/cuda/kernel/hsv_rgb.hpp
+++ b/src/backend/cuda/kernel/hsv_rgb.hpp
@@ -21,9 +21,9 @@ static const int THREADS_Y = 16;
 
 template<typename T>
 void hsv2rgb_convert(Param<T> out, CParam<T> in, bool isHSV2RGB) {
-    auto hsvrgbConverter =
-        common::getKernel("cuda::hsvrgbConverter", {hsv_rgb_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(isHSV2RGB)});
+    auto hsvrgbConverter = common::getKernel(
+        "cuda::hsvrgbConverter", std::array{hsv_rgb_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(isHSV2RGB)));
 
     const dim3 threads(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/identity.hpp b/src/backend/cuda/kernel/identity.hpp
index ae92d7535c..58e369823b 100644
--- a/src/backend/cuda/kernel/identity.hpp
+++ b/src/backend/cuda/kernel/identity.hpp
@@ -20,8 +20,9 @@ namespace kernel {
 
 template<typename T>
 void identity(Param<T> out) {
-    auto identity = common::getKernel("cuda::identity", {identity_cuh_src},
-                                      {TemplateTypename<T>()});
+    auto identity =
+        common::getKernel("cuda::identity", std::array{identity_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(32, 8);
     int blocks_x = divup(out.dims[0], threads.x);
diff --git a/src/backend/cuda/kernel/iir.hpp b/src/backend/cuda/kernel/iir.hpp
index 985e623249..38b9ece04d 100644
--- a/src/backend/cuda/kernel/iir.hpp
+++ b/src/backend/cuda/kernel/iir.hpp
@@ -22,9 +22,10 @@ template<typename T, bool batch_a>
 void iir(Param<T> y, CParam<T> c, CParam<T> a) {
     constexpr int MAX_A_SIZE = 1024;
 
-    auto iir = common::getKernel("cuda::iir", {iir_cuh_src},
-                                 {TemplateTypename<T>(), TemplateArg(batch_a)},
-                                 {DefineValue(MAX_A_SIZE)});
+    auto iir = common::getKernel(
+        "cuda::iir", std::array{iir_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(batch_a)),
+        std::array{DefineValue(MAX_A_SIZE)});
 
     const int blocks_y = y.dims[1];
     const int blocks_x = y.dims[2];
diff --git a/src/backend/cuda/kernel/index.hpp b/src/backend/cuda/kernel/index.hpp
index 589245213f..5a44f4be6f 100644
--- a/src/backend/cuda/kernel/index.hpp
+++ b/src/backend/cuda/kernel/index.hpp
@@ -21,8 +21,8 @@ namespace kernel {
 
 template<typename T>
 void index(Param<T> out, CParam<T> in, const IndexKernelParam& p) {
-    auto index = common::getKernel("cuda::index", {index_cuh_src},
-                                   {TemplateTypename<T>()});
+    auto index = common::getKernel("cuda::index", std::array{index_cuh_src},
+                                   TemplateArgs(TemplateTypename<T>()));
     dim3 threads;
     switch (out.dims[1]) {
         case 1: threads.y = 1; break;
diff --git a/src/backend/cuda/kernel/iota.hpp b/src/backend/cuda/kernel/iota.hpp
index 0b5cd61b78..d108bc2a25 100644
--- a/src/backend/cuda/kernel/iota.hpp
+++ b/src/backend/cuda/kernel/iota.hpp
@@ -26,8 +26,8 @@ void iota(Param<T> out, const af::dim4 &sdims) {
     constexpr unsigned TILEX   = 512;
     constexpr unsigned TILEY   = 32;
 
-    auto iota = common::getKernel("cuda::iota", {iota_cuh_src},
-                                  {TemplateTypename<T>()});
+    auto iota = common::getKernel("cuda::iota", std::array{iota_cuh_src},
+                                  TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(IOTA_TX, IOTA_TY, 1);
 
diff --git a/src/backend/cuda/kernel/ireduce.hpp b/src/backend/cuda/kernel/ireduce.hpp
index f1fd13d054..b57ba5d29b 100644
--- a/src/backend/cuda/kernel/ireduce.hpp
+++ b/src/backend/cuda/kernel/ireduce.hpp
@@ -37,10 +37,10 @@ void ireduce_dim_launcher(Param<T> out, uint *olptr, CParam<T> in,
     blocks.y = divup(blocks.y, blocks.z);
 
     auto ireduceDim = common::getKernel(
-        "cuda::ireduceDim", {ireduce_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(op), TemplateArg(dim),
-         TemplateArg(is_first), TemplateArg(threads_y)},
-        {DefineValue(THREADS_X)});
+        "cuda::ireduceDim", std::array{ireduce_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(op), TemplateArg(dim),
+                     TemplateArg(is_first), TemplateArg(threads_y)),
+        std::array{DefineValue(THREADS_X)});
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -104,11 +104,11 @@ void ireduce_first_launcher(Param<T> out, uint *olptr, CParam<T> in,
     uint repeat = divup(in.dims[0], (blocks_x * threads_x));
 
     // threads_x can take values 32, 64, 128, 256
-    auto ireduceFirst =
-        common::getKernel("cuda::ireduceFirst", {ireduce_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(op),
-                           TemplateArg(is_first), TemplateArg(threads_x)},
-                          {DefineValue(THREADS_PER_BLOCK)});
+    auto ireduceFirst = common::getKernel(
+        "cuda::ireduceFirst", std::array{ireduce_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(op),
+                     TemplateArg(is_first), TemplateArg(threads_x)),
+        std::array{DefineValue(THREADS_PER_BLOCK)});
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
diff --git a/src/backend/cuda/kernel/lookup.hpp b/src/backend/cuda/kernel/lookup.hpp
index 4f4758dca3..bca81cdebc 100644
--- a/src/backend/cuda/kernel/lookup.hpp
+++ b/src/backend/cuda/kernel/lookup.hpp
@@ -43,9 +43,9 @@ void lookup(Param<in_t> out, CParam<in_t> in, CParam<idx_t> indices, int nDims,
         dim3 blocks(blks, 1);
 
         auto lookup1d = common::getKernel(
-            "cuda::lookup1D", {lookup_cuh_src},
-            {TemplateTypename<in_t>(), TemplateTypename<idx_t>()},
-            {DefineValue(THREADS), DefineValue(THRD_LOAD)});
+            "cuda::lookup1D", std::array{lookup_cuh_src},
+            TemplateArgs(TemplateTypename<in_t>(), TemplateTypename<idx_t>()),
+            std::array{DefineValue(THREADS), DefineValue(THRD_LOAD)});
 
         EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -63,10 +63,10 @@ void lookup(Param<in_t> out, CParam<in_t> in, CParam<idx_t> indices, int nDims,
         blocks.z = divup(blocks.y, maxBlocksY);
         blocks.y = divup(blocks.y, blocks.z);
 
-        auto lookupnd =
-            common::getKernel("cuda::lookupND", {lookup_cuh_src},
-                              {TemplateTypename<in_t>(),
-                               TemplateTypename<idx_t>(), TemplateArg(dim)});
+        auto lookupnd = common::getKernel(
+            "cuda::lookupND", std::array{lookup_cuh_src},
+            TemplateArgs(TemplateTypename<in_t>(), TemplateTypename<idx_t>(),
+                         TemplateArg(dim)));
         EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
         lookupnd(qArgs, out, in, indices, blks_x, blks_y);
diff --git a/src/backend/cuda/kernel/lu_split.hpp b/src/backend/cuda/kernel/lu_split.hpp
index 72def543e3..8e74c6fbe5 100644
--- a/src/backend/cuda/kernel/lu_split.hpp
+++ b/src/backend/cuda/kernel/lu_split.hpp
@@ -15,6 +15,8 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/lu_split_cuh.hpp>
 
+#include <array>
+
 namespace cuda {
 namespace kernel {
 
@@ -28,9 +30,9 @@ void lu_split(Param<T> lower, Param<T> upper, Param<T> in) {
     const bool sameDims =
         lower.dims[0] == in.dims[0] && lower.dims[1] == in.dims[1];
 
-    auto luSplit =
-        common::getKernel("cuda::luSplit", {lu_split_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(sameDims)});
+    auto luSplit = common::getKernel(
+        "cuda::luSplit", std::array{lu_split_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(sameDims)));
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/match_template.hpp b/src/backend/cuda/kernel/match_template.hpp
index 31d75e1bd6..3969bfd453 100644
--- a/src/backend/cuda/kernel/match_template.hpp
+++ b/src/backend/cuda/kernel/match_template.hpp
@@ -25,9 +25,9 @@ void matchTemplate(Param<outType> out, CParam<inType> srch,
                    CParam<inType> tmplt, const af::matchType mType,
                    bool needMean) {
     auto matchTemplate = common::getKernel(
-        "cuda::matchTemplate", {match_template_cuh_src},
-        {TemplateTypename<inType>(), TemplateTypename<outType>(),
-         TemplateArg(mType), TemplateArg(needMean)});
+        "cuda::matchTemplate", std::array{match_template_cuh_src},
+        TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>(),
+                     TemplateArg(mType), TemplateArg(needMean)));
 
     const dim3 threads(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/meanshift.hpp b/src/backend/cuda/kernel/meanshift.hpp
index ffa3cba76b..530279fd1b 100644
--- a/src/backend/cuda/kernel/meanshift.hpp
+++ b/src/backend/cuda/kernel/meanshift.hpp
@@ -13,6 +13,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/meanshift_cuh.hpp>
 
+#include <array>
 #include <type_traits>
 
 namespace cuda {
@@ -27,11 +28,10 @@ void meanshift(Param<T> out, CParam<T> in, const float spatialSigma,
     typedef typename std::conditional<std::is_same<T, double>::value, double,
                                       float>::type AccType;
     auto meanshift = common::getKernel(
-        "cuda::meanshift", {meanshift_cuh_src},
-        {
-            TemplateTypename<AccType>(), TemplateTypename<T>(),
-            TemplateArg((IsColor ? 3 : 1))  // channels
-        });
+        "cuda::meanshift", std::array{meanshift_cuh_src},
+        TemplateArgs(TemplateTypename<AccType>(), TemplateTypename<T>(),
+                     TemplateArg((IsColor ? 3 : 1))  // channels
+                     ));
 
     static dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/medfilt.hpp b/src/backend/cuda/kernel/medfilt.hpp
index 3095db1a46..c0062ccc2f 100644
--- a/src/backend/cuda/kernel/medfilt.hpp
+++ b/src/backend/cuda/kernel/medfilt.hpp
@@ -26,11 +26,11 @@ template<typename T>
 void medfilt2(Param<T> out, CParam<T> in, const af::borderType pad, int w_len,
               int w_wid) {
     UNUSED(w_wid);
-    auto medfilt2 =
-        common::getKernel("cuda::medfilt2", {medfilt_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(pad),
-                           TemplateArg(w_len), TemplateArg(w_wid)},
-                          {DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+    auto medfilt2 = common::getKernel(
+        "cuda::medfilt2", std::array{medfilt_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
+                     TemplateArg(w_len), TemplateArg(w_wid)),
+        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
 
     const dim3 threads(THREADS_X, THREADS_Y);
 
@@ -46,9 +46,10 @@ void medfilt2(Param<T> out, CParam<T> in, const af::borderType pad, int w_len,
 
 template<typename T>
 void medfilt1(Param<T> out, CParam<T> in, const af::borderType pad, int w_wid) {
-    auto medfilt1 = common::getKernel(
-        "cuda::medfilt1", {medfilt_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(pad), TemplateArg(w_wid)});
+    auto medfilt1 =
+        common::getKernel("cuda::medfilt1", std::array{medfilt_cuh_src},
+                          TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
+                                       TemplateArg(w_wid)));
 
     const dim3 threads(THREADS_X);
 
diff --git a/src/backend/cuda/kernel/memcopy.hpp b/src/backend/cuda/kernel/memcopy.hpp
index f37252c633..7a971bddb0 100644
--- a/src/backend/cuda/kernel/memcopy.hpp
+++ b/src/backend/cuda/kernel/memcopy.hpp
@@ -126,35 +126,40 @@ void memcopy(Param<T> out, CParam<T> in, dim_t indims) {
     // Conversion to cuda base vector types.
     switch (sizeofNewT) {
         case 1: {
-            auto memCopy{
-                common::getKernel(kernelName, {memcopy_cuh_src}, {"char"})};
+            auto memCopy{common::getKernel(kernelName,
+                                           std::array{memcopy_cuh_src},
+                                           TemplateArgs(TemplateArg("char")))};
             memCopy(qArgs, Param<char>((char *)out.ptr, out.dims, out.strides),
                     CParam<char>((const char *)in.ptr, in.dims, in.strides));
         } break;
         case 2: {
-            auto memCopy{
-                common::getKernel(kernelName, {memcopy_cuh_src}, {"short"})};
+            auto memCopy{common::getKernel(kernelName,
+                                           std::array{memcopy_cuh_src},
+                                           TemplateArgs(TemplateArg("short")))};
             memCopy(qArgs,
                     Param<short>((short *)out.ptr, out.dims, out.strides),
                     CParam<short>((const short *)in.ptr, in.dims, in.strides));
         } break;
         case 4: {
-            auto memCopy{
-                common::getKernel(kernelName, {memcopy_cuh_src}, {"float"})};
+            auto memCopy{common::getKernel(kernelName,
+                                           std::array{memcopy_cuh_src},
+                                           TemplateArgs(TemplateArg("float")))};
             memCopy(qArgs,
                     Param<float>((float *)out.ptr, out.dims, out.strides),
                     CParam<float>((const float *)in.ptr, in.dims, in.strides));
         } break;
         case 8: {
             auto memCopy{
-                common::getKernel(kernelName, {memcopy_cuh_src}, {"float2"})};
+                common::getKernel(kernelName, std::array{memcopy_cuh_src},
+                                  TemplateArgs(TemplateArg("float2")))};
             memCopy(
                 qArgs, Param<float2>((float2 *)out.ptr, out.dims, out.strides),
                 CParam<float2>((const float2 *)in.ptr, in.dims, in.strides));
         } break;
         case 16: {
             auto memCopy{
-                common::getKernel(kernelName, {memcopy_cuh_src}, {"float4"})};
+                common::getKernel(kernelName, std::array{memcopy_cuh_src},
+                                  TemplateArgs(TemplateArg("float4")))};
             memCopy(
                 qArgs, Param<float4>((float4 *)out.ptr, out.dims, out.strides),
                 CParam<float4>((const float4 *)in.ptr, in.dims, in.strides));
@@ -188,18 +193,14 @@ void copy(Param<outType> dst, CParam<inType> src, dim_t ondims,
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
-    auto copy{common::getKernel(th.loop0 ? "cuda::scaledCopyLoop0"
-                                : th.loop2 | th.loop3
-                                    ? "cuda::scaledCopyLoop123"
-                                : th.loop1 ? "cuda::scaledCopyLoop1"
-                                           : "cuda::scaledCopy",
-                                {copy_cuh_src},
-                                {
-                                    TemplateTypename<inType>(),
-                                    TemplateTypename<outType>(),
-                                    TemplateArg(same_dims),
-                                    TemplateArg(factor != 1.0),
-                                })};
+    auto copy{common::getKernel(
+        th.loop0              ? "cuda::scaledCopyLoop0"
+        : th.loop2 | th.loop3 ? "cuda::scaledCopyLoop123"
+        : th.loop1            ? "cuda::scaledCopyLoop1"
+                              : "cuda::scaledCopy",
+        std::array{copy_cuh_src},
+        TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>(),
+                     TemplateArg(same_dims), TemplateArg(factor != 1.0)))};
 
     copy(qArgs, dst, src, default_value, factor);
 
diff --git a/src/backend/cuda/kernel/moments.hpp b/src/backend/cuda/kernel/moments.hpp
index 03f536eaeb..2af86afef6 100644
--- a/src/backend/cuda/kernel/moments.hpp
+++ b/src/backend/cuda/kernel/moments.hpp
@@ -21,8 +21,9 @@ static const int THREADS = 128;
 
 template<typename T>
 void moments(Param<float> out, CParam<T> in, const af::momentType moment) {
-    auto moments = common::getKernel("cuda::moments", {moments_cuh_src},
-                                     {TemplateTypename<T>()});
+    auto moments =
+        common::getKernel("cuda::moments", std::array{moments_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(THREADS, 1, 1);
     dim3 blocks(in.dims[1], in.dims[2] * in.dims[3]);
diff --git a/src/backend/cuda/kernel/morph.hpp b/src/backend/cuda/kernel/morph.hpp
index d9ae0ea37f..1202850f40 100644
--- a/src/backend/cuda/kernel/morph.hpp
+++ b/src/backend/cuda/kernel/morph.hpp
@@ -31,11 +31,10 @@ void morph(Param<T> out, CParam<T> in, CParam<T> mask, bool isDilation) {
     const int SeLength = (windLen <= 10 ? windLen : 0);
 
     auto morph = common::getKernel(
-        "cuda::morph", {morph_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(isDilation), TemplateArg(SeLength)},
-        {
-            DefineValue(MAX_MORPH_FILTER_LEN),
-        });
+        "cuda::morph", std::array{morph_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(isDilation),
+                     TemplateArg(SeLength)),
+        std::array{DefineValue(MAX_MORPH_FILTER_LEN)});
 
     morph.copyToReadOnly(morph.getDevPtr("cFilter"),
                          reinterpret_cast<CUdeviceptr>(mask.ptr),
@@ -68,11 +67,10 @@ void morph3d(Param<T> out, CParam<T> in, CParam<T> mask, bool isDilation) {
     }
 
     auto morph3D = common::getKernel(
-        "cuda::morph3D", {morph_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(isDilation), TemplateArg(windLen)},
-        {
-            DefineValue(MAX_MORPH_FILTER_LEN),
-        });
+        "cuda::morph3D", std::array{morph_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(isDilation),
+                     TemplateArg(windLen)),
+        std::array{DefineValue(MAX_MORPH_FILTER_LEN)});
 
     morph3D.copyToReadOnly(
         morph3D.getDevPtr("cFilter"), reinterpret_cast<CUdeviceptr>(mask.ptr),
diff --git a/src/backend/cuda/kernel/pad_array_borders.hpp b/src/backend/cuda/kernel/pad_array_borders.hpp
index decc7a5ae2..b55bd419c5 100644
--- a/src/backend/cuda/kernel/pad_array_borders.hpp
+++ b/src/backend/cuda/kernel/pad_array_borders.hpp
@@ -16,6 +16,8 @@
 #include <nvrtc_kernel_headers/pad_array_borders_cuh.hpp>
 #include <af/defines.h>
 
+#include <array>
+
 namespace cuda {
 namespace kernel {
 
@@ -25,9 +27,9 @@ static const int PADB_THREADS_Y = 8;
 template<typename T>
 void padBorders(Param<T> out, CParam<T> in, dim4 const lBoundPadding,
                 const af::borderType btype) {
-    auto padBorders =
-        common::getKernel("cuda::padBorders", {pad_array_borders_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(btype)});
+    auto padBorders = common::getKernel(
+        "cuda::padBorders", std::array{pad_array_borders_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(btype)));
 
     dim3 threads(kernel::PADB_THREADS_X, kernel::PADB_THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/range.hpp b/src/backend/cuda/kernel/range.hpp
index 4364d3e6a6..cb1f8e13e4 100644
--- a/src/backend/cuda/kernel/range.hpp
+++ b/src/backend/cuda/kernel/range.hpp
@@ -25,8 +25,8 @@ void range(Param<T> out, const int dim) {
     constexpr unsigned RANGE_TILEX = 512;
     constexpr unsigned RANGE_TILEY = 32;
 
-    auto range = common::getKernel("cuda::range", {range_cuh_src},
-                                   {TemplateTypename<T>()});
+    auto range = common::getKernel("cuda::range", std::array{range_cuh_src},
+                                   TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(RANGE_TX, RANGE_TY, 1);
 
diff --git a/src/backend/cuda/kernel/reorder.hpp b/src/backend/cuda/kernel/reorder.hpp
index fc6920ab7f..cb10ad3cb0 100644
--- a/src/backend/cuda/kernel/reorder.hpp
+++ b/src/backend/cuda/kernel/reorder.hpp
@@ -25,8 +25,9 @@ void reorder(Param<T> out, CParam<T> in, const dim_t *rdims) {
     constexpr unsigned TILEX = 512;
     constexpr unsigned TILEY = 32;
 
-    auto reorder = common::getKernel("cuda::reorder", {reorder_cuh_src},
-                                     {TemplateTypename<T>()});
+    auto reorder =
+        common::getKernel("cuda::reorder", std::array{reorder_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/resize.hpp b/src/backend/cuda/kernel/resize.hpp
index 7c5504c75b..231dab781b 100644
--- a/src/backend/cuda/kernel/resize.hpp
+++ b/src/backend/cuda/kernel/resize.hpp
@@ -23,9 +23,9 @@ static const unsigned TY = 16;
 
 template<typename T>
 void resize(Param<T> out, CParam<T> in, af_interp_type method) {
-    auto resize =
-        common::getKernel("cuda::resize", {resize_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(method)});
+    auto resize = common::getKernel(
+        "cuda::resize", std::array{resize_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(method)));
 
     dim3 threads(TX, TY, 1);
     dim3 blocks(divup(out.dims[0], threads.x), divup(out.dims[1], threads.y));
diff --git a/src/backend/cuda/kernel/rotate.hpp b/src/backend/cuda/kernel/rotate.hpp
index 648e126230..5c86b57edf 100644
--- a/src/backend/cuda/kernel/rotate.hpp
+++ b/src/backend/cuda/kernel/rotate.hpp
@@ -32,9 +32,9 @@ typedef struct {
 template<typename T>
 void rotate(Param<T> out, CParam<T> in, const float theta,
             const af::interpType method, const int order) {
-    auto rotate =
-        common::getKernel("cuda::rotate", {rotate_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(order)});
+    auto rotate = common::getKernel(
+        "cuda::rotate", std::array{rotate_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(order)));
 
     const float c = cos(-theta), s = sin(-theta);
     float tx, ty;
diff --git a/src/backend/cuda/kernel/scan_dim.hpp b/src/backend/cuda/kernel/scan_dim.hpp
index dafa280267..88c62e175e 100644
--- a/src/backend/cuda/kernel/scan_dim.hpp
+++ b/src/backend/cuda/kernel/scan_dim.hpp
@@ -25,11 +25,12 @@ static void scan_dim_launcher(Param<To> out, Param<To> tmp, CParam<Ti> in,
                               const uint threads_y, const dim_t blocks_all[4],
                               int dim, bool isFinalPass, bool inclusive_scan) {
     auto scan_dim = common::getKernel(
-        "cuda::scan_dim", {scan_dim_cuh_src},
-        {TemplateTypename<Ti>(), TemplateTypename<To>(), TemplateArg(op),
-         TemplateArg(dim), TemplateArg(isFinalPass), TemplateArg(threads_y),
-         TemplateArg(inclusive_scan)},
-        {DefineValue(THREADS_X)});
+        "cuda::scan_dim", std::array{scan_dim_cuh_src},
+        TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>(),
+                     TemplateArg(op), TemplateArg(dim),
+                     TemplateArg(isFinalPass), TemplateArg(threads_y),
+                     TemplateArg(inclusive_scan)),
+        std::array{DefineValue(THREADS_X)});
 
     dim3 threads(THREADS_X, threads_y);
 
@@ -52,9 +53,10 @@ template<typename To, af_op_t op>
 static void bcast_dim_launcher(Param<To> out, CParam<To> tmp,
                                const uint threads_y, const dim_t blocks_all[4],
                                int dim, bool inclusive_scan) {
-    auto scan_dim_bcast = common::getKernel(
-        "cuda::scan_dim_bcast", {scan_dim_cuh_src},
-        {TemplateTypename<To>(), TemplateArg(op), TemplateArg(dim)});
+    auto scan_dim_bcast =
+        common::getKernel("cuda::scan_dim_bcast", std::array{scan_dim_cuh_src},
+                          TemplateArgs(TemplateTypename<To>(), TemplateArg(op),
+                                       TemplateArg(dim)));
 
     dim3 threads(THREADS_X, threads_y);
 
diff --git a/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp b/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
index e3a618d125..0754e1fc22 100644
--- a/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
+++ b/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
@@ -32,10 +32,10 @@ static void scan_dim_nonfinal_launcher(Param<To> out, Param<To> tmp,
                                        const dim_t blocks_all[4],
                                        bool inclusive_scan) {
     auto scanbykey_dim_nonfinal = common::getKernel(
-        "cuda::scanbykey_dim_nonfinal", {scan_dim_by_key_cuh_src},
-        {TemplateTypename<Ti>(), TemplateTypename<Tk>(), TemplateTypename<To>(),
-         TemplateArg(op)},
-        {DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)});
+        "cuda::scanbykey_dim_nonfinal", std::array{scan_dim_by_key_cuh_src},
+        TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
+                     TemplateTypename<To>(), TemplateArg(op)),
+        std::array{DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)});
 
     dim3 threads(THREADS_X, threads_y);
 
@@ -56,10 +56,10 @@ static void scan_dim_final_launcher(Param<To> out, CParam<Ti> in,
                                     const dim_t blocks_all[4],
                                     bool calculateFlags, bool inclusive_scan) {
     auto scanbykey_dim_final = common::getKernel(
-        "cuda::scanbykey_dim_final", {scan_dim_by_key_cuh_src},
-        {TemplateTypename<Ti>(), TemplateTypename<Tk>(), TemplateTypename<To>(),
-         TemplateArg(op)},
-        {DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)});
+        "cuda::scanbykey_dim_final", std::array{scan_dim_by_key_cuh_src},
+        TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
+                     TemplateTypename<To>(), TemplateArg(op)),
+        std::array{DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)});
 
     dim3 threads(THREADS_X, threads_y);
 
@@ -78,8 +78,8 @@ static void bcast_dim_launcher(Param<To> out, CParam<To> tmp, Param<int> tlid,
                                const int dim, const uint threads_y,
                                const dim_t blocks_all[4]) {
     auto scanbykey_dim_bcast = common::getKernel(
-        "cuda::scanbykey_dim_bcast", {scan_dim_by_key_cuh_src},
-        {TemplateTypename<To>(), TemplateArg(op)});
+        "cuda::scanbykey_dim_bcast", std::array{scan_dim_by_key_cuh_src},
+        TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
     dim3 threads(THREADS_X, threads_y);
     dim3 blocks(blocks_all[0] * blocks_all[2], blocks_all[1] * blocks_all[3]);
 
diff --git a/src/backend/cuda/kernel/scan_first.hpp b/src/backend/cuda/kernel/scan_first.hpp
index f400f4b5d3..0fe6ce1d5f 100644
--- a/src/backend/cuda/kernel/scan_first.hpp
+++ b/src/backend/cuda/kernel/scan_first.hpp
@@ -25,12 +25,12 @@ static void scan_first_launcher(Param<To> out, Param<To> tmp, CParam<Ti> in,
                                 const uint blocks_x, const uint blocks_y,
                                 const uint threads_x, bool isFinalPass,
                                 bool inclusive_scan) {
-    auto scan_first =
-        common::getKernel("cuda::scan_first", {scan_first_cuh_src},
-                          {TemplateTypename<Ti>(), TemplateTypename<To>(),
-                           TemplateArg(op), TemplateArg(isFinalPass),
-                           TemplateArg(threads_x), TemplateArg(inclusive_scan)},
-                          {DefineValue(THREADS_PER_BLOCK)});
+    auto scan_first = common::getKernel(
+        "cuda::scan_first", std::array{scan_first_cuh_src},
+        TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>(),
+                     TemplateArg(op), TemplateArg(isFinalPass),
+                     TemplateArg(threads_x), TemplateArg(inclusive_scan)),
+        std::array{DefineValue(THREADS_PER_BLOCK)});
 
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
@@ -51,9 +51,9 @@ template<typename To, af_op_t op>
 static void bcast_first_launcher(Param<To> out, CParam<To> tmp,
                                  const uint blocks_x, const uint blocks_y,
                                  const uint threads_x, bool inclusive_scan) {
-    auto scan_first_bcast =
-        common::getKernel("cuda::scan_first_bcast", {scan_first_cuh_src},
-                          {TemplateTypename<To>(), TemplateArg(op)});
+    auto scan_first_bcast = common::getKernel(
+        "cuda::scan_first_bcast", std::array{scan_first_cuh_src},
+        TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
 
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
diff --git a/src/backend/cuda/kernel/scan_first_by_key_impl.hpp b/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
index b5e2d070e1..6f9fbd36dd 100644
--- a/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
+++ b/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
@@ -30,10 +30,11 @@ static void scan_nonfinal_launcher(Param<To> out, Param<To> tmp,
                                    const uint blocks_x, const uint blocks_y,
                                    const uint threads_x, bool inclusive_scan) {
     auto scanbykey_first_nonfinal = common::getKernel(
-        "cuda::scanbykey_first_nonfinal", {scan_first_by_key_cuh_src},
-        {TemplateTypename<Ti>(), TemplateTypename<Tk>(), TemplateTypename<To>(),
-         TemplateArg(op)},
-        {DefineValue(THREADS_PER_BLOCK), DefineKeyValue(DIMX, threads_x)});
+        "cuda::scanbykey_first_nonfinal", std::array{scan_first_by_key_cuh_src},
+        TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
+                     TemplateTypename<To>(), TemplateArg(op)),
+        std::array{DefineValue(THREADS_PER_BLOCK),
+                   DefineKeyValue(DIMX, threads_x)});
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
@@ -51,10 +52,11 @@ static void scan_final_launcher(Param<To> out, CParam<Ti> in, CParam<Tk> key,
                                 const uint threads_x, bool calculateFlags,
                                 bool inclusive_scan) {
     auto scanbykey_first_final = common::getKernel(
-        "cuda::scanbykey_first_final", {scan_first_by_key_cuh_src},
-        {TemplateTypename<Ti>(), TemplateTypename<Tk>(), TemplateTypename<To>(),
-         TemplateArg(op)},
-        {DefineValue(THREADS_PER_BLOCK), DefineKeyValue(DIMX, threads_x)});
+        "cuda::scanbykey_first_final", std::array{scan_first_by_key_cuh_src},
+        TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
+                     TemplateTypename<To>(), TemplateArg(op)),
+        std::array{DefineValue(THREADS_PER_BLOCK),
+                   DefineKeyValue(DIMX, threads_x)});
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
@@ -71,8 +73,8 @@ static void bcast_first_launcher(Param<To> out, Param<To> tmp, Param<int> tlid,
                                  const dim_t blocks_x, const dim_t blocks_y,
                                  const uint threads_x) {
     auto scanbykey_first_bcast = common::getKernel(
-        "cuda::scanbykey_first_bcast", {scan_first_by_key_cuh_src},
-        {TemplateTypename<To>(), TemplateArg(op)});
+        "cuda::scanbykey_first_bcast", std::array{scan_first_by_key_cuh_src},
+        TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
     uint lim = divup(out.dims[0], (threads_x * blocks_x));
diff --git a/src/backend/cuda/kernel/select.hpp b/src/backend/cuda/kernel/select.hpp
index 6f8972e04f..ceec068e96 100644
--- a/src/backend/cuda/kernel/select.hpp
+++ b/src/backend/cuda/kernel/select.hpp
@@ -29,9 +29,9 @@ void select(Param<T> out, CParam<char> cond, CParam<T> a, CParam<T> b,
     bool is_same = true;
     for (int i = 0; i < 4; i++) { is_same &= (a.dims[i] == b.dims[i]); }
 
-    auto select =
-        common::getKernel("cuda::select", {select_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(is_same)});
+    auto select = common::getKernel(
+        "cuda::select", std::array{select_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(is_same)));
 
     dim3 threads(DIMX, DIMY);
 
@@ -59,9 +59,9 @@ void select(Param<T> out, CParam<char> cond, CParam<T> a, CParam<T> b,
 template<typename T>
 void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const T b,
                    int ndims, bool flip) {
-    auto selectScalar =
-        common::getKernel("cuda::selectScalar", {select_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(flip)});
+    auto selectScalar = common::getKernel(
+        "cuda::selectScalar", std::array{select_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(flip)));
 
     dim3 threads(DIMX, DIMY);
 
diff --git a/src/backend/cuda/kernel/sobel.hpp b/src/backend/cuda/kernel/sobel.hpp
index 0c2f5a5324..943d8d520e 100644
--- a/src/backend/cuda/kernel/sobel.hpp
+++ b/src/backend/cuda/kernel/sobel.hpp
@@ -26,13 +26,10 @@ void sobel(Param<To> dx, Param<To> dy, CParam<Ti> in,
            const unsigned& ker_size) {
     UNUSED(ker_size);
 
-    auto sobel3x3 =
-        common::getKernel("cuda::sobel3x3", {sobel_cuh_src},
-                          {
-                              TemplateTypename<Ti>(),
-                              TemplateTypename<To>(),
-                          },
-                          {DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+    auto sobel3x3 = common::getKernel(
+        "cuda::sobel3x3", std::array{sobel_cuh_src},
+        TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>()),
+        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
 
     const dim3 threads(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/sparse.hpp b/src/backend/cuda/kernel/sparse.hpp
index 797b7fec5f..66109b2934 100644
--- a/src/backend/cuda/kernel/sparse.hpp
+++ b/src/backend/cuda/kernel/sparse.hpp
@@ -23,9 +23,9 @@ void coo2dense(Param<T> output, CParam<T> values, CParam<int> rowIdx,
                CParam<int> colIdx) {
     constexpr int reps = 4;
 
-    auto coo2Dense =
-        common::getKernel("cuda::coo2Dense", {sparse_cuh_src},
-                          {TemplateTypename<T>()}, {DefineValue(reps)});
+    auto coo2Dense = common::getKernel(
+        "cuda::coo2Dense", std::array{sparse_cuh_src},
+        TemplateArgs(TemplateTypename<T>()), std::array{DefineValue(reps)});
 
     dim3 threads(256, 1, 1);
 
diff --git a/src/backend/cuda/kernel/sparse_arith.hpp b/src/backend/cuda/kernel/sparse_arith.hpp
index 0f2f4ac70d..fb66e19a79 100644
--- a/src/backend/cuda/kernel/sparse_arith.hpp
+++ b/src/backend/cuda/kernel/sparse_arith.hpp
@@ -27,9 +27,9 @@ template<typename T, af_op_t op>
 void sparseArithOpCSR(Param<T> out, CParam<T> values, CParam<int> rowIdx,
                       CParam<int> colIdx, CParam<T> rhs, const bool reverse) {
     auto csrArithDSD =
-        common::getKernel("cuda::csrArithDSD", {sparse_arith_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(op)},
-                          {DefineValue(TX), DefineValue(TY)});
+        common::getKernel("cuda::csrArithDSD", std::array{sparse_arith_cuh_src},
+                          TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
+                          std::array{DefineValue(TX), DefineValue(TY)});
 
     // Each Y for threads does one row
     dim3 threads(TX, TY, 1);
@@ -46,9 +46,10 @@ void sparseArithOpCSR(Param<T> out, CParam<T> values, CParam<int> rowIdx,
 template<typename T, af_op_t op>
 void sparseArithOpCOO(Param<T> out, CParam<T> values, CParam<int> rowIdx,
                       CParam<int> colIdx, CParam<T> rhs, const bool reverse) {
-    auto cooArithDSD = common::getKernel(
-        "cuda::cooArithDSD", {sparse_arith_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(op)}, {DefineValue(THREADS)});
+    auto cooArithDSD =
+        common::getKernel("cuda::cooArithDSD", std::array{sparse_arith_cuh_src},
+                          TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
+                          std::array{DefineValue(THREADS)});
 
     // Linear indexing with one elements per thread
     dim3 threads(THREADS, 1, 1);
@@ -66,9 +67,9 @@ template<typename T, af_op_t op>
 void sparseArithOpCSR(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
                       CParam<T> rhs, const bool reverse) {
     auto csrArithSSD =
-        common::getKernel("cuda::csrArithSSD", {sparse_arith_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(op)},
-                          {DefineValue(TX), DefineValue(TY)});
+        common::getKernel("cuda::csrArithSSD", std::array{sparse_arith_cuh_src},
+                          TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
+                          std::array{DefineValue(TX), DefineValue(TY)});
 
     // Each Y for threads does one row
     dim3 threads(TX, TY, 1);
@@ -85,9 +86,10 @@ void sparseArithOpCSR(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
 template<typename T, af_op_t op>
 void sparseArithOpCOO(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
                       CParam<T> rhs, const bool reverse) {
-    auto cooArithSSD = common::getKernel(
-        "cuda::cooArithSSD", {sparse_arith_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(op)}, {DefineValue(THREADS)});
+    auto cooArithSSD =
+        common::getKernel("cuda::cooArithSSD", std::array{sparse_arith_cuh_src},
+                          TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
+                          std::array{DefineValue(THREADS)});
 
     // Linear indexing with one elements per thread
     dim3 threads(THREADS, 1, 1);
diff --git a/src/backend/cuda/kernel/susan.hpp b/src/backend/cuda/kernel/susan.hpp
index 6d45a41058..e8246b5249 100644
--- a/src/backend/cuda/kernel/susan.hpp
+++ b/src/backend/cuda/kernel/susan.hpp
@@ -26,8 +26,9 @@ void susan_responses(T* out, const T* in, const unsigned idim0,
                      const unsigned idim1, const int radius, const float t,
                      const float g, const unsigned edge) {
     auto susan = common::getKernel(
-        "cuda::susan", {susan_cuh_src}, {TemplateTypename<T>()},
-        {DefineValue(BLOCK_X), DefineValue(BLOCK_Y)});
+        "cuda::susan", std::array{susan_cuh_src},
+        TemplateArgs(TemplateTypename<T>()),
+        std::array{DefineValue(BLOCK_X), DefineValue(BLOCK_Y)});
 
     dim3 threads(BLOCK_X, BLOCK_Y);
     dim3 blocks(divup(idim0 - edge * 2, BLOCK_X),
@@ -45,8 +46,8 @@ template<typename T>
 void nonMaximal(float* x_out, float* y_out, float* resp_out, unsigned* count,
                 const unsigned idim0, const unsigned idim1, const T* resp_in,
                 const unsigned edge, const unsigned max_corners) {
-    auto nonMax = common::getKernel("cuda::nonMax", {susan_cuh_src},
-                                    {TemplateTypename<T>()});
+    auto nonMax = common::getKernel("cuda::nonMax", std::array{susan_cuh_src},
+                                    TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(BLOCK_X, BLOCK_Y);
     dim3 blocks(divup(idim0 - edge * 2, BLOCK_X),
diff --git a/src/backend/cuda/kernel/tile.hpp b/src/backend/cuda/kernel/tile.hpp
index 8edebf3991..5656fcf8e1 100644
--- a/src/backend/cuda/kernel/tile.hpp
+++ b/src/backend/cuda/kernel/tile.hpp
@@ -25,8 +25,8 @@ void tile(Param<T> out, CParam<T> in) {
     constexpr unsigned TILEX = 512;
     constexpr unsigned TILEY = 32;
 
-    auto tile = common::getKernel("cuda::tile", {tile_cuh_src},
-                                  {TemplateTypename<T>()});
+    auto tile = common::getKernel("cuda::tile", std::array{tile_cuh_src},
+                                  TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/transform.hpp b/src/backend/cuda/kernel/transform.hpp
index df9bf32c8b..489063cc8a 100644
--- a/src/backend/cuda/kernel/transform.hpp
+++ b/src/backend/cuda/kernel/transform.hpp
@@ -31,8 +31,9 @@ template<typename T>
 void transform(Param<T> out, CParam<T> in, CParam<float> tf, const bool inverse,
                const bool perspective, const af::interpType method, int order) {
     auto transform = common::getKernel(
-        "cuda::transform", {transform_cuh_src},
-        {TemplateTypename<T>(), TemplateArg(inverse), TemplateArg(order)});
+        "cuda::transform", std::array{transform_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(inverse),
+                     TemplateArg(order)));
 
     const unsigned int nImg2  = in.dims[2];
     const unsigned int nImg3  = in.dims[3];
diff --git a/src/backend/cuda/kernel/transpose.hpp b/src/backend/cuda/kernel/transpose.hpp
index 3a5101a37d..aca9efb9c6 100644
--- a/src/backend/cuda/kernel/transpose.hpp
+++ b/src/backend/cuda/kernel/transpose.hpp
@@ -25,11 +25,11 @@ static const int THREADS_Y = 256 / TILE_DIM;
 template<typename T>
 void transpose(Param<T> out, CParam<T> in, const bool conjugate,
                const bool is32multiple) {
-    auto transpose =
-        common::getKernel("cuda::transpose", {transpose_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(conjugate),
-                           TemplateArg(is32multiple)},
-                          {DefineValue(TILE_DIM), DefineValue(THREADS_Y)});
+    auto transpose = common::getKernel(
+        "cuda::transpose", std::array{transpose_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(conjugate),
+                     TemplateArg(is32multiple)),
+        std::array{DefineValue(TILE_DIM), DefineValue(THREADS_Y)});
 
     dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/transpose_inplace.hpp b/src/backend/cuda/kernel/transpose_inplace.hpp
index 0ba76f19da..d603a08653 100644
--- a/src/backend/cuda/kernel/transpose_inplace.hpp
+++ b/src/backend/cuda/kernel/transpose_inplace.hpp
@@ -25,11 +25,11 @@ static const int THREADS_Y = 256 / TILE_DIM;
 template<typename T>
 void transpose_inplace(Param<T> in, const bool conjugate,
                        const bool is32multiple) {
-    auto transposeIP =
-        common::getKernel("cuda::transposeIP", {transpose_inplace_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(conjugate),
-                           TemplateArg(is32multiple)},
-                          {DefineValue(TILE_DIM), DefineValue(THREADS_Y)});
+    auto transposeIP = common::getKernel(
+        "cuda::transposeIP", std::array{transpose_inplace_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(conjugate),
+                     TemplateArg(is32multiple)),
+        std::array{DefineValue(TILE_DIM), DefineValue(THREADS_Y)});
 
     // dimensions passed to this function should be input dimensions
     // any necessary transformations and dimension related calculations are
diff --git a/src/backend/cuda/kernel/triangle.hpp b/src/backend/cuda/kernel/triangle.hpp
index b49601ce51..e6efac7be6 100644
--- a/src/backend/cuda/kernel/triangle.hpp
+++ b/src/backend/cuda/kernel/triangle.hpp
@@ -25,10 +25,10 @@ void triangle(Param<T> r, CParam<T> in, bool is_upper, bool is_unit_diag) {
     constexpr unsigned TILEX = 128;
     constexpr unsigned TILEY = 32;
 
-    auto triangle =
-        common::getKernel("cuda::triangle", {triangle_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(is_upper),
-                           TemplateArg(is_unit_diag)});
+    auto triangle = common::getKernel(
+        "cuda::triangle", std::array{triangle_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(is_upper),
+                     TemplateArg(is_unit_diag)));
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/unwrap.hpp b/src/backend/cuda/kernel/unwrap.hpp
index 8e171ac816..15f74df963 100644
--- a/src/backend/cuda/kernel/unwrap.hpp
+++ b/src/backend/cuda/kernel/unwrap.hpp
@@ -23,9 +23,9 @@ template<typename T>
 void unwrap(Param<T> out, CParam<T> in, const int wx, const int wy,
             const int sx, const int sy, const int px, const int py,
             const int dx, const int dy, const int nx, const bool is_column) {
-    auto unwrap =
-        common::getKernel("cuda::unwrap", {unwrap_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(is_column)});
+    auto unwrap = common::getKernel(
+        "cuda::unwrap", std::array{unwrap_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     dim3 threads, blocks;
     int reps;
diff --git a/src/backend/cuda/kernel/where.hpp b/src/backend/cuda/kernel/where.hpp
index 66555253c0..bf992648d3 100644
--- a/src/backend/cuda/kernel/where.hpp
+++ b/src/backend/cuda/kernel/where.hpp
@@ -23,8 +23,8 @@ namespace kernel {
 
 template<typename T>
 static void where(Param<uint> &out, CParam<T> in) {
-    auto where = common::getKernel("cuda::where", {where_cuh_src},
-                                   {TemplateTypename<T>()});
+    auto where = common::getKernel("cuda::where", std::array{where_cuh_src},
+                                   TemplateArgs(TemplateTypename<T>()));
 
     uint threads_x = nextpow2(std::max(32u, (uint)in.dims[0]));
     threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
diff --git a/src/backend/cuda/kernel/wrap.hpp b/src/backend/cuda/kernel/wrap.hpp
index 33a32a6ef3..7185ea38bb 100644
--- a/src/backend/cuda/kernel/wrap.hpp
+++ b/src/backend/cuda/kernel/wrap.hpp
@@ -22,9 +22,9 @@ namespace kernel {
 template<typename T>
 void wrap(Param<T> out, CParam<T> in, const int wx, const int wy, const int sx,
           const int sy, const int px, const int py, const bool is_column) {
-    auto wrap =
-        common::getKernel("cuda::wrap", {wrap_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(is_column)});
+    auto wrap = common::getKernel(
+        "cuda::wrap", std::array{wrap_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     int nx = (out.dims[0] + 2 * px - wx) / sx + 1;
     int ny = (out.dims[1] + 2 * py - wy) / sy + 1;
@@ -51,9 +51,9 @@ void wrap_dilated(Param<T> out, CParam<T> in, const dim_t wx, const dim_t wy,
                   const dim_t sx, const dim_t sy, const dim_t px,
                   const dim_t py, const dim_t dx, const dim_t dy,
                   const bool is_column) {
-    auto wrap =
-        common::getKernel("cuda::wrap_dilated", {wrap_cuh_src},
-                          {TemplateTypename<T>(), TemplateArg(is_column)});
+    auto wrap = common::getKernel(
+        "cuda::wrap_dilated", std::array{wrap_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     int nx = 1 + (out.dims[0] + 2 * px - (((wx - 1) * dx) + 1)) / sx;
     int ny = 1 + (out.dims[1] + 2 * py - (((wy - 1) * dy) + 1)) / sy;
diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp
index 999632d55a..4a85ce292e 100644
--- a/src/backend/opencl/compile_module.cpp
+++ b/src/backend/opencl/compile_module.cpp
@@ -17,6 +17,7 @@
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
 #include <kernel_headers/KParam.hpp>
+#include <nonstd/span.hpp>
 #include <platform.hpp>
 #include <traits.hpp>
 
@@ -32,6 +33,7 @@ using cl::Error;
 using cl::Program;
 using common::loggerFactory;
 using fmt::format;
+using nonstd::span;
 using opencl::getActiveDeviceId;
 using opencl::getDevice;
 using opencl::Kernel;
@@ -99,8 +101,8 @@ const static string DEFAULT_MACROS_STR(
                                            #endif\n                     \
                                            ");
 
-Program buildProgram(const vector<string> &kernelSources,
-                     const vector<string> &compileOpts) {
+Program buildProgram(span<const string> kernelSources,
+                     span<const string> compileOpts) {
     Program retVal;
     try {
         static const string defaults =
@@ -151,9 +153,9 @@ string getKernelCacheFilename(const int device, const string &key) {
 
 namespace common {
 
-Module compileModule(const string &moduleKey, const vector<string> &sources,
-                     const vector<string> &options,
-                     const vector<string> &kInstances, const bool isJIT) {
+Module compileModule(const string &moduleKey, span<const string> sources,
+                     span<const string> options, span<const string> kInstances,
+                     const bool isJIT) {
     UNUSED(kInstances);
     UNUSED(isJIT);
 
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 18a89e00a7..d475f32b71 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -278,7 +278,8 @@ cl::Kernel getKernel(const vector<Node*>& output_nodes,
         if (isHalfSupported(device)) {
             options.emplace_back(DefineKey(USE_HALF));
         }
-        return common::getKernel(funcName, {jit_cl_src, jitKer_cl_src}, {},
+        return common::getKernel(funcName,
+                                 std::array{jit_cl_src, jitKer_cl_src}, {},
                                  options, true)
             .get();
     }
diff --git a/src/backend/opencl/kernel/anisotropic_diffusion.hpp b/src/backend/opencl/kernel/anisotropic_diffusion.hpp
index e7d18136dd..84af9db4a7 100644
--- a/src/backend/opencl/kernel/anisotropic_diffusion.hpp
+++ b/src/backend/opencl/kernel/anisotropic_diffusion.hpp
@@ -49,9 +49,9 @@ void anisotropicDiffusion(Param inout, const float dt, const float mct,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto diffUpdate =
-        common::getKernel("aisoDiffUpdate", {anisotropic_diffusion_cl_src},
-                          tmpltArgs, compileOpts);
+    auto diffUpdate = common::getKernel(
+        "aisoDiffUpdate", std::array{anisotropic_diffusion_cl_src}, tmpltArgs,
+        compileOpts);
 
     NDRange local(THREADS_X, THREADS_Y, 1);
 
diff --git a/src/backend/opencl/kernel/approx.hpp b/src/backend/opencl/kernel/approx.hpp
index be569fbf61..1d702ed090 100644
--- a/src/backend/opencl/kernel/approx.hpp
+++ b/src/backend/opencl/kernel/approx.hpp
@@ -72,8 +72,9 @@ void approx1(Param yo, const Param yi, const Param xo, const int xdim,
     };
     auto compileOpts = genCompileOptions<Ty, Tp>(order, xdim);
 
-    auto approx1 = common::getKernel("approx1", {interp_cl_src, approx1_cl_src},
-                                     tmpltArgs, compileOpts);
+    auto approx1 =
+        common::getKernel("approx1", std::array{interp_cl_src, approx1_cl_src},
+                          tmpltArgs, compileOpts);
 
     NDRange local(THREADS, 1, 1);
     dim_t blocksPerMat = divup(yo.info.dims[0], local[0]);
@@ -110,8 +111,9 @@ void approx2(Param zo, const Param zi, const Param xo, const int xdim,
     };
     auto compileOpts = genCompileOptions<Ty, Tp>(order, xdim, ydim);
 
-    auto approx2 = common::getKernel("approx2", {interp_cl_src, approx2_cl_src},
-                                     tmpltArgs, compileOpts);
+    auto approx2 =
+        common::getKernel("approx2", std::array{interp_cl_src, approx2_cl_src},
+                          tmpltArgs, compileOpts);
 
     NDRange local(TX, TY, 1);
     dim_t blocksPerMatX = divup(zo.info.dims[0], local[0]);
diff --git a/src/backend/opencl/kernel/assign.hpp b/src/backend/opencl/kernel/assign.hpp
index 568ec9b185..0b9ae34472 100644
--- a/src/backend/opencl/kernel/assign.hpp
+++ b/src/backend/opencl/kernel/assign.hpp
@@ -34,16 +34,15 @@ void assign(Param out, const Param in, const AssignKernelParam_t& p,
     constexpr int THREADS_X = 32;
     constexpr int THREADS_Y = 8;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 2> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto assign =
-        common::getKernel("assignKernel", {assign_cl_src}, targs, options);
+    auto assign = common::getKernel("assignKernel", std::array{assign_cl_src},
+                                    targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/bilateral.hpp b/src/backend/opencl/kernel/bilateral.hpp
index 168fbcea6d..a191d53815 100644
--- a/src/backend/opencl/kernel/bilateral.hpp
+++ b/src/backend/opencl/kernel/bilateral.hpp
@@ -32,7 +32,7 @@ void bilateral(Param out, const Param in, const float s_sigma,
     constexpr bool UseNativeExp = !std::is_same<inType, double>::value ||
                                   std::is_same<inType, cdouble>::value;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<inType>(),
         TemplateTypename<outType>(),
     };
@@ -43,8 +43,8 @@ void bilateral(Param out, const Param in, const float s_sigma,
     if (UseNativeExp) { options.emplace_back(DefineKey(USE_NATIVE_EXP)); }
     options.emplace_back(getTypeBuildDefinition<inType>());
 
-    auto bilateralOp =
-        common::getKernel("bilateral", {bilateral_cl_src}, targs, options);
+    auto bilateralOp = common::getKernel(
+        "bilateral", std::array{bilateral_cl_src}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/canny.hpp b/src/backend/opencl/kernel/canny.hpp
index 3c82b9df4f..7444ac00aa 100644
--- a/src/backend/opencl/kernel/canny.hpp
+++ b/src/backend/opencl/kernel/canny.hpp
@@ -41,9 +41,9 @@ void nonMaxSuppression(Param output, const Param magnitude, const Param dx,
     };
     options.emplace_back(getTypeBuildDefinition<T>());
 
-    auto nonMaxOp = common::getKernel("nonMaxSuppressionKernel",
-                                      {nonmax_suppression_cl_src},
-                                      {TemplateTypename<T>()}, options);
+    auto nonMaxOp = common::getKernel(
+        "nonMaxSuppressionKernel", std::array{nonmax_suppression_cl_src},
+        TemplateArgs(TemplateTypename<T>()), options);
 
     NDRange threads(kernel::THREADS_X, kernel::THREADS_Y, 1);
 
@@ -74,8 +74,9 @@ void initEdgeOut(Param output, const Param strong, const Param weak) {
     };
     options.emplace_back(getTypeBuildDefinition<T>());
 
-    auto initOp = common::getKernel("initEdgeOutKernel", {trace_edge_cl_src},
-                                    {TemplateTypename<T>()}, options);
+    auto initOp =
+        common::getKernel("initEdgeOutKernel", std::array{trace_edge_cl_src},
+                          TemplateArgs(TemplateTypename<T>()), options);
 
     NDRange threads(kernel::THREADS_X, kernel::THREADS_Y, 1);
 
@@ -106,9 +107,9 @@ void suppressLeftOver(Param output) {
     };
     options.emplace_back(getTypeBuildDefinition<T>());
 
-    auto finalOp =
-        common::getKernel("suppressLeftOverKernel", {trace_edge_cl_src},
-                          {TemplateTypename<T>()}, options);
+    auto finalOp = common::getKernel(
+        "suppressLeftOverKernel", std::array{trace_edge_cl_src},
+        TemplateArgs(TemplateTypename<T>()), options);
 
     NDRange threads(kernel::THREADS_X, kernel::THREADS_Y, 1);
 
@@ -142,8 +143,9 @@ void edgeTrackingHysteresis(Param output, const Param strong,
     };
     options.emplace_back(getTypeBuildDefinition<T>());
 
-    auto edgeTraceOp = common::getKernel("edgeTrackKernel", {trace_edge_cl_src},
-                                         {TemplateTypename<T>()}, options);
+    auto edgeTraceOp =
+        common::getKernel("edgeTrackKernel", std::array{trace_edge_cl_src},
+                          TemplateArgs(TemplateTypename<T>()), options);
 
     NDRange threads(kernel::THREADS_X, kernel::THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/convolve/conv2_impl.hpp b/src/backend/opencl/kernel/convolve/conv2_impl.hpp
index abe95ae896..61f9d1d56d 100644
--- a/src/backend/opencl/kernel/convolve/conv2_impl.hpp
+++ b/src/backend/opencl/kernel/convolve/conv2_impl.hpp
@@ -50,8 +50,9 @@ void conv2Helper(const conv_kparam_t& param, Param out, const Param signal,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto convolve = common::getKernel("convolve", {ops_cl_src, convolve_cl_src},
-                                      tmpltArgs, compileOpts);
+    auto convolve =
+        common::getKernel("convolve", std::array{ops_cl_src, convolve_cl_src},
+                          tmpltArgs, compileOpts);
 
     convolve(EnqueueArgs(getQueue(), param.global, param.local), *out.data,
              out.info, *signal.data, signal.info, *param.impulse, filter.info,
diff --git a/src/backend/opencl/kernel/convolve/conv_common.hpp b/src/backend/opencl/kernel/convolve/conv_common.hpp
index 92cf5858e7..987e623dcf 100644
--- a/src/backend/opencl/kernel/convolve/conv_common.hpp
+++ b/src/backend/opencl/kernel/convolve/conv_common.hpp
@@ -113,8 +113,9 @@ void convNHelper(const conv_kparam_t& param, Param& out, const Param& signal,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto convolve = common::getKernel("convolve", {ops_cl_src, convolve_cl_src},
-                                      tmpltArgs, compileOpts);
+    auto convolve =
+        common::getKernel("convolve", std::array{ops_cl_src, convolve_cl_src},
+                          tmpltArgs, compileOpts);
 
     convolve(EnqueueArgs(getQueue(), param.global, param.local), *out.data,
              out.info, *signal.data, signal.info, cl::Local(param.loc_size),
diff --git a/src/backend/opencl/kernel/convolve_separable.cpp b/src/backend/opencl/kernel/convolve_separable.cpp
index 85b9bfadb9..7017170e41 100644
--- a/src/backend/opencl/kernel/convolve_separable.cpp
+++ b/src/backend/opencl/kernel/convolve_separable.cpp
@@ -44,12 +44,12 @@ void convSep(Param out, const Param signal, const Param filter,
     const size_t C1_SIZE = (THREADS_Y + 2 * (fLen - 1)) * THREADS_X;
     size_t locSize       = (conv_dim == 0 ? C0_SIZE : C1_SIZE);
 
-    std::vector<TemplateArg> tmpltArgs = {
+    std::array<TemplateArg, 5> tmpltArgs = {
         TemplateTypename<T>(), TemplateTypename<accType>(),
         TemplateArg(conv_dim), TemplateArg(expand),
         TemplateArg(fLen),
     };
-    std::vector<std::string> compileOpts = {
+    std::array<std::string, 11> compileOpts = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(Ti, dtype_traits<T>::getName()),
         DefineKeyValue(To, dtype_traits<accType>::getName()),
@@ -60,12 +60,11 @@ void convSep(Param out, const Param signal, const Param filter,
         DefineKeyFromStr(binOpName<af_mul_t>()),
         DefineKeyValue(IS_CPLX, (IsComplex ? 1 : 0)),
         DefineKeyValue(LOCAL_MEM_SIZE, locSize),
-    };
-    compileOpts.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto conv =
-        common::getKernel("convolve", {ops_cl_src, convolve_separable_cl_src},
-                          tmpltArgs, compileOpts);
+    auto conv = common::getKernel(
+        "convolve", std::array{ops_cl_src, convolve_separable_cl_src},
+        tmpltArgs, compileOpts);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/cscmm.hpp b/src/backend/opencl/kernel/cscmm.hpp
index 7047af13aa..9857133f9d 100644
--- a/src/backend/opencl/kernel/cscmm.hpp
+++ b/src/backend/opencl/kernel/cscmm.hpp
@@ -38,13 +38,13 @@ void cscmm_nn(Param out, const Param &values, const Param &colIdx,
     const bool use_alpha = (alpha != scalar<T>(1.0));
     const bool use_beta  = (beta != scalar<T>(0.0));
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 7> targs = {
         TemplateTypename<T>(),       TemplateArg(use_alpha),
         TemplateArg(use_beta),       TemplateArg(is_conj),
         TemplateArg(rows_per_group), TemplateArg(cols_per_group),
         TemplateArg(threads),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 9> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(USE_ALPHA, use_alpha),
         DefineKeyValue(USE_BETA, use_beta),
@@ -53,11 +53,10 @@ void cscmm_nn(Param out, const Param &values, const Param &colIdx,
         DefineKeyValue(ROWS_PER_GROUP, rows_per_group),
         DefineKeyValue(COLS_PER_GROUP, cols_per_group),
         DefineKeyValue(IS_CPLX, (af::iscplx<T>() ? 1 : 0)),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
     auto cscmmNN =
-        common::getKernel("cscmm_nn", {cscmm_cl_src}, targs, options);
+        common::getKernel("cscmm_nn", std::array{cscmm_cl_src}, targs, options);
 
     cl::NDRange local(threads, 1);
     int M = out.info.dims[0];
diff --git a/src/backend/opencl/kernel/cscmv.hpp b/src/backend/opencl/kernel/cscmv.hpp
index 5d948783fb..a3b66714c3 100644
--- a/src/backend/opencl/kernel/cscmv.hpp
+++ b/src/backend/opencl/kernel/cscmv.hpp
@@ -38,12 +38,12 @@ void cscmv(Param out, const Param &values, const Param &colIdx,
 
     cl::NDRange local(THREADS_PER_GROUP);
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 6> targs = {
         TemplateTypename<T>(),       TemplateArg(use_alpha),
         TemplateArg(use_beta),       TemplateArg(is_conj),
         TemplateArg(rows_per_group), TemplateArg(local[0]),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 8> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(USE_ALPHA, use_alpha),
         DefineKeyValue(USE_BETA, use_beta),
@@ -51,11 +51,10 @@ void cscmv(Param out, const Param &values, const Param &colIdx,
         DefineKeyValue(THREADS, local[0]),
         DefineKeyValue(ROWS_PER_GROUP, rows_per_group),
         DefineKeyValue(IS_CPLX, (af::iscplx<T>() ? 1 : 0)),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto cscmvBlock =
-        common::getKernel("cscmv_block", {cscmv_cl_src}, targs, options);
+    auto cscmvBlock = common::getKernel("cscmv_block", std::array{cscmv_cl_src},
+                                        targs, options);
 
     int K        = colIdx.info.dims[0] - 1;
     int M        = out.info.dims[0];
diff --git a/src/backend/opencl/kernel/csrmm.hpp b/src/backend/opencl/kernel/csrmm.hpp
index a9b7b8fb95..42b5cc093a 100644
--- a/src/backend/opencl/kernel/csrmm.hpp
+++ b/src/backend/opencl/kernel/csrmm.hpp
@@ -38,25 +38,24 @@ void csrmm_nt(Param out, const Param &values, const Param &rowIdx,
     const bool use_alpha = (alpha != scalar<T>(1.0));
     const bool use_beta  = (beta != scalar<T>(0.0));
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 4> targs = {
         TemplateTypename<T>(),
         TemplateArg(use_alpha),
         TemplateArg(use_beta),
         TemplateArg(use_greedy),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 7> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(USE_ALPHA, use_alpha),
         DefineKeyValue(USE_BETA, use_beta),
         DefineKeyValue(USE_GREEDY, use_greedy),
         DefineValue(THREADS_PER_GROUP),
         DefineKeyValue(IS_CPLX, (af::iscplx<T>() ? 1 : 0)),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
     // FIXME: Switch to perf (thread vs block) baesd kernel
     auto csrmm_nt_func =
-        common::getKernel("csrmm_nt", {csrmm_cl_src}, targs, options);
+        common::getKernel("csrmm_nt", std::array{csrmm_cl_src}, targs, options);
 
     cl::NDRange local(THREADS_PER_GROUP, 1);
     int M = rowIdx.info.dims[0] - 1;
diff --git a/src/backend/opencl/kernel/csrmv.hpp b/src/backend/opencl/kernel/csrmv.hpp
index d6b52ff6b4..2d7abaa190 100644
--- a/src/backend/opencl/kernel/csrmv.hpp
+++ b/src/backend/opencl/kernel/csrmv.hpp
@@ -43,24 +43,24 @@ void csrmv(Param out, const Param &values, const Param &rowIdx,
 
     cl::NDRange local(THREADS_PER_GROUP);
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 5> targs = {
         TemplateTypename<T>(),   TemplateArg(use_alpha), TemplateArg(use_beta),
         TemplateArg(use_greedy), TemplateArg(local[0]),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 7> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(USE_ALPHA, use_alpha),
         DefineKeyValue(USE_BETA, use_beta),
         DefineKeyValue(USE_GREEDY, use_greedy),
         DefineKeyValue(THREADS, local[0]),
         DefineKeyValue(IS_CPLX, (af::iscplx<T>() ? 1 : 0)),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
     auto csrmv =
         (is_csrmv_block
-             ? common::getKernel("csrmv_thread", {csrmv_cl_src}, targs, options)
-             : common::getKernel("csrmv_block", {csrmv_cl_src}, targs,
+             ? common::getKernel("csrmv_thread", std::array{csrmv_cl_src},
+                                 targs, options)
+             : common::getKernel("csrmv_block", std::array{csrmv_cl_src}, targs,
                                  options));
 
     int M = rowIdx.info.dims[0] - 1;
diff --git a/src/backend/opencl/kernel/diagonal.hpp b/src/backend/opencl/kernel/diagonal.hpp
index 4ed94e2ba6..e4320aa6dc 100644
--- a/src/backend/opencl/kernel/diagonal.hpp
+++ b/src/backend/opencl/kernel/diagonal.hpp
@@ -27,17 +27,16 @@ namespace kernel {
 
 template<typename T>
 static void diagCreate(Param out, Param in, int num) {
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 3> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto diagCreate = common::getKernel("diagCreateKernel",
-                                        {diag_create_cl_src}, targs, options);
+    auto diagCreate = common::getKernel(
+        "diagCreateKernel", std::array{diag_create_cl_src}, targs, options);
 
     cl::NDRange local(32, 8);
     int groups_x = divup(out.info.dims[0], local[0]);
@@ -52,17 +51,16 @@ static void diagCreate(Param out, Param in, int num) {
 
 template<typename T>
 static void diagExtract(Param out, Param in, int num) {
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 3> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto diagExtract = common::getKernel("diagExtractKernel",
-                                         {diag_extract_cl_src}, targs, options);
+    auto diagExtract = common::getKernel(
+        "diagExtractKernel", std::array{diag_extract_cl_src}, targs, options);
 
     cl::NDRange local(256, 1);
     int groups_x = divup(out.info.dims[0], local[0]);
diff --git a/src/backend/opencl/kernel/diff.hpp b/src/backend/opencl/kernel/diff.hpp
index 02251f6d41..c249e55d94 100644
--- a/src/backend/opencl/kernel/diff.hpp
+++ b/src/backend/opencl/kernel/diff.hpp
@@ -28,20 +28,18 @@ void diff(Param out, const Param in, const unsigned indims, const unsigned dim,
     constexpr int TX = 16;
     constexpr int TY = 16;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 3> targs = {
         TemplateTypename<T>(),
         TemplateArg(dim),
         TemplateArg(isDiff2),
     };
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineKeyValue(DIM, dim),
+    std::array<std::string, 4> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineKeyValue(DIM, dim),
         DefineKeyValue(isDiff2, (isDiff2 ? 1 : 0)),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto diffOp =
-        common::getKernel("diff_kernel", {diff_cl_src}, targs, options);
+    auto diffOp = common::getKernel("diff_kernel", std::array{diff_cl_src},
+                                    targs, options);
 
     cl::NDRange local(TX, TY, 1);
     if (dim == 0 && indims == 1) { local = cl::NDRange(TX * TY, 1, 1); }
diff --git a/src/backend/opencl/kernel/exampleFunction.hpp b/src/backend/opencl/kernel/exampleFunction.hpp
index 98ff024060..4b5e506c13 100644
--- a/src/backend/opencl/kernel/exampleFunction.hpp
+++ b/src/backend/opencl/kernel/exampleFunction.hpp
@@ -43,25 +43,25 @@ template<typename T>
 void exampleFunc(Param c, const Param a, const Param b, const af_someenum_t p) {
     // Compilation options for compiling OpenCL kernel.
     // Go to common/kernel_cache.hpp to find details on this.
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
 
     // Compilation options for compiling OpenCL kernel.
     // Go to common/kernel_cache.hpp to find details on this.
-    std::vector<std::string> options = {
+    std::array<std::string, 2> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
 
-    // The following templated function can take variable
-    // number of template parameters and if one of them is double
-    // precision, it will enable necessary constants, flags, ops
-    // in opencl kernel compilation stage
-    options.emplace_back(getTypeBuildDefinition<T>());
+        // The following templated function can take variable
+        // number of template parameters and if one of them is double
+        // precision, it will enable necessary constants, flags, ops
+        // in opencl kernel compilation stage
+        getTypeBuildDefinition<T>()};
 
     // Fetch the Kernel functor, go to common/kernel_cache.hpp
     // to find details of this function
-    auto exOp = common::getKernel("example", {example_cl_src}, targs, options);
+    auto exOp = common::getKernel("example", std::array{example_cl_src}, targs,
+                                  options);
 
     // configure work group parameters
     cl::NDRange local(THREADS_X, THREADS_Y);
diff --git a/src/backend/opencl/kernel/fast.hpp b/src/backend/opencl/kernel/fast.hpp
index 1ef1ca46ff..9b4fc4341f 100644
--- a/src/backend/opencl/kernel/fast.hpp
+++ b/src/backend/opencl/kernel/fast.hpp
@@ -33,24 +33,23 @@ void fast(const unsigned arc_length, unsigned *out_feat, Param &x_out,
     constexpr int FAST_THREADS_NONMAX_X = 32;
     constexpr int FAST_THREADS_NONMAX_Y = 8;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 3> targs = {
         TemplateTypename<T>(),
         TemplateArg(arc_length),
         TemplateArg(nonmax),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 4> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(ARC_LENGTH, arc_length),
         DefineKeyValue(NONMAX, static_cast<unsigned>(nonmax)),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
-
-    auto locate =
-        common::getKernel("locate_features", {fast_cl_src}, targs, options);
-    auto nonMax =
-        common::getKernel("non_max_counts", {fast_cl_src}, targs, options);
-    auto getFeat =
-        common::getKernel("get_features", {fast_cl_src}, targs, options);
+        getTypeBuildDefinition<T>()};
+
+    auto locate  = common::getKernel("locate_features", std::array{fast_cl_src},
+                                     targs, options);
+    auto nonMax  = common::getKernel("non_max_counts", std::array{fast_cl_src},
+                                     targs, options);
+    auto getFeat = common::getKernel("get_features", std::array{fast_cl_src},
+                                     targs, options);
 
     const unsigned max_feat =
         ceil(in.info.dims[0] * in.info.dims[1] * feature_ratio);
diff --git a/src/backend/opencl/kernel/fftconvolve.hpp b/src/backend/opencl/kernel/fftconvolve.hpp
index 157c779936..222bde02e8 100644
--- a/src/backend/opencl/kernel/fftconvolve.hpp
+++ b/src/backend/opencl/kernel/fftconvolve.hpp
@@ -70,25 +70,24 @@ void packDataHelper(Param packed, Param sig, Param filter, const int rank,
     constexpr auto ctDType =
         static_cast<af_dtype>(dtype_traits<convT>::af_type);
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 3> targs = {
         TemplateTypename<T>(),
         TemplateTypename<convT>(),
         TemplateArg(IsTypeDouble),
     };
     std::vector<std::string> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
+        getTypeBuildDefinition<T, convT>()};
     if (ctDType == c32) {
         options.emplace_back(DefineKeyValue(CONVT, "float"));
     } else if (ctDType == c64 && IsTypeDouble) {
         options.emplace_back(DefineKeyValue(CONVT, "double"));
     }
-    options.emplace_back(getTypeBuildDefinition<T, convT>());
 
-    auto packData = common::getKernel("pack_data", {fftconvolve_pack_cl_src},
-                                      targs, options);
-    auto padArray = common::getKernel("pad_array", {fftconvolve_pack_cl_src},
-                                      targs, options);
+    auto packData = common::getKernel(
+        "pack_data", std::array{fftconvolve_pack_cl_src}, targs, options);
+    auto padArray = common::getKernel(
+        "pad_array", std::array{fftconvolve_pack_cl_src}, targs, options);
 
     Param sig_tmp, filter_tmp;
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
@@ -129,7 +128,7 @@ void complexMultiplyHelper(Param packed, Param sig, Param filter,
     constexpr auto ctDType =
         static_cast<af_dtype>(dtype_traits<convT>::af_type);
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 3> targs = {
         TemplateTypename<T>(),
         TemplateTypename<convT>(),
         TemplateArg(IsTypeDouble),
@@ -140,16 +139,16 @@ void complexMultiplyHelper(Param packed, Param sig, Param filter,
         DefineKeyValue(AF_BATCH_LHS, static_cast<int>(AF_BATCH_LHS)),
         DefineKeyValue(AF_BATCH_RHS, static_cast<int>(AF_BATCH_RHS)),
         DefineKeyValue(AF_BATCH_SAME, static_cast<int>(AF_BATCH_SAME)),
-    };
+        getTypeBuildDefinition<T, convT>()};
     if (ctDType == c32) {
         options.emplace_back(DefineKeyValue(CONVT, "float"));
     } else if (ctDType == c64 && IsTypeDouble) {
         options.emplace_back(DefineKeyValue(CONVT, "double"));
     }
-    options.emplace_back(getTypeBuildDefinition<T, convT>());
 
-    auto cplxMul = common::getKernel(
-        "complex_multiply", {fftconvolve_multiply_cl_src}, targs, options);
+    auto cplxMul = common::getKernel("complex_multiply",
+                                     std::array{fftconvolve_multiply_cl_src},
+                                     targs, options);
 
     Param sig_tmp, filter_tmp;
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
@@ -179,7 +178,7 @@ void reorderOutputHelper(Param out, Param packed, Param sig, Param filter,
         static_cast<af_dtype>(dtype_traits<convT>::af_type);
     constexpr bool RoundResult = std::is_integral<T>::value;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 5> targs = {
         TemplateTypename<T>(),     TemplateTypename<convT>(),
         TemplateArg(IsTypeDouble), TemplateArg(RoundResult),
         TemplateArg(expand),
@@ -188,16 +187,16 @@ void reorderOutputHelper(Param out, Param packed, Param sig, Param filter,
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(ROUND_OUT, static_cast<int>(RoundResult)),
         DefineKeyValue(EXPAND, static_cast<int>(expand)),
-    };
+        getTypeBuildDefinition<T, convT>()};
     if (ctDType == c32) {
         options.emplace_back(DefineKeyValue(CONVT, "float"));
     } else if (ctDType == c64 && IsTypeDouble) {
         options.emplace_back(DefineKeyValue(CONVT, "double"));
     }
-    options.emplace_back(getTypeBuildDefinition<T, convT>());
 
-    auto reorder = common::getKernel(
-        "reorder_output", {fftconvolve_reorder_cl_src}, targs, options);
+    auto reorder = common::getKernel("reorder_output",
+                                     std::array{fftconvolve_reorder_cl_src},
+                                     targs, options);
 
     int fftScale = 1;
 
diff --git a/src/backend/opencl/kernel/flood_fill.hpp b/src/backend/opencl/kernel/flood_fill.hpp
index 4061db1472..45b8dc7bf7 100644
--- a/src/backend/opencl/kernel/flood_fill.hpp
+++ b/src/backend/opencl/kernel/flood_fill.hpp
@@ -33,15 +33,13 @@ constexpr int ZERO      = 0;
 
 template<typename T>
 void initSeeds(Param out, const Param seedsx, const Param seedsy) {
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineValue(VALID),
-        DefineKey(INIT_SEEDS),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+    std::array<std::string, 4> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(VALID),
+        DefineKey(INIT_SEEDS), getTypeBuildDefinition<T>()};
 
-    auto initSeeds = common::getKernel("init_seeds", {flood_fill_cl_src},
-                                       {TemplateTypename<T>()}, options);
+    auto initSeeds =
+        common::getKernel("init_seeds", std::array{flood_fill_cl_src},
+                          TemplateArgs(TemplateTypename<T>()), options);
     cl::NDRange local(kernel::THREADS, 1, 1);
     cl::NDRange global(divup(seedsx.info.dims[0], local[0]) * local[0], 1, 1);
 
@@ -52,16 +50,14 @@ void initSeeds(Param out, const Param seedsx, const Param seedsy) {
 
 template<typename T>
 void finalizeOutput(Param out, const T newValue) {
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineValue(VALID),
-        DefineValue(ZERO),
-        DefineKey(FINALIZE_OUTPUT),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
-
-    auto finalizeOut = common::getKernel("finalize_output", {flood_fill_cl_src},
-                                         {TemplateTypename<T>()}, options);
+    std::array<std::string, 5> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(VALID),
+        DefineValue(ZERO), DefineKey(FINALIZE_OUTPUT),
+        getTypeBuildDefinition<T>()};
+
+    auto finalizeOut =
+        common::getKernel("finalize_output", std::array{flood_fill_cl_src},
+                          TemplateArgs(TemplateTypename<T>()), options);
     cl::NDRange local(kernel::THREADS_X, kernel::THREADS_Y, 1);
     cl::NDRange global(divup(out.info.dims[0], local[0]) * local[0],
                        divup(out.info.dims[1], local[1]) * local[1], 1);
@@ -77,7 +73,7 @@ void floodFill(Param out, const Param image, const Param seedsx,
     constexpr int RADIUS = 1;
 
     UNUSED(nlookup);
-    std::vector<std::string> options = {
+    std::array<std::string, 11> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineValue(RADIUS),
         DefineValue(VALID),
@@ -89,11 +85,11 @@ void floodFill(Param out, const Param image, const Param seedsx,
         DefineKeyValue(GROUP_SIZE, (THREADS_Y * THREADS_X)),
         DefineKeyValue(AF_IS_PLATFORM_NVIDIA,
                        (int)(AFCL_PLATFORM_NVIDIA == getActivePlatform())),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto floodStep = common::getKernel("flood_step", {flood_fill_cl_src},
-                                       {TemplateTypename<T>()}, options);
+    auto floodStep =
+        common::getKernel("flood_step", std::array{flood_fill_cl_src},
+                          TemplateArgs(TemplateTypename<T>()), options);
     cl::NDRange local(kernel::THREADS_X, kernel::THREADS_Y, 1);
     cl::NDRange global(divup(out.info.dims[0], local[0]) * local[0],
                        divup(out.info.dims[1], local[1]) * local[1], 1);
diff --git a/src/backend/opencl/kernel/gradient.hpp b/src/backend/opencl/kernel/gradient.hpp
index f18e2a965f..ad7ce75c84 100644
--- a/src/backend/opencl/kernel/gradient.hpp
+++ b/src/backend/opencl/kernel/gradient.hpp
@@ -29,20 +29,19 @@ void gradient(Param grad0, Param grad1, const Param in) {
     constexpr int TX = 32;
     constexpr int TY = 8;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 6> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineValue(TX),
         DefineValue(TY),
         DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
         DefineKeyValue(CPLX, static_cast<int>(af::iscplx<T>())),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto gradOp =
-        common::getKernel("gradient", {gradient_cl_src}, targs, options);
+    auto gradOp = common::getKernel("gradient", std::array{gradient_cl_src},
+                                    targs, options);
 
     cl::NDRange local(TX, TY, 1);
 
diff --git a/src/backend/opencl/kernel/harris.hpp b/src/backend/opencl/kernel/harris.hpp
index 3b3bedb3a9..eb57c8ad71 100644
--- a/src/backend/opencl/kernel/harris.hpp
+++ b/src/backend/opencl/kernel/harris.hpp
@@ -62,20 +62,22 @@ void conv_helper(Array<T> &ixx, Array<T> &ixy, Array<T> &iyy,
 
 template<typename T>
 std::array<Kernel, 4> getHarrisKernels() {
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 2> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
     return {
-        common::getKernel("second_order_deriv", {harris_cl_src}, targs,
+        common::getKernel("second_order_deriv", std::array{harris_cl_src},
+                          targs, options),
+        common::getKernel("keep_corners", std::array{harris_cl_src}, targs,
+                          options),
+        common::getKernel("harris_responses", std::array{harris_cl_src}, targs,
+                          options),
+        common::getKernel("non_maximal", std::array{harris_cl_src}, targs,
                           options),
-        common::getKernel("keep_corners", {harris_cl_src}, targs, options),
-        common::getKernel("harris_responses", {harris_cl_src}, targs, options),
-        common::getKernel("non_maximal", {harris_cl_src}, targs, options),
     };
 }
 
diff --git a/src/backend/opencl/kernel/histogram.hpp b/src/backend/opencl/kernel/histogram.hpp
index b14fe5c0b3..03a2c2c892 100644
--- a/src/backend/opencl/kernel/histogram.hpp
+++ b/src/backend/opencl/kernel/histogram.hpp
@@ -29,7 +29,7 @@ void histogram(Param out, const Param in, int nbins, float minval, float maxval,
     constexpr int THREADS_X = 256;
     constexpr int THRD_LOAD = 16;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(isLinear),
     };
@@ -41,8 +41,8 @@ void histogram(Param out, const Param in, int nbins, float minval, float maxval,
     options.emplace_back(getTypeBuildDefinition<T>());
     if (isLinear) { options.emplace_back(DefineKey(IS_LINEAR)); }
 
-    auto histogram =
-        common::getKernel("histogram", {histogram_cl_src}, targs, options);
+    auto histogram = common::getKernel(
+        "histogram", std::array{histogram_cl_src}, targs, options);
 
     int nElems  = in.info.dims[0] * in.info.dims[1];
     int blk_x   = divup(nElems, THRD_LOAD * THREADS_X);
diff --git a/src/backend/opencl/kernel/homography.hpp b/src/backend/opencl/kernel/homography.hpp
index 4585d7636e..34f1b2c7e9 100644
--- a/src/backend/opencl/kernel/homography.hpp
+++ b/src/backend/opencl/kernel/homography.hpp
@@ -31,16 +31,14 @@ constexpr int HG_THREADS   = 256;
 
 template<typename T>
 std::array<Kernel, 5> getHomographyKernels(const af_homography_type htype) {
-    std::vector<TemplateArg> targs   = {TemplateTypename<T>(),
+    std::array<TemplateArg, 2> targs = {TemplateTypename<T>(),
                                         TemplateArg(htype)};
     std::vector<std::string> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
-    options.emplace_back(
+        getTypeBuildDefinition<T>(),
         DefineKeyValue(EPS, (std::is_same<T, double>::value
                                  ? std::numeric_limits<double>::epsilon()
-                                 : std::numeric_limits<float>::epsilon())));
+                                 : std::numeric_limits<float>::epsilon()))};
     if (htype == AF_HOMOGRAPHY_RANSAC) {
         options.emplace_back(DefineKey(RANSAC));
     }
@@ -51,16 +49,16 @@ std::array<Kernel, 5> getHomographyKernels(const af_homography_type htype) {
         options.emplace_back(DefineKey(IS_CPU));
     }
     return {
-        common::getKernel("compute_homography", {homography_cl_src}, targs,
-                          options),
-        common::getKernel("eval_homography", {homography_cl_src}, targs,
-                          options),
-        common::getKernel("compute_median", {homography_cl_src}, targs,
-                          options),
-        common::getKernel("find_min_median", {homography_cl_src}, targs,
-                          options),
-        common::getKernel("compute_lmeds_inliers", {homography_cl_src}, targs,
-                          options),
+        common::getKernel("compute_homography", std::array{homography_cl_src},
+                          targs, options),
+        common::getKernel("eval_homography", std::array{homography_cl_src},
+                          targs, options),
+        common::getKernel("compute_median", std::array{homography_cl_src},
+                          targs, options),
+        common::getKernel("find_min_median", std::array{homography_cl_src},
+                          targs, options),
+        common::getKernel("compute_lmeds_inliers",
+                          std::array{homography_cl_src}, targs, options),
     };
 }
 
diff --git a/src/backend/opencl/kernel/hsv_rgb.hpp b/src/backend/opencl/kernel/hsv_rgb.hpp
index e0afe9f14e..5e30938b17 100644
--- a/src/backend/opencl/kernel/hsv_rgb.hpp
+++ b/src/backend/opencl/kernel/hsv_rgb.hpp
@@ -27,18 +27,17 @@ void hsv2rgb_convert(Param out, const Param in, bool isHSV2RGB) {
     constexpr int THREADS_X = 16;
     constexpr int THREADS_Y = 16;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(isHSV2RGB),
     };
     std::vector<std::string> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
     if (isHSV2RGB) { options.emplace_back(DefineKey(isHSV2RGB)); }
 
-    auto convert =
-        common::getKernel("hsvrgbConvert", {hsv_rgb_cl_src}, targs, options);
+    auto convert = common::getKernel(
+        "hsvrgbConvert", std::array{hsv_rgb_cl_src}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/identity.hpp b/src/backend/opencl/kernel/identity.hpp
index 6ae1aa2eb0..6369beb3ce 100644
--- a/src/backend/opencl/kernel/identity.hpp
+++ b/src/backend/opencl/kernel/identity.hpp
@@ -27,18 +27,17 @@ namespace kernel {
 
 template<typename T>
 static void identity(Param out) {
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 4> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(ONE, af::scalar_to_option(scalar<T>(1))),
         DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto identityOp =
-        common::getKernel("identity_kernel", {identity_cl_src}, targs, options);
+    auto identityOp = common::getKernel(
+        "identity_kernel", std::array{identity_cl_src}, targs, options);
 
     cl::NDRange local(32, 8);
     int groups_x = divup(out.info.dims[0], local[0]);
diff --git a/src/backend/opencl/kernel/iir.hpp b/src/backend/opencl/kernel/iir.hpp
index a2b3942b81..2bbb407fe9 100644
--- a/src/backend/opencl/kernel/iir.hpp
+++ b/src/backend/opencl/kernel/iir.hpp
@@ -29,19 +29,18 @@ void iir(Param y, Param c, Param a) {
     // allocted outside
     constexpr int MAX_A_SIZE = (1024 * sizeof(double)) / sizeof(T);
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(batch_a),
     };
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineValue(MAX_A_SIZE),
+    std::array<std::string, 5> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(MAX_A_SIZE),
         DefineKeyValue(BATCH_A, batch_a),
         DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto iir = common::getKernel("iir_kernel", {iir_cl_src}, targs, options);
+    auto iir =
+        common::getKernel("iir_kernel", std::array{iir_cl_src}, targs, options);
 
     const int groups_y = y.info.dims[1];
     const int groups_x = y.info.dims[2];
diff --git a/src/backend/opencl/kernel/index.hpp b/src/backend/opencl/kernel/index.hpp
index 3215ee22b5..881f000697 100644
--- a/src/backend/opencl/kernel/index.hpp
+++ b/src/backend/opencl/kernel/index.hpp
@@ -31,13 +31,13 @@ typedef struct {
 template<typename T>
 void index(Param out, const Param in, const IndexKernelParam_t& p,
            cl::Buffer* bPtr[4]) {
-    std::vector<std::string> options = {
+    std::array<std::string, 2> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto index    = common::getKernel("indexKernel", {index_cl_src},
-                                      {TemplateTypename<T>()}, options);
+    auto index =
+        common::getKernel("indexKernel", std::array{index_cl_src},
+                          TemplateArgs(TemplateTypename<T>()), options);
     int threads_x = 256;
     int threads_y = 1;
     cl::NDRange local(threads_x, threads_y);
diff --git a/src/backend/opencl/kernel/iota.hpp b/src/backend/opencl/kernel/iota.hpp
index b0aced9524..cbf490fbf0 100644
--- a/src/backend/opencl/kernel/iota.hpp
+++ b/src/backend/opencl/kernel/iota.hpp
@@ -31,13 +31,12 @@ void iota(Param out, const af::dim4& sdims) {
     constexpr int TILEX   = 512;
     constexpr int TILEY   = 32;
 
-    std::vector<std::string> options = {
+    std::array<std::string, 2> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto iota = common::getKernel("iota_kernel", {iota_cl_src},
-                                  {TemplateTypename<T>()}, options);
+    auto iota = common::getKernel("iota_kernel", std::array{iota_cl_src},
+                                  TemplateArgs(TemplateTypename<T>()), options);
     cl::NDRange local(IOTA_TX, IOTA_TY, 1);
 
     int blocksPerMatX = divup(out.info.dims[0], TILEX);
diff --git a/src/backend/opencl/kernel/ireduce.hpp b/src/backend/opencl/kernel/ireduce.hpp
index d6a89f03d5..5bdd55c180 100644
--- a/src/backend/opencl/kernel/ireduce.hpp
+++ b/src/backend/opencl/kernel/ireduce.hpp
@@ -33,11 +33,11 @@ void ireduceDimLauncher(Param out, cl::Buffer *oidx, Param in, cl::Buffer *iidx,
                         const int dim, const int threads_y, const bool is_first,
                         const uint groups_all[4], Param rlen) {
     ToNumStr<T> toNumStr;
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 5> targs = {
         TemplateTypename<T>(), TemplateArg(dim),       TemplateArg(op),
         TemplateArg(is_first), TemplateArg(threads_y),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 9> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(kDim, dim),
         DefineKeyValue(DIMY, threads_y),
@@ -46,12 +46,11 @@ void ireduceDimLauncher(Param out, cl::Buffer *oidx, Param in, cl::Buffer *iidx,
         DefineKeyFromStr(binOpName<op>()),
         DefineKeyValue(CPLX, af::iscplx<T>()),
         DefineKeyValue(IS_FIRST, is_first),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto ireduceDim =
-        common::getKernel("ireduce_dim_kernel",
-                          {iops_cl_src, ireduce_dim_cl_src}, targs, options);
+    auto ireduceDim = common::getKernel(
+        "ireduce_dim_kernel", std::array{iops_cl_src, ireduce_dim_cl_src},
+        targs, options);
 
     cl::NDRange local(THREADS_X, threads_y);
     cl::NDRange global(groups_all[0] * groups_all[2] * local[0],
@@ -109,13 +108,13 @@ void ireduceFirstLauncher(Param out, cl::Buffer *oidx, Param in,
                           const bool is_first, const uint groups_x,
                           const uint groups_y, Param rlen) {
     ToNumStr<T> toNumStr;
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 4> targs = {
         TemplateTypename<T>(),
         TemplateArg(op),
         TemplateArg(is_first),
         TemplateArg(threads_x),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 8> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(DIMX, threads_x),
         DefineValue(THREADS_PER_GROUP),
@@ -123,12 +122,11 @@ void ireduceFirstLauncher(Param out, cl::Buffer *oidx, Param in,
         DefineKeyFromStr(binOpName<op>()),
         DefineKeyValue(CPLX, af::iscplx<T>()),
         DefineKeyValue(IS_FIRST, is_first),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto ireduceFirst =
-        common::getKernel("ireduce_first_kernel",
-                          {iops_cl_src, ireduce_first_cl_src}, targs, options);
+    auto ireduceFirst = common::getKernel(
+        "ireduce_first_kernel", std::array{iops_cl_src, ireduce_first_cl_src},
+        targs, options);
 
     cl::NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     cl::NDRange global(groups_x * in.info.dims[2] * local[0],
diff --git a/src/backend/opencl/kernel/laset.hpp b/src/backend/opencl/kernel/laset.hpp
index 07399511e6..fb52f3571f 100644
--- a/src/backend/opencl/kernel/laset.hpp
+++ b/src/backend/opencl/kernel/laset.hpp
@@ -46,20 +46,18 @@ void laset(int m, int n, T offdiag, T diag, cl_mem dA, size_t dA_offset,
     constexpr int BLK_X = 64;
     constexpr int BLK_Y = 32;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(uplo),
     };
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineValue(BLK_X),
+    std::array<std::string, 5> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(BLK_X),
         DefineValue(BLK_Y),
         DefineKeyValue(IS_CPLX, static_cast<int>(af::iscplx<T>())),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto lasetOp =
-        common::getKernel(laset_name<uplo>(), {laset_cl_src}, targs, options);
+    auto lasetOp = common::getKernel(laset_name<uplo>(),
+                                     std::array{laset_cl_src}, targs, options);
 
     int groups_x = (m - 1) / BLK_X + 1;
     int groups_y = (n - 1) / BLK_Y + 1;
diff --git a/src/backend/opencl/kernel/laset_band.hpp b/src/backend/opencl/kernel/laset_band.hpp
index 1043310f70..9ceffec9e0 100644
--- a/src/backend/opencl/kernel/laset_band.hpp
+++ b/src/backend/opencl/kernel/laset_band.hpp
@@ -36,15 +36,15 @@ void laset_band(int m, int  n, int k,
 {
     static const std::string src(laset_band_cl, laset_band_cl_len);
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(), TemplateArg(uplo),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 4> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineValue(NB),
         DefineKeyValue(IS_CPLX, static_cast<int>(af::iscplx<T>())),
+        getTypeBuildDefinition<T>()
     };
-    options.emplace_back(getTypeBuildDefinition<T>());
 
     auto lasetBandOp = common::getKernel(laset_band_name<uplo>(), {src}, targs, options);
 
diff --git a/src/backend/opencl/kernel/laswp.hpp b/src/backend/opencl/kernel/laswp.hpp
index ace55aacfe..0fd58eb961 100644
--- a/src/backend/opencl/kernel/laswp.hpp
+++ b/src/backend/opencl/kernel/laswp.hpp
@@ -34,16 +34,15 @@ void laswp(int n, cl_mem in, size_t offset, int ldda, int k1, int k2,
            const int *ipiv, int inci, cl::CommandQueue &queue) {
     constexpr int NTHREADS = 256;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineValue(MAX_PIVOTS),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+    std::array<std::string, 3> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(MAX_PIVOTS),
+        getTypeBuildDefinition<T>()};
 
-    auto laswpOp = common::getKernel("laswp", {laswp_cl_src}, targs, options);
+    auto laswpOp =
+        common::getKernel("laswp", std::array{laswp_cl_src}, targs, options);
 
     int groups = divup(n, NTHREADS);
     cl::NDRange local(NTHREADS);
diff --git a/src/backend/opencl/kernel/lookup.hpp b/src/backend/opencl/kernel/lookup.hpp
index f00ef8a8bb..ed82d58b6a 100644
--- a/src/backend/opencl/kernel/lookup.hpp
+++ b/src/backend/opencl/kernel/lookup.hpp
@@ -29,17 +29,15 @@ void lookup(Param out, const Param in, const Param indices,
     constexpr int THREADS_X = 32;
     constexpr int THREADS_Y = 8;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 3> targs = {
         TemplateTypename<in_t>(),
         TemplateTypename<idx_t>(),
         TemplateArg(dim),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 4> options = {
         DefineKeyValue(in_t, dtype_traits<in_t>::getName()),
         DefineKeyValue(idx_t, dtype_traits<idx_t>::getName()),
-        DefineKeyValue(DIM, dim),
-    };
-    options.emplace_back(getTypeBuildDefinition<in_t, idx_t>());
+        DefineKeyValue(DIM, dim), getTypeBuildDefinition<in_t, idx_t>()};
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
@@ -49,8 +47,8 @@ void lookup(Param out, const Param in, const Param indices,
     cl::NDRange global(blk_x * out.info.dims[2] * THREADS_X,
                        blk_y * out.info.dims[3] * THREADS_Y);
 
-    auto arrIdxOp =
-        common::getKernel("lookupND", {lookup_cl_src}, targs, options);
+    auto arrIdxOp = common::getKernel("lookupND", std::array{lookup_cl_src},
+                                      targs, options);
 
     arrIdxOp(cl::EnqueueArgs(getQueue(), global, local), *out.data, out.info,
              *in.data, in.info, *indices.data, indices.info, blk_x, blk_y);
diff --git a/src/backend/opencl/kernel/lu_split.hpp b/src/backend/opencl/kernel/lu_split.hpp
index f2ac2d983d..e27eb78955 100644
--- a/src/backend/opencl/kernel/lu_split.hpp
+++ b/src/backend/opencl/kernel/lu_split.hpp
@@ -30,20 +30,18 @@ void luSplitLauncher(Param lower, Param upper, const Param in, bool same_dims) {
     constexpr unsigned TILEX = 128;
     constexpr unsigned TILEY = 32;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(same_dims),
     };
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineValue(same_dims),
+    std::array<std::string, 5> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(same_dims),
         DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
         DefineKeyValue(ONE, af::scalar_to_option(scalar<T>(1))),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto luSplit =
-        common::getKernel("luSplit", {lu_split_cl_src}, targs, options);
+    auto luSplit = common::getKernel("luSplit", std::array{lu_split_cl_src},
+                                     targs, options);
 
     cl::NDRange local(TX, TY);
 
diff --git a/src/backend/opencl/kernel/match_template.hpp b/src/backend/opencl/kernel/match_template.hpp
index f32fd722ef..5b7c471c33 100644
--- a/src/backend/opencl/kernel/match_template.hpp
+++ b/src/backend/opencl/kernel/match_template.hpp
@@ -28,13 +28,13 @@ void matchTemplate(Param out, const Param srch, const Param tmplt,
     constexpr int THREADS_X = 16;
     constexpr int THREADS_Y = 16;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 4> targs = {
         TemplateTypename<inType>(),
         TemplateTypename<outType>(),
         TemplateArg(mType),
         TemplateArg(needMean),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 14> options = {
         DefineKeyValue(inType, dtype_traits<inType>::getName()),
         DefineKeyValue(outType, dtype_traits<outType>::getName()),
         DefineKeyValue(MATCH_T, static_cast<int>(mType)),
@@ -48,11 +48,10 @@ void matchTemplate(Param out, const Param srch, const Param tmplt,
         DefineKeyValue(AF_NCC, static_cast<int>(AF_NCC)),
         DefineKeyValue(AF_ZNCC, static_cast<int>(AF_ZNCC)),
         DefineKeyValue(AF_SHD, static_cast<int>(AF_SHD)),
-    };
-    options.emplace_back(getTypeBuildDefinition<outType>());
+        getTypeBuildDefinition<outType>()};
 
-    auto matchImgOp = common::getKernel("matchTemplate", {matchTemplate_cl_src},
-                                        targs, options);
+    auto matchImgOp = common::getKernel(
+        "matchTemplate", std::array{matchTemplate_cl_src}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/mean.hpp b/src/backend/opencl/kernel/mean.hpp
index 35bcee0fef..3149da3280 100644
--- a/src/backend/opencl/kernel/mean.hpp
+++ b/src/backend/opencl/kernel/mean.hpp
@@ -108,7 +108,7 @@ void meanDimLauncher(Param out, Param owt, Param in, Param inWeight,
     ToNumStr<Tw> twNumStr;
     common::Transform<uint, Tw, af_add_t> transform_weight;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 7> targs = {
         TemplateTypename<Ti>(),     TemplateTypename<To>(),
         TemplateTypename<Tw>(),     TemplateArg(dim),
         TemplateArg(threads_y),     TemplateArg(input_weight),
@@ -124,13 +124,13 @@ void meanDimLauncher(Param out, Param owt, Param in, Param inWeight,
         DefineKeyValue(init_To, toNumStr(common::Binary<To, af_add_t>::init())),
         DefineKeyValue(init_Tw, twNumStr(transform_weight(0))),
         DefineKeyValue(one_Tw, twNumStr(transform_weight(1))),
-    };
-    options.emplace_back(getTypeBuildDefinition<Ti, To>());
+        getTypeBuildDefinition<Ti, To>()};
     if (input_weight) { options.emplace_back(DefineKey(INPUT_WEIGHT)); }
     if (output_weight) { options.emplace_back(DefineKey(OUTPUT_WEIGHT)); }
 
     auto meanOp = common::getKernel(
-        "meanDim", {mean_ops_cl_src, mean_dim_cl_src}, targs, options);
+        "meanDim", std::array{mean_ops_cl_src, mean_dim_cl_src}, targs,
+        options);
 
     NDRange local(THREADS_X, threads_y);
     NDRange global(groups_all[0] * groups_all[2] * local[0],
@@ -202,7 +202,7 @@ void meanFirstLauncher(Param out, Param owt, Param in, Param inWeight,
     ToNumStr<Tw> twNumStr;
     common::Transform<uint, Tw, af_add_t> transform_weight;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 6> targs = {
         TemplateTypename<Ti>(),    TemplateTypename<To>(),
         TemplateTypename<Tw>(),    TemplateArg(threads_x),
         TemplateArg(input_weight), TemplateArg(output_weight),
@@ -222,7 +222,8 @@ void meanFirstLauncher(Param out, Param owt, Param in, Param inWeight,
     if (output_weight) { options.emplace_back(DefineKey(OUTPUT_WEIGHT)); }
 
     auto meanOp = common::getKernel(
-        "meanFirst", {mean_ops_cl_src, mean_first_cl_src}, targs, options);
+        "meanFirst", std::array{mean_ops_cl_src, mean_first_cl_src}, targs,
+        options);
 
     NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     NDRange global(groups_x * in.info.dims[2] * local[0],
diff --git a/src/backend/opencl/kernel/meanshift.hpp b/src/backend/opencl/kernel/meanshift.hpp
index a616f6abc0..fb92f18866 100644
--- a/src/backend/opencl/kernel/meanshift.hpp
+++ b/src/backend/opencl/kernel/meanshift.hpp
@@ -32,19 +32,18 @@ void meanshift(Param out, const Param in, const float spatialSigma,
     constexpr int THREADS_X = 16;
     constexpr int THREADS_Y = 16;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(is_color),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 4> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(AccType, dtype_traits<AccType>::getName()),
         DefineKeyValue(MAX_CHANNELS, (is_color ? 3 : 1)),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto meanshiftOp =
-        common::getKernel("meanshift", {meanshift_cl_src}, targs, options);
+    auto meanshiftOp = common::getKernel(
+        "meanshift", std::array{meanshift_cl_src}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/medfilt.hpp b/src/backend/opencl/kernel/medfilt.hpp
index af1d4f3615..e8af452eda 100644
--- a/src/backend/opencl/kernel/medfilt.hpp
+++ b/src/backend/opencl/kernel/medfilt.hpp
@@ -35,22 +35,21 @@ void medfilt1(Param out, const Param in, const unsigned w_wid,
     const int ARR_SIZE = (w_wid - w_wid / 2) + 1;
     size_t loc_size    = (THREADS_X + w_wid - 1) * sizeof(T);
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(pad),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 7> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(pad, static_cast<int>(pad)),
         DefineKeyValue(AF_PAD_ZERO, static_cast<int>(AF_PAD_ZERO)),
         DefineKeyValue(AF_PAD_SYM, static_cast<int>(AF_PAD_SYM)),
         DefineValue(ARR_SIZE),
         DefineValue(w_wid),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto medfiltOp =
-        common::getKernel("medfilt1", {medfilt1_cl_src}, targs, options);
+    auto medfiltOp = common::getKernel("medfilt1", std::array{medfilt1_cl_src},
+                                       targs, options);
 
     cl::NDRange local(THREADS_X, 1, 1);
 
@@ -71,13 +70,13 @@ void medfilt2(Param out, const Param in, const af_border_type pad,
     const size_t loc_size =
         (THREADS_X + w_len - 1) * (THREADS_Y + w_wid - 1) * sizeof(T);
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 4> targs = {
         TemplateTypename<T>(),
         TemplateArg(pad),
         TemplateArg(w_len),
         TemplateArg(w_wid),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 8> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(pad, static_cast<int>(pad)),
         DefineKeyValue(AF_PAD_ZERO, static_cast<int>(AF_PAD_ZERO)),
@@ -85,11 +84,10 @@ void medfilt2(Param out, const Param in, const af_border_type pad,
         DefineValue(ARR_SIZE),
         DefineValue(w_wid),
         DefineValue(w_len),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto medfiltOp =
-        common::getKernel("medfilt2", {medfilt2_cl_src}, targs, options);
+    auto medfiltOp = common::getKernel("medfilt2", std::array{medfilt2_cl_src},
+                                       targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/memcopy.hpp b/src/backend/opencl/kernel/memcopy.hpp
index 159fe4d35a..e4091fea53 100644
--- a/src/backend/opencl/kernel/memcopy.hpp
+++ b/src/backend/opencl/kernel/memcopy.hpp
@@ -23,9 +23,6 @@
 #include <string>
 #include <vector>
 
-using std::string;
-using std::vector;
-
 namespace opencl {
 namespace kernel {
 typedef struct {
@@ -149,7 +146,7 @@ void memcopy(const cl::Buffer& b_out, const dim4& ostrides,
         : th.loop1 ? th.loop3 ? "memCopyLoop13" : "memCopyLoop1"
         : th.loop3 ? "memCopyLoop3"
                    : "memCopy"};  // Conversion to  base vector types.
-    const char* tArg{
+    TemplateArg tArg{
         sizeofNewT == 1   ? "char"
         : sizeofNewT == 2 ? "short"
         : sizeofNewT == 4 ? "float"
@@ -157,8 +154,9 @@ void memcopy(const cl::Buffer& b_out, const dim4& ostrides,
         : sizeofNewT == 16
             ? "float4"
             : "type is larger than 16 bytes, which is unsupported"};
-    auto memCopy{common::getKernel(kernelName, {memcopy_cl_src}, {tArg},
-                                   {DefineKeyValue(T, tArg)})};
+    auto memCopy{common::getKernel(kernelName, std::array{memcopy_cl_src},
+                                   std::array{tArg},
+                                   std::array{DefineKeyValue(T, tArg)})};
     const cl::NDRange local{th.genLocal(memCopy.get())};
     const cl::NDRange global{th.genGlobal(local)};
 
@@ -209,12 +207,12 @@ void copy(const Param out, const Param in, dim_t ondims,
                                 std::is_same<inType, cdouble>::value};
     const char* factorType[]{"float", "double"};
 
-    const std::vector<TemplateArg> targs{
+    const std::array<TemplateArg, 5> targs{
         TemplateTypename<inType>(), TemplateTypename<outType>(),
         TemplateArg(same_dims),     TemplateArg(factorType[factorTypeIdx]),
         TemplateArg(factor != 1.0),
     };
-    const std::vector<std::string> options{
+    const std::array<std::string, 8> options{
         DefineKeyValue(inType, dtype_traits<inType>::getName()),
         DefineKeyValue(outType, dtype_traits<outType>::getName()),
         std::string(" -D inType_") + dtype_traits<inType>::getName(),
@@ -222,7 +220,7 @@ void copy(const Param out, const Param in, dim_t ondims,
         DefineKeyValue(SAME_DIMS, static_cast<int>(same_dims)),
         std::string(" -D factorType=") + factorType[factorTypeIdx],
         std::string((factor != 1.0) ? " -D FACTOR" : " -D NOFACTOR"),
-        {getTypeBuildDefinition<inType, outType>()},
+        getTypeBuildDefinition<inType, outType>(),
     };
 
     threadsMgt<int> th(odims_.dims, ondims_, 1, 1, totalSize, sizeof(outType));
@@ -230,7 +228,7 @@ void copy(const Param out, const Param in, dim_t ondims,
                                   : th.loop3 ? "scaledCopyLoop13"
                                   : th.loop1 ? "scaledCopyLoop1"
                                              : "scaledCopy",
-                                  {copy_cl_src}, targs, options);
+                                  std::array{copy_cl_src}, targs, options);
     const cl::NDRange local{th.genLocal(copy.get())};
     const cl::NDRange global{th.genGlobal(local)};
 
diff --git a/src/backend/opencl/kernel/moments.hpp b/src/backend/opencl/kernel/moments.hpp
index facabba3ff..6da71b9833 100644
--- a/src/backend/opencl/kernel/moments.hpp
+++ b/src/backend/opencl/kernel/moments.hpp
@@ -28,18 +28,17 @@ template<typename T>
 void moments(Param out, const Param in, af_moment_type moment) {
     constexpr int THREADS = 128;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(out.info.dims[0]),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 3> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(MOMENTS_SZ, out.info.dims[0]),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto momentsOp =
-        common::getKernel("moments", {moments_cl_src}, targs, options);
+    auto momentsOp = common::getKernel("moments", std::array{moments_cl_src},
+                                       targs, options);
 
     cl::NDRange local(THREADS, 1, 1);
     cl::NDRange global(in.info.dims[1] * local[0],
diff --git a/src/backend/opencl/kernel/morph.hpp b/src/backend/opencl/kernel/morph.hpp
index a89b729613..43b5d6d443 100644
--- a/src/backend/opencl/kernel/morph.hpp
+++ b/src/backend/opencl/kernel/morph.hpp
@@ -55,7 +55,8 @@ void morph(Param out, const Param in, const Param mask, bool isDilation) {
     };
     options.emplace_back(getTypeBuildDefinition<T>());
 
-    auto morphOp = common::getKernel("morph", {morph_cl_src}, targs, options);
+    auto morphOp =
+        common::getKernel("morph", std::array{morph_cl_src}, targs, options);
 
     NDRange local(THREADS_X, THREADS_Y);
 
@@ -114,7 +115,8 @@ void morph3d(Param out, const Param in, const Param mask, bool isDilation) {
     };
     options.emplace_back(getTypeBuildDefinition<T>());
 
-    auto morphOp = common::getKernel("morph3d", {morph_cl_src}, targs, options);
+    auto morphOp =
+        common::getKernel("morph3d", std::array{morph_cl_src}, targs, options);
 
     NDRange local(CUBE_X, CUBE_Y, CUBE_Z);
 
diff --git a/src/backend/opencl/kernel/nearest_neighbour.hpp b/src/backend/opencl/kernel/nearest_neighbour.hpp
index f8e523f03c..841a844038 100644
--- a/src/backend/opencl/kernel/nearest_neighbour.hpp
+++ b/src/backend/opencl/kernel/nearest_neighbour.hpp
@@ -45,7 +45,7 @@ void allDistances(Param dist, Param query, Param train, const dim_t dist_dim,
     unsigned unroll_len = nextpow2(feat_len);
     if (unroll_len != feat_len) unroll_len = 0;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 4> targs = {
         TemplateTypename<T>(),
         TemplateArg(dist_type),
         TemplateArg(use_lmem),
@@ -70,8 +70,9 @@ void allDistances(Param dist, Param query, Param train, const dim_t dist_dim,
         options.emplace_back(DefineKeyValue(DISTOP, "_shd_"));
         options.emplace_back(DefineKey(__SHD__));
     }
-    auto hmOp = common::getKernel("knnAllDistances", {nearest_neighbour_cl_src},
-                                  targs, options);
+    auto hmOp =
+        common::getKernel("knnAllDistances",
+                          std::array{nearest_neighbour_cl_src}, targs, options);
 
     const dim_t sample_dim = (dist_dim == 0) ? 1 : 0;
 
diff --git a/src/backend/opencl/kernel/orb.hpp b/src/backend/opencl/kernel/orb.hpp
index b755644e37..f2e72c7317 100644
--- a/src/backend/opencl/kernel/orb.hpp
+++ b/src/backend/opencl/kernel/orb.hpp
@@ -87,10 +87,14 @@ std::array<Kernel, 4> getOrbKernels() {
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
     return {
-        common::getKernel("harris_response", {orb_cl_src}, targs, compileOpts),
-        common::getKernel("keep_features", {orb_cl_src}, targs, compileOpts),
-        common::getKernel("centroid_angle", {orb_cl_src}, targs, compileOpts),
-        common::getKernel("extract_orb", {orb_cl_src}, targs, compileOpts),
+        common::getKernel("harris_response", std::array{orb_cl_src}, targs,
+                          compileOpts),
+        common::getKernel("keep_features", std::array{orb_cl_src}, targs,
+                          compileOpts),
+        common::getKernel("centroid_angle", std::array{orb_cl_src}, targs,
+                          compileOpts),
+        common::getKernel("extract_orb", std::array{orb_cl_src}, targs,
+                          compileOpts),
     };
 }
 
diff --git a/src/backend/opencl/kernel/pad_array_borders.hpp b/src/backend/opencl/kernel/pad_array_borders.hpp
index 567f2d33b4..4d18b06099 100644
--- a/src/backend/opencl/kernel/pad_array_borders.hpp
+++ b/src/backend/opencl/kernel/pad_array_borders.hpp
@@ -45,8 +45,9 @@ void padBorders(Param out, const Param in, dim4 const& lBPadding,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto pad = common::getKernel("padBorders", {pad_array_borders_cl_src},
-                                 tmpltArgs, compileOpts);
+    auto pad =
+        common::getKernel("padBorders", std::array{pad_array_borders_cl_src},
+                          tmpltArgs, compileOpts);
 
     NDRange local(PADB_THREADS_X, PADB_THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/random_engine.hpp b/src/backend/opencl/kernel/random_engine.hpp
index 21f932ba28..c15f9e292f 100644
--- a/src/backend/opencl/kernel/random_engine.hpp
+++ b/src/backend/opencl/kernel/random_engine.hpp
@@ -56,7 +56,7 @@ static Kernel getRandomEngineKernel(const af_random_engine_type type,
         default:
             AF_ERROR("Random Engine Type Not Supported", AF_ERR_NOT_SUPPORTED);
     }
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(kerIdx),
     };
@@ -162,8 +162,9 @@ void initMersenneState(cl::Buffer state, cl::Buffer table, const uintl &seed) {
     cl::NDRange local(THREADS_PER_GROUP, 1);
     cl::NDRange global(local[0] * MAX_BLOCKS, 1);
 
-    auto initOp = common::getKernel("mersenneInitState",
-                                    {random_engine_mersenne_init_cl_src}, {});
+    auto initOp =
+        common::getKernel("mersenneInitState",
+                          std::array{random_engine_mersenne_init_cl_src}, {});
     initOp(cl::EnqueueArgs(getQueue(), global, local), state, table, seed);
     CL_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/opencl/kernel/range.hpp b/src/backend/opencl/kernel/range.hpp
index b8eb75dfe6..d4a5acbd33 100644
--- a/src/backend/opencl/kernel/range.hpp
+++ b/src/backend/opencl/kernel/range.hpp
@@ -30,14 +30,13 @@ void range(Param out, const int dim) {
     constexpr int RANGE_TILEX = 512;
     constexpr int RANGE_TILEY = 32;
 
-    std::vector<TemplateArg> targs   = {TemplateTypename<T>()};
-    std::vector<std::string> options = {
+    std::array<TemplateArg, 1> targs   = {TemplateTypename<T>()};
+    std::array<std::string, 2> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto rangeOp =
-        common::getKernel("range_kernel", {range_cl_src}, targs, options);
+    auto rangeOp = common::getKernel("range_kernel", std::array{range_cl_src},
+                                     targs, options);
 
     cl::NDRange local(RANGE_TX, RANGE_TY, 1);
 
diff --git a/src/backend/opencl/kernel/reduce.hpp b/src/backend/opencl/kernel/reduce.hpp
index f3c8022b71..f52d044bcb 100644
--- a/src/backend/opencl/kernel/reduce.hpp
+++ b/src/backend/opencl/kernel/reduce.hpp
@@ -38,11 +38,11 @@ void reduceDimLauncher(Param out, Param in, const int dim, const uint threads_y,
                        const uint groups_all[4], int change_nan,
                        double nanval) {
     ToNumStr<To> toNumStr;
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 5> targs = {
         TemplateTypename<Ti>(), TemplateTypename<To>(), TemplateArg(dim),
         TemplateArg(op),        TemplateArg(threads_y),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 10> options = {
         DefineKeyValue(Ti, dtype_traits<Ti>::getName()),
         DefineKeyValue(To, dtype_traits<To>::getName()),
         DefineKeyValue(T, "To"),
@@ -52,11 +52,11 @@ void reduceDimLauncher(Param out, Param in, const int dim, const uint threads_y,
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
         DefineKeyValue(CPLX, af::iscplx<Ti>()),
-    };
-    options.emplace_back(getTypeBuildDefinition<Ti, To>());
+        getTypeBuildDefinition<Ti, To>()};
 
     auto reduceDim = common::getKernel(
-        "reduce_dim_kernel", {ops_cl_src, reduce_dim_cl_src}, targs, options);
+        "reduce_dim_kernel", std::array{ops_cl_src, reduce_dim_cl_src}, targs,
+        options);
 
     cl::NDRange local(THREADS_X, threads_y);
     cl::NDRange global(groups_all[0] * groups_all[2] * local[0],
@@ -115,13 +115,13 @@ void reduceAllLauncher(Param out, Param in, const uint groups_x,
                        const uint groups_y, const uint threads_x,
                        int change_nan, double nanval) {
     ToNumStr<To> toNumStr;
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 4> targs = {
         TemplateTypename<Ti>(),
         TemplateTypename<To>(),
         TemplateArg(op),
         TemplateArg(threads_x),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 9> options = {
         DefineKeyValue(Ti, dtype_traits<Ti>::getName()),
         DefineKeyValue(To, dtype_traits<To>::getName()),
         DefineKeyValue(T, "To"),
@@ -130,11 +130,11 @@ void reduceAllLauncher(Param out, Param in, const uint groups_x,
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
         DefineKeyValue(CPLX, af::iscplx<Ti>()),
-    };
-    options.emplace_back(getTypeBuildDefinition<Ti, To>());
+        getTypeBuildDefinition<Ti, To>()};
 
     auto reduceAll = common::getKernel(
-        "reduce_all_kernel", {ops_cl_src, reduce_all_cl_src}, targs, options);
+        "reduce_all_kernel", std::array{ops_cl_src, reduce_all_cl_src}, targs,
+        options);
 
     cl::NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     cl::NDRange global(groups_x * in.info.dims[2] * local[0],
@@ -163,13 +163,13 @@ void reduceFirstLauncher(Param out, Param in, const uint groups_x,
                          const uint groups_y, const uint threads_x,
                          int change_nan, double nanval) {
     ToNumStr<To> toNumStr;
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 4> targs = {
         TemplateTypename<Ti>(),
         TemplateTypename<To>(),
         TemplateArg(op),
         TemplateArg(threads_x),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 9> options = {
         DefineKeyValue(Ti, dtype_traits<Ti>::getName()),
         DefineKeyValue(To, dtype_traits<To>::getName()),
         DefineKeyValue(T, "To"),
@@ -178,12 +178,11 @@ void reduceFirstLauncher(Param out, Param in, const uint groups_x,
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
         DefineKeyValue(CPLX, af::iscplx<Ti>()),
-    };
-    options.emplace_back(getTypeBuildDefinition<Ti, To>());
+        getTypeBuildDefinition<Ti, To>()};
 
-    auto reduceFirst =
-        common::getKernel("reduce_first_kernel",
-                          {ops_cl_src, reduce_first_cl_src}, targs, options);
+    auto reduceFirst = common::getKernel(
+        "reduce_first_kernel", std::array{ops_cl_src, reduce_first_cl_src},
+        targs, options);
 
     cl::NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     cl::NDRange global(groups_x * in.info.dims[2] * local[0],
diff --git a/src/backend/opencl/kernel/reduce_by_key.hpp b/src/backend/opencl/kernel/reduce_by_key.hpp
index ec841dafc4..79779ca320 100644
--- a/src/backend/opencl/kernel/reduce_by_key.hpp
+++ b/src/backend/opencl/kernel/reduce_by_key.hpp
@@ -65,7 +65,8 @@ void reduceBlocksByKeyDim(cl::Buffer *reduced_block_sizes, Param keys_out,
 
     auto reduceBlocksByKeyDim = common::getKernel(
         "reduce_blocks_by_key_dim",
-        {ops_cl_src, reduce_blocks_by_key_dim_cl_src}, tmpltArgs, compileOpts);
+        std::array{ops_cl_src, reduce_blocks_by_key_dim_cl_src}, tmpltArgs,
+        compileOpts);
     int numBlocks = divup(n, threads_x);
 
     cl::NDRange local(threads_x);
@@ -105,10 +106,10 @@ void reduceBlocksByKey(cl::Buffer *reduced_block_sizes, Param keys_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    auto reduceBlocksByKeyFirst =
-        common::getKernel("reduce_blocks_by_key_first",
-                          {ops_cl_src, reduce_blocks_by_key_first_cl_src},
-                          tmpltArgs, compileOpts);
+    auto reduceBlocksByKeyFirst = common::getKernel(
+        "reduce_blocks_by_key_first",
+        std::array{ops_cl_src, reduce_blocks_by_key_first_cl_src}, tmpltArgs,
+        compileOpts);
     int numBlocks = divup(n, threads_x);
 
     cl::NDRange local(threads_x);
@@ -146,9 +147,10 @@ void finalBoundaryReduce(cl::Buffer *reduced_block_sizes, Param keys_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
-    auto finalBoundaryReduce = common::getKernel(
-        "final_boundary_reduce", {ops_cl_src, reduce_by_key_boundary_cl_src},
-        tmpltArgs, compileOpts);
+    auto finalBoundaryReduce =
+        common::getKernel("final_boundary_reduce",
+                          std::array{ops_cl_src, reduce_by_key_boundary_cl_src},
+                          tmpltArgs, compileOpts);
 
     cl::NDRange local(threads_x);
     cl::NDRange global(threads_x * numBlocks);
@@ -184,10 +186,10 @@ void finalBoundaryReduceDim(cl::Buffer *reduced_block_sizes, Param keys_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
-    auto finalBoundaryReduceDim =
-        common::getKernel("final_boundary_reduce_dim",
-                          {ops_cl_src, reduce_by_key_boundary_dim_cl_src},
-                          tmpltArgs, compileOpts);
+    auto finalBoundaryReduceDim = common::getKernel(
+        "final_boundary_reduce_dim",
+        std::array{ops_cl_src, reduce_by_key_boundary_dim_cl_src}, tmpltArgs,
+        compileOpts);
 
     cl::NDRange local(threads_x);
     cl::NDRange global(threads_x * numBlocks,
@@ -220,9 +222,9 @@ void compact(cl::Buffer *reduced_block_sizes, Param keys_out, Param vals_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
-    auto compact =
-        common::getKernel("compact", {ops_cl_src, reduce_by_key_compact_cl_src},
-                          tmpltArgs, compileOpts);
+    auto compact = common::getKernel(
+        "compact", std::array{ops_cl_src, reduce_by_key_compact_cl_src},
+        tmpltArgs, compileOpts);
 
     cl::NDRange local(threads_x);
     cl::NDRange global(threads_x * numBlocks, vals_out.info.dims[1],
@@ -256,7 +258,7 @@ void compactDim(cl::Buffer *reduced_block_sizes, Param keys_out, Param vals_out,
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
     auto compactDim = common::getKernel(
-        "compact_dim", {ops_cl_src, reduce_by_key_compact_dim_cl_src},
+        "compact_dim", std::array{ops_cl_src, reduce_by_key_compact_dim_cl_src},
         tmpltArgs, compileOpts);
 
     cl::NDRange local(threads_x);
@@ -285,10 +287,10 @@ void testNeedsReduction(cl::Buffer needs_reduction, cl::Buffer needs_boundary,
         DefineKeyValue(DIMX, threads_x),
     };
 
-    auto testIfNeedsReduction =
-        common::getKernel("test_needs_reduction",
-                          {ops_cl_src, reduce_by_key_needs_reduction_cl_src},
-                          tmpltArgs, compileOpts);
+    auto testIfNeedsReduction = common::getKernel(
+        "test_needs_reduction",
+        std::array{ops_cl_src, reduce_by_key_needs_reduction_cl_src}, tmpltArgs,
+        compileOpts);
 
     cl::NDRange local(threads_x);
     cl::NDRange global(threads_x * numBlocks);
diff --git a/src/backend/opencl/kernel/regions.hpp b/src/backend/opencl/kernel/regions.hpp
index 0baa0abfaf..710ccdf64b 100644
--- a/src/backend/opencl/kernel/regions.hpp
+++ b/src/backend/opencl/kernel/regions.hpp
@@ -66,9 +66,12 @@ std::array<Kernel, 3> getRegionsKernels(const bool full_conn,
     options.emplace_back(getTypeBuildDefinition<T>());
 
     return {
-        common::getKernel("initial_label", {regions_cl_src}, targs, options),
-        common::getKernel("final_relabel", {regions_cl_src}, targs, options),
-        common::getKernel("update_equiv", {regions_cl_src}, targs, options),
+        common::getKernel("initial_label", std::array{regions_cl_src}, targs,
+                          options),
+        common::getKernel("final_relabel", std::array{regions_cl_src}, targs,
+                          options),
+        common::getKernel("update_equiv", std::array{regions_cl_src}, targs,
+                          options),
     };
 }
 
diff --git a/src/backend/opencl/kernel/reorder.hpp b/src/backend/opencl/kernel/reorder.hpp
index 550ff127cc..e2dc87f481 100644
--- a/src/backend/opencl/kernel/reorder.hpp
+++ b/src/backend/opencl/kernel/reorder.hpp
@@ -28,16 +28,15 @@ void reorder(Param out, const Param in, const dim_t* rdims) {
     constexpr int TILEX = 512;
     constexpr int TILEY = 32;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 1> targs = {
         TemplateTypename<T>(),
     };
-    std::vector<std::string> options = {
+    std::array<std::string, 2> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+        getTypeBuildDefinition<T>()};
 
-    auto reorderOp =
-        common::getKernel("reorder_kernel", {reorder_cl_src}, targs, options);
+    auto reorderOp = common::getKernel(
+        "reorder_kernel", std::array{reorder_cl_src}, targs, options);
 
     cl::NDRange local(TX, TY, 1);
 
diff --git a/src/backend/opencl/kernel/resize.hpp b/src/backend/opencl/kernel/resize.hpp
index 0e55caa4e7..ae0184a4a1 100644
--- a/src/backend/opencl/kernel/resize.hpp
+++ b/src/backend/opencl/kernel/resize.hpp
@@ -40,7 +40,7 @@ void resize(Param out, const Param in, const af_interp_type method) {
     constexpr bool IsComplex =
         std::is_same<T, cfloat>::value || std::is_same<T, cdouble>::value;
 
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(method),
     };
@@ -48,12 +48,10 @@ void resize(Param out, const Param in, const af_interp_type method) {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(VT, dtype_traits<vtype_t<T>>::getName()),
         DefineKeyValue(WT, dtype_traits<wtype_t<BT>>::getName()),
-        DefineKeyValue(CPLX, (IsComplex ? 1 : 0)),
-    };
+        DefineKeyValue(CPLX, (IsComplex ? 1 : 0)), getTypeBuildDefinition<T>()};
     if (IsComplex) {
         options.emplace_back(DefineKeyValue(TB, dtype_traits<BT>::getName()));
     }
-    options.emplace_back(getTypeBuildDefinition<T>());
 
     switch (method) {
         case AF_INTERP_NEAREST:
@@ -68,8 +66,8 @@ void resize(Param out, const Param in, const af_interp_type method) {
         default: break;
     }
 
-    auto resizeOp =
-        common::getKernel("resize_kernel", {resize_cl_src}, targs, options);
+    auto resizeOp = common::getKernel(
+        "resize_kernel", std::array{resize_cl_src}, targs, options);
 
     cl::NDRange local(RESIZE_TX, RESIZE_TY, 1);
 
diff --git a/src/backend/opencl/kernel/rotate.hpp b/src/backend/opencl/kernel/rotate.hpp
index 2edf47cf91..999a7f25a5 100644
--- a/src/backend/opencl/kernel/rotate.hpp
+++ b/src/backend/opencl/kernel/rotate.hpp
@@ -79,8 +79,9 @@ void rotate(Param out, const Param in, const float theta, af_interp_type method,
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
     addInterpEnumOptions(compileOpts);
 
-    auto rotate = common::getKernel(
-        "rotateKernel", {interp_cl_src, rotate_cl_src}, tmpltArgs, compileOpts);
+    auto rotate = common::getKernel("rotateKernel",
+                                    std::array{interp_cl_src, rotate_cl_src},
+                                    tmpltArgs, compileOpts);
 
     const float c = cos(-theta), s = sin(-theta);
     float tx, ty;
diff --git a/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt b/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
index a59904cfe7..e5d0de3a97 100644
--- a/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
+++ b/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
@@ -40,6 +40,7 @@ foreach(SBK_BINARY_OP ${SBK_BINARY_OPS})
         $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>
         $<TARGET_PROPERTY:OpenCL::cl2hpp,INTERFACE_INCLUDE_DIRECTORIES>
         $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
+        $<TARGET_PROPERTY:nonstd::span-lite,INTERFACE_INCLUDE_DIRECTORIES>
         ${ArrayFire_BINARY_DIR}/include
       )
     if(TARGET Forge::forge)
@@ -80,6 +81,7 @@ foreach(SBK_BINARY_OP ${SBK_BINARY_OPS})
         ${opencl_compile_definitions}
         $<TARGET_PROPERTY:Boost::boost,INTERFACE_COMPILE_DEFINITIONS>
         $<TARGET_PROPERTY:af_spdlog,INTERFACE_COMPILE_DEFINITIONS>
+        $<TARGET_PROPERTY:nonstd::span-lite,INTERFACE_COMPILE_DEFINITIONS>
         TYPE=${SBK_BINARY_OP} AFDLL)
     target_sources(opencl_scan_by_key
       INTERFACE $<TARGET_OBJECTS:opencl_scan_by_key_${SBK_BINARY_OP}>)
diff --git a/src/backend/opencl/kernel/scan_dim.hpp b/src/backend/opencl/kernel/scan_dim.hpp
index c246711c47..00c4cfc8ef 100644
--- a/src/backend/opencl/kernel/scan_dim.hpp
+++ b/src/backend/opencl/kernel/scan_dim.hpp
@@ -57,8 +57,8 @@ static opencl::Kernel getScanDimKernel(const std::string key, int dim,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    return common::getKernel(key, {ops_cl_src, scan_dim_cl_src}, tmpltArgs,
-                             compileOpts);
+    return common::getKernel(key, std::array{ops_cl_src, scan_dim_cl_src},
+                             tmpltArgs, compileOpts);
 }
 
 template<typename Ti, typename To, af_op_t op>
diff --git a/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp b/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
index b73c30ec07..8376c3a876 100644
--- a/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
@@ -57,7 +57,8 @@ static opencl::Kernel getScanDimKernel(const std::string key, int dim,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    return common::getKernel(key, {ops_cl_src, scan_dim_by_key_cl_src},
+    return common::getKernel(key,
+                             std::array{ops_cl_src, scan_dim_by_key_cl_src},
                              tmpltArgs, compileOpts);
 }
 
diff --git a/src/backend/opencl/kernel/scan_first.hpp b/src/backend/opencl/kernel/scan_first.hpp
index d4c03d041c..a8031ecc5e 100644
--- a/src/backend/opencl/kernel/scan_first.hpp
+++ b/src/backend/opencl/kernel/scan_first.hpp
@@ -58,8 +58,8 @@ static opencl::Kernel getScanFirstKernel(const std::string key,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    return common::getKernel(key, {ops_cl_src, scan_first_cl_src}, tmpltArgs,
-                             compileOpts);
+    return common::getKernel(key, std::array{ops_cl_src, scan_first_cl_src},
+                             tmpltArgs, compileOpts);
 }
 
 template<typename Ti, typename To, af_op_t op>
diff --git a/src/backend/opencl/kernel/scan_first_by_key_impl.hpp b/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
index 3deee884b3..f8835e18a8 100644
--- a/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
@@ -61,7 +61,8 @@ static opencl::Kernel getScanFirstKernel(const std::string key,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    return common::getKernel(key, {ops_cl_src, scan_first_by_key_cl_src},
+    return common::getKernel(key,
+                             std::array{ops_cl_src, scan_first_by_key_cl_src},
                              tmpltArgs, compileOpts);
 }
 
diff --git a/src/backend/opencl/kernel/select.hpp b/src/backend/opencl/kernel/select.hpp
index 743f200d5c..69602817a9 100644
--- a/src/backend/opencl/kernel/select.hpp
+++ b/src/backend/opencl/kernel/select.hpp
@@ -29,18 +29,16 @@ constexpr int REPEAT = 64;
 template<typename T>
 void selectLauncher(Param out, Param cond, Param a, Param b, const int ndims,
                     const bool is_same) {
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(is_same),
     };
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineValue(is_same),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+    std::array<std::string, 3> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(is_same),
+        getTypeBuildDefinition<T>()};
 
-    auto selectOp =
-        common::getKernel("select_kernel", {select_cl_src}, targs, options);
+    auto selectOp = common::getKernel(
+        "select_kernel", std::array{select_cl_src}, targs, options);
 
     int threads[] = {DIMX, DIMY};
 
@@ -74,18 +72,16 @@ void select(Param out, Param cond, Param a, Param b, int ndims) {
 template<typename T>
 void select_scalar(Param out, Param cond, Param a, const T b, const int ndims,
                    const bool flip) {
-    std::vector<TemplateArg> targs = {
+    std::array<TemplateArg, 2> targs = {
         TemplateTypename<T>(),
         TemplateArg(flip),
     };
-    std::vector<std::string> options = {
-        DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineValue(flip),
-    };
-    options.emplace_back(getTypeBuildDefinition<T>());
+    std::array<std::string, 3> options = {
+        DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(flip),
+        getTypeBuildDefinition<T>()};
 
-    auto selectOp = common::getKernel("select_scalar_kernel", {select_cl_src},
-                                      targs, options);
+    auto selectOp = common::getKernel(
+        "select_scalar_kernel", std::array{select_cl_src}, targs, options);
 
     int threads[] = {DIMX, DIMY};
 
diff --git a/src/backend/opencl/kernel/sift.hpp b/src/backend/opencl/kernel/sift.hpp
index 4b1609514e..90b063b2d0 100644
--- a/src/backend/opencl/kernel/sift.hpp
+++ b/src/backend/opencl/kernel/sift.hpp
@@ -355,19 +355,20 @@ std::array<Kernel, 7> getSiftKernels() {
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
     return {
-        common::getKernel("sub", {sift_nonfree_cl_src}, targs, compileOpts),
-        common::getKernel("detectExtrema", {sift_nonfree_cl_src}, targs,
-                          compileOpts),
-        common::getKernel("interpolateExtrema", {sift_nonfree_cl_src}, targs,
-                          compileOpts),
-        common::getKernel("calcOrientation", {sift_nonfree_cl_src}, targs,
-                          compileOpts),
-        common::getKernel("removeDuplicates", {sift_nonfree_cl_src}, targs,
-                          compileOpts),
-        common::getKernel("computeDescriptor", {sift_nonfree_cl_src}, targs,
-                          compileOpts),
-        common::getKernel("computeGLOHDescriptor", {sift_nonfree_cl_src}, targs,
+        common::getKernel("sub", std::array{sift_nonfree_cl_src}, targs,
                           compileOpts),
+        common::getKernel("detectExtrema", std::array{sift_nonfree_cl_src},
+                          targs, compileOpts),
+        common::getKernel("interpolateExtrema", std::array{sift_nonfree_cl_src},
+                          targs, compileOpts),
+        common::getKernel("calcOrientation", std::array{sift_nonfree_cl_src},
+                          targs, compileOpts),
+        common::getKernel("removeDuplicates", std::array{sift_nonfree_cl_src},
+                          targs, compileOpts),
+        common::getKernel("computeDescriptor", std::array{sift_nonfree_cl_src},
+                          targs, compileOpts),
+        common::getKernel("computeGLOHDescriptor",
+                          std::array{sift_nonfree_cl_src}, targs, compileOpts),
     };
 }
 
diff --git a/src/backend/opencl/kernel/sobel.hpp b/src/backend/opencl/kernel/sobel.hpp
index d68b2dc933..8e0c406f4a 100644
--- a/src/backend/opencl/kernel/sobel.hpp
+++ b/src/backend/opencl/kernel/sobel.hpp
@@ -38,8 +38,8 @@ void sobel(Param dx, Param dy, const Param in) {
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    auto sobel =
-        common::getKernel("sobel3x3", {sobel_cl_src}, targs, compileOpts);
+    auto sobel = common::getKernel("sobel3x3", std::array{sobel_cl_src}, targs,
+                                   compileOpts);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/sparse.hpp b/src/backend/opencl/kernel/sparse.hpp
index e938ed2f46..6cfed4b554 100644
--- a/src/backend/opencl/kernel/sparse.hpp
+++ b/src/backend/opencl/kernel/sparse.hpp
@@ -42,8 +42,8 @@ void coo2dense(Param out, const Param values, const Param rowIdx,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto coo2dense = common::getKernel("coo2Dense", {coo2dense_cl_src},
-                                       tmpltArgs, compileOpts);
+    auto coo2dense = common::getKernel(
+        "coo2Dense", std::array{coo2dense_cl_src}, tmpltArgs, compileOpts);
 
     cl::NDRange local(THREADS_PER_GROUP, 1, 1);
 
@@ -75,8 +75,8 @@ void csr2dense(Param output, const Param values, const Param rowIdx,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto csr2dense = common::getKernel("csr2Dense", {csr2dense_cl_src},
-                                       tmpltArgs, compileOpts);
+    auto csr2dense = common::getKernel(
+        "csr2Dense", std::array{csr2dense_cl_src}, tmpltArgs, compileOpts);
 
     cl::NDRange local(threads, 1);
     int groups_x = std::min((int)(divup(M, local[0])), MAX_GROUPS);
@@ -101,8 +101,8 @@ void dense2csr(Param values, Param rowIdx, Param colIdx, const Param dense) {
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto dense2Csr = common::getKernel("dense2Csr", {dense2csr_cl_src},
-                                       tmpltArgs, compileOpts);
+    auto dense2Csr = common::getKernel(
+        "dense2Csr", std::array{dense2csr_cl_src}, tmpltArgs, compileOpts);
 
     int num_rows = dense.info.dims[0];
     int num_cols = dense.info.dims[1];
@@ -146,8 +146,8 @@ void swapIndex(Param ovalues, Param oindex, const Param ivalues,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto swapIndex = common::getKernel("swapIndex", {csr2coo_cl_src}, tmpltArgs,
-                                       compileOpts);
+    auto swapIndex = common::getKernel("swapIndex", std::array{csr2coo_cl_src},
+                                       tmpltArgs, compileOpts);
 
     cl::NDRange global(ovalues.info.dims[0], 1, 1);
 
@@ -168,8 +168,8 @@ void csr2coo(Param ovalues, Param orowIdx, Param ocolIdx, const Param ivalues,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto csr2coo =
-        common::getKernel("csr2Coo", {csr2coo_cl_src}, tmpltArgs, compileOpts);
+    auto csr2coo = common::getKernel("csr2Coo", std::array{csr2coo_cl_src},
+                                     tmpltArgs, compileOpts);
 
     const int MAX_GROUPS = 4096;
     int M                = irowIdx.info.dims[0] - 1;
@@ -208,8 +208,8 @@ void coo2csr(Param ovalues, Param orowIdx, Param ocolIdx, const Param ivalues,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto csrReduce = common::getKernel("csrReduce", {csr2coo_cl_src}, tmpltArgs,
-                                       compileOpts);
+    auto csrReduce = common::getKernel("csrReduce", std::array{csr2coo_cl_src},
+                                       tmpltArgs, compileOpts);
 
     // Now we need to sort this into column major
     kernel::sort0ByKeyIterative<int, int>(rowCopy, index, true);
diff --git a/src/backend/opencl/kernel/sparse_arith.hpp b/src/backend/opencl/kernel/sparse_arith.hpp
index 25ae4e3db5..f10b3327a0 100644
--- a/src/backend/opencl/kernel/sparse_arith.hpp
+++ b/src/backend/opencl/kernel/sparse_arith.hpp
@@ -50,7 +50,7 @@ auto fetchKernel(const std::string key, const common::Source &additionalSrc,
     constexpr bool IsComplex =
         std::is_same<T, cfloat>::value || std::is_same<T, cdouble>::value;
 
-    std::vector<TemplateArg> tmpltArgs = {
+    std::array<TemplateArg, 2> tmpltArgs = {
         TemplateTypename<T>(),
         TemplateArg(op),
     };
@@ -62,8 +62,9 @@ auto fetchKernel(const std::string key, const common::Source &additionalSrc,
     options.emplace_back(getTypeBuildDefinition<T>());
     options.insert(std::end(options), std::begin(additionalOptions),
                    std::end(additionalOptions));
-    return common::getKernel(key, {sparse_arith_common_cl_src, additionalSrc},
-                             tmpltArgs, options);
+    return common::getKernel(
+        key, std::array{sparse_arith_common_cl_src, additionalSrc}, tmpltArgs,
+        options);
 }
 
 template<typename T, af_op_t op>
@@ -142,8 +143,9 @@ static void csrCalcOutNNZ(Param outRowIdx, unsigned &nnzC, const uint M,
         TemplateTypename<uint>(),
     };
 
-    auto calcNNZ = common::getKernel(
-        "csr_calc_out_nnz", {ssarith_calc_out_nnz_cl_src}, tmpltArgs, {});
+    auto calcNNZ = common::getKernel("csr_calc_out_nnz",
+                                     std::array{ssarith_calc_out_nnz_cl_src},
+                                     tmpltArgs, {});
 
     cl::NDRange local(256, 1);
     cl::NDRange global(divup(M, local[0]) * local[0], 1, 1);
diff --git a/src/backend/opencl/kernel/susan.hpp b/src/backend/opencl/kernel/susan.hpp
index 7ebb1a20ec..d3cdfb8af2 100644
--- a/src/backend/opencl/kernel/susan.hpp
+++ b/src/backend/opencl/kernel/susan.hpp
@@ -48,8 +48,8 @@ void susan(cl::Buffer* out, const cl::Buffer* in, const unsigned in_off,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto susan = common::getKernel("susan_responses", {susan_cl_src}, targs,
-                                   compileOpts);
+    auto susan = common::getKernel("susan_responses", std::array{susan_cl_src},
+                                   targs, compileOpts);
 
     cl::NDRange local(SUSAN_THREADS_X, SUSAN_THREADS_Y);
     cl::NDRange global(divup(idim0 - 2 * edge, local[0]) * local[0],
@@ -74,8 +74,8 @@ unsigned nonMaximal(cl::Buffer* x_out, cl::Buffer* y_out, cl::Buffer* resp_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto nonMax =
-        common::getKernel("non_maximal", {susan_cl_src}, targs, compileOpts);
+    auto nonMax = common::getKernel("non_maximal", std::array{susan_cl_src},
+                                    targs, compileOpts);
 
     unsigned corners_found = 0;
     auto d_corners_found   = memAlloc<unsigned>(1);
diff --git a/src/backend/opencl/kernel/swapdblk.hpp b/src/backend/opencl/kernel/swapdblk.hpp
index 106db3c4d2..ff875e25da 100644
--- a/src/backend/opencl/kernel/swapdblk.hpp
+++ b/src/backend/opencl/kernel/swapdblk.hpp
@@ -41,8 +41,8 @@ void swapdblk(int n, int nb, cl_mem dA, size_t dA_offset, int ldda, int inca,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto swapdblk =
-        common::getKernel("swapdblk", {swapdblk_cl_src}, targs, compileOpts);
+    auto swapdblk = common::getKernel("swapdblk", std::array{swapdblk_cl_src},
+                                      targs, compileOpts);
 
     int nblocks = n / nb;
 
diff --git a/src/backend/opencl/kernel/tile.hpp b/src/backend/opencl/kernel/tile.hpp
index e0b268e594..cc65a1fc54 100644
--- a/src/backend/opencl/kernel/tile.hpp
+++ b/src/backend/opencl/kernel/tile.hpp
@@ -41,7 +41,8 @@ void tile(Param out, const Param in) {
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto tile = common::getKernel("tile", {tile_cl_src}, targs, compileOpts);
+    auto tile =
+        common::getKernel("tile", std::array{tile_cl_src}, targs, compileOpts);
 
     NDRange local(TX, TY, 1);
 
diff --git a/src/backend/opencl/kernel/transform.hpp b/src/backend/opencl/kernel/transform.hpp
index c107361771..a64468ea26 100644
--- a/src/backend/opencl/kernel/transform.hpp
+++ b/src/backend/opencl/kernel/transform.hpp
@@ -79,9 +79,9 @@ void transform(Param out, const Param in, const Param tf, bool isInverse,
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
     addInterpEnumOptions(compileOpts);
 
-    auto transform =
-        common::getKernel("transformKernel", {interp_cl_src, transform_cl_src},
-                          tmpltArgs, compileOpts);
+    auto transform = common::getKernel(
+        "transformKernel", std::array{interp_cl_src, transform_cl_src},
+        tmpltArgs, compileOpts);
 
     const int nImg2 = in.info.dims[2];
     const int nImg3 = in.info.dims[3];
diff --git a/src/backend/opencl/kernel/transpose.hpp b/src/backend/opencl/kernel/transpose.hpp
index 39b775d0cc..87e6b65fee 100644
--- a/src/backend/opencl/kernel/transpose.hpp
+++ b/src/backend/opencl/kernel/transpose.hpp
@@ -48,8 +48,8 @@ void transpose(Param out, const Param in, cl::CommandQueue queue,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto transpose = common::getKernel("transpose", {transpose_cl_src},
-                                       tmpltArgs, compileOpts);
+    auto transpose = common::getKernel(
+        "transpose", std::array{transpose_cl_src}, tmpltArgs, compileOpts);
 
     NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/transpose_inplace.hpp b/src/backend/opencl/kernel/transpose_inplace.hpp
index f53340fd26..06020a6e3c 100644
--- a/src/backend/opencl/kernel/transpose_inplace.hpp
+++ b/src/backend/opencl/kernel/transpose_inplace.hpp
@@ -48,9 +48,9 @@ void transpose_inplace(Param in, cl::CommandQueue& queue, const bool conjugate,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto transpose =
-        common::getKernel("transpose_inplace", {transpose_inplace_cl_src},
-                          tmpltArgs, compileOpts);
+    auto transpose = common::getKernel("transpose_inplace",
+                                       std::array{transpose_inplace_cl_src},
+                                       tmpltArgs, compileOpts);
 
     NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/triangle.hpp b/src/backend/opencl/kernel/triangle.hpp
index 0421b09e8d..8380894b07 100644
--- a/src/backend/opencl/kernel/triangle.hpp
+++ b/src/backend/opencl/kernel/triangle.hpp
@@ -51,8 +51,8 @@ void triangle(Param out, const Param in, bool is_upper, bool is_unit_diag) {
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto triangle = common::getKernel("triangle", {triangle_cl_src}, tmpltArgs,
-                                      compileOpts);
+    auto triangle = common::getKernel("triangle", std::array{triangle_cl_src},
+                                      tmpltArgs, compileOpts);
 
     NDRange local(TX, TY);
 
diff --git a/src/backend/opencl/kernel/unwrap.hpp b/src/backend/opencl/kernel/unwrap.hpp
index d525015772..68d6846893 100644
--- a/src/backend/opencl/kernel/unwrap.hpp
+++ b/src/backend/opencl/kernel/unwrap.hpp
@@ -46,8 +46,8 @@ void unwrap(Param out, const Param in, const dim_t wx, const dim_t wy,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto unwrap =
-        common::getKernel("unwrap", {unwrap_cl_src}, tmpltArgs, compileOpts);
+    auto unwrap = common::getKernel("unwrap", std::array{unwrap_cl_src},
+                                    tmpltArgs, compileOpts);
 
     dim_t TX = 1, TY = 1;
     dim_t BX       = 1;
diff --git a/src/backend/opencl/kernel/where.hpp b/src/backend/opencl/kernel/where.hpp
index 3cc9601e4d..9c17143398 100644
--- a/src/backend/opencl/kernel/where.hpp
+++ b/src/backend/opencl/kernel/where.hpp
@@ -45,8 +45,8 @@ static void get_out_idx(cl::Buffer *out_data, Param &otmp, Param &rtmp,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto getIdx = common::getKernel("get_out_idx", {where_cl_src}, tmpltArgs,
-                                    compileOpts);
+    auto getIdx = common::getKernel("get_out_idx", std::array{where_cl_src},
+                                    tmpltArgs, compileOpts);
 
     NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     NDRange global(local[0] * groups_x * in.info.dims[2],
diff --git a/src/backend/opencl/kernel/wrap.hpp b/src/backend/opencl/kernel/wrap.hpp
index ba202a48c3..72797bd5f5 100644
--- a/src/backend/opencl/kernel/wrap.hpp
+++ b/src/backend/opencl/kernel/wrap.hpp
@@ -46,8 +46,8 @@ void wrap(Param out, const Param in, const dim_t wx, const dim_t wy,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto wrap =
-        common::getKernel("wrap", {wrap_cl_src}, tmpltArgs, compileOpts);
+    auto wrap = common::getKernel("wrap", std::array{wrap_cl_src}, tmpltArgs,
+                                  compileOpts);
 
     dim_t nx = (out.info.dims[0] + 2 * px - wx) / sx + 1;
     dim_t ny = (out.info.dims[1] + 2 * py - wy) / sy + 1;
@@ -91,8 +91,9 @@ void wrap_dilated(Param out, const Param in, const dim_t wx, const dim_t wy,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto dilatedWrap = common::getKernel("wrap_dilated", {wrap_dilated_cl_src},
-                                         tmpltArgs, compileOpts);
+    auto dilatedWrap =
+        common::getKernel("wrap_dilated", std::array{wrap_dilated_cl_src},
+                          tmpltArgs, compileOpts);
 
     dim_t nx = 1 + (out.info.dims[0] + 2 * px - (((wx - 1) * dx) + 1)) / sx;
     dim_t ny = 1 + (out.info.dims[1] + 2 * py - (((wy - 1) * dy) + 1)) / sy;

From 416bb5bbb0d425cfe827d9a90faf5f80d86595a0 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 17 Oct 2022 12:23:36 -0400
Subject: [PATCH 112/473] Fix the way we encode backendId for unified backend

The way we were formatting the backend ID was incorrect and failed when we had
more than 3 backends. With the new oneAPI backend, this mechanism was failing
and causing errors.
---
 src/backend/common/ArrayInfo.cpp | 18 +++++-------------
 src/backend/common/ArrayInfo.hpp |  3 ++-
 2 files changed, 7 insertions(+), 14 deletions(-)

diff --git a/src/backend/common/ArrayInfo.cpp b/src/backend/common/ArrayInfo.cpp
index 6a0ca86086..c2c6a842f2 100644
--- a/src/backend/common/ArrayInfo.cpp
+++ b/src/backend/common/ArrayInfo.cpp
@@ -38,26 +38,18 @@ unsigned ArrayInfo::getDevId() const {
 }
 
 void ArrayInfo::setId(int id) const {
-    // 1 << (backendId + 8) sets the 9th, 10th or 11th bit of devId to 1
-    // for CPU, CUDA and OpenCL respectively
-    // See ArrayInfo.hpp for more
-    unsigned backendId =
-        detail::getBackend() >> 1U;  // Convert enums 1, 2, 4 to ints 0, 1, 2
-    const_cast<ArrayInfo *>(this)->setId(id | 1 << (backendId + 8U));
+    const_cast<ArrayInfo *>(this)->setId(id);
 }
 
 void ArrayInfo::setId(int id) {
-    // 1 << (backendId + 8) sets the 9th, 10th or 11th bit of devId to 1
-    // for CPU, CUDA and OpenCL respectively
-    // See ArrayInfo.hpp for more
-    unsigned backendId =
-        detail::getBackend() >> 1U;  // Convert enums 1, 2, 4 to ints 0, 1, 2
-    devId = id | 1U << (backendId + 8U);
+    /// Shift the backend flag to the end of the devId integer
+    unsigned backendId = detail::getBackend();
+    devId              = id | backendId << 8U;
 }
 
 af_backend ArrayInfo::getBackendId() const {
     // devId >> 8 converts the backend info to 1, 2, 4 which are enums
-    // for CPU, CUDA and OpenCL respectively
+    // for CPU, CUDA, OpenCL, and oneAPI respectively
     // See ArrayInfo.hpp for more
     unsigned backendId = devId >> 8U;
     return static_cast<af_backend>(backendId);
diff --git a/src/backend/common/ArrayInfo.hpp b/src/backend/common/ArrayInfo.hpp
index 7f5516e5a4..f2a99c0b1e 100644
--- a/src/backend/common/ArrayInfo.hpp
+++ b/src/backend/common/ArrayInfo.hpp
@@ -28,7 +28,8 @@ class ArrayInfo {
     // The devId variable stores information about the deviceId as well as the
     // backend. The 8 LSBs (0-7) are used to store the device ID. The 09th LSB
     // is set to 1 if backend is CPU The 10th LSB is set to 1 if backend is CUDA
-    // The 11th LSB is set to 1 if backend is OpenCL
+    // The 11th LSB is set to 1 if backend is OpenCL The 12th LSB is set to 1
+    // for oneAPI
     // This information can be retrieved directly from an af_array by doing
     //     int* devId = reinterpret_cast<int*>(a); // a is an af_array
     //     af_backend backendID = *devId >> 8;   // Returns 1, 2, 4 for CPU,

From 0bb2f7de2a3804cced3b75f28adc5b857c1407b4 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 17 Oct 2022 12:26:09 -0400
Subject: [PATCH 113/473] Remove extra print from the memcpy kernel in oneAPI

---
 src/backend/oneapi/kernel/memcopy.hpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 701060820f..2bb2443cb2 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -57,9 +57,6 @@ class memCopy {
         const int id0        = group_id_0 * gg.get_local_range(0) + lid0;
         const int id1        = group_id_1 * gg.get_local_range(1) + lid1;
 
-        debug_ << "[" << id0 << "," << id1 << "," << id2 << "," << id3 << "]"
-               << sycl::stream_manipulator::endl;
-
         T *iptr = in_.get_pointer();
         iptr += offset_;
         // FIXME: Do more work per work group

From 9f0829b737bf2828e3a00b37ddd1d268ef382936 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 20 Oct 2022 16:27:37 -0400
Subject: [PATCH 114/473] Fix backend_index and NUM_BACKENDS constants in
 unified

---
 src/api/unified/symbol_manager.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/api/unified/symbol_manager.hpp b/src/api/unified/symbol_manager.hpp
index b77f7e9bbe..3106bfa2ae 100644
--- a/src/api/unified/symbol_manager.hpp
+++ b/src/api/unified/symbol_manager.hpp
@@ -23,7 +23,7 @@
 
 namespace unified {
 
-const int NUM_BACKENDS = 3;
+const int NUM_BACKENDS = 4;
 
 #define UNIFIED_ERROR_LOAD_LIB()                                       \
     AF_RETURN_ERROR(                                                   \
@@ -37,6 +37,7 @@ static inline int backend_index(af::Backend be) {
         case AF_BACKEND_CPU: return 0;
         case AF_BACKEND_CUDA: return 1;
         case AF_BACKEND_OPENCL: return 2;
+        case AF_BACKEND_ONEAPI: return 3;
         default: return -1;
     }
 }

From 3b9b820f4ed1ea5fa54015602fc91cfd6dce4dbc Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Fri, 21 Oct 2022 15:04:18 -0400
Subject: [PATCH 115/473] histogram ported to oneapi. (#3305)

* histogram ported to oneapi. 50/62 tests pass. see below

    Batch tests fail with value errors:
      Histogram/1.40Bins0min100max_Batch, where TypeParam = float
      Histogram/3.40Bins0min100max_Batch, where TypeParam = int
      Histogram/4.40Bins0min100max_Batch, where TypeParam = unsigned int
      Histogram/5.40Bins0min100max_Batch, where TypeParam = char
      Histogram/6.40Bins0min100max_Batch, where TypeParam = unsigned char
      Histogram/7.40Bins0min100max_Batch, where TypeParam = short
      Histogram/8.40Bins0min100max_Batch, where TypeParam = unsigned short
      Histogram/9.40Bins0min100max_Batch, where TypeParam = long long
      Histogram/10.40Bins0min100max_Batch, where TypeParam = unsigned long long

    Tests fail because reductions do not function for test harness:
      Histogram.SNIPPET_hist_nominmax
      Histogram.SNIPPET_histequal

    GFOR, LargeBins expected to fail (getMaxMemorySize not OneAPI supported):
      histogram.GFOR
      histogram.LargeBins

    IndexedArray expected to fail without JIT support:
      LargeBins

Authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
---
 src/backend/oneapi/CMakeLists.txt       |   1 +
 src/backend/oneapi/histogram.cpp        |   7 +-
 src/backend/oneapi/kernel/histogram.hpp | 167 ++++++++++++++++++++++++
 3 files changed, 173 insertions(+), 2 deletions(-)
 create mode 100755 src/backend/oneapi/kernel/histogram.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 67f9ec8b23..2ab7314581 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -209,6 +209,7 @@ target_sources(afoneapi
     kernel/diagonal.hpp
     kernel/diff.hpp
     kernel/iota.hpp
+    kernel/histogram.hpp
     kernel/memcopy.hpp
     kernel/random_engine.hpp
     kernel/random_engine_write.hpp
diff --git a/src/backend/oneapi/histogram.cpp b/src/backend/oneapi/histogram.cpp
index cf85c4e844..62ccd879af 100644
--- a/src/backend/oneapi/histogram.cpp
+++ b/src/backend/oneapi/histogram.cpp
@@ -11,6 +11,7 @@
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
 #include <histogram.hpp>
+#include <kernel/histogram.hpp>
 #include <af/dim4.hpp>
 
 using af::dim4;
@@ -22,10 +23,12 @@ template<typename T>
 Array<uint> histogram(const Array<T> &in, const unsigned &nbins,
                       const double &minval, const double &maxval,
                       const bool isLinear) {
-    ONEAPI_NOT_SUPPORTED("");
     const dim4 &dims = in.dims();
     dim4 outDims     = dim4(nbins, 1, dims[2], dims[3]);
-    Array<uint> out  = createValueArray<uint>(outDims, uint(0));
+    // Array<uint> out  = createValueArray<uint>(outDims, uint(0));
+    // \TODO revert createEmptyArray to createValueArray once JIT functions
+    Array<uint> out = createEmptyArray<uint>(outDims);
+    kernel::histogram<T>(out, in, nbins, minval, maxval, isLinear);
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/histogram.hpp b/src/backend/oneapi/kernel/histogram.hpp
new file mode 100755
index 0000000000..bc9f74f88c
--- /dev/null
+++ b/src/backend/oneapi/kernel/histogram.hpp
@@ -0,0 +1,167 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+#define MAX_BINS 4000
+#define THREADS_X 256
+#define THRD_LOAD 16
+
+// using memory_order = memory_order;
+// using memory_scope = memory_scope;
+
+template<typename T>
+using local_atomic_ref =
+    sycl::atomic_ref<T, sycl::memory_order::relaxed,
+                     sycl::memory_scope::work_group,
+                     sycl::access::address_space::local_space>;
+
+template<typename T>
+using global_atomic_ref =
+    sycl::atomic_ref<T, sycl::memory_order::relaxed, sycl::memory_scope::system,
+                     sycl::access::address_space::global_space>;
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename T>
+class histogramKernel {
+   public:
+    histogramKernel(sycl::accessor<uint> d_dst, KParam oInfo,
+                    const sycl::accessor<T> d_src, KParam iInfo,
+                    local_accessor<uint, 1> localMemAcc, int len, int nbins,
+                    float minval, float maxval, int nBBS, const bool isLinear)
+        : d_dst_(d_dst)
+        , oInfo_(oInfo)
+        , d_src_(d_src)
+        , iInfo_(iInfo)
+        , localMemAcc_(localMemAcc)
+        , len_(len)
+        , nbins_(nbins)
+        , minval_(minval)
+        , maxval_(maxval)
+        , nBBS_(nBBS)
+        , isLinear_(isLinear) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        unsigned b2   = g.get_group_id(0) / nBBS_;
+        int start     = (g.get_group_id(0) - b2 * nBBS_) * THRD_LOAD *
+                        g.get_local_range(0) +
+                    it.get_local_id(0);
+        int end = fmin((int)(start + THRD_LOAD * g.get_local_range(0)), len_);
+
+        // offset input and output to account for batch ops
+        const T *in = d_src_.get_pointer() + b2 * iInfo_.strides[2] +
+                      g.get_group_id(1) * iInfo_.strides[3] + iInfo_.offset;
+        uint outOffset =
+            b2 * oInfo_.strides[2] + g.get_group_id(1) * oInfo_.strides[3];
+
+        float dx = (maxval_ - minval_) / (float)nbins_;
+
+        bool use_global = nbins_ > MAX_BINS;
+
+        if (!use_global) {
+            for (int i = it.get_local_id(0); i < nbins_;
+                 i += g.get_local_range(0))
+                localMemAcc_[i] = 0;
+            it.barrier();
+        }
+
+        for (int row = start; row < end; row += g.get_local_range(0)) {
+            const int i0  = row % iInfo_.dims[0];
+            const int i1  = row / iInfo_.dims[0];
+            const int idx = isLinear_ ? row : i0 + i1 * iInfo_.strides[1];
+
+            int bin = (int)(((float)in[idx] - minval_) / dx);
+            bin     = fmax(bin, 0);
+            bin     = fmin(bin, (int)nbins_ - 1);
+
+            if (use_global) {
+                global_atomic_ref<uint>(d_dst_[outOffset + bin])++;
+            } else {
+                local_atomic_ref<uint>(localMemAcc_[bin])++;
+            }
+        }
+
+        if (!use_global) {
+            it.barrier();
+            for (int i = it.get_local_id(0); i < nbins_;
+                 i += g.get_local_range(0)) {
+                global_atomic_ref<uint>(d_dst_[outOffset + i]) +=
+                    localMemAcc_[i];
+            }
+        }
+    }
+
+   private:
+    sycl::accessor<uint> d_dst_;
+    KParam oInfo_;
+    sycl::accessor<T> d_src_;
+    KParam iInfo_;
+    local_accessor<uint, 1> localMemAcc_;
+    int len_;
+    int nbins_;
+    float minval_;
+    float maxval_;
+    int nBBS_;
+    bool isLinear_;
+};
+
+template<typename T>
+void histogram(Param<uint> out, const Param<T> in, int nbins, float minval,
+               float maxval, bool isLinear) {
+    int nElems  = in.info.dims[0] * in.info.dims[1];
+    int blk_x   = divup(nElems, THRD_LOAD * THREADS_X);
+    int locSize = nbins <= MAX_BINS ? (nbins * sizeof(uint)) : 1;
+
+    auto local           = sycl::range{THREADS_X, 1};
+    const size_t global0 = blk_x * in.info.dims[2] * THREADS_X;
+    const size_t global1 = in.info.dims[3];
+    auto global          = sycl::range{global0, global1};
+
+    // \TODO drop this first memset once createEmptyArray is reverted back to
+    //       createValueArray in ../histogram.cpp
+    getQueue()
+        .submit([&](sycl::handler &h) {
+            auto outAcc = out.data->get_access(h);
+            h.parallel_for(sycl::range<1>{(size_t)nbins},
+                           [=](sycl::id<1> idx) { outAcc[idx[0]] = 0; });
+        })
+        .wait();
+    getQueue().submit([&](sycl::handler &h) {
+        auto inAcc  = in.data->get_access(h);
+        auto outAcc = out.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        auto localMem = local_accessor<uint, 1>(locSize, h);
+
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            histogramKernel<T>(outAcc, out.info, inAcc, in.info, localMem,
+                               nElems, nbins, minval, maxval, blk_x, isLinear));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi

From 0f9a29b678fab650d2ab22bce1988342daa8c392 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 21 Oct 2022 14:53:40 -0400
Subject: [PATCH 116/473] Add driver minimums for CUDA 11.8 toolkit

---
 src/backend/cuda/device_manager.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 354a216741..221534f6dc 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -96,6 +96,7 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {11080, 9, 0, 0},
     {11070, 8, 7, 0},
     {11060, 8, 6, 0},
     {11050, 8, 6, 0},
@@ -131,6 +132,7 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
+        {11080, 450.80f, 452.39f},
         {11070, 450.80f, 452.39f},
         {11060, 450.80f, 452.39f},
         {11050, 450.80f, 452.39f},
@@ -159,7 +161,7 @@ static ComputeCapabilityToStreamingProcessors gpus[] = {
     {0x21, 48},  {0x30, 192}, {0x32, 192}, {0x35, 192}, {0x37, 192},
     {0x50, 128}, {0x52, 128}, {0x53, 128}, {0x60, 64},  {0x61, 128},
     {0x62, 128}, {0x70, 64},  {0x75, 64},  {0x80, 64},  {0x86, 128},
-    {0x87, 128}, {-1, -1},
+    {0x87, 128}, {0x89, 128}, {0x90, 128}, {-1, -1},
 };
 
 // pulled from CUTIL from CUDA SDK

From f79efb9330044db1355ea0d6983c7ae24b76cc9a Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 24 Oct 2022 21:10:43 -0400
Subject: [PATCH 117/473] adds shared memory based reduction to oneapi backend

---
 src/api/c/optypes.hpp                       |   4 +-
 src/backend/common/Binary.hpp               |  16 +-
 src/backend/oneapi/CMakeLists.txt           |   4 +
 src/backend/oneapi/copy.cpp                 |  14 +-
 src/backend/oneapi/device_manager.cpp       |  33 ++-
 src/backend/oneapi/kernel/reduce.hpp        | 112 ++++++++
 src/backend/oneapi/kernel/reduce_all.hpp    | 288 ++++++++++++++++++++
 src/backend/oneapi/kernel/reduce_config.hpp |  22 ++
 src/backend/oneapi/kernel/reduce_dim.hpp    | 236 ++++++++++++++++
 src/backend/oneapi/kernel/reduce_first.hpp  | 236 ++++++++++++++++
 src/backend/oneapi/platform.cpp             |   6 +-
 src/backend/oneapi/reduce_impl.hpp          |  12 +-
 12 files changed, 945 insertions(+), 38 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/reduce.hpp
 create mode 100644 src/backend/oneapi/kernel/reduce_all.hpp
 create mode 100644 src/backend/oneapi/kernel/reduce_config.hpp
 create mode 100644 src/backend/oneapi/kernel/reduce_dim.hpp
 create mode 100644 src/backend/oneapi/kernel/reduce_first.hpp

diff --git a/src/api/c/optypes.hpp b/src/api/c/optypes.hpp
index aeb90e1dcd..696ae07668 100644
--- a/src/api/c/optypes.hpp
+++ b/src/api/c/optypes.hpp
@@ -9,7 +9,7 @@
 
 #pragma once
 
-typedef enum {
+typedef enum af_op_t : int {
     af_none_t = -1,
     af_add_t  = 0,
     af_sub_t,
@@ -100,4 +100,4 @@ typedef enum {
     af_rsqrt_t,
 
     af_moddims_t
-} af_op_t;
+};
diff --git a/src/backend/common/Binary.hpp b/src/backend/common/Binary.hpp
index 6eeaad2058..ca500ac865 100644
--- a/src/backend/common/Binary.hpp
+++ b/src/backend/common/Binary.hpp
@@ -78,15 +78,17 @@ template<>
 struct Binary<char, af_min_t> {
     static __DH__ char init() { return 1; }
 
-    __DH__ char operator()(char lhs, char rhs) { return min(lhs > 0, rhs > 0); }
+    __DH__ char operator()(char lhs, char rhs) {
+        return detail::min(lhs > 0, rhs > 0);
+    }
 };
 
-#define SPECIALIZE_COMPLEX_MIN(T, Tr)                               \
-    template<>                                                      \
-    struct Binary<T, af_min_t> {                                    \
-        static __DH__ T init() { return scalar<T>(maxval<Tr>()); }  \
-                                                                    \
-        __DH__ T operator()(T lhs, T rhs) { return min(lhs, rhs); } \
+#define SPECIALIZE_COMPLEX_MIN(T, Tr)                                       \
+    template<>                                                              \
+    struct Binary<T, af_min_t> {                                            \
+        static __DH__ T init() { return scalar<T>(maxval<Tr>()); }          \
+                                                                            \
+        __DH__ T operator()(T lhs, T rhs) { return detail::min(lhs, rhs); } \
     };
 
 SPECIALIZE_COMPLEX_MIN(cfloat, float)
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 2ab7314581..d9ae78f742 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -217,6 +217,10 @@ target_sources(afoneapi
     kernel/random_engine_philox.hpp
     kernel/random_engine_threefry.hpp
     kernel/range.hpp
+    kernel/reduce.hpp
+    kernel/reduce_all.hpp
+    kernel/reduce_first.hpp
+    kernel/reduce_dim.hpp
     kernel/transpose.hpp
     kernel/transpose_inplace.hpp
     kernel/triangle.hpp
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index d852480342..f24db5650c 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -213,13 +213,15 @@ template<typename T>
 T getScalar(const Array<T> &in) {
     T retVal{};
 
+    sycl::buffer retBuffer(&retVal, {1},
+                           {sycl::property::buffer::use_host_ptr()});
+
     getQueue()
-        .submit([=](sycl::handler &h) {
-            sycl::range rr(1);
-            sycl::id offset_id(in.getOffset());
-            auto acc_in = const_cast<sycl::buffer<T> *>(in.get())->get_access(
-                h, rr, offset_id);
-            h.copy(acc_in, (void *)&retVal);
+        .submit([&](sycl::handler &h) {
+            auto acc_in  = in.getData()->get_access(h, sycl::range{1},
+                                                    sycl::id{in.getOffset()});
+            auto acc_out = retBuffer.get_access();
+            h.copy(acc_in, acc_out);
         })
         .wait();
 
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index d8315eac38..ed97248dcb 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -47,8 +47,8 @@ static inline bool compare_default(const unique_ptr<sycl::device>& ldev,
                                    const unique_ptr<sycl::device>& rdev) {
     // TODO: update sorting criteria
     // select according to something applicable to oneapi backend
-    auto l_mem = ldev->get_info<sycl::info::device::local_mem_size>();
-    auto r_mem = rdev->get_info<sycl::info::device::local_mem_size>();
+    auto l_mem = ldev->get_info<sycl::info::device::global_mem_size>();
+    auto r_mem = rdev->get_info<sycl::info::device::global_mem_size>();
     return l_mem > r_mem;
 }
 
@@ -103,19 +103,22 @@ DeviceManager::DeviceManager()
 
     // Create contexts and queues once the sort is done
     for (int i = 0; i < nDevices; i++) {
-        try {
-            mContexts.push_back(make_unique<sycl::context>(*devices[i]));
-            mQueues.push_back(
-                make_unique<sycl::queue>(*mContexts.back(), *devices[i]));
-            mIsGLSharingOn.push_back(false);
-            // TODO:
-            // mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
-            // mPlatforms.push_back(getPlatformEnum(*devices[i]));
-            mDevices.emplace_back(std::move(devices[i]));
-        } catch (sycl::exception& err) {
-            AF_TRACE("Error creating context for device {} with error {}\n",
-                     devices[i]->get_info<sycl::info::device::name>(),
-                     err.what());
+        if (devices[i]->is_gpu() || devices[i]->is_cpu() ||
+            !devices[i]->is_accelerator()) {
+            try {
+                mContexts.push_back(make_unique<sycl::context>(*devices[i]));
+                mQueues.push_back(
+                    make_unique<sycl::queue>(*mContexts.back(), *devices[i]));
+                mIsGLSharingOn.push_back(false);
+                // TODO:
+                // mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
+                // mPlatforms.push_back(getPlatformEnum(*devices[i]));
+                mDevices.emplace_back(std::move(devices[i]));
+            } catch (sycl::exception& err) {
+                AF_TRACE("Error creating context for device {} with error {}\n",
+                         devices[i]->get_info<sycl::info::device::name>(),
+                         err.what());
+            }
         }
     }
     nDevices = mDevices.size();
diff --git a/src/backend/oneapi/kernel/reduce.hpp b/src/backend/oneapi/kernel/reduce.hpp
new file mode 100644
index 0000000000..9db0561b0a
--- /dev/null
+++ b/src/backend/oneapi/kernel/reduce.hpp
@@ -0,0 +1,112 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/Transform.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/reduce_all.hpp>
+#include <kernel/reduce_config.hpp>
+#include <kernel/reduce_dim.hpp>
+#include <kernel/reduce_first.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+
+#include <algorithm>
+#include <climits>
+#include <complex>
+#include <iostream>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_cpu_dispatch(Param<To> out, Param<Ti> in, int dim, bool change_nan,
+                         double nanval) {
+    // TODO: use kernels optimized for SIMD-based subgroup sizes
+    reduce_default_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
+}
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_gpu_dispatch(Param<To> out, Param<Ti> in, int dim, bool change_nan,
+                         double nanval) {
+    // TODO: use kernels optimized for gpu subgroup sizes
+    reduce_default_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
+}
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_default_dispatch(Param<To> out, Param<Ti> in, int dim,
+                             bool change_nan, double nanval) {
+    switch (dim) {
+        case 0:
+            return reduce_first_default<Ti, To, op>(out, in, change_nan,
+                                                    nanval);
+        case 1:
+            return reduce_dim_default<Ti, To, op, 1>(out, in, change_nan,
+                                                     nanval);
+        case 2:
+            return reduce_dim_default<Ti, To, op, 2>(out, in, change_nan,
+                                                     nanval);
+        case 3:
+            return reduce_dim_default<Ti, To, op, 3>(out, in, change_nan,
+                                                     nanval);
+    }
+}
+
+template<typename Ti, typename To, af_op_t op>
+void reduce(Param<To> out, Param<Ti> in, int dim, bool change_nan,
+            double nanval) {
+    // TODO: logic to dispatch to different kernels depending on device type
+    if (getQueue().get_device().is_cpu()) {
+        reduce_cpu_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
+    } else if (getQueue().get_device().is_gpu()) {
+        reduce_gpu_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
+    } else {
+        reduce_default_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
+    }
+}
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_all(Param<To> out, Param<Ti> in, bool change_nan, double nanval) {
+    int in_elements =
+        in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
+    bool is_linear = (in.info.strides[0] == 1);
+    for (int k = 1; k < 4; k++) {
+        is_linear &= (in.info.strides[k] ==
+                      (in.info.strides[k - 1] * in.info.dims[k - 1]));
+    }
+
+    if (is_linear) {
+        in.info.dims[0] = in_elements;
+        for (int k = 1; k < 4; k++) {
+            in.info.dims[k]    = 1;
+            in.info.strides[k] = in_elements;
+        }
+    }
+
+    uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+    threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
+    uint threads_y = THREADS_PER_BLOCK / threads_x;
+
+    // TODO: perf REPEAT, consider removing or runtime eval
+    // max problem size < SM resident threads, don't use REPEAT
+    uint blocks_x = divup(in.info.dims[0], threads_x * REPEAT);
+    uint blocks_y = divup(in.info.dims[1], threads_y);
+
+    reduce_all_launcher_default<Ti, To, op>(out, in, blocks_x, blocks_y,
+                                            threads_x, change_nan, nanval);
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
new file mode 100644
index 0000000000..be22e94c90
--- /dev/null
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -0,0 +1,288 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/Transform.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/reduce_config.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+
+#include <algorithm>
+#include <climits>
+#include <complex>
+#include <iostream>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename T>
+using global_atomic_ref =
+    sycl::atomic_ref<T, sycl::memory_order::relaxed, sycl::memory_scope::system,
+                     sycl::access::address_space::global_space>;
+
+template<typename Ti, typename To, af_op_t op>
+class reduceAllKernelSMEM {
+   public:
+    reduceAllKernelSMEM(sycl::accessor<To> out, KParam oInfo,
+                        sycl::accessor<unsigned> retCount,
+                        sycl::accessor<To> tmp, KParam tmpInfo,
+                        sycl::accessor<Ti> in, KParam iInfo, uint DIMX,
+                        uint groups_x, uint groups_y, uint repeat,
+                        bool change_nan, To nanval,
+                        local_accessor<compute_t<To>, 1> s_ptr,
+                        local_accessor<bool, 1> amLast, sycl::stream debug)
+        : out_(out)
+        , oInfo_(oInfo)
+        , retCount_(retCount)
+        , tmp_(tmp)
+        , tmpInfo_(tmpInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , DIMX_(DIMX)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , repeat_(repeat)
+        , change_nan_(change_nan)
+        , nanval_(nanval)
+        , s_ptr_(s_ptr)
+        , amLast_(amLast)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * DIMX_ + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid = groupId_x * g.get_local_range(0) * repeat_ + lidx;
+        const uint yid = groupId_y * g.get_local_range(1) + lidy;
+
+        common::Binary<compute_t<To>, op> reduce;
+        common::Transform<Ti, compute_t<To>, op> transform;
+
+        const data_t<Ti> *const iptr =
+            in_.get_pointer() + wid * iInfo_.strides[3] +
+            zid * iInfo_.strides[2] + yid * iInfo_.strides[1] + iInfo_.offset;
+
+        bool cond = (yid < iInfo_.dims[1]) && (zid < iInfo_.dims[2]) &&
+                    (wid < iInfo_.dims[3]);
+
+        dim_t last = (xid + repeat_ * DIMX_);
+        int lim    = sycl::min(last, iInfo_.dims[0]);
+
+        compute_t<To> out_val = common::Binary<compute_t<To>, op>::init();
+        for (int id = xid; cond && id < lim; id += DIMX_) {
+            compute_t<To> in_val = transform(iptr[id]);
+            if (change_nan_)
+                in_val = !IS_NAN(in_val) ? in_val
+                                         : static_cast<compute_t<To>>(nanval_);
+            out_val = reduce(in_val, out_val);
+        }
+
+        s_ptr_[lid] = out_val;
+
+        group_barrier(g);
+
+        if (THREADS_PER_BLOCK == 256) {
+            if (lid < 128) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 128]);
+            group_barrier(g);
+        }
+
+        if (THREADS_PER_BLOCK >= 128) {
+            if (lid < 64) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 64]);
+            group_barrier(g);
+        }
+
+        if (THREADS_PER_BLOCK >= 64) {
+            if (lid < 32) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 32]);
+            group_barrier(g);
+        }
+
+        // TODO: replace with subgroup operations in optimized kernels
+        if (lid < 16) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 16]);
+        group_barrier(g);
+
+        if (lid < 8) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 8]);
+        group_barrier(g);
+
+        if (lid < 4) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 4]);
+        group_barrier(g);
+
+        if (lid < 2) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 2]);
+        group_barrier(g);
+
+        if (lid < 1) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 1]);
+        group_barrier(g);
+
+        const unsigned total_blocks =
+            (g.get_group_range(0) * g.get_group_range(1));
+        const int uubidx =
+            (g.get_group_range(0) * g.get_group_id(1)) + g.get_group_id(0);
+        if (cond && lid == 0) {
+            if (total_blocks != 1) {
+                tmp_[uubidx] = s_ptr_[0];
+            } else {
+                out_[0] = s_ptr_[0];
+            }
+        }
+
+        // Last block to perform final reduction
+        if (total_blocks > 1) {
+            sycl::atomic_fence(sycl::memory_order::seq_cst,
+                               sycl::memory_scope::device);
+
+            // thread 0 takes a ticket
+            if (lid == 0) {
+                unsigned int ticket = global_atomic_ref<uint>(retCount_[0])++;
+                // If the ticket ID == number of blocks, we are the last block
+                amLast_[0] = (ticket == (total_blocks - 1));
+            }
+            group_barrier(g);
+
+            if (amLast_[0]) {
+                int i   = lid;
+                out_val = common::Binary<compute_t<To>, op>::init();
+
+                while (i < total_blocks) {
+                    compute_t<To> in_val = compute_t<To>(tmp_[i]);
+                    out_val              = reduce(in_val, out_val);
+                    i += THREADS_PER_BLOCK;
+                }
+
+                s_ptr_[lid] = out_val;
+                group_barrier(g);
+
+                // reduce final block
+                if (THREADS_PER_BLOCK == 256) {
+                    if (lid < 128)
+                        s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 128]);
+                    group_barrier(g);
+                }
+
+                if (THREADS_PER_BLOCK >= 128) {
+                    if (lid < 64)
+                        s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 64]);
+                    group_barrier(g);
+                }
+
+                if (THREADS_PER_BLOCK >= 64) {
+                    if (lid < 32)
+                        s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 32]);
+                    group_barrier(g);
+                }
+
+                if (lid < 16)
+                    s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 16]);
+                group_barrier(g);
+
+                if (lid < 8) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 8]);
+                group_barrier(g);
+
+                if (lid < 4) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 4]);
+                group_barrier(g);
+
+                if (lid < 2) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 2]);
+                group_barrier(g);
+
+                if (lid < 1) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 1]);
+                group_barrier(g);
+
+                if (lid == 0) {
+                    out_[0] = s_ptr_[0];
+
+                    // reset retirement count so that next run succeeds
+                    retCount_[0] = 0;
+                }
+            }
+        }
+    }
+
+   protected:
+    sycl::accessor<To> out_;
+    sycl::accessor<unsigned> retCount_;
+    sycl::accessor<To> tmp_;
+    KParam oInfo_, tmpInfo_, iInfo_;
+    sycl::accessor<Ti> in_;
+    uint DIMX_, repeat_;
+    uint groups_x_, groups_y_;
+    bool change_nan_;
+    To nanval_;
+    local_accessor<compute_t<To>, 1> s_ptr_;
+    local_accessor<bool, 1> amLast_;
+    sycl::stream debug_;
+};
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
+                                 const uint groups_x, const uint groups_y,
+                                 const uint threads_x, bool change_nan,
+                                 double nanval) {
+    sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
+                          groups_y * in.info.dims[3] * local[1]);
+
+    uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
+
+    long tmp_elements = groups_x * in.info.dims[2] * groups_y * in.info.dims[3];
+    if (tmp_elements > UINT_MAX) {
+        AF_ERROR(
+            "Too many blocks requested (typeof(retirementCount) == unsigned)",
+            AF_ERR_RUNTIME);
+    }
+    Array<To> tmp = createEmptyArray<To>(tmp_elements);
+    // TODO: JIT dependency
+    // Array<unsigned> retirementCount = createValueArray<unsigned>(1, 0);
+    Array<unsigned> retirementCount = createEmptyArray<unsigned>(1);
+    getQueue().submit([=](sycl::handler &h) {
+        auto acc = retirementCount.getData()->get_access(h);
+        h.single_task([=] { acc[0] = 0; });
+    });
+
+    getQueue()
+        .submit([=](sycl::handler &h) {
+            auto out_acc      = out.data->get_access(h);
+            auto retCount_acc = retirementCount.getData()->get_access(h);
+            auto tmp_acc      = tmp.getData()->get_access(h);
+            auto in_acc       = in.data->get_access(h);
+
+            sycl::stream debug_stream(2048 * 256, 128, h);
+
+            auto shrdMem =
+                local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
+            auto amLast = local_accessor<bool, 1>(1, h);
+            h.parallel_for(
+                sycl::nd_range<2>(global, local),
+                reduceAllKernelSMEM<Ti, To, op>(
+                    out_acc, out.info, retCount_acc, tmp_acc, (KParam)tmp,
+                    in_acc, in.info, threads_x, groups_x, groups_y, repeat,
+                    change_nan, scalar<To>(nanval), shrdMem, amLast,
+                    debug_stream));
+        })
+        .wait_and_throw();
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
\ No newline at end of file
diff --git a/src/backend/oneapi/kernel/reduce_config.hpp b/src/backend/oneapi/kernel/reduce_config.hpp
new file mode 100644
index 0000000000..827497967b
--- /dev/null
+++ b/src/backend/oneapi/kernel/reduce_config.hpp
@@ -0,0 +1,22 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+namespace oneapi {
+namespace kernel {
+
+// TODO: are different values more appropriate for reduce on oneapi?
+static const uint THREADS_PER_BLOCK = 256;
+static const uint THREADS_X         = 32;
+static const uint THREADS_Y         = THREADS_PER_BLOCK / THREADS_X;
+static const uint REPEAT            = 32;
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
new file mode 100644
index 0000000000..5105fb8b1c
--- /dev/null
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -0,0 +1,236 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/Transform.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/reduce_config.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+
+#include <algorithm>
+#include <climits>
+#include <complex>
+#include <iostream>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename Ti, typename To, af_op_t op, uint dim, uint DIMY>
+class reduceDimKernelSMEM {
+   public:
+    reduceDimKernelSMEM(sycl::accessor<To> out, KParam oInfo,
+                        sycl::accessor<Ti> in, KParam iInfo, uint groups_x,
+                        uint groups_y, uint offset_dim, bool change_nan,
+                        To nanval, local_accessor<compute_t<To>, 1> s_val,
+                        sycl::stream debug)
+        : out_(out)
+        , oInfo_(oInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , offset_dim_(offset_dim)
+        , change_nan_(change_nan)
+        , nanval_(nanval)
+        , s_val_(s_val)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid       = groupId_x * g.get_local_range(0) + lidx;
+        const uint yid       = groupId_y;
+
+        uint ids[4] = {xid, yid, zid, wid};
+
+        data_t<To> *const optr =
+            out_.get_pointer() + ids[3] * oInfo_.strides[3] +
+            ids[2] * oInfo_.strides[2] + ids[1] * oInfo_.strides[1] + ids[0];
+
+        const uint groupIdx_dim = ids[dim];
+        ids[dim]                = ids[dim] * g.get_local_range(1) + lidy;
+
+        const data_t<Ti> *iptr =
+            in_.get_pointer() + ids[3] * iInfo_.strides[3] +
+            ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + ids[0];
+
+        const uint id_dim_in   = ids[dim];
+        const uint istride_dim = iInfo_.strides[dim];
+        bool is_valid          = (ids[0] < iInfo_.dims[0]) &&
+                        (ids[1] < iInfo_.dims[1]) &&
+                        (ids[2] < iInfo_.dims[2]) && (ids[3] < iInfo_.dims[3]);
+
+        common::Binary<compute_t<To>, op> reduce;
+        common::Transform<Ti, compute_t<To>, op> transform;
+
+        compute_t<To> out_val = common::Binary<compute_t<To>, op>::init();
+        for (int id = id_dim_in; is_valid && (id < iInfo_.dims[dim]);
+             id += offset_dim_ * g.get_local_range(1)) {
+            compute_t<To> in_val = transform(*iptr);
+            if (change_nan_)
+                in_val = !IS_NAN(in_val) ? in_val
+                                         : static_cast<compute_t<To>>(nanval_);
+            out_val = reduce(in_val, out_val);
+            iptr += offset_dim_ * g.get_local_range(1) * istride_dim;
+        }
+
+        s_val_[lid] = out_val;
+
+        it.barrier();
+        compute_t<To> *s_ptr = s_val_.get_pointer() + lid;
+
+        if (DIMY == 8) {
+            if (lidy < 4) *s_ptr = reduce(*s_ptr, s_ptr[THREADS_X * 4]);
+            it.barrier();
+        }
+
+        if (DIMY >= 4) {
+            if (lidy < 2) *s_ptr = reduce(*s_ptr, s_ptr[THREADS_X * 2]);
+            it.barrier();
+        }
+
+        if (DIMY >= 2) {
+            if (lidy < 1) *s_ptr = reduce(*s_ptr, s_ptr[THREADS_X * 1]);
+            it.barrier();
+        }
+
+        if (lidy == 0 && is_valid && (groupIdx_dim < oInfo_.dims[dim])) {
+            *optr = data_t<To>(*s_ptr);
+        }
+    }
+
+   protected:
+    sycl::accessor<To> out_;
+    KParam oInfo_, iInfo_;
+    sycl::accessor<Ti> in_;
+    uint groups_x_, groups_y_, offset_dim_;
+    bool change_nan_;
+    To nanval_;
+    local_accessor<compute_t<To>, 1> s_val_;
+    sycl::stream debug_;
+};
+
+template<typename Ti, typename To, af_op_t op, uint dim>
+void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
+                                 const uint threads_y,
+                                 const dim_t blocks_dim[4], bool change_nan,
+                                 double nanval) {
+    sycl::range<2> local(THREADS_X, threads_y);
+    sycl::range<2> global(blocks_dim[0] * blocks_dim[2] * local[0],
+                          blocks_dim[1] * blocks_dim[3] * local[1]);
+
+    getQueue().submit([=](sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto in_acc  = in.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 256, 128, h);
+
+        auto shrdMem =
+            local_accessor<compute_t<To>, 1>(THREADS_X * threads_y, h);
+
+        switch (threads_y) {
+            case 8:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    reduceDimKernelSMEM<Ti, To, op, dim, 8>(
+                        out_acc, out.info, in_acc, in.info, blocks_dim[0],
+                        blocks_dim[1], blocks_dim[dim], change_nan,
+                        scalar<To>(nanval), shrdMem, debug_stream));
+                break;
+            case 4:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    reduceDimKernelSMEM<Ti, To, op, dim, 4>(
+                        out_acc, out.info, in_acc, in.info, blocks_dim[0],
+                        blocks_dim[1], blocks_dim[dim], change_nan,
+                        scalar<To>(nanval), shrdMem, debug_stream));
+                break;
+            case 2:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    reduceDimKernelSMEM<Ti, To, op, dim, 2>(
+                        out_acc, out.info, in_acc, in.info, blocks_dim[0],
+                        blocks_dim[1], blocks_dim[dim], change_nan,
+                        scalar<To>(nanval), shrdMem, debug_stream));
+                break;
+            case 1:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    reduceDimKernelSMEM<Ti, To, op, dim, 1>(
+                        out_acc, out.info, in_acc, in.info, blocks_dim[0],
+                        blocks_dim[1], blocks_dim[dim], change_nan,
+                        scalar<To>(nanval), shrdMem, debug_stream));
+                break;
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Ti, typename To, af_op_t op, int dim>
+void reduce_dim_default(Param<To> out, Param<Ti> in, bool change_nan,
+                        double nanval) {
+    uint threads_y = std::min(THREADS_Y, nextpow2(in.info.dims[dim]));
+    uint threads_x = THREADS_X;
+
+    dim_t blocks_dim[] = {divup(in.info.dims[0], threads_x), in.info.dims[1],
+                          in.info.dims[2], in.info.dims[3]};
+    blocks_dim[dim]    = divup(in.info.dims[dim], threads_y * REPEAT);
+
+    Param<To> tmp = out;
+    bufptr<To> tmp_alloc;
+    if (blocks_dim[dim] > 1) {
+        tmp.info.dims[dim] = blocks_dim[dim];
+        int tmp_elements   = tmp.info.dims[0] * tmp.info.dims[1] *
+                           tmp.info.dims[2] * tmp.info.dims[3];
+
+        tmp_alloc = memAlloc<To>(tmp_elements);
+        tmp.data  = tmp_alloc.get();
+
+        tmp.info.dims[dim] = blocks_dim[dim];
+        for (int k = dim + 1; k < 4; k++)
+            tmp.info.strides[k] *= blocks_dim[dim];
+    }
+
+    reduce_dim_launcher_default<Ti, To, op, dim>(tmp, in, threads_y, blocks_dim,
+                                                 change_nan, nanval);
+
+    if (blocks_dim[dim] > 1) {
+        blocks_dim[dim] = 1;
+
+        if (op == af_notzero_t) {
+            reduce_dim_launcher_default<To, To, af_add_t, dim>(
+                out, tmp, threads_y, blocks_dim, change_nan, nanval);
+        } else {
+            reduce_dim_launcher_default<To, To, op, dim>(
+                out, tmp, threads_y, blocks_dim, change_nan, nanval);
+        }
+    }
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
new file mode 100644
index 0000000000..cd096e69e1
--- /dev/null
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -0,0 +1,236 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/Transform.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/reduce_config.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+
+#include <algorithm>
+#include <climits>
+#include <complex>
+#include <iostream>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename Ti, typename To, af_op_t op, uint DIMX>
+class reduceFirstKernelSMEM {
+   public:
+    reduceFirstKernelSMEM(sycl::accessor<To> out, KParam oInfo,
+                          sycl::accessor<Ti> in, KParam iInfo, uint groups_x,
+                          uint groups_y, uint repeat, bool change_nan,
+                          To nanval, local_accessor<compute_t<To>, 1> s_val,
+                          sycl::stream debug)
+        : out_(out)
+        , oInfo_(oInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , repeat_(repeat)
+        , change_nan_(change_nan)
+        , nanval_(nanval)
+        , s_val_(s_val)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid = groupId_x * g.get_local_range(0) * repeat_ + lidx;
+        const uint yid = groupId_y * g.get_local_range(1) + lidy;
+
+        common::Binary<compute_t<To>, op> reduce;
+        common::Transform<Ti, compute_t<To>, op> transform;
+
+        const data_t<Ti> *const iptr =
+            in_.get_pointer() + wid * iInfo_.strides[3] +
+            zid * iInfo_.strides[2] + yid * iInfo_.strides[1] + iInfo_.offset;
+
+        data_t<To> *const optr = out_.get_pointer() + wid * oInfo_.strides[3] +
+                                 zid * oInfo_.strides[2] +
+                                 yid * oInfo_.strides[1];
+
+        bool cond = (yid < iInfo_.dims[1]) && (zid < iInfo_.dims[2]) &&
+                    (wid < iInfo_.dims[3]);
+
+        dim_t last = (xid + repeat_ * DIMX);
+        int lim    = sycl::min(last, iInfo_.dims[0]);
+
+        compute_t<To> out_val = common::Binary<compute_t<To>, op>::init();
+        for (int id = xid; cond && id < lim; id += DIMX) {
+            compute_t<To> in_val = transform(iptr[id]);
+            if (change_nan_)
+                in_val = !IS_NAN(in_val) ? in_val
+                                         : static_cast<compute_t<To>>(nanval_);
+            out_val = reduce(in_val, out_val);
+        }
+
+        s_val_[lid] = out_val;
+
+        it.barrier();
+        compute_t<To> *s_ptr = s_val_.get_pointer() + lidy * DIMX;
+
+        if (DIMX == 256) {
+            if (lidx < 128)
+                s_ptr[lidx] = reduce(s_ptr[lidx], s_ptr[lidx + 128]);
+            it.barrier();
+        }
+
+        if (DIMX >= 128) {
+            if (lidx < 64) s_ptr[lidx] = reduce(s_ptr[lidx], s_ptr[lidx + 64]);
+            it.barrier();
+        }
+
+        if (DIMX >= 64) {
+            if (lidx < 32) s_ptr[lidx] = reduce(s_ptr[lidx], s_ptr[lidx + 32]);
+            it.barrier();
+        }
+
+        // TODO: replace with subgroup operations in optimized kernels
+        if (lidx < 16) s_ptr[lidx] = reduce(s_ptr[lidx], s_ptr[lidx + 16]);
+        it.barrier();
+
+        if (lidx < 8) s_ptr[lidx] = reduce(s_ptr[lidx], s_ptr[lidx + 8]);
+        it.barrier();
+
+        if (lidx < 4) s_ptr[lidx] = reduce(s_ptr[lidx], s_ptr[lidx + 4]);
+        it.barrier();
+
+        if (lidx < 2) s_ptr[lidx] = reduce(s_ptr[lidx], s_ptr[lidx + 2]);
+        it.barrier();
+
+        if (lidx < 1) s_ptr[lidx] = reduce(s_ptr[lidx], s_ptr[lidx + 1]);
+        it.barrier();
+
+        if (cond && lidx == 0) optr[groupId_x] = data_t<To>(s_ptr[lidx]);
+    }
+
+   protected:
+    sycl::accessor<To> out_;
+    KParam oInfo_, iInfo_;
+    sycl::accessor<Ti> in_;
+    uint groups_x_, groups_y_, repeat_;
+    bool change_nan_;
+    To nanval_;
+    local_accessor<compute_t<To>, 1> s_val_;
+    sycl::stream debug_;
+};
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_first_launcher_default(Param<To> out, Param<Ti> in,
+                                   const uint groups_x, const uint groups_y,
+                                   const uint threads_x, bool change_nan,
+                                   double nanval) {
+    sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
+                          groups_y * in.info.dims[3] * local[1]);
+
+    uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
+
+    getQueue().submit([=](sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto in_acc  = in.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 256, 128, h);
+
+        auto shrdMem = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
+
+        switch (threads_x) {
+            case 32:
+                h.parallel_for(sycl::nd_range<2>(global, local),
+                               reduceFirstKernelSMEM<Ti, To, op, 32>(
+                                   out_acc, out.info, in_acc, in.info, groups_x,
+                                   groups_y, repeat, change_nan,
+                                   scalar<To>(nanval), shrdMem, debug_stream));
+                break;
+            case 64:
+                h.parallel_for(sycl::nd_range<2>(global, local),
+                               reduceFirstKernelSMEM<Ti, To, op, 64>(
+                                   out_acc, out.info, in_acc, in.info, groups_x,
+                                   groups_y, repeat, change_nan,
+                                   scalar<To>(nanval), shrdMem, debug_stream));
+                break;
+            case 128:
+                h.parallel_for(sycl::nd_range<2>(global, local),
+                               reduceFirstKernelSMEM<Ti, To, op, 128>(
+                                   out_acc, out.info, in_acc, in.info, groups_x,
+                                   groups_y, repeat, change_nan,
+                                   scalar<To>(nanval), shrdMem, debug_stream));
+                break;
+            case 256:
+                h.parallel_for(sycl::nd_range<2>(global, local),
+                               reduceFirstKernelSMEM<Ti, To, op, 256>(
+                                   out_acc, out.info, in_acc, in.info, groups_x,
+                                   groups_y, repeat, change_nan,
+                                   scalar<To>(nanval), shrdMem, debug_stream));
+                break;
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_first_default(Param<To> out, Param<Ti> in, bool change_nan,
+                          double nanval) {
+    uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+    threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
+    uint threads_y = THREADS_PER_BLOCK / threads_x;
+
+    uint blocks_x = divup(in.info.dims[0], threads_x * REPEAT);
+    uint blocks_y = divup(in.info.dims[1], threads_y);
+
+    Param<To> tmp = out;
+    bufptr<To> tmp_alloc;
+    if (blocks_x > 1) {
+        tmp_alloc = memAlloc<To>(blocks_x * in.info.dims[1] * in.info.dims[2] *
+                                 in.info.dims[3]);
+        tmp.data  = tmp_alloc.get();
+
+        tmp.info.dims[0] = blocks_x;
+        for (int k = 1; k < 4; k++) tmp.info.strides[k] *= blocks_x;
+    }
+
+    reduce_first_launcher_default<Ti, To, op>(tmp, in, blocks_x, blocks_y,
+                                              threads_x, change_nan, nanval);
+
+    if (blocks_x > 1) {
+        // FIXME: Is there an alternative to the if condition?
+        if (op == af_notzero_t) {
+            reduce_first_launcher_default<To, To, af_add_t>(
+                out, tmp, 1, blocks_y, threads_x, change_nan, nanval);
+        } else {
+            reduce_first_launcher_default<To, To, op>(
+                out, tmp, 1, blocks_y, threads_x, change_nan, nanval);
+        }
+    }
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index f2128c5ac5..d32d9e8d46 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -355,13 +355,11 @@ bool isGLSharingSupported() {
 
 bool isDoubleSupported(unsigned device) {
     DeviceManager& devMngr = DeviceManager::getInstance();
-
-    sycl::device dev;
     {
         common::lock_guard_t lock(devMngr.deviceMutex);
-        dev = *devMngr.mDevices[device];
+        sycl::device& dev = *devMngr.mDevices[device];
+        return dev.has(sycl::aspect::fp64);
     }
-    return dev.has(sycl::aspect::fp64);
 }
 
 bool isHalfSupported(unsigned device) {
diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp
index 0300fa99b0..007fbccac4 100644
--- a/src/backend/oneapi/reduce_impl.hpp
+++ b/src/backend/oneapi/reduce_impl.hpp
@@ -9,7 +9,7 @@
 
 #include <Array.hpp>
 #include <err_oneapi.hpp>
-//#include <kernel/reduce.hpp>
+#include <kernel/reduce.hpp>
 //#include <kernel/reduce_by_key.hpp>
 #include <reduce.hpp>
 #include <af/dim4.hpp>
@@ -17,12 +17,16 @@
 
 using af::dim4;
 using std::swap;
+
 namespace oneapi {
+
 template<af_op_t op, typename Ti, typename To>
 Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan,
                  double nanval) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<To> out = createEmptyArray<To>(1);
+    dim4 odims    = in.dims();
+    odims[dim]    = 1;
+    Array<To> out = createEmptyArray<To>(odims);
+    kernel::reduce<Ti, To, op>(out, in, dim, change_nan, nanval);
     return out;
 }
 
@@ -35,8 +39,8 @@ void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
 
 template<af_op_t op, typename Ti, typename To>
 Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
-    ONEAPI_NOT_SUPPORTED("");
     Array<To> out = createEmptyArray<To>(1);
+    kernel::reduce_all<Ti, To, op>(out, in, change_nan, nanval);
     return out;
 }
 

From a4c022081456c7c96df4ae102d63b1760b72f66c Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 1 Nov 2022 19:43:22 -0400
Subject: [PATCH 118/473] adds mean kernel

---
 src/backend/oneapi/CMakeLists.txt        |   1 +
 src/backend/oneapi/kernel/mean.hpp       | 789 +++++++++++++++++++++++
 src/backend/oneapi/kernel/reduce_all.hpp |  40 +-
 src/backend/oneapi/mean.cpp              |  20 +-
 4 files changed, 813 insertions(+), 37 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/mean.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index d9ae78f742..e8df95c5ed 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -211,6 +211,7 @@ target_sources(afoneapi
     kernel/iota.hpp
     kernel/histogram.hpp
     kernel/memcopy.hpp
+    kernel/mean.hpp
     kernel/random_engine.hpp
     kernel/random_engine_write.hpp
     kernel/random_engine_mersenne.hpp
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
new file mode 100644
index 0000000000..f63f46096c
--- /dev/null
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -0,0 +1,789 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/Transform.hpp>
+#include <common/dispatch.hpp>
+#include <common/half.hpp>
+//#include <copy.hpp>?
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/reduce_config.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+
+#include <iostream>
+#include <memory>
+#include <vector>
+
+namespace oneapi {
+
+/*
+TODO: port half
+__device__ auto operator*(float lhs, __half rhs) -> __half {
+    return __float2half(lhs * __half2float(rhs));
+}
+
+__device__ auto operator/(__half lhs, float rhs) -> __half {
+    return __float2half(__half2float(lhs) / rhs);
+}
+*/
+
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename To, typename Tw>
+void stable_mean(To *lhs, Tw *l_wt, To rhs, Tw r_wt) {
+    if (((*l_wt) != (Tw)0) || (r_wt != (Tw)0)) {
+        Tw l_scale = (*l_wt);
+        (*l_wt) += r_wt;
+        l_scale = l_scale / (*l_wt);
+
+        Tw r_scale = r_wt / (*l_wt);
+        (*lhs)     = (l_scale * *lhs) + (r_scale * rhs);
+    }
+}
+
+template<typename Ti, typename Tw, typename To, uint dim, uint DIMY>
+class meanDimKernelSMEM {
+   public:
+    meanDimKernelSMEM(sycl::accessor<To> out, KParam oInfo,
+                      sycl::accessor<Tw> owt, KParam owInfo,
+                      sycl::accessor<Ti> in, KParam iInfo,
+                      sycl::accessor<Tw> iwt, KParam iwInfo, uint groups_x,
+                      uint groups_y, uint offset_dim,
+                      local_accessor<compute_t<To>, 1> s_val,
+                      local_accessor<compute_t<Tw>, 1> s_idx,
+                      sycl::stream debug, bool input_weight, bool output_weight)
+        : out_(out)
+        , oInfo_(oInfo)
+        , owt_(owt)
+        , owInfo_(owInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , iwt_(iwt)
+        , iwInfo_(iwInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , offset_dim_(offset_dim)
+        , s_val_(s_val)
+        , s_idx_(s_idx)
+        , debug_(debug)
+        , input_weight_(input_weight)
+        , output_weight_(output_weight) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid        = g.get_group_id(0) / groups_x_;
+        const uint wid        = g.get_group_id(1) / groups_y_;
+        const uint groupIdx_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupIdx_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid        = groupIdx_x * g.get_local_range(0) + lidx;
+        const uint yid =
+            groupIdx_y;  // yid  of output. updated for input later.
+
+        uint ids[4] = {xid, yid, zid, wid};
+
+        const Ti *iptr = in_.get_pointer();
+        To *optr       = out_.get_pointer();
+
+        uint ooffset = ids[3] * oInfo_.strides[3] + ids[2] * oInfo_.strides[2] +
+                       ids[1] * oInfo_.strides[1] + ids[0];
+        // There is only one element per block for out
+        // There are blockDim.y elements per block for in
+        // Hence increment ids[dim] just after offseting out and before
+        // offsetting in
+        optr += ooffset;
+
+        const uint blockIdx_dim = ids[dim];
+        ids[dim]                = ids[dim] * g.get_local_range(1) + lidy;
+
+        uint ioffset = ids[3] * iInfo_.strides[3] + ids[2] * iInfo_.strides[2] +
+                       ids[1] * iInfo_.strides[1] + ids[0];
+        iptr += ioffset;
+
+        const Tw *iwptr;
+        Tw *owptr;
+
+        if (output_weight_) owptr = owt_.get_pointer() + ooffset;
+        if (input_weight_) iwptr = iwt_.get_pointer() + ioffset;
+
+        const uint id_dim_in   = ids[dim];
+        const uint istride_dim = iInfo_.strides[dim];
+
+        bool is_valid = (ids[0] < iInfo_.dims[0]) &&
+                        (ids[1] < iInfo_.dims[1]) &&
+                        (ids[2] < iInfo_.dims[2]) && (ids[3] < iInfo_.dims[3]);
+
+        common::Transform<Ti, compute_t<To>, af_add_t> transform;
+
+        compute_t<To> val    = common::Binary<compute_t<To>, af_add_t>::init();
+        compute_t<Tw> weight = common::Binary<compute_t<Tw>, af_add_t>::init();
+
+        if (is_valid && id_dim_in < iInfo_.dims[dim]) {
+            val = transform(*iptr);
+            if (iwptr != NULL) {
+                weight = *iwptr;
+            } else {
+                weight = (Tw)1;
+            }
+        }
+
+        const uint id_dim_in_start =
+            id_dim_in + offset_dim_ * g.get_local_range(0);
+
+        for (int id = id_dim_in_start; is_valid && (id < iInfo_.dims[dim]);
+             id += offset_dim_ * g.get_local_range(0)) {
+            iptr = iptr + offset_dim_ * g.get_local_range(0) * istride_dim;
+            if (input_weight_) {
+                iwptr =
+                    iwptr + offset_dim_ * g.get_local_range(0) * istride_dim;
+                stable_mean(&val, &weight, transform(*iptr),
+                            compute_t<Tw>(*iwptr));
+            } else {
+                // Faster version of stable_mean when iwptr is NULL
+                val    = val + (transform(*iptr) - val) / (weight + (Tw)1);
+                weight = weight + (Tw)1;
+            }
+        }
+
+        s_val_[lid] = val;
+        s_idx_[lid] = weight;
+
+        compute_t<To> *s_vptr = s_val_.get_pointer() + lid;
+        compute_t<Tw> *s_iptr = s_idx_.get_pointer() + lid;
+        group_barrier(g);
+
+        if (DIMY == 8) {
+            if (lidy < 4) {
+                stable_mean(s_vptr, s_iptr, s_vptr[THREADS_X * 4],
+                            s_iptr[THREADS_X * 4]);
+            }
+            group_barrier(g);
+        }
+
+        if (DIMY >= 4) {
+            if (lidy < 2) {
+                stable_mean(s_vptr, s_iptr, s_vptr[THREADS_X * 2],
+                            s_iptr[THREADS_X * 2]);
+            }
+            group_barrier(g);
+        }
+
+        if (DIMY >= 2) {
+            if (lidy < 1) {
+                stable_mean(s_vptr, s_iptr, s_vptr[THREADS_X * 1],
+                            s_iptr[THREADS_X * 1]);
+            }
+            group_barrier(g);
+        }
+
+        if (lidy == 0 && is_valid && (blockIdx_dim < oInfo_.dims[dim])) {
+            *optr = *s_vptr;
+            if (output_weight_) *owptr = *s_iptr;
+        }
+    }
+
+   protected:
+    sycl::accessor<To> out_;
+    sycl::accessor<Tw> owt_;
+    sycl::accessor<Ti> in_;
+    sycl::accessor<Tw> iwt_;
+    KParam oInfo_, owInfo_, iInfo_, iwInfo_;
+    const uint groups_x_, groups_y_, offset_dim_;
+    local_accessor<compute_t<To>, 1> s_val_;
+    local_accessor<compute_t<Tw>, 1> s_idx_;
+    bool input_weight_, output_weight_;
+    sycl::stream debug_;
+};
+
+template<typename Ti, typename Tw, typename To, int dim>
+void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
+                       Param<Tw> iwt, const uint threads_y,
+                       const dim_t blocks_dim[4]) {
+    sycl::range<2> local(THREADS_X, threads_y);
+    sycl::range<2> global(blocks_dim[0] * blocks_dim[2] * local[0],
+                          blocks_dim[1] * blocks_dim[3] * local[1]);
+
+    sycl::buffer<Tw, 1> empty(sycl::range<1>{1});
+    getQueue().submit([&](sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto in_acc  = in.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 2048, 2048, h);
+
+        auto s_val = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
+        auto s_idx = local_accessor<compute_t<Tw>, 1>(THREADS_PER_BLOCK, h);
+
+        bool input_weight = ((iwt.info.dims[0] * iwt.info.dims[1] *
+                              iwt.info.dims[2] * iwt.info.dims[3]) != 0);
+
+        bool output_weight = ((owt.info.dims[0] * owt.info.dims[1] *
+                               owt.info.dims[2] * owt.info.dims[3]) != 0);
+
+        auto owt_acc =
+            (output_weight) ? owt.data->get_access(h) : empty.get_access(h);
+        auto iwt_acc =
+            (input_weight) ? iwt.data->get_access(h) : empty.get_access(h);
+
+        switch (threads_y) {
+            case 8:
+                h.parallel_for(sycl::nd_range<2>(global, local),
+                               meanDimKernelSMEM<Ti, Tw, To, dim, 8>(
+                                   out_acc, out.info, owt_acc, owt.info, in_acc,
+                                   in.info, iwt_acc, iwt.info, blocks_dim[0],
+                                   blocks_dim[1], blocks_dim[dim], s_val, s_idx,
+                                   debug_stream, input_weight, output_weight));
+                break;
+            case 4:
+                h.parallel_for(sycl::nd_range<2>(global, local),
+                               meanDimKernelSMEM<Ti, Tw, To, dim, 8>(
+                                   out_acc, out.info, owt_acc, owt.info, in_acc,
+                                   in.info, iwt_acc, iwt.info, blocks_dim[0],
+                                   blocks_dim[1], blocks_dim[dim], s_val, s_idx,
+                                   debug_stream, input_weight, output_weight));
+                break;
+            case 2:
+                h.parallel_for(sycl::nd_range<2>(global, local),
+                               meanDimKernelSMEM<Ti, Tw, To, dim, 8>(
+                                   out_acc, out.info, owt_acc, owt.info, in_acc,
+                                   in.info, iwt_acc, iwt.info, blocks_dim[0],
+                                   blocks_dim[1], blocks_dim[dim], s_val, s_idx,
+                                   debug_stream, input_weight, output_weight));
+                break;
+            case 1:
+                h.parallel_for(sycl::nd_range<2>(global, local),
+                               meanDimKernelSMEM<Ti, Tw, To, dim, 8>(
+                                   out_acc, out.info, owt_acc, owt.info, in_acc,
+                                   in.info, iwt_acc, iwt.info, blocks_dim[0],
+                                   blocks_dim[1], blocks_dim[dim], s_val, s_idx,
+                                   debug_stream, input_weight, output_weight));
+                break;
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Ti, typename Tw, typename To, int dim>
+void mean_dim(Param<To> out, Param<Ti> in, Param<Tw> iwt) {
+    uint threads_y = std::min(THREADS_Y, nextpow2(in.info.dims[dim]));
+    uint threads_x = THREADS_X;
+
+    dim_t blocks_dim[] = {divup(in.info.dims[0], threads_x), in.info.dims[1],
+                          in.info.dims[2], in.info.dims[3]};
+
+    blocks_dim[dim] = divup(in.info.dims[dim], threads_y * REPEAT);
+
+    Array<To> tmpOut = createEmptyArray<To>(dim4());
+    Array<Tw> tmpWt  = createEmptyArray<Tw>(dim4());
+
+    if (blocks_dim[dim] > 1) {
+        dim4 dims(4, out.info.dims);
+        dims[dim] = blocks_dim[dim];
+        tmpOut    = createEmptyArray<To>(dims);
+        tmpWt     = createEmptyArray<Tw>(dims);
+    } else {
+        tmpOut = createParamArray(out, false);
+    }
+
+    mean_dim_launcher<Ti, Tw, To, dim>(tmpOut, tmpWt, in, iwt, threads_y,
+                                       blocks_dim);
+
+    if (blocks_dim[dim] > 1) {
+        blocks_dim[dim] = 1;
+
+        Array<Tw> owt = createEmptyArray<Tw>(dim4());
+        mean_dim_launcher<To, Tw, To, dim>(out, owt, tmpOut, tmpWt, threads_y,
+                                           blocks_dim);
+    }
+}
+
+// Calculate mean along the first dimension. If wt is an empty Param, use
+// weight as 1 and treat it as count. If owt is empty Param, do not write
+// temporary reduced counts/weights to it.
+template<typename Ti, typename Tw, typename To>
+class meanFirstKernelSMEM {
+   public:
+    meanFirstKernelSMEM(sycl::accessor<To> out, KParam oInfo,
+                        sycl::accessor<Tw> owt, KParam owInfo,
+                        sycl::accessor<Ti> in, KParam iInfo,
+                        sycl::accessor<Tw> iwt, KParam iwInfo, const uint DIMX,
+                        const uint groups_x, const uint groups_y,
+                        const uint repeat,
+                        local_accessor<compute_t<To>, 1> s_val,
+                        local_accessor<compute_t<Tw>, 1> s_idx,
+                        sycl::stream debug, bool input_weight,
+                        bool output_weight)
+        : out_(out)
+        , oInfo_(oInfo)
+        , owt_(owt)
+        , owInfo_(owInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , iwt_(iwt)
+        , iwInfo_(iwInfo)
+        , DIMX_(DIMX)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , repeat_(repeat)
+        , s_val_(s_val)
+        , s_idx_(s_idx)
+        , debug_(debug)
+        , input_weight_(input_weight)
+        , output_weight_(output_weight) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * DIMX_ + lidx;
+
+        const uint zid        = g.get_group_id(0) / groups_x_;
+        const uint wid        = g.get_group_id(1) / groups_y_;
+        const uint groupIdx_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupIdx_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid = groupIdx_x * g.get_local_range(0) * repeat_ + lidx;
+        const uint yid = groupIdx_y * g.get_local_range(1) + lidy;
+
+        const Ti *iptr = in_.get_pointer();
+        To *optr       = out_.get_pointer();
+
+        iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] +
+                yid * iInfo_.strides[1];
+        optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] +
+                yid * oInfo_.strides[1];
+
+        const Tw *iwptr;
+        Tw *owptr;
+        if (input_weight_)
+            iwptr = iwt_.get_pointer() + wid * iwInfo_.strides[3] +
+                    zid * iwInfo_.strides[2] + yid * iwInfo_.strides[1];
+
+        if (output_weight_)
+            owptr = owt_.get_pointer() + wid * oInfo_.strides[3] +
+                    zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
+
+        bool cond = (yid < iInfo_.dims[1] && zid < iInfo_.dims[2] &&
+                     wid < iInfo_.dims[3]);
+
+        int lim = sycl::min((dim_t)(xid + repeat_ * DIMX_), iInfo_.dims[0]);
+
+        common::Transform<Ti, compute_t<To>, af_add_t> transform;
+
+        compute_t<To> val    = common::Binary<compute_t<To>, af_add_t>::init();
+        compute_t<Tw> weight = common::Binary<compute_t<Tw>, af_add_t>::init();
+
+        if (cond && xid < lim) {
+            val = transform(iptr[xid]);
+            if (input_weight_) {
+                weight = iwptr[xid];
+            } else {
+                weight = (Tw)1;
+            }
+        }
+
+        if (input_weight_) {
+            for (int id = xid + DIMX_; cond && id < lim; id += DIMX_) {
+                stable_mean(&val, &weight, transform(iptr[id]),
+                            compute_t<Tw>(iwptr[id]));
+            }
+        } else {
+            for (int id = xid + DIMX_; cond && id < lim; id += DIMX_) {
+                // Faster version of stable_mean when iwptr is NULL
+                val    = val + (transform(iptr[id]) - val) / (weight + (Tw)1);
+                weight = weight + (Tw)1;
+            }
+        }
+
+        s_val_[lid] = val;
+        s_idx_[lid] = weight;
+        group_barrier(g);
+
+        compute_t<To> *s_vptr = s_val_.get_pointer() + lidy * DIMX_;
+        compute_t<Tw> *s_iptr = s_idx_.get_pointer() + lidy * DIMX_;
+
+        if (DIMX_ == 256) {
+            if (lidx < 128) {
+                stable_mean(s_vptr + lidx, s_iptr + lidx, s_vptr[lidx + 128],
+                            s_iptr[lidx + 128]);
+            }
+            group_barrier(g);
+        }
+
+        if (DIMX_ >= 128) {
+            if (lidx < 64) {
+                stable_mean(s_vptr + lidx, s_iptr + lidx, s_vptr[lidx + 64],
+                            s_iptr[lidx + 64]);
+            }
+            group_barrier(g);
+        }
+
+        if (DIMX_ >= 64) {
+            if (lidx < 32) {
+                stable_mean(s_vptr + lidx, s_iptr + lidx, s_vptr[lidx + 32],
+                            s_iptr[lidx + 32]);
+            }
+            group_barrier(g);
+        }
+
+        if (lidx < 16) {
+            stable_mean(s_vptr + lidx, s_iptr + lidx, s_vptr[lidx + 16],
+                        s_iptr[lidx + 16]);
+        }
+        group_barrier(g);
+
+        if (lidx < 8) {
+            stable_mean(s_vptr + lidx, s_iptr + lidx, s_vptr[lidx + 8],
+                        s_iptr[lidx + 8]);
+        }
+        group_barrier(g);
+
+        if (lidx < 4) {
+            stable_mean(s_vptr + lidx, s_iptr + lidx, s_vptr[lidx + 4],
+                        s_iptr[lidx + 4]);
+        }
+        group_barrier(g);
+
+        if (lidx < 2) {
+            stable_mean(s_vptr + lidx, s_iptr + lidx, s_vptr[lidx + 2],
+                        s_iptr[lidx + 2]);
+        }
+        group_barrier(g);
+
+        if (lidx < 1) {
+            stable_mean(s_vptr + lidx, s_iptr + lidx, s_vptr[lidx + 1],
+                        s_iptr[lidx + 1]);
+        }
+        group_barrier(g);
+
+        if (cond && lidx == 0) {
+            optr[groupIdx_x] = s_vptr[0];
+            if (output_weight_) owptr[groupIdx_x] = s_iptr[0];
+        }
+    }
+
+   protected:
+    sycl::accessor<To> out_;
+    sycl::accessor<Tw> owt_;
+    sycl::accessor<Ti> in_;
+    sycl::accessor<Tw> iwt_;
+    KParam oInfo_, owInfo_, iInfo_, iwInfo_;
+    const uint DIMX_, groups_x_, groups_y_, repeat_;
+    local_accessor<compute_t<To>, 1> s_val_;
+    local_accessor<compute_t<Tw>, 1> s_idx_;
+    bool input_weight_, output_weight_;
+    sycl::stream debug_;
+};
+
+template<typename Ti, typename Tw, typename To>
+void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
+                         Param<Tw> iwt, const uint groups_x,
+                         const uint groups_y, const uint threads_x) {
+    sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
+                          groups_y * in.info.dims[3] * local[1]);
+
+    uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
+
+    sycl::buffer<Tw, 1> empty(sycl::range<1>{1});
+    getQueue().submit([&](sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto in_acc  = in.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 2048, 2048, h);
+
+        auto s_val = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
+        auto s_idx = local_accessor<compute_t<Tw>, 1>(THREADS_PER_BLOCK, h);
+
+        bool input_weight = ((iwt.info.dims[0] * iwt.info.dims[1] *
+                              iwt.info.dims[2] * iwt.info.dims[3]) != 0);
+
+        bool output_weight = ((owt.info.dims[0] * owt.info.dims[1] *
+                               owt.info.dims[2] * owt.info.dims[3]) != 0);
+
+        auto owt_acc =
+            (output_weight) ? owt.data->get_access(h) : empty.get_access(h);
+        auto iwt_acc =
+            (input_weight) ? iwt.data->get_access(h) : empty.get_access(h);
+
+        h.parallel_for(
+            sycl::nd_range<2>(global, local),
+            meanFirstKernelSMEM<Ti, Tw, To>(
+                out_acc, out.info, owt_acc, owt.info, in_acc, in.info, iwt_acc,
+                iwt.info, threads_x, groups_x, groups_y, repeat, s_val, s_idx,
+                debug_stream, input_weight, output_weight));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Ti, typename Tw, typename To>
+void mean_first(Param<To> out, Param<Ti> in, Param<Tw> iwt) {
+    uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+    threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
+    uint threads_y = THREADS_PER_BLOCK / threads_x;
+
+    uint blocks_x = divup(in.info.dims[0], threads_x * REPEAT);
+    uint blocks_y = divup(in.info.dims[1], threads_y);
+
+    Array<To> tmpOut = createEmptyArray<To>(dim4());
+    Array<Tw> tmpWt  = createEmptyArray<Tw>(dim4());
+    if (blocks_x > 1) {
+        tmpOut = createEmptyArray<To>(
+            {blocks_x, in.info.dims[1], in.info.dims[2], in.info.dims[3]});
+        tmpWt = createEmptyArray<Tw>(
+            {blocks_x, in.info.dims[1], in.info.dims[2], in.info.dims[3]});
+    } else {
+        tmpOut = createParamArray(out, false);
+    }
+
+    mean_first_launcher<Ti, Tw, To>(tmpOut, tmpWt, in, iwt, blocks_x, blocks_y,
+                                    threads_x);
+
+    if (blocks_x > 1) {
+        Param<Tw> owt;
+        owt.data = nullptr;
+        mean_first_launcher<To, Tw, To>(out, owt, tmpOut, tmpWt, 1, blocks_y,
+                                        threads_x);
+    }
+}
+
+template<typename Ti, typename Tw, typename To>
+void mean_weighted(Param<To> out, Param<Ti> in, Param<Tw> iwt, int dim) {
+    switch (dim) {
+        case 0: return mean_first<Ti, Tw, To>(out, in, iwt);
+        case 1: return mean_dim<Ti, Tw, To, 1>(out, in, iwt);
+        case 2: return mean_dim<Ti, Tw, To, 2>(out, in, iwt);
+        case 3: return mean_dim<Ti, Tw, To, 3>(out, in, iwt);
+    }
+}
+
+template<typename Ti, typename Tw, typename To>
+void mean(Param<To> out, Param<Ti> in, int dim) {
+    Param<Tw> dummy_weight;
+    mean_weighted<Ti, Tw, To>(out, in, dummy_weight, dim);
+}
+
+template<typename T, typename Tw>
+T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
+    int in_elements =
+        in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
+    // FIXME: Use better heuristics to get to the optimum number
+    if (in_elements > 4096) {
+        bool in_is_linear = (in.info.strides[0] == 1);
+        bool wt_is_linear = (iwt.info.strides[0] == 1);
+        for (int k = 1; k < 4; k++) {
+            in_is_linear &= (in.info.strides[k] ==
+                             (in.info.strides[k - 1] * in.info.dims[k - 1]));
+            wt_is_linear &= (iwt.info.strides[k] ==
+                             (iwt.info.strides[k - 1] * iwt.info.dims[k - 1]));
+        }
+
+        if (in_is_linear && wt_is_linear) {
+            in.info.dims[0] = in_elements;
+            for (int k = 1; k < 4; k++) {
+                in.info.dims[k]    = 1;
+                in.info.strides[k] = in_elements;
+            }
+
+            for (int k = 0; k < 4; k++) {
+                iwt.info.dims[k]    = in.info.dims[k];
+                iwt.info.strides[k] = in.info.strides[k];
+            }
+        }
+
+        uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+        threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
+        uint threads_y = THREADS_PER_BLOCK / threads_x;
+
+        uint blocks_x = divup(in.info.dims[0], threads_x * REPEAT);
+        uint blocks_y = divup(in.info.dims[1], threads_y);
+
+        Array<T> tmpOut = createEmptyArray<T>(
+            {blocks_x, in.info.dims[1], in.info.dims[2], in.info.dims[3]});
+        Array<Tw> tmpWt = createEmptyArray<Tw>(
+            {blocks_x, in.info.dims[1], in.info.dims[2], in.info.dims[3]});
+
+        int tmp_elements = tmpOut.elements();
+
+        mean_first_launcher<T, Tw, T>(tmpOut, tmpWt, in, iwt, blocks_x,
+                                      blocks_y, threads_x);
+
+        std::vector<T> h_ptr(tmp_elements);
+        std::vector<Tw> h_wptr(tmp_elements);
+        sycl::buffer hBuffer(h_ptr.data(), {tmp_elements},
+                             {sycl::property::buffer::use_host_ptr()});
+        sycl::buffer hwBuffer(h_wptr.data(), {tmp_elements},
+                              {sycl::property::buffer::use_host_ptr()});
+
+        auto e1 = getQueue().submit([&](sycl::handler &h) {
+            auto acc_in =
+                tmpOut.getData()->get_access(h, sycl::range{tmp_elements});
+            auto acc_out = hBuffer.get_access();
+            h.copy(acc_in, acc_out);
+        });
+        auto e2 = getQueue().submit([&](sycl::handler &h) {
+            auto acc_in =
+                tmpWt.getData()->get_access(h, sycl::range{tmp_elements});
+            auto acc_out = hwBuffer.get_access();
+            h.copy(acc_in, acc_out);
+        });
+        e1.wait();
+        e2.wait();
+
+        compute_t<T> val     = static_cast<compute_t<T>>(h_ptr[0]);
+        compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_wptr[0]);
+
+        for (int i = 1; i < tmp_elements; i++) {
+            stable_mean(&val, &weight, compute_t<T>(h_ptr[i]),
+                        compute_t<Tw>(h_wptr[i]));
+        }
+
+        return static_cast<T>(val);
+    } else {
+        std::vector<T> h_ptr(in_elements);
+        std::vector<Tw> h_wptr(in_elements);
+
+        sycl::buffer hBuffer(h_ptr.data(), {in_elements},
+                             {sycl::property::buffer::use_host_ptr()});
+        sycl::buffer hwBuffer(h_wptr.data(), {in_elements},
+                              {sycl::property::buffer::use_host_ptr()});
+
+        auto e1 = getQueue().submit([&](sycl::handler &h) {
+            auto acc_in  = in.data->get_access(h, sycl::range{in_elements});
+            auto acc_out = hBuffer.get_access();
+            h.copy(acc_in, acc_out);
+        });
+        auto e2 = getQueue().submit([&](sycl::handler &h) {
+            auto acc_in  = iwt.data->get_access(h, sycl::range{in_elements});
+            auto acc_out = hwBuffer.get_access();
+            h.copy(acc_in, acc_out);
+        });
+        e1.wait();
+        e2.wait();
+
+        compute_t<T> val     = static_cast<compute_t<T>>(h_ptr[0]);
+        compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_wptr[0]);
+        for (int i = 1; i < in_elements; i++) {
+            stable_mean(&val, &weight, compute_t<T>(h_ptr[i]),
+                        compute_t<Tw>(h_wptr[i]));
+        }
+
+        return static_cast<T>(val);
+    }
+}
+
+template<typename Ti, typename Tw, typename To>
+To mean_all(Param<Ti> in) {
+    using std::unique_ptr;
+    int in_elements =
+        in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
+    bool is_linear = (in.info.strides[0] == 1);
+    for (int k = 1; k < 4; k++) {
+        is_linear &= (in.info.strides[k] ==
+                      (in.info.strides[k - 1] * in.info.dims[k - 1]));
+    }
+
+    // FIXME: Use better heuristics to get to the optimum number
+    if (in_elements > 4096 || !is_linear) {
+        if (is_linear) {
+            in.info.dims[0] = in_elements;
+            for (int k = 1; k < 4; k++) {
+                in.info.dims[k]    = 1;
+                in.info.strides[k] = in_elements;
+            }
+        }
+
+        uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+        threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
+        uint threads_y = THREADS_PER_BLOCK / threads_x;
+
+        uint blocks_x = divup(in.info.dims[0], threads_x * REPEAT);
+        uint blocks_y = divup(in.info.dims[1], threads_y);
+
+        dim4 outDims(blocks_x, in.info.dims[1], in.info.dims[2],
+                     in.info.dims[3]);
+
+        Array<To> tmpOut = createEmptyArray<To>(outDims);
+        Array<Tw> tmpCt  = createEmptyArray<Tw>(outDims);
+
+        Param<Tw> iwt;
+        mean_first_launcher<Ti, Tw, To>(tmpOut, tmpCt, in, iwt, blocks_x,
+                                        blocks_y, threads_x);
+
+        int tmp_elements = tmpOut.elements();
+        std::vector<To> h_ptr(tmp_elements);
+        std::vector<Tw> h_cptr(tmp_elements);
+
+        sycl::buffer hBuffer(h_ptr.data(), {tmp_elements},
+                             {sycl::property::buffer::use_host_ptr()});
+        sycl::buffer hcBuffer(h_cptr.data(), {tmp_elements},
+                              {sycl::property::buffer::use_host_ptr()});
+
+        auto e1 = getQueue().submit([&](sycl::handler &h) {
+            auto acc_in =
+                tmpOut.getData()->get_access(h, sycl::range{tmp_elements});
+            auto acc_out = hBuffer.get_access();
+            h.copy(acc_in, acc_out);
+        });
+        auto e2 = getQueue().submit([&](sycl::handler &h) {
+            auto acc_in =
+                tmpCt.getData()->get_access(h, sycl::range{tmp_elements});
+            auto acc_out = hcBuffer.get_access();
+            h.copy(acc_in, acc_out);
+        });
+        e1.wait();
+        e2.wait();
+
+        compute_t<To> val    = static_cast<compute_t<To>>(h_ptr[0]);
+        compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_cptr[0]);
+
+        for (int i = 1; i < tmp_elements; i++) {
+            stable_mean(&val, &weight, compute_t<To>(h_ptr[i]),
+                        compute_t<Tw>(h_cptr[i]));
+        }
+
+        return static_cast<To>(val);
+    } else {
+        std::vector<Ti> h_ptr(in_elements);
+        sycl::buffer outBuffer(h_ptr.data(), {in_elements},
+                               {sycl::property::buffer::use_host_ptr()});
+
+        getQueue()
+            .submit([&](sycl::handler &h) {
+                auto acc_in  = in.data->get_access(h);
+                auto acc_out = outBuffer.get_access();
+                h.copy(acc_in, acc_out);
+            })
+            .wait();
+
+        common::Transform<Ti, compute_t<To>, af_add_t> transform;
+        compute_t<Tw> count = static_cast<compute_t<Tw>>(1);
+
+        compute_t<To> val    = transform(h_ptr[0]);
+        compute_t<Tw> weight = count;
+        for (int i = 1; i < in_elements; i++) {
+            stable_mean(&val, &weight, transform(h_ptr[i]), count);
+        }
+
+        return static_cast<To>(val);
+    }
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index be22e94c90..372dc931fb 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -260,29 +260,25 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
         h.single_task([=] { acc[0] = 0; });
     });
 
-    getQueue()
-        .submit([=](sycl::handler &h) {
-            auto out_acc      = out.data->get_access(h);
-            auto retCount_acc = retirementCount.getData()->get_access(h);
-            auto tmp_acc      = tmp.getData()->get_access(h);
-            auto in_acc       = in.data->get_access(h);
-
-            sycl::stream debug_stream(2048 * 256, 128, h);
-
-            auto shrdMem =
-                local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
-            auto amLast = local_accessor<bool, 1>(1, h);
-            h.parallel_for(
-                sycl::nd_range<2>(global, local),
-                reduceAllKernelSMEM<Ti, To, op>(
-                    out_acc, out.info, retCount_acc, tmp_acc, (KParam)tmp,
-                    in_acc, in.info, threads_x, groups_x, groups_y, repeat,
-                    change_nan, scalar<To>(nanval), shrdMem, amLast,
-                    debug_stream));
-        })
-        .wait_and_throw();
+    getQueue().submit([=](sycl::handler &h) {
+        auto out_acc      = out.data->get_access(h);
+        auto retCount_acc = retirementCount.getData()->get_access(h);
+        auto tmp_acc      = tmp.getData()->get_access(h);
+        auto in_acc       = in.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 256, 128, h);
+
+        auto shrdMem = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
+        auto amLast  = local_accessor<bool, 1>(1, h);
+        h.parallel_for(
+            sycl::nd_range<2>(global, local),
+            reduceAllKernelSMEM<Ti, To, op>(
+                out_acc, out.info, retCount_acc, tmp_acc, (KParam)tmp, in_acc,
+                in.info, threads_x, groups_x, groups_y, repeat, change_nan,
+                scalar<To>(nanval), shrdMem, amLast, debug_stream));
+    });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 }  // namespace kernel
-}  // namespace oneapi
\ No newline at end of file
+}  // namespace oneapi
diff --git a/src/backend/oneapi/mean.cpp b/src/backend/oneapi/mean.cpp
index 41d72a547e..85c4bc0576 100644
--- a/src/backend/oneapi/mean.cpp
+++ b/src/backend/oneapi/mean.cpp
@@ -11,7 +11,7 @@
 #include <mean.hpp>
 
 #include <common/half.hpp>
-// #include <kernel/mean.hpp>
+#include <kernel/mean.hpp>
 #include <af/dim4.hpp>
 
 using af::dim4;
@@ -21,39 +21,29 @@ using std::swap;
 namespace oneapi {
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in) {
-    ONEAPI_NOT_SUPPORTED("mean Not supported");
-
-    return To(0);
-    // return kernel::meanAll<Ti, Tw, To>(in);
+    return kernel::mean_all<Ti, Tw, To>(in);
 }
 
 template<typename T, typename Tw>
 T mean(const Array<T>& in, const Array<Tw>& wts) {
-    ONEAPI_NOT_SUPPORTED("mean Not supported");
-
-    return T(0);
-    // return kernel::meanAllWeighted<T, Tw>(in, wts);
+    return kernel::mean_all_weighted<T, Tw>(in, wts);
 }
 
 template<typename Ti, typename Tw, typename To>
 Array<To> mean(const Array<Ti>& in, const int dim) {
-    ONEAPI_NOT_SUPPORTED("mean Not supported");
-
     dim4 odims    = in.dims();
     odims[dim]    = 1;
     Array<To> out = createEmptyArray<To>(odims);
-    // kernel::mean<Ti, Tw, To>(out, in, dim);
+    kernel::mean<Ti, Tw, To>(out, in, dim);
     return out;
 }
 
 template<typename T, typename Tw>
 Array<T> mean(const Array<T>& in, const Array<Tw>& wts, const int dim) {
-    ONEAPI_NOT_SUPPORTED("mean Not supported");
-
     dim4 odims   = in.dims();
     odims[dim]   = 1;
     Array<T> out = createEmptyArray<T>(odims);
-    // kernel::meanWeighted<T, Tw, T>(out, in, wts, dim);
+    kernel::mean_weighted<T, Tw, T>(out, in, wts, dim);
     return out;
 }
 

From bb9edfdaf639a8891d4cafc2b5483946fd1aafa2 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 3 Nov 2022 00:47:45 -0400
Subject: [PATCH 119/473] adds where kernel, scan_first dependency

---
 src/backend/oneapi/CMakeLists.txt        |   2 +
 src/backend/oneapi/kernel/scan_first.hpp | 303 +++++++++++++++++++++++
 src/backend/oneapi/kernel/where.hpp      | 180 ++++++++++++++
 src/backend/oneapi/scan.cpp              |  16 +-
 src/backend/oneapi/where.cpp             |  12 +-
 5 files changed, 497 insertions(+), 16 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/scan_first.hpp
 create mode 100644 src/backend/oneapi/kernel/where.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index e8df95c5ed..9b866a729f 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -222,9 +222,11 @@ target_sources(afoneapi
     kernel/reduce_all.hpp
     kernel/reduce_first.hpp
     kernel/reduce_dim.hpp
+    kernel/scan_first.hpp
     kernel/transpose.hpp
     kernel/transpose_inplace.hpp
     kernel/triangle.hpp
+    kernel/where.hpp
 )
 
 add_library(ArrayFire::afoneapi ALIAS afoneapi)
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
new file mode 100644
index 0000000000..886cd2f977
--- /dev/null
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -0,0 +1,303 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/Transform.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/reduce_config.hpp>
+#include <memory.hpp>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename Ti, typename To, af_op_t op>
+class scanFirstKernel {
+public:
+    scanFirstKernel(sycl::accessor<To> out_acc, KParam oInfo,
+                    sycl::accessor<To> tmp_acc, KParam tInfo,
+                    sycl::accessor<Ti> in_acc, KParam iInfo,
+                    const uint groups_x, const uint groups_y, const uint lim,
+                    const bool isFinalPass, const uint DIMX, const bool inclusive_scan,
+                    local_accessor<To, 1> s_val, local_accessor<To, 1> s_tmp,
+                    sycl::stream debug_stream) :
+        out_acc_(out_acc), oInfo_(oInfo),
+        tmp_acc_(tmp_acc), tInfo_(tInfo),
+        in_acc_(in_acc),   iInfo_(iInfo),
+        groups_x_(groups_x), groups_y_(groups_y), lim_(lim),
+        isFinalPass_(isFinalPass), DIMX_(DIMX), inclusive_scan_(inclusive_scan),
+        s_val_(s_val), s_tmp_(s_tmp), debug_stream_(debug_stream) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid       = groupId_x * g.get_local_range(0) * lim_ + lidx;
+        const uint yid       = groupId_y * g.get_local_range(1) + lidy;
+
+        bool cond_yzw =
+            (yid < oInfo_.dims[1]) && (zid < oInfo_.dims[2]) && (wid < oInfo_.dims[3]);
+
+        //if (!cond_yzw) return;  // retire warps early TODO: move
+
+        const Ti *iptr = in_acc_.get_pointer();
+        To *optr       = out_acc_.get_pointer();
+        To *tptr       = tmp_acc_.get_pointer();
+
+        iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + yid * iInfo_.strides[1];
+        optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
+        tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] + yid * tInfo_.strides[1];
+
+        To *sptr = s_val_.get_pointer() + lidy * (2 * DIMX_ + 1);
+
+        common::Transform<Ti, To, op> transform;
+        common::Binary<To, op> binop;
+
+        const To init = common::Binary<To, op>::init();
+        int id        = xid;
+        To val        = init;
+
+        const bool isLast = (lidx == (DIMX_ - 1));
+        for (int k = 0; k < lim_; k++) {
+            if (isLast) s_tmp_[lidy] = val;
+
+            bool cond  = (id < oInfo_.dims[0]) && cond_yzw;
+            val        = cond ? transform(iptr[id]) : init;
+            /*
+            if constexpr(std::is_fundamental<To>::value) {
+                debug_stream_ << id<<":"<<val <<", "<< sycl::stream_manipulator::endl;
+            }
+            */
+            sptr[lidx] = val;
+            group_barrier(g);
+
+            int start = 0;
+            #pragma unroll
+            for (int off = 1; off < DIMX_; off *= 2) {
+                if (lidx >= off) val = binop(val, sptr[(start - off) + lidx]);
+                start              = DIMX_ - start;
+                sptr[start + lidx] = val;
+
+                group_barrier(g);
+            }
+
+            val = binop(val, s_tmp_[lidy]);
+
+            if (inclusive_scan_) {
+                if (cond && cond_yzw) {
+                    //debug_stream_ << "oi0 ";
+                     optr[id] = val; }
+            } else {
+                if (cond_yzw && id == (oInfo_.dims[0] - 1)) {
+                    optr[0] = init;
+                } else if (cond_yzw && id < (oInfo_.dims[0] - 1)) {
+                    //debug_stream_ << "oe0 ";
+                    optr[id + 1] = val;
+                }
+            }
+            id += g.get_local_range(0);
+            group_barrier(g);
+        }
+
+        if (!isFinalPass_ && isLast && cond_yzw) {
+            //debug_stream_ << "ot ";
+             tptr[groupId_x] = val; }
+    }
+
+protected:
+    sycl::accessor<To> out_acc_;
+    sycl::accessor<To> tmp_acc_;
+    sycl::accessor<Ti> in_acc_; 
+    KParam oInfo_, tInfo_, iInfo_;
+    const uint groups_x_, groups_y_, lim_, DIMX_;
+    const bool isFinalPass_, inclusive_scan_;
+    local_accessor<To, 1> s_val_;
+    local_accessor<To, 1> s_tmp_;
+    sycl::stream debug_stream_;
+};
+
+template<typename To, af_op_t op>
+class scanFirstBcastKernel {
+public: 
+    scanFirstBcastKernel(sycl::accessor<To> out_acc, KParam oInfo,
+                         sycl::accessor<To> tmp_acc, KParam tInfo,
+                         const uint groups_x, const uint groups_y, const uint lim,
+                         const bool inclusive_scan, sycl::stream debug_stream) :
+        out_acc_(out_acc), oInfo_(oInfo),
+        tmp_acc_(tmp_acc), tInfo_(tInfo),
+        groups_x_(groups_x), groups_y_(groups_y), lim_(lim),
+        inclusive_scan_(inclusive_scan), debug_stream_(debug_stream) {}
+    
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid       = groupId_x * g.get_local_range(0) * lim_ + lidx;
+        const uint yid       = groupId_y * g.get_local_range(1) + lidy;
+
+        if (groupId_x == 0) return;
+
+        bool cond =
+            (yid < oInfo_.dims[1]) && (zid < oInfo_.dims[2]) && (wid < oInfo_.dims[3]);
+        if (!cond) return;
+
+        To *optr       = out_acc_.get_pointer();
+        const To *tptr = tmp_acc_.get_pointer();
+
+        optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
+        tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] + yid * tInfo_.strides[1];
+
+        common::Binary<To, op> binop;
+        To accum = tptr[groupId_x - 1];
+
+        // Shift broadcast one step to the right for exclusive scan (#2366)
+        int offset = !inclusive_scan_;
+        for (int k = 0, id = xid + offset; k < lim_ && id < oInfo_.dims[0];
+            k++, id += g.get_group_range(0)) {
+            optr[id] = binop(accum, optr[id]);
+        }
+    }
+protected:
+    sycl::accessor<To> out_acc_; 
+    sycl::accessor<To> tmp_acc_;
+    KParam oInfo_, tInfo_;
+    const uint groups_x_, groups_y_, lim_;
+    const bool inclusive_scan_;
+    sycl::stream debug_stream_;
+};
+
+
+template<typename Ti, typename To, af_op_t op>
+static void scan_first_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
+                                const uint groups_x, const uint groups_y,
+                                const uint threads_x, bool isFinalPass,
+                                bool inclusive_scan) {
+    sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> global(groups_x * out.info.dims[2] * local[0], 
+                          groups_y * out.info.dims[3] * local[1]);
+    uint lim = divup(out.info.dims[0], (threads_x * groups_x));
+
+    getQueue().submit([&] (sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto tmp_acc = tmp.data->get_access(h);
+        auto in_acc  = in.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 256, 128, h);
+
+        const int DIMY            = THREADS_PER_BLOCK / threads_x;
+        const int SHARED_MEM_SIZE = (2 * threads_x + 1) * (DIMY);
+        auto s_val = local_accessor<compute_t<To>, 1>(SHARED_MEM_SIZE, h);
+        auto s_tmp = local_accessor<compute_t<To>, 1>(DIMY, h);
+
+        //TODO threads_x as template arg for #pragma unroll?
+        h.parallel_for(sycl::nd_range<2>(global, local), 
+            scanFirstKernel<Ti, To, op>(
+                    out_acc, out.info,
+                    tmp_acc, tmp.info,
+                    in_acc, in.info,
+                    groups_x, groups_y, lim,
+                    isFinalPass, threads_x, inclusive_scan,
+                    s_val, s_tmp, debug_stream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename To, af_op_t op>
+static void bcast_first_launcher(Param<To> out, Param<To> tmp,
+                                 const uint groups_x, const uint groups_y,
+                                 const uint threads_x, bool inclusive_scan) {
+    sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> global(groups_x * out.info.dims[2] * local[0], 
+                          groups_y * out.info.dims[3] * local[1]);
+    uint lim = divup(out.info.dims[0], (threads_x * groups_x));
+
+    getQueue().submit([&] (sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto tmp_acc = tmp.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 256, 128, h);
+
+        const int DIMY            = THREADS_PER_BLOCK / threads_x;
+        const int SHARED_MEM_SIZE = (2 * threads_x + 1) * (DIMY);
+        auto s_val = local_accessor<compute_t<To>, 1>(SHARED_MEM_SIZE, h);
+        auto s_tmp = local_accessor<compute_t<To>, 1>(DIMY, h);
+
+        h.parallel_for(sycl::nd_range<2>(global, local), 
+            scanFirstBcastKernel<To, op>(
+                    out_acc, out.info,
+                    tmp_acc, tmp.info,
+                    groups_x, groups_y, lim,
+                    inclusive_scan, debug_stream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Ti, typename To, af_op_t op>
+static void scan_first(Param<To> out, Param<Ti> in, bool inclusive_scan) {
+    uint threads_x = nextpow2(std::max(32u, (uint)out.info.dims[0]));
+    threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
+    uint threads_y = THREADS_PER_BLOCK / threads_x;
+
+    uint groups_x = divup(out.info.dims[0], threads_x * REPEAT);
+    uint groups_y = divup(out.info.dims[1], threads_y);
+
+    if (groups_x == 1) {
+        scan_first_launcher<Ti, To, op>(out, out, in, groups_x, groups_y,
+                                        threads_x, true, inclusive_scan);
+    } else {
+        Param<To> tmp = out;
+
+        tmp.info.dims[0]    = groups_x;
+        tmp.info.strides[0] = 1;
+        for (int k = 1; k < 4; k++)
+            tmp.info.strides[k] = tmp.info.strides[k - 1] * tmp.info.dims[k - 1];
+
+        int tmp_elements = tmp.info.strides[3] * tmp.info.dims[3];
+        auto tmp_alloc   = memAlloc<To>(tmp_elements);
+        tmp.data          = tmp_alloc.get();
+
+        scan_first_launcher<Ti, To, op>(out, tmp, in, groups_x, groups_y,
+                                        threads_x, false, inclusive_scan);
+
+        // FIXME: Is there an alternative to the if condition ?
+        if (op == af_notzero_t) {
+            scan_first_launcher<To, To, af_add_t>(tmp, tmp, tmp, 1, groups_y,
+                                                  threads_x, true, true);
+        } else {
+            scan_first_launcher<To, To, op>(tmp, tmp, tmp, 1, groups_y,
+                                            threads_x, true, true);
+        }
+
+        bcast_first_launcher<To, op>(out, tmp, groups_x, groups_y, threads_x,
+                                     inclusive_scan);
+    }
+}
+
+}  // namespace kernel
+}  // namespace oneapi
\ No newline at end of file
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
new file mode 100644
index 0000000000..67c7500ee6
--- /dev/null
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -0,0 +1,180 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <memory.hpp>
+#include <kernel/reduce_config.hpp>
+#include <kernel/scan_first.hpp>
+
+#include <Param.hpp>
+#include <backend.hpp>
+#include <math.hpp>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+class whereKernel {
+public:
+    whereKernel(sycl::accessor<uint> out_acc,  KParam oInfo,
+                sycl::accessor<uint> otmp_acc, KParam otInfo,
+                sycl::accessor<uint> rtmp_acc, KParam rtInfo,
+                sycl::accessor<T> in_acc,      KParam iInfo,
+                uint groups_x, uint groups_y, uint lim, sycl::stream debug) : 
+        out_acc_(out_acc), oInfo_(oInfo), otmp_acc_(otmp_acc), otInfo_(otInfo),
+        rtmp_acc_(rtmp_acc), rtInfo_(rtInfo), in_acc_(in_acc), iInfo_(iInfo),
+        groups_x_(groups_x), groups_y_(groups_y), lim_(lim), debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid       = groupId_x * g.get_local_range(0) * lim_ + lidx;
+        const uint yid       = groupId_y * g.get_local_range(1) + lidy;
+
+        const uint *otptr = otmp_acc_.get_pointer();
+        const uint *rtptr = rtmp_acc_.get_pointer();
+        const T *iptr     = in_acc_.get_pointer();
+
+        const uint off =
+            wid * otInfo_.strides[3] + zid * otInfo_.strides[2] + yid * otInfo_.strides[1];
+        const uint bid = wid * rtInfo_.strides[3] + zid * rtInfo_.strides[2] +
+                        yid * rtInfo_.strides[1] + groupId_x;
+
+        otptr +=
+            wid * otInfo_.strides[3] + zid * otInfo_.strides[2] + yid * otInfo_.strides[1];
+        iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + yid * iInfo_.strides[1];
+
+        bool cond =
+            (yid < otInfo_.dims[1]) && (zid < otInfo_.dims[2]) && (wid < otInfo_.dims[3]);
+        T zero = scalar<T>(0);
+
+        if (!cond) return;
+
+        uint accum = (bid == 0) ? 0 : rtptr[bid - 1];
+
+        for (uint k = 0, id = xid; k < lim_ && id < otInfo_.dims[0];
+            k++, id += g.get_local_range(0)) {
+            uint idx = otptr[id] + accum;
+            if (iptr[id] != zero) out_acc_[idx - 1] = (off + id);
+        }
+    }
+protected:
+    sycl::accessor<uint> out_acc_;
+    sycl::accessor<uint> otmp_acc_;
+    sycl::accessor<uint> rtmp_acc_;
+    sycl::accessor<T> in_acc_;
+    KParam oInfo_, otInfo_, rtInfo_, iInfo_;
+    uint groups_x_, groups_y_, lim_;
+    sycl::stream debug_;
+};
+
+template<typename T>
+static void where(Param<uint> &out, Param<T> in) {
+    uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+    threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
+    uint threads_y = THREADS_PER_BLOCK / threads_x;
+
+    uint groups_x = divup((uint)in.info.dims[0], (uint)(threads_x * REPEAT));
+    uint groups_y = divup(in.info.dims[1], threads_y);
+
+    Param<uint> rtmp;
+    Param<uint> otmp;
+    rtmp.info.dims[0]    = groups_x;
+    otmp.info.dims[0]    = in.info.dims[0];
+    rtmp.info.strides[0] = 1;
+    otmp.info.strides[0] = 1;
+
+    for (int k = 1; k < 4; k++) {
+        rtmp.info.dims[k]    = in.info.dims[k];
+        rtmp.info.strides[k] = rtmp.info.strides[k - 1] * rtmp.info.dims[k - 1];
+
+        otmp.info.dims[k]    = in.info.dims[k];
+        otmp.info.strides[k] = otmp.info.strides[k - 1] * otmp.info.dims[k - 1];
+    }
+
+    int rtmp_elements = rtmp.info.strides[3] * rtmp.info.dims[3];
+    int otmp_elements = otmp.info.strides[3] * otmp.info.dims[3];
+    auto rtmp_alloc   = memAlloc<uint>(rtmp_elements);
+    auto otmp_alloc   = memAlloc<uint>(otmp_elements);
+    rtmp.data          = rtmp_alloc.get();
+    otmp.data          = otmp_alloc.get();
+
+    scan_first_launcher<T, uint, af_notzero_t>(
+        otmp, rtmp, in, groups_x, groups_y, threads_x, false, true);
+
+    // Linearize the dimensions and perform scan
+    Param<uint> ltmp = rtmp;
+    ltmp.info.dims[0]     = rtmp_elements;
+    for (int k = 1; k < 4; k++) {
+        ltmp.info.dims[k]    = 1;
+        ltmp.info.strides[k] = rtmp_elements;
+    }
+
+    scan_first<uint, uint, af_add_t>(ltmp, ltmp, true);
+
+    // Get output size and allocate output
+    uint total;
+    sycl::buffer retBuffer(&total, {1},
+        {sycl::property::buffer::use_host_ptr()});
+
+    getQueue()
+        .submit([&](sycl::handler &h) {
+            auto acc_in = rtmp.data->get_access(h, sycl::range{1}, sycl::id{rtmp_elements - 1});
+            auto acc_out = retBuffer.get_access();
+            h.copy(acc_in, acc_out);
+    }).wait();
+
+    auto out_alloc = memAlloc<uint>(total);
+    out.data        = out_alloc.get();
+
+    out.info.dims[0]    = total;
+    out.info.strides[0] = 1;
+    for (int k = 1; k < 4; k++) {
+        out.info.dims[k]    = 1;
+        out.info.strides[k] = total;
+    }
+
+    sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> global(groups_x * in.info.dims[2] * local[0], 
+                          groups_y * in.info.dims[3] * local[1]);
+    uint lim = divup(otmp.info.dims[0], (threads_x * groups_x));
+
+    getQueue().submit([&] (sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto otmp_acc = otmp.data->get_access(h);
+        auto rtmp_acc = rtmp.data->get_access(h);
+        auto in_acc  = in.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 256, 128, h);
+        h.parallel_for(sycl::nd_range<2>(global, local), 
+            whereKernel<T>(
+                    out_acc, out.info,
+                    otmp_acc, otmp.info,
+                    rtmp_acc, rtmp.info,
+                    in_acc, in.info,
+                    groups_x, groups_y, lim,
+                    debug_stream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+    out_alloc.release();
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/scan.cpp b/src/backend/oneapi/scan.cpp
index c71564cc65..51183cfe8c 100644
--- a/src/backend/oneapi/scan.cpp
+++ b/src/backend/oneapi/scan.cpp
@@ -11,23 +11,21 @@
 #include <scan.hpp>
 
 // #include <kernel/scan_dim.hpp>
-// #include <kernel/scan_first.hpp>
+#include <kernel/scan_first.hpp>
 
 namespace oneapi {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusiveScan) {
-    ONEAPI_NOT_SUPPORTED("scan Not supported");
-
     Array<To> out = createEmptyArray<To>(in.dims());
 
-    // Param Out = out;
-    // Param In  = in;
+    Param<To> Out = out;
+    Param<Ti> In  = in;
 
-    // if (dim == 0) {
-    //     kernel::scanFirst<Ti, To, op>(Out, In, inclusiveScan);
-    // } else {
+    if (dim == 0) {
+        kernel::scan_first<Ti, To, op>(Out, In, inclusiveScan);
+    } else {
     //     kernel::scanDim<Ti, To, op>(Out, In, dim, inclusiveScan);
-    // }
+    }
 
     return out;
 }
diff --git a/src/backend/oneapi/where.cpp b/src/backend/oneapi/where.cpp
index df9267df72..2965cbe883 100644
--- a/src/backend/oneapi/where.cpp
+++ b/src/backend/oneapi/where.cpp
@@ -9,7 +9,7 @@
 
 #include <Array.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/where.hpp>
+#include <kernel/where.hpp>
 #include <where.hpp>
 #include <af/dim4.hpp>
 #include <complex>
@@ -18,12 +18,10 @@ namespace oneapi {
 
 template<typename T>
 Array<uint> where(const Array<T> &in) {
-    // Param<uint> Out;
-    // Param<T> In = in;
-    ONEAPI_NOT_SUPPORTED("where Not supported");
-    // kernel::where<T>(Out, In);
-    // return createParamArray<uint>(Out, true);
-    return createEmptyArray<uint>(af::dim4(1));
+    Param<uint> Out;
+    Param<T> In = in;
+    kernel::where<T>(Out, In);
+    return createParamArray<uint>(Out, true);
 }
 
 #define INSTANTIATE(T) template Array<uint> where<T>(const Array<T> &in);

From 9469aee07a33984ad4a6647165fb1c290c89e07b Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 4 Nov 2022 14:12:24 -0400
Subject: [PATCH 120/473] adds scan_dim kernels

---
 src/backend/oneapi/CMakeLists.txt            |   1 +
 src/backend/oneapi/kernel/default_config.hpp |  21 ++
 src/backend/oneapi/kernel/mean.hpp           |   7 +-
 src/backend/oneapi/kernel/reduce.hpp         |   6 +-
 src/backend/oneapi/kernel/reduce_all.hpp     |  21 +-
 src/backend/oneapi/kernel/reduce_config.hpp  |   3 +
 src/backend/oneapi/kernel/reduce_dim.hpp     |  19 +-
 src/backend/oneapi/kernel/reduce_first.hpp   |  11 +-
 src/backend/oneapi/kernel/scan_dim.hpp       | 349 +++++++++++++++++++
 src/backend/oneapi/kernel/scan_first.hpp     | 154 ++++----
 src/backend/oneapi/kernel/where.hpp          |  89 ++---
 src/backend/oneapi/scan.cpp                  |  11 +-
 12 files changed, 544 insertions(+), 148 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/default_config.hpp
 create mode 100644 src/backend/oneapi/kernel/scan_dim.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 9b866a729f..4b0f9f0a29 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -223,6 +223,7 @@ target_sources(afoneapi
     kernel/reduce_first.hpp
     kernel/reduce_dim.hpp
     kernel/scan_first.hpp
+    kernel/scan_dim.hpp
     kernel/transpose.hpp
     kernel/transpose_inplace.hpp
     kernel/triangle.hpp
diff --git a/src/backend/oneapi/kernel/default_config.hpp b/src/backend/oneapi/kernel/default_config.hpp
new file mode 100644
index 0000000000..c279fd98bb
--- /dev/null
+++ b/src/backend/oneapi/kernel/default_config.hpp
@@ -0,0 +1,21 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+namespace oneapi {
+namespace kernel {
+
+static const uint THREADS_PER_BLOCK = 256;
+static const uint THREADS_X         = 32;
+static const uint THREADS_Y         = THREADS_PER_BLOCK / THREADS_X;
+static const uint REPEAT            = 32;
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index f63f46096c..3f7b5c3fee 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -16,6 +16,7 @@
 //#include <copy.hpp>?
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/default_config.hpp>
 #include <kernel/reduce_config.hpp>
 #include <math.hpp>
 #include <memory.hpp>
@@ -253,7 +254,7 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                 break;
             case 4:
                 h.parallel_for(sycl::nd_range<2>(global, local),
-                               meanDimKernelSMEM<Ti, Tw, To, dim, 8>(
+                               meanDimKernelSMEM<Ti, Tw, To, dim, 4>(
                                    out_acc, out.info, owt_acc, owt.info, in_acc,
                                    in.info, iwt_acc, iwt.info, blocks_dim[0],
                                    blocks_dim[1], blocks_dim[dim], s_val, s_idx,
@@ -261,7 +262,7 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                 break;
             case 2:
                 h.parallel_for(sycl::nd_range<2>(global, local),
-                               meanDimKernelSMEM<Ti, Tw, To, dim, 8>(
+                               meanDimKernelSMEM<Ti, Tw, To, dim, 2>(
                                    out_acc, out.info, owt_acc, owt.info, in_acc,
                                    in.info, iwt_acc, iwt.info, blocks_dim[0],
                                    blocks_dim[1], blocks_dim[dim], s_val, s_idx,
@@ -269,7 +270,7 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                 break;
             case 1:
                 h.parallel_for(sycl::nd_range<2>(global, local),
-                               meanDimKernelSMEM<Ti, Tw, To, dim, 8>(
+                               meanDimKernelSMEM<Ti, Tw, To, dim, 1>(
                                    out_acc, out.info, owt_acc, owt.info, in_acc,
                                    in.info, iwt_acc, iwt.info, blocks_dim[0],
                                    blocks_dim[1], blocks_dim[dim], s_val, s_idx,
diff --git a/src/backend/oneapi/kernel/reduce.hpp b/src/backend/oneapi/kernel/reduce.hpp
index 9db0561b0a..cae5c9854c 100644
--- a/src/backend/oneapi/kernel/reduce.hpp
+++ b/src/backend/oneapi/kernel/reduce.hpp
@@ -96,12 +96,12 @@ void reduce_all(Param<To> out, Param<Ti> in, bool change_nan, double nanval) {
     }
 
     uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
-    threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
-    uint threads_y = THREADS_PER_BLOCK / threads_x;
+    threads_x      = std::min(threads_x, creduce::THREADS_PER_BLOCK);
+    uint threads_y = creduce::THREADS_PER_BLOCK / threads_x;
 
     // TODO: perf REPEAT, consider removing or runtime eval
     // max problem size < SM resident threads, don't use REPEAT
-    uint blocks_x = divup(in.info.dims[0], threads_x * REPEAT);
+    uint blocks_x = divup(in.info.dims[0], threads_x * creduce::REPEAT);
     uint blocks_y = divup(in.info.dims[1], threads_y);
 
     reduce_all_launcher_default<Ti, To, op>(out, in, blocks_x, blocks_y,
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index 372dc931fb..6fdf008e69 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -105,17 +105,17 @@ class reduceAllKernelSMEM {
 
         group_barrier(g);
 
-        if (THREADS_PER_BLOCK == 256) {
+        if (creduce::THREADS_PER_BLOCK == 256) {
             if (lid < 128) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 128]);
             group_barrier(g);
         }
 
-        if (THREADS_PER_BLOCK >= 128) {
+        if (creduce::THREADS_PER_BLOCK >= 128) {
             if (lid < 64) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 64]);
             group_barrier(g);
         }
 
-        if (THREADS_PER_BLOCK >= 64) {
+        if (creduce::THREADS_PER_BLOCK >= 64) {
             if (lid < 32) s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 32]);
             group_barrier(g);
         }
@@ -168,26 +168,26 @@ class reduceAllKernelSMEM {
                 while (i < total_blocks) {
                     compute_t<To> in_val = compute_t<To>(tmp_[i]);
                     out_val              = reduce(in_val, out_val);
-                    i += THREADS_PER_BLOCK;
+                    i += creduce::THREADS_PER_BLOCK;
                 }
 
                 s_ptr_[lid] = out_val;
                 group_barrier(g);
 
                 // reduce final block
-                if (THREADS_PER_BLOCK == 256) {
+                if (creduce::THREADS_PER_BLOCK == 256) {
                     if (lid < 128)
                         s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 128]);
                     group_barrier(g);
                 }
 
-                if (THREADS_PER_BLOCK >= 128) {
+                if (creduce::THREADS_PER_BLOCK >= 128) {
                     if (lid < 64)
                         s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 64]);
                     group_barrier(g);
                 }
 
-                if (THREADS_PER_BLOCK >= 64) {
+                if (creduce::THREADS_PER_BLOCK >= 64) {
                     if (lid < 32)
                         s_ptr_[lid] = reduce(s_ptr_[lid], s_ptr_[lid + 32]);
                     group_barrier(g);
@@ -239,7 +239,7 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
                                  const uint groups_x, const uint groups_y,
                                  const uint threads_x, bool change_nan,
                                  double nanval) {
-    sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> local(threads_x, creduce::THREADS_PER_BLOCK / threads_x);
     sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
                           groups_y * in.info.dims[3] * local[1]);
 
@@ -268,8 +268,9 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
-        auto shrdMem = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
-        auto amLast  = local_accessor<bool, 1>(1, h);
+        auto shrdMem =
+            local_accessor<compute_t<To>, 1>(creduce::THREADS_PER_BLOCK, h);
+        auto amLast = local_accessor<bool, 1>(1, h);
         h.parallel_for(
             sycl::nd_range<2>(global, local),
             reduceAllKernelSMEM<Ti, To, op>(
diff --git a/src/backend/oneapi/kernel/reduce_config.hpp b/src/backend/oneapi/kernel/reduce_config.hpp
index 827497967b..a7d185de75 100644
--- a/src/backend/oneapi/kernel/reduce_config.hpp
+++ b/src/backend/oneapi/kernel/reduce_config.hpp
@@ -12,11 +12,14 @@
 namespace oneapi {
 namespace kernel {
 
+namespace creduce {
 // TODO: are different values more appropriate for reduce on oneapi?
 static const uint THREADS_PER_BLOCK = 256;
 static const uint THREADS_X         = 32;
 static const uint THREADS_Y         = THREADS_PER_BLOCK / THREADS_X;
 static const uint REPEAT            = 32;
 
+}  // namespace creduce
+
 }  // namespace kernel
 }  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index 5105fb8b1c..3f9e365b8b 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -105,17 +105,20 @@ class reduceDimKernelSMEM {
         compute_t<To> *s_ptr = s_val_.get_pointer() + lid;
 
         if (DIMY == 8) {
-            if (lidy < 4) *s_ptr = reduce(*s_ptr, s_ptr[THREADS_X * 4]);
+            if (lidy < 4)
+                *s_ptr = reduce(*s_ptr, s_ptr[creduce::THREADS_X * 4]);
             it.barrier();
         }
 
         if (DIMY >= 4) {
-            if (lidy < 2) *s_ptr = reduce(*s_ptr, s_ptr[THREADS_X * 2]);
+            if (lidy < 2)
+                *s_ptr = reduce(*s_ptr, s_ptr[creduce::THREADS_X * 2]);
             it.barrier();
         }
 
         if (DIMY >= 2) {
-            if (lidy < 1) *s_ptr = reduce(*s_ptr, s_ptr[THREADS_X * 1]);
+            if (lidy < 1)
+                *s_ptr = reduce(*s_ptr, s_ptr[creduce::THREADS_X * 1]);
             it.barrier();
         }
 
@@ -140,7 +143,7 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                                  const uint threads_y,
                                  const dim_t blocks_dim[4], bool change_nan,
                                  double nanval) {
-    sycl::range<2> local(THREADS_X, threads_y);
+    sycl::range<2> local(creduce::THREADS_X, threads_y);
     sycl::range<2> global(blocks_dim[0] * blocks_dim[2] * local[0],
                           blocks_dim[1] * blocks_dim[3] * local[1]);
 
@@ -151,7 +154,7 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
         sycl::stream debug_stream(2048 * 256, 128, h);
 
         auto shrdMem =
-            local_accessor<compute_t<To>, 1>(THREADS_X * threads_y, h);
+            local_accessor<compute_t<To>, 1>(creduce::THREADS_X * threads_y, h);
 
         switch (threads_y) {
             case 8:
@@ -194,12 +197,12 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
 template<typename Ti, typename To, af_op_t op, int dim>
 void reduce_dim_default(Param<To> out, Param<Ti> in, bool change_nan,
                         double nanval) {
-    uint threads_y = std::min(THREADS_Y, nextpow2(in.info.dims[dim]));
-    uint threads_x = THREADS_X;
+    uint threads_y = std::min(creduce::THREADS_Y, nextpow2(in.info.dims[dim]));
+    uint threads_x = creduce::THREADS_X;
 
     dim_t blocks_dim[] = {divup(in.info.dims[0], threads_x), in.info.dims[1],
                           in.info.dims[2], in.info.dims[3]};
-    blocks_dim[dim]    = divup(in.info.dims[dim], threads_y * REPEAT);
+    blocks_dim[dim]    = divup(in.info.dims[dim], threads_y * creduce::REPEAT);
 
     Param<To> tmp = out;
     bufptr<To> tmp_alloc;
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index cd096e69e1..ebf55fb63e 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -148,7 +148,7 @@ void reduce_first_launcher_default(Param<To> out, Param<Ti> in,
                                    const uint groups_x, const uint groups_y,
                                    const uint threads_x, bool change_nan,
                                    double nanval) {
-    sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> local(threads_x, creduce::THREADS_PER_BLOCK / threads_x);
     sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
                           groups_y * in.info.dims[3] * local[1]);
 
@@ -160,7 +160,8 @@ void reduce_first_launcher_default(Param<To> out, Param<Ti> in,
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
-        auto shrdMem = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
+        auto shrdMem =
+            local_accessor<compute_t<To>, 1>(creduce::THREADS_PER_BLOCK, h);
 
         switch (threads_x) {
             case 32:
@@ -200,10 +201,10 @@ template<typename Ti, typename To, af_op_t op>
 void reduce_first_default(Param<To> out, Param<Ti> in, bool change_nan,
                           double nanval) {
     uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
-    threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
-    uint threads_y = THREADS_PER_BLOCK / threads_x;
+    threads_x      = std::min(threads_x, creduce::THREADS_PER_BLOCK);
+    uint threads_y = creduce::THREADS_PER_BLOCK / threads_x;
 
-    uint blocks_x = divup(in.info.dims[0], threads_x * REPEAT);
+    uint blocks_x = divup(in.info.dims[0], threads_x * creduce::REPEAT);
     uint blocks_y = divup(in.info.dims[1], threads_y);
 
     Param<To> tmp = out;
diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp
new file mode 100644
index 0000000000..1db981ca9a
--- /dev/null
+++ b/src/backend/oneapi/kernel/scan_dim.hpp
@@ -0,0 +1,349 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/Transform.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/default_config.hpp>
+#include <memory.hpp>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename Ti, typename To, af_op_t op, int dim>
+class scanDimKernel {
+   public:
+    scanDimKernel(sycl::accessor<To> out_acc, KParam oInfo,
+                  sycl::accessor<To> tmp_acc, KParam tInfo,
+                  sycl::accessor<Ti> in_acc, KParam iInfo, const uint groups_x,
+                  const uint groups_y, const uint blocks_dim, const uint lim,
+                  const bool isFinalPass, const uint DIMY,
+                  const bool inclusive_scan, local_accessor<To, 1> s_val,
+                  local_accessor<To, 1> s_tmp, sycl::stream debug)
+        : out_acc_(out_acc)
+        , oInfo_(oInfo)
+        , tmp_acc_(tmp_acc)
+        , tInfo_(tInfo)
+        , in_acc_(in_acc)
+        , iInfo_(iInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , blocks_dim_(blocks_dim)
+        , lim_(lim)
+        , isFinalPass_(isFinalPass)
+        , DIMY_(DIMY)
+        , inclusive_scan_(inclusive_scan)
+        , s_val_(s_val)
+        , s_tmp_(s_tmp)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid       = groupId_x * g.get_local_range(0) + lidx;
+        const uint yid       = groupId_y;
+
+        int ids[4] = {xid, yid, zid, wid};
+
+        const Ti *iptr = in_acc_.get_pointer();
+        To *optr       = out_acc_.get_pointer();
+        To *tptr       = tmp_acc_.get_pointer();
+
+        // There is only one element per block for out
+        // There are blockDim.y elements per block for in
+        // Hence increment ids[dim] just after offseting out and before
+        // offsetting in
+        tptr += ids[3] * tInfo_.strides[3] + ids[2] * tInfo_.strides[2] +
+                ids[1] * tInfo_.strides[1] + ids[0];
+
+        const int groupIdx_dim = ids[dim];
+        ids[dim]               = ids[dim] * g.get_local_range(1) * lim_ + lidy;
+
+        optr += ids[3] * oInfo_.strides[3] + ids[2] * oInfo_.strides[2] +
+                ids[1] * oInfo_.strides[1] + ids[0];
+        iptr += ids[3] * iInfo_.strides[3] + ids[2] * iInfo_.strides[2] +
+                ids[1] * iInfo_.strides[1] + ids[0];
+        int id_dim        = ids[dim];
+        const int out_dim = oInfo_.dims[dim];
+
+        bool is_valid = (ids[0] < oInfo_.dims[0]) &&
+                        (ids[1] < oInfo_.dims[1]) &&
+                        (ids[2] < oInfo_.dims[2]) && (ids[3] < oInfo_.dims[3]);
+
+        const int ostride_dim = oInfo_.strides[dim];
+        const int istride_dim = iInfo_.strides[dim];
+
+        To *sptr = s_val_.get_pointer() + lid;
+
+        common::Transform<Ti, To, op> transform;
+        common::Binary<To, op> binop;
+
+        const To init = common::Binary<To, op>::init();
+        To val        = init;
+
+        const bool isLast = (lidy == (DIMY_ - 1));
+
+        for (int k = 0; k < lim_; k++) {
+            if (isLast) s_tmp_[lidx] = val;
+
+            bool cond = (is_valid) && (id_dim < out_dim);
+            val       = cond ? transform(*iptr) : init;
+            *sptr     = val;
+            group_barrier(g);
+
+            int start = 0;
+#pragma unroll
+            for (int off = 1; off < DIMY_; off *= 2) {
+                if (lidy >= off)
+                    val = binop(val, sptr[(start - off) * (int)THREADS_X]);
+                start                   = DIMY_ - start;
+                sptr[start * THREADS_X] = val;
+
+                group_barrier(g);
+            }
+
+            val = binop(val, s_tmp_[lidx]);
+            if (inclusive_scan_) {
+                if (cond) { *optr = val; }
+            } else if (is_valid) {
+                if (id_dim == (out_dim - 1)) {
+                    *(optr - (id_dim * ostride_dim)) = init;
+                } else if (id_dim < (out_dim - 1)) {
+                    *(optr + ostride_dim) = val;
+                }
+            }
+            id_dim += g.get_local_range(1);
+            iptr += g.get_local_range(1) * istride_dim;
+            optr += g.get_local_range(1) * ostride_dim;
+            group_barrier(g);
+        }
+
+        if (!isFinalPass_ && is_valid && (groupIdx_dim < tInfo_.dims[dim]) &&
+            isLast) {
+            *tptr = val;
+        }
+    }
+
+   protected:
+    sycl::accessor<To> out_acc_;
+    sycl::accessor<To> tmp_acc_;
+    sycl::accessor<Ti> in_acc_;
+    KParam oInfo_, tInfo_, iInfo_;
+    const uint groups_x_, groups_y_, blocks_dim_, lim_, DIMY_;
+    const bool isFinalPass_, inclusive_scan_;
+    local_accessor<To, 1> s_val_;
+    local_accessor<To, 1> s_tmp_;
+    sycl::stream debug_;
+};
+
+template<typename To, af_op_t op, int dim>
+class scanDimBcastKernel {
+   public:
+    scanDimBcastKernel(sycl::accessor<To> out_acc, KParam oInfo,
+                       sycl::accessor<To> tmp_acc, KParam tInfo,
+                       const uint groups_x, const uint groups_y,
+                       const uint groups_dim, const uint lim,
+                       const bool inclusive_scan, sycl::stream debug)
+        : out_acc_(out_acc)
+        , oInfo_(oInfo)
+        , tmp_acc_(tmp_acc)
+        , tInfo_(tInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , groups_dim_(groups_dim)
+        , lim_(lim)
+        , inclusive_scan_(inclusive_scan)
+        , debug_(debug) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid       = groupId_x * g.get_local_range(0) + lidx;
+        const uint yid       = groupId_y;
+
+        int ids[4] = {xid, yid, zid, wid};
+
+        const To *tptr = tmp_acc_.get_pointer();
+        To *optr       = out_acc_.get_pointer();
+
+        // There is only one element per block for out
+        // There are blockDim.y elements per block for in
+        // Hence increment ids[dim] just after offseting out and before
+        // offsetting in
+        tptr += ids[3] * tInfo_.strides[3] + ids[2] * tInfo_.strides[2] +
+                ids[1] * tInfo_.strides[1] + ids[0];
+
+        const int groupIdx_dim = ids[dim];
+        ids[dim]               = ids[dim] * g.get_local_range(1) * lim_ + lidy;
+
+        optr += ids[3] * oInfo_.strides[3] + ids[2] * oInfo_.strides[2] +
+                ids[1] * oInfo_.strides[1] + ids[0];
+        const int id_dim  = ids[dim];
+        const int out_dim = oInfo_.dims[dim];
+
+        // Shift broadcast one step to the right for exclusive scan (#2366)
+        int offset = inclusive_scan_ ? 0 : oInfo_.strides[dim];
+        optr += offset;
+
+        bool is_valid = (ids[0] < oInfo_.dims[0]) &&
+                        (ids[1] < oInfo_.dims[1]) &&
+                        (ids[2] < oInfo_.dims[2]) && (ids[3] < oInfo_.dims[3]);
+
+        if (!is_valid) return;
+        if (groupIdx_dim == 0) return;
+
+        To accum = *(tptr - tInfo_.strides[dim]);
+
+        common::Binary<To, op> binop;
+        const int ostride_dim = oInfo_.strides[dim];
+
+        for (int k = 0, id = id_dim; is_valid && k < lim_ && (id < out_dim);
+             k++, id += g.get_local_range(1)) {
+            *optr = binop(*optr, accum);
+            optr += g.get_local_range(1) * ostride_dim;
+        }
+    }
+
+   protected:
+    sycl::accessor<To> out_acc_;
+    sycl::accessor<To> tmp_acc_;
+    KParam oInfo_, tInfo_;
+    const uint groups_x_, groups_y_, groups_dim_, lim_;
+    const bool inclusive_scan_;
+    sycl::stream debug_;
+};
+
+template<typename Ti, typename To, af_op_t op, int dim>
+static void scan_dim_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
+                              const uint threads_y, const dim_t blocks_all[4],
+                              bool isFinalPass, bool inclusive_scan) {
+    sycl::range<2> local(THREADS_X, threads_y);
+    sycl::range<2> global(blocks_all[0] * blocks_all[2] * local[0],
+                          blocks_all[1] * blocks_all[3] * local[1]);
+
+    uint lim = divup(out.info.dims[dim], (threads_y * blocks_all[dim]));
+
+    getQueue().submit([&](sycl::handler &h) {
+        // TODO: specify access modes in all kernels
+        auto out_acc = out.data->get_access(h);
+        auto tmp_acc = tmp.data->get_access(h);
+        auto in_acc  = in.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 256, 128, h);
+
+        auto s_val =
+            local_accessor<compute_t<To>, 1>(THREADS_X * threads_y * 2, h);
+        auto s_tmp = local_accessor<compute_t<To>, 1>(THREADS_X, h);
+
+        h.parallel_for(
+            sycl::nd_range<2>(global, local),
+            scanDimKernel<Ti, To, op, dim>(
+                out_acc, out.info, tmp_acc, tmp.info, in_acc, in.info,
+                blocks_all[0], blocks_all[1], blocks_all[dim], lim, isFinalPass,
+                threads_y, inclusive_scan, s_val, s_tmp, debug_stream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename To, af_op_t op, int dim>
+static void bcast_dim_launcher(Param<To> out, Param<To> tmp,
+                               const uint threads_y, const dim_t blocks_all[4],
+                               bool inclusive_scan) {
+    sycl::range<2> local(THREADS_X, threads_y);
+    sycl::range<2> global(blocks_all[0] * blocks_all[2] * local[0],
+                          blocks_all[1] * blocks_all[3] * local[1]);
+
+    uint lim = divup(out.info.dims[dim], (threads_y * blocks_all[dim]));
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto out_acc = out.data->get_access(h);
+        auto tmp_acc = tmp.data->get_access(h);
+
+        sycl::stream debug_stream(2048 * 256, 128, h);
+
+        h.parallel_for(sycl::nd_range<2>(global, local),
+                       scanDimBcastKernel<To, op, dim>(
+                           out_acc, out.info, tmp_acc, tmp.info, blocks_all[0],
+                           blocks_all[1], blocks_all[dim], lim, inclusive_scan,
+                           debug_stream));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Ti, typename To, af_op_t op, int dim>
+static void scan_dim(Param<To> out, Param<Ti> in, bool inclusive_scan) {
+    uint threads_y = std::min(THREADS_Y, nextpow2(out.info.dims[dim]));
+    uint threads_x = THREADS_X;
+
+    dim_t blocks_all[] = {divup(out.info.dims[0], threads_x), out.info.dims[1],
+                          out.info.dims[2], out.info.dims[3]};
+
+    blocks_all[dim] = divup(out.info.dims[dim], threads_y * REPEAT);
+
+    if (blocks_all[dim] == 1) {
+        scan_dim_launcher<Ti, To, op, dim>(out, out, in, threads_y, blocks_all,
+                                           true, inclusive_scan);
+    } else {
+        Param<To> tmp = out;
+
+        tmp.info.dims[dim]  = blocks_all[dim];
+        tmp.info.strides[0] = 1;
+        for (int k = 1; k < 4; k++)
+            tmp.info.strides[k] =
+                tmp.info.strides[k - 1] * tmp.info.dims[k - 1];
+
+        int tmp_elements = tmp.info.strides[3] * tmp.info.dims[3];
+        auto tmp_alloc   = memAlloc<To>(tmp_elements);
+        tmp.data         = tmp_alloc.get();
+
+        scan_dim_launcher<Ti, To, op, dim>(out, tmp, in, threads_y, blocks_all,
+                                           false, inclusive_scan);
+
+        int bdim        = blocks_all[dim];
+        blocks_all[dim] = 1;
+
+        // FIXME: Is there an alternative to the if condition ?
+        if (op == af_notzero_t) {
+            scan_dim_launcher<To, To, af_add_t, dim>(tmp, tmp, tmp, threads_y,
+                                                     blocks_all, true, true);
+        } else {
+            scan_dim_launcher<To, To, op, dim>(tmp, tmp, tmp, threads_y,
+                                               blocks_all, true, true);
+        }
+
+        blocks_all[dim] = bdim;
+        bcast_dim_launcher<To, op, dim>(out, tmp, threads_y, blocks_all,
+                                        inclusive_scan);
+    }
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index 886cd2f977..0efed3de48 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -14,7 +14,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
-#include <kernel/reduce_config.hpp>
+#include <kernel/default_config.hpp>
 #include <memory.hpp>
 
 namespace oneapi {
@@ -27,23 +27,32 @@ using local_accessor =
 
 template<typename Ti, typename To, af_op_t op>
 class scanFirstKernel {
-public:
+   public:
     scanFirstKernel(sycl::accessor<To> out_acc, KParam oInfo,
                     sycl::accessor<To> tmp_acc, KParam tInfo,
                     sycl::accessor<Ti> in_acc, KParam iInfo,
                     const uint groups_x, const uint groups_y, const uint lim,
-                    const bool isFinalPass, const uint DIMX, const bool inclusive_scan,
-                    local_accessor<To, 1> s_val, local_accessor<To, 1> s_tmp,
-                    sycl::stream debug_stream) :
-        out_acc_(out_acc), oInfo_(oInfo),
-        tmp_acc_(tmp_acc), tInfo_(tInfo),
-        in_acc_(in_acc),   iInfo_(iInfo),
-        groups_x_(groups_x), groups_y_(groups_y), lim_(lim),
-        isFinalPass_(isFinalPass), DIMX_(DIMX), inclusive_scan_(inclusive_scan),
-        s_val_(s_val), s_tmp_(s_tmp), debug_stream_(debug_stream) {}
+                    const bool isFinalPass, const uint DIMX,
+                    const bool inclusive_scan, local_accessor<To, 1> s_val,
+                    local_accessor<To, 1> s_tmp, sycl::stream debug_stream)
+        : out_acc_(out_acc)
+        , oInfo_(oInfo)
+        , tmp_acc_(tmp_acc)
+        , tInfo_(tInfo)
+        , in_acc_(in_acc)
+        , iInfo_(iInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , lim_(lim)
+        , isFinalPass_(isFinalPass)
+        , DIMX_(DIMX)
+        , inclusive_scan_(inclusive_scan)
+        , s_val_(s_val)
+        , s_tmp_(s_tmp)
+        , debug_stream_(debug_stream) {}
 
     void operator()(sycl::nd_item<2> it) const {
-        sycl::group g = it.get_group();
+        sycl::group g   = it.get_group();
         const uint lidx = it.get_local_id(0);
         const uint lidy = it.get_local_id(1);
         const uint lid  = lidy * g.get_local_range(0) + lidx;
@@ -55,18 +64,21 @@ class scanFirstKernel {
         const uint xid       = groupId_x * g.get_local_range(0) * lim_ + lidx;
         const uint yid       = groupId_y * g.get_local_range(1) + lidy;
 
-        bool cond_yzw =
-            (yid < oInfo_.dims[1]) && (zid < oInfo_.dims[2]) && (wid < oInfo_.dims[3]);
+        bool cond_yzw = (yid < oInfo_.dims[1]) && (zid < oInfo_.dims[2]) &&
+                        (wid < oInfo_.dims[3]);
 
-        //if (!cond_yzw) return;  // retire warps early TODO: move
+        // if (!cond_yzw) return;  // retire warps early TODO: move
 
         const Ti *iptr = in_acc_.get_pointer();
         To *optr       = out_acc_.get_pointer();
         To *tptr       = tmp_acc_.get_pointer();
 
-        iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + yid * iInfo_.strides[1];
-        optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
-        tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] + yid * tInfo_.strides[1];
+        iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] +
+                yid * iInfo_.strides[1];
+        optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] +
+                yid * oInfo_.strides[1];
+        tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] +
+                yid * tInfo_.strides[1];
 
         To *sptr = s_val_.get_pointer() + lidy * (2 * DIMX_ + 1);
 
@@ -83,16 +95,11 @@ class scanFirstKernel {
 
             bool cond  = (id < oInfo_.dims[0]) && cond_yzw;
             val        = cond ? transform(iptr[id]) : init;
-            /*
-            if constexpr(std::is_fundamental<To>::value) {
-                debug_stream_ << id<<":"<<val <<", "<< sycl::stream_manipulator::endl;
-            }
-            */
             sptr[lidx] = val;
             group_barrier(g);
 
             int start = 0;
-            #pragma unroll
+#pragma unroll
             for (int off = 1; off < DIMX_; off *= 2) {
                 if (lidx >= off) val = binop(val, sptr[(start - off) + lidx]);
                 start              = DIMX_ - start;
@@ -104,14 +111,12 @@ class scanFirstKernel {
             val = binop(val, s_tmp_[lidy]);
 
             if (inclusive_scan_) {
-                if (cond && cond_yzw) {
-                    //debug_stream_ << "oi0 ";
-                     optr[id] = val; }
+                if (cond) { optr[id] = val; }
             } else {
                 if (cond_yzw && id == (oInfo_.dims[0] - 1)) {
                     optr[0] = init;
                 } else if (cond_yzw && id < (oInfo_.dims[0] - 1)) {
-                    //debug_stream_ << "oe0 ";
+                    // debug_stream_ << "oe0 ";
                     optr[id + 1] = val;
                 }
             }
@@ -120,14 +125,15 @@ class scanFirstKernel {
         }
 
         if (!isFinalPass_ && isLast && cond_yzw) {
-            //debug_stream_ << "ot ";
-             tptr[groupId_x] = val; }
+            // debug_stream_ << "ot ";
+            tptr[groupId_x] = val;
+        }
     }
 
-protected:
+   protected:
     sycl::accessor<To> out_acc_;
     sycl::accessor<To> tmp_acc_;
-    sycl::accessor<Ti> in_acc_; 
+    sycl::accessor<Ti> in_acc_;
     KParam oInfo_, tInfo_, iInfo_;
     const uint groups_x_, groups_y_, lim_, DIMX_;
     const bool isFinalPass_, inclusive_scan_;
@@ -138,18 +144,24 @@ class scanFirstKernel {
 
 template<typename To, af_op_t op>
 class scanFirstBcastKernel {
-public: 
+   public:
     scanFirstBcastKernel(sycl::accessor<To> out_acc, KParam oInfo,
                          sycl::accessor<To> tmp_acc, KParam tInfo,
-                         const uint groups_x, const uint groups_y, const uint lim,
-                         const bool inclusive_scan, sycl::stream debug_stream) :
-        out_acc_(out_acc), oInfo_(oInfo),
-        tmp_acc_(tmp_acc), tInfo_(tInfo),
-        groups_x_(groups_x), groups_y_(groups_y), lim_(lim),
-        inclusive_scan_(inclusive_scan), debug_stream_(debug_stream) {}
-    
+                         const uint groups_x, const uint groups_y,
+                         const uint lim, const bool inclusive_scan,
+                         sycl::stream debug_stream)
+        : out_acc_(out_acc)
+        , oInfo_(oInfo)
+        , tmp_acc_(tmp_acc)
+        , tInfo_(tInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , lim_(lim)
+        , inclusive_scan_(inclusive_scan)
+        , debug_stream_(debug_stream) {}
+
     void operator()(sycl::nd_item<2> it) const {
-        sycl::group g = it.get_group();
+        sycl::group g   = it.get_group();
         const uint lidx = it.get_local_id(0);
         const uint lidy = it.get_local_id(1);
         const uint lid  = lidy * g.get_local_range(0) + lidx;
@@ -163,15 +175,17 @@ class scanFirstBcastKernel {
 
         if (groupId_x == 0) return;
 
-        bool cond =
-            (yid < oInfo_.dims[1]) && (zid < oInfo_.dims[2]) && (wid < oInfo_.dims[3]);
+        bool cond = (yid < oInfo_.dims[1]) && (zid < oInfo_.dims[2]) &&
+                    (wid < oInfo_.dims[3]);
         if (!cond) return;
 
         To *optr       = out_acc_.get_pointer();
         const To *tptr = tmp_acc_.get_pointer();
 
-        optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
-        tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] + yid * tInfo_.strides[1];
+        optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] +
+                yid * oInfo_.strides[1];
+        tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] +
+                yid * tInfo_.strides[1];
 
         common::Binary<To, op> binop;
         To accum = tptr[groupId_x - 1];
@@ -179,12 +193,13 @@ class scanFirstBcastKernel {
         // Shift broadcast one step to the right for exclusive scan (#2366)
         int offset = !inclusive_scan_;
         for (int k = 0, id = xid + offset; k < lim_ && id < oInfo_.dims[0];
-            k++, id += g.get_group_range(0)) {
+             k++, id += g.get_group_range(0)) {
             optr[id] = binop(accum, optr[id]);
         }
     }
-protected:
-    sycl::accessor<To> out_acc_; 
+
+   protected:
+    sycl::accessor<To> out_acc_;
     sycl::accessor<To> tmp_acc_;
     KParam oInfo_, tInfo_;
     const uint groups_x_, groups_y_, lim_;
@@ -192,18 +207,17 @@ class scanFirstBcastKernel {
     sycl::stream debug_stream_;
 };
 
-
 template<typename Ti, typename To, af_op_t op>
 static void scan_first_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
                                 const uint groups_x, const uint groups_y,
                                 const uint threads_x, bool isFinalPass,
                                 bool inclusive_scan) {
     sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
-    sycl::range<2> global(groups_x * out.info.dims[2] * local[0], 
+    sycl::range<2> global(groups_x * out.info.dims[2] * local[0],
                           groups_y * out.info.dims[3] * local[1]);
     uint lim = divup(out.info.dims[0], (threads_x * groups_x));
 
-    getQueue().submit([&] (sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         auto out_acc = out.data->get_access(h);
         auto tmp_acc = tmp.data->get_access(h);
         auto in_acc  = in.data->get_access(h);
@@ -215,15 +229,13 @@ static void scan_first_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
         auto s_val = local_accessor<compute_t<To>, 1>(SHARED_MEM_SIZE, h);
         auto s_tmp = local_accessor<compute_t<To>, 1>(DIMY, h);
 
-        //TODO threads_x as template arg for #pragma unroll?
-        h.parallel_for(sycl::nd_range<2>(global, local), 
+        // TODO threads_x as template arg for #pragma unroll?
+        h.parallel_for(
+            sycl::nd_range<2>(global, local),
             scanFirstKernel<Ti, To, op>(
-                    out_acc, out.info,
-                    tmp_acc, tmp.info,
-                    in_acc, in.info,
-                    groups_x, groups_y, lim,
-                    isFinalPass, threads_x, inclusive_scan,
-                    s_val, s_tmp, debug_stream));
+                out_acc, out.info, tmp_acc, tmp.info, in_acc, in.info, groups_x,
+                groups_y, lim, isFinalPass, threads_x, inclusive_scan, s_val,
+                s_tmp, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
@@ -233,27 +245,20 @@ static void bcast_first_launcher(Param<To> out, Param<To> tmp,
                                  const uint groups_x, const uint groups_y,
                                  const uint threads_x, bool inclusive_scan) {
     sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
-    sycl::range<2> global(groups_x * out.info.dims[2] * local[0], 
+    sycl::range<2> global(groups_x * out.info.dims[2] * local[0],
                           groups_y * out.info.dims[3] * local[1]);
     uint lim = divup(out.info.dims[0], (threads_x * groups_x));
 
-    getQueue().submit([&] (sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         auto out_acc = out.data->get_access(h);
         auto tmp_acc = tmp.data->get_access(h);
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
-        const int DIMY            = THREADS_PER_BLOCK / threads_x;
-        const int SHARED_MEM_SIZE = (2 * threads_x + 1) * (DIMY);
-        auto s_val = local_accessor<compute_t<To>, 1>(SHARED_MEM_SIZE, h);
-        auto s_tmp = local_accessor<compute_t<To>, 1>(DIMY, h);
-
-        h.parallel_for(sycl::nd_range<2>(global, local), 
-            scanFirstBcastKernel<To, op>(
-                    out_acc, out.info,
-                    tmp_acc, tmp.info,
-                    groups_x, groups_y, lim,
-                    inclusive_scan, debug_stream));
+        h.parallel_for(sycl::nd_range<2>(global, local),
+                       scanFirstBcastKernel<To, op>(
+                           out_acc, out.info, tmp_acc, tmp.info, groups_x,
+                           groups_y, lim, inclusive_scan, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
@@ -276,11 +281,12 @@ static void scan_first(Param<To> out, Param<Ti> in, bool inclusive_scan) {
         tmp.info.dims[0]    = groups_x;
         tmp.info.strides[0] = 1;
         for (int k = 1; k < 4; k++)
-            tmp.info.strides[k] = tmp.info.strides[k - 1] * tmp.info.dims[k - 1];
+            tmp.info.strides[k] =
+                tmp.info.strides[k - 1] * tmp.info.dims[k - 1];
 
         int tmp_elements = tmp.info.strides[3] * tmp.info.dims[3];
         auto tmp_alloc   = memAlloc<To>(tmp_elements);
-        tmp.data          = tmp_alloc.get();
+        tmp.data         = tmp_alloc.get();
 
         scan_first_launcher<Ti, To, op>(out, tmp, in, groups_x, groups_y,
                                         threads_x, false, inclusive_scan);
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index 67c7500ee6..c927c995da 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -12,9 +12,9 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
-#include <memory.hpp>
-#include <kernel/reduce_config.hpp>
+#include <kernel/default_config.hpp>
 #include <kernel/scan_first.hpp>
+#include <memory.hpp>
 
 #include <Param.hpp>
 #include <backend.hpp>
@@ -25,18 +25,27 @@ namespace kernel {
 
 template<typename T>
 class whereKernel {
-public:
-    whereKernel(sycl::accessor<uint> out_acc,  KParam oInfo,
+   public:
+    whereKernel(sycl::accessor<uint> out_acc, KParam oInfo,
                 sycl::accessor<uint> otmp_acc, KParam otInfo,
                 sycl::accessor<uint> rtmp_acc, KParam rtInfo,
-                sycl::accessor<T> in_acc,      KParam iInfo,
-                uint groups_x, uint groups_y, uint lim, sycl::stream debug) : 
-        out_acc_(out_acc), oInfo_(oInfo), otmp_acc_(otmp_acc), otInfo_(otInfo),
-        rtmp_acc_(rtmp_acc), rtInfo_(rtInfo), in_acc_(in_acc), iInfo_(iInfo),
-        groups_x_(groups_x), groups_y_(groups_y), lim_(lim), debug_(debug) {}
+                sycl::accessor<T> in_acc, KParam iInfo, uint groups_x,
+                uint groups_y, uint lim, sycl::stream debug)
+        : out_acc_(out_acc)
+        , oInfo_(oInfo)
+        , otmp_acc_(otmp_acc)
+        , otInfo_(otInfo)
+        , rtmp_acc_(rtmp_acc)
+        , rtInfo_(rtInfo)
+        , in_acc_(in_acc)
+        , iInfo_(iInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , lim_(lim)
+        , debug_(debug) {}
 
     void operator()(sycl::nd_item<2> it) const {
-        sycl::group g = it.get_group();
+        sycl::group g   = it.get_group();
         const uint lidx = it.get_local_id(0);
         const uint lidy = it.get_local_id(1);
         const uint lid  = lidy * g.get_local_range(0) + lidx;
@@ -52,17 +61,18 @@ class whereKernel {
         const uint *rtptr = rtmp_acc_.get_pointer();
         const T *iptr     = in_acc_.get_pointer();
 
-        const uint off =
-            wid * otInfo_.strides[3] + zid * otInfo_.strides[2] + yid * otInfo_.strides[1];
+        const uint off = wid * otInfo_.strides[3] + zid * otInfo_.strides[2] +
+                         yid * otInfo_.strides[1];
         const uint bid = wid * rtInfo_.strides[3] + zid * rtInfo_.strides[2] +
-                        yid * rtInfo_.strides[1] + groupId_x;
+                         yid * rtInfo_.strides[1] + groupId_x;
 
-        otptr +=
-            wid * otInfo_.strides[3] + zid * otInfo_.strides[2] + yid * otInfo_.strides[1];
-        iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] + yid * iInfo_.strides[1];
+        otptr += wid * otInfo_.strides[3] + zid * otInfo_.strides[2] +
+                 yid * otInfo_.strides[1];
+        iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] +
+                yid * iInfo_.strides[1];
 
-        bool cond =
-            (yid < otInfo_.dims[1]) && (zid < otInfo_.dims[2]) && (wid < otInfo_.dims[3]);
+        bool cond = (yid < otInfo_.dims[1]) && (zid < otInfo_.dims[2]) &&
+                    (wid < otInfo_.dims[3]);
         T zero = scalar<T>(0);
 
         if (!cond) return;
@@ -70,12 +80,13 @@ class whereKernel {
         uint accum = (bid == 0) ? 0 : rtptr[bid - 1];
 
         for (uint k = 0, id = xid; k < lim_ && id < otInfo_.dims[0];
-            k++, id += g.get_local_range(0)) {
+             k++, id += g.get_local_range(0)) {
             uint idx = otptr[id] + accum;
             if (iptr[id] != zero) out_acc_[idx - 1] = (off + id);
         }
     }
-protected:
+
+   protected:
     sycl::accessor<uint> out_acc_;
     sycl::accessor<uint> otmp_acc_;
     sycl::accessor<uint> rtmp_acc_;
@@ -113,15 +124,15 @@ static void where(Param<uint> &out, Param<T> in) {
     int otmp_elements = otmp.info.strides[3] * otmp.info.dims[3];
     auto rtmp_alloc   = memAlloc<uint>(rtmp_elements);
     auto otmp_alloc   = memAlloc<uint>(otmp_elements);
-    rtmp.data          = rtmp_alloc.get();
-    otmp.data          = otmp_alloc.get();
+    rtmp.data         = rtmp_alloc.get();
+    otmp.data         = otmp_alloc.get();
 
     scan_first_launcher<T, uint, af_notzero_t>(
         otmp, rtmp, in, groups_x, groups_y, threads_x, false, true);
 
     // Linearize the dimensions and perform scan
-    Param<uint> ltmp = rtmp;
-    ltmp.info.dims[0]     = rtmp_elements;
+    Param<uint> ltmp  = rtmp;
+    ltmp.info.dims[0] = rtmp_elements;
     for (int k = 1; k < 4; k++) {
         ltmp.info.dims[k]    = 1;
         ltmp.info.strides[k] = rtmp_elements;
@@ -132,17 +143,19 @@ static void where(Param<uint> &out, Param<T> in) {
     // Get output size and allocate output
     uint total;
     sycl::buffer retBuffer(&total, {1},
-        {sycl::property::buffer::use_host_ptr()});
+                           {sycl::property::buffer::use_host_ptr()});
 
     getQueue()
         .submit([&](sycl::handler &h) {
-            auto acc_in = rtmp.data->get_access(h, sycl::range{1}, sycl::id{rtmp_elements - 1});
+            auto acc_in  = rtmp.data->get_access(h, sycl::range{1},
+                                                 sycl::id{rtmp_elements - 1});
             auto acc_out = retBuffer.get_access();
             h.copy(acc_in, acc_out);
-    }).wait();
+        })
+        .wait();
 
     auto out_alloc = memAlloc<uint>(total);
-    out.data        = out_alloc.get();
+    out.data       = out_alloc.get();
 
     out.info.dims[0]    = total;
     out.info.strides[0] = 1;
@@ -152,25 +165,21 @@ static void where(Param<uint> &out, Param<T> in) {
     }
 
     sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
-    sycl::range<2> global(groups_x * in.info.dims[2] * local[0], 
+    sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
                           groups_y * in.info.dims[3] * local[1]);
     uint lim = divup(otmp.info.dims[0], (threads_x * groups_x));
 
-    getQueue().submit([&] (sycl::handler &h) {
-        auto out_acc = out.data->get_access(h);
+    getQueue().submit([&](sycl::handler &h) {
+        auto out_acc  = out.data->get_access(h);
         auto otmp_acc = otmp.data->get_access(h);
         auto rtmp_acc = rtmp.data->get_access(h);
-        auto in_acc  = in.data->get_access(h);
+        auto in_acc   = in.data->get_access(h);
 
         sycl::stream debug_stream(2048 * 256, 128, h);
-        h.parallel_for(sycl::nd_range<2>(global, local), 
-            whereKernel<T>(
-                    out_acc, out.info,
-                    otmp_acc, otmp.info,
-                    rtmp_acc, rtmp.info,
-                    in_acc, in.info,
-                    groups_x, groups_y, lim,
-                    debug_stream));
+        h.parallel_for(sycl::nd_range<2>(global, local),
+                       whereKernel<T>(out_acc, out.info, otmp_acc, otmp.info,
+                                      rtmp_acc, rtmp.info, in_acc, in.info,
+                                      groups_x, groups_y, lim, debug_stream));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
     out_alloc.release();
diff --git a/src/backend/oneapi/scan.cpp b/src/backend/oneapi/scan.cpp
index 51183cfe8c..81b7494d68 100644
--- a/src/backend/oneapi/scan.cpp
+++ b/src/backend/oneapi/scan.cpp
@@ -10,7 +10,7 @@
 #include <err_oneapi.hpp>
 #include <scan.hpp>
 
-// #include <kernel/scan_dim.hpp>
+#include <kernel/scan_dim.hpp>
 #include <kernel/scan_first.hpp>
 
 namespace oneapi {
@@ -21,10 +21,11 @@ Array<To> scan(const Array<Ti>& in, const int dim, bool inclusiveScan) {
     Param<To> Out = out;
     Param<Ti> In  = in;
 
-    if (dim == 0) {
-        kernel::scan_first<Ti, To, op>(Out, In, inclusiveScan);
-    } else {
-    //     kernel::scanDim<Ti, To, op>(Out, In, dim, inclusiveScan);
+    switch (dim) {
+        case 0: kernel::scan_first<Ti, To, op>(Out, In, inclusiveScan); break;
+        case 1: kernel::scan_dim<Ti, To, op, 1>(Out, In, inclusiveScan); break;
+        case 2: kernel::scan_dim<Ti, To, op, 2>(Out, In, inclusiveScan); break;
+        case 3: kernel::scan_dim<Ti, To, op, 3>(Out, In, inclusiveScan); break;
     }
 
     return out;

From 16f22445a1faf4c6a7e211bdd354d3db17d137bf Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 4 Nov 2022 18:43:18 -0400
Subject: [PATCH 121/473] corrects accessor types

---
 src/backend/oneapi/kernel/iota.hpp         | 38 +++++++-------
 src/backend/oneapi/kernel/mean.hpp         | 61 +++++++++++-----------
 src/backend/oneapi/kernel/reduce_all.hpp   | 18 ++++---
 src/backend/oneapi/kernel/reduce_dim.hpp   | 18 ++++---
 src/backend/oneapi/kernel/reduce_first.hpp | 18 ++++---
 src/backend/oneapi/kernel/scan_dim.hpp     | 36 +++++++------
 src/backend/oneapi/kernel/scan_first.hpp   | 46 +++++++++-------
 src/backend/oneapi/kernel/where.hpp        | 30 ++++++-----
 8 files changed, 152 insertions(+), 113 deletions(-)

diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index d4672dfd0d..ee0b16d23a 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -12,7 +12,6 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
-#include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
 #include <traits.hpp>
 #include <af/dim4.hpp>
@@ -40,10 +39,6 @@ class iotaKernel {
         , debug_(debug) {}
 
     void operator()(sycl::nd_item<2> it) const {
-        // printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
-        // debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) <<
-        // "]" << sycl::stream_manipulator::endl;
-
         sycl::group gg = it.get_group();
         const int oz   = gg.get_group_id(0) / blocksPerMatX_;
         const int ow   = gg.get_group_id(1) / blocksPerMatY_;
@@ -99,19 +94,26 @@ void iota(Param<T> out, const af::dim4& sdims) {
                           local[1] * blocksPerMatY * out.info.dims[3]);
     sycl::nd_range<2> ndrange(global, local);
 
-    getQueue().submit([=](sycl::handler& h) {
-        auto out_acc = out.data->get_access(h);
-
-        sycl::stream debug_stream(2048, 128, h);
-
-        h.parallel_for(
-            ndrange, iotaKernel<T>(
-                         out_acc, out.info, static_cast<int>(sdims[0]),
-                         static_cast<int>(sdims[1]), static_cast<int>(sdims[2]),
-                         static_cast<int>(sdims[3]), blocksPerMatX,
-                         blocksPerMatY, debug_stream));
-    });
-    ONEAPI_DEBUG_FINISH(getQueue());
+    try {
+        getQueue()
+            .submit([=](sycl::handler& h) {
+                auto out_acc = out.data->get_access(h);
+
+                sycl::stream debug_stream(2048, 128, h);
+
+                h.parallel_for(
+                    ndrange,
+                    iotaKernel<T>(out_acc, out.info, static_cast<int>(sdims[0]),
+                                  static_cast<int>(sdims[1]),
+                                  static_cast<int>(sdims[2]),
+                                  static_cast<int>(sdims[3]), blocksPerMatX,
+                                  blocksPerMatY, debug_stream));
+            })
+            .wait();
+        ONEAPI_DEBUG_FINISH(getQueue());
+    } catch (sycl::exception& e) {
+        std::cout << e.what() << std::endl;
+    } catch (std::exception& e) { std::cout << e.what() << std::endl; }
 }
 
 }  // namespace kernel
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 3f7b5c3fee..f64fc94b96 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -45,6 +45,12 @@ using local_accessor =
     sycl::accessor<T, dimensions, sycl::access::mode::read_write,
                    sycl::access::target::local>;
 
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
 template<typename To, typename Tw>
 void stable_mean(To *lhs, Tw *l_wt, To rhs, Tw r_wt) {
     if (((*l_wt) != (Tw)0) || (r_wt != (Tw)0)) {
@@ -60,12 +66,11 @@ void stable_mean(To *lhs, Tw *l_wt, To rhs, Tw r_wt) {
 template<typename Ti, typename Tw, typename To, uint dim, uint DIMY>
 class meanDimKernelSMEM {
    public:
-    meanDimKernelSMEM(sycl::accessor<To> out, KParam oInfo,
-                      sycl::accessor<Tw> owt, KParam owInfo,
-                      sycl::accessor<Ti> in, KParam iInfo,
-                      sycl::accessor<Tw> iwt, KParam iwInfo, uint groups_x,
-                      uint groups_y, uint offset_dim,
-                      local_accessor<compute_t<To>, 1> s_val,
+    meanDimKernelSMEM(write_accessor<To> out, KParam oInfo,
+                      write_accessor<Tw> owt, KParam owInfo,
+                      read_accessor<Ti> in, KParam iInfo, read_accessor<Tw> iwt,
+                      KParam iwInfo, uint groups_x, uint groups_y,
+                      uint offset_dim, local_accessor<compute_t<To>, 1> s_val,
                       local_accessor<compute_t<Tw>, 1> s_idx,
                       sycl::stream debug, bool input_weight, bool output_weight)
         : out_(out)
@@ -202,10 +207,10 @@ class meanDimKernelSMEM {
     }
 
    protected:
-    sycl::accessor<To> out_;
-    sycl::accessor<Tw> owt_;
-    sycl::accessor<Ti> in_;
-    sycl::accessor<Tw> iwt_;
+    write_accessor<To> out_;
+    write_accessor<Tw> owt_;
+    read_accessor<Ti> in_;
+    read_accessor<Tw> iwt_;
     KParam oInfo_, owInfo_, iInfo_, iwInfo_;
     const uint groups_x_, groups_y_, offset_dim_;
     local_accessor<compute_t<To>, 1> s_val_;
@@ -224,8 +229,8 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
 
     sycl::buffer<Tw, 1> empty(sycl::range<1>{1});
     getQueue().submit([&](sycl::handler &h) {
-        auto out_acc = out.data->get_access(h);
-        auto in_acc  = in.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
+        read_accessor<Ti> in_acc{*in.data, h};
 
         sycl::stream debug_stream(2048 * 2048, 2048, h);
 
@@ -238,10 +243,8 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
         bool output_weight = ((owt.info.dims[0] * owt.info.dims[1] *
                                owt.info.dims[2] * owt.info.dims[3]) != 0);
 
-        auto owt_acc =
-            (output_weight) ? owt.data->get_access(h) : empty.get_access(h);
-        auto iwt_acc =
-            (input_weight) ? iwt.data->get_access(h) : empty.get_access(h);
+        write_accessor<Tw> owt_acc{(output_weight) ? *owt.data : empty, h};
+        read_accessor<Tw> iwt_acc{(input_weight) ? *iwt.data : empty, h};
 
         switch (threads_y) {
             case 8:
@@ -321,10 +324,10 @@ void mean_dim(Param<To> out, Param<Ti> in, Param<Tw> iwt) {
 template<typename Ti, typename Tw, typename To>
 class meanFirstKernelSMEM {
    public:
-    meanFirstKernelSMEM(sycl::accessor<To> out, KParam oInfo,
-                        sycl::accessor<Tw> owt, KParam owInfo,
-                        sycl::accessor<Ti> in, KParam iInfo,
-                        sycl::accessor<Tw> iwt, KParam iwInfo, const uint DIMX,
+    meanFirstKernelSMEM(write_accessor<To> out, KParam oInfo,
+                        write_accessor<Tw> owt, KParam owInfo,
+                        read_accessor<Ti> in, KParam iInfo,
+                        read_accessor<Tw> iwt, KParam iwInfo, const uint DIMX,
                         const uint groups_x, const uint groups_y,
                         const uint repeat,
                         local_accessor<compute_t<To>, 1> s_val,
@@ -480,10 +483,10 @@ class meanFirstKernelSMEM {
     }
 
    protected:
-    sycl::accessor<To> out_;
-    sycl::accessor<Tw> owt_;
-    sycl::accessor<Ti> in_;
-    sycl::accessor<Tw> iwt_;
+    write_accessor<To> out_;
+    write_accessor<Tw> owt_;
+    read_accessor<Ti> in_;
+    read_accessor<Tw> iwt_;
     KParam oInfo_, owInfo_, iInfo_, iwInfo_;
     const uint DIMX_, groups_x_, groups_y_, repeat_;
     local_accessor<compute_t<To>, 1> s_val_;
@@ -504,8 +507,8 @@ void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
 
     sycl::buffer<Tw, 1> empty(sycl::range<1>{1});
     getQueue().submit([&](sycl::handler &h) {
-        auto out_acc = out.data->get_access(h);
-        auto in_acc  = in.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
+        read_accessor<Ti> in_acc{*in.data, h};
 
         sycl::stream debug_stream(2048 * 2048, 2048, h);
 
@@ -518,10 +521,8 @@ void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
         bool output_weight = ((owt.info.dims[0] * owt.info.dims[1] *
                                owt.info.dims[2] * owt.info.dims[3]) != 0);
 
-        auto owt_acc =
-            (output_weight) ? owt.data->get_access(h) : empty.get_access(h);
-        auto iwt_acc =
-            (input_weight) ? iwt.data->get_access(h) : empty.get_access(h);
+        write_accessor<Tw> owt_acc{(output_weight) ? *owt.data : empty, h};
+        read_accessor<Tw> iwt_acc{(input_weight) ? *iwt.data : empty, h};
 
         h.parallel_for(
             sycl::nd_range<2>(global, local),
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index 6fdf008e69..8ad65d7948 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -38,13 +38,19 @@ using global_atomic_ref =
     sycl::atomic_ref<T, sycl::memory_order::relaxed, sycl::memory_scope::system,
                      sycl::access::address_space::global_space>;
 
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
 template<typename Ti, typename To, af_op_t op>
 class reduceAllKernelSMEM {
    public:
-    reduceAllKernelSMEM(sycl::accessor<To> out, KParam oInfo,
+    reduceAllKernelSMEM(write_accessor<To> out, KParam oInfo,
                         sycl::accessor<unsigned> retCount,
                         sycl::accessor<To> tmp, KParam tmpInfo,
-                        sycl::accessor<Ti> in, KParam iInfo, uint DIMX,
+                        read_accessor<Ti> in, KParam iInfo, uint DIMX,
                         uint groups_x, uint groups_y, uint repeat,
                         bool change_nan, To nanval,
                         local_accessor<compute_t<To>, 1> s_ptr,
@@ -220,11 +226,11 @@ class reduceAllKernelSMEM {
     }
 
    protected:
-    sycl::accessor<To> out_;
+    write_accessor<To> out_;
     sycl::accessor<unsigned> retCount_;
     sycl::accessor<To> tmp_;
+    read_accessor<Ti> in_;
     KParam oInfo_, tmpInfo_, iInfo_;
-    sycl::accessor<Ti> in_;
     uint DIMX_, repeat_;
     uint groups_x_, groups_y_;
     bool change_nan_;
@@ -261,10 +267,10 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
     });
 
     getQueue().submit([=](sycl::handler &h) {
-        auto out_acc      = out.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
         auto retCount_acc = retirementCount.getData()->get_access(h);
         auto tmp_acc      = tmp.getData()->get_access(h);
-        auto in_acc       = in.data->get_access(h);
+        read_accessor<Ti> in_acc{*in.data, h};
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index 3f9e365b8b..b5e4252651 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -33,11 +33,17 @@ using local_accessor =
     sycl::accessor<T, dimensions, sycl::access::mode::read_write,
                    sycl::access::target::local>;
 
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
 template<typename Ti, typename To, af_op_t op, uint dim, uint DIMY>
 class reduceDimKernelSMEM {
    public:
-    reduceDimKernelSMEM(sycl::accessor<To> out, KParam oInfo,
-                        sycl::accessor<Ti> in, KParam iInfo, uint groups_x,
+    reduceDimKernelSMEM(write_accessor<To> out, KParam oInfo,
+                        read_accessor<Ti> in, KParam iInfo, uint groups_x,
                         uint groups_y, uint offset_dim, bool change_nan,
                         To nanval, local_accessor<compute_t<To>, 1> s_val,
                         sycl::stream debug)
@@ -128,9 +134,9 @@ class reduceDimKernelSMEM {
     }
 
    protected:
-    sycl::accessor<To> out_;
+    write_accessor<To> out_;
     KParam oInfo_, iInfo_;
-    sycl::accessor<Ti> in_;
+    read_accessor<Ti> in_;
     uint groups_x_, groups_y_, offset_dim_;
     bool change_nan_;
     To nanval_;
@@ -148,8 +154,8 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                           blocks_dim[1] * blocks_dim[3] * local[1]);
 
     getQueue().submit([=](sycl::handler &h) {
-        auto out_acc = out.data->get_access(h);
-        auto in_acc  = in.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
+        read_accessor<Ti> in_acc{*in.data, h};
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index ebf55fb63e..6bfe177148 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -33,11 +33,17 @@ using local_accessor =
     sycl::accessor<T, dimensions, sycl::access::mode::read_write,
                    sycl::access::target::local>;
 
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
 template<typename Ti, typename To, af_op_t op, uint DIMX>
 class reduceFirstKernelSMEM {
    public:
-    reduceFirstKernelSMEM(sycl::accessor<To> out, KParam oInfo,
-                          sycl::accessor<Ti> in, KParam iInfo, uint groups_x,
+    reduceFirstKernelSMEM(write_accessor<To> out, KParam oInfo,
+                          read_accessor<Ti> in, KParam iInfo, uint groups_x,
                           uint groups_y, uint repeat, bool change_nan,
                           To nanval, local_accessor<compute_t<To>, 1> s_val,
                           sycl::stream debug)
@@ -133,9 +139,9 @@ class reduceFirstKernelSMEM {
     }
 
    protected:
-    sycl::accessor<To> out_;
+    write_accessor<To> out_;
     KParam oInfo_, iInfo_;
-    sycl::accessor<Ti> in_;
+    read_accessor<Ti> in_;
     uint groups_x_, groups_y_, repeat_;
     bool change_nan_;
     To nanval_;
@@ -155,8 +161,8 @@ void reduce_first_launcher_default(Param<To> out, Param<Ti> in,
     uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
 
     getQueue().submit([=](sycl::handler &h) {
-        auto out_acc = out.data->get_access(h);
-        auto in_acc  = in.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
+        read_accessor<Ti> in_acc{*in.data, h};
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp
index 1db981ca9a..f617c782dc 100644
--- a/src/backend/oneapi/kernel/scan_dim.hpp
+++ b/src/backend/oneapi/kernel/scan_dim.hpp
@@ -25,12 +25,18 @@ using local_accessor =
     sycl::accessor<T, dimensions, sycl::access::mode::read_write,
                    sycl::access::target::local>;
 
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
 template<typename Ti, typename To, af_op_t op, int dim>
 class scanDimKernel {
    public:
-    scanDimKernel(sycl::accessor<To> out_acc, KParam oInfo,
-                  sycl::accessor<To> tmp_acc, KParam tInfo,
-                  sycl::accessor<Ti> in_acc, KParam iInfo, const uint groups_x,
+    scanDimKernel(write_accessor<To> out_acc, KParam oInfo,
+                  write_accessor<To> tmp_acc, KParam tInfo,
+                  read_accessor<Ti> in_acc, KParam iInfo, const uint groups_x,
                   const uint groups_y, const uint blocks_dim, const uint lim,
                   const bool isFinalPass, const uint DIMY,
                   const bool inclusive_scan, local_accessor<To, 1> s_val,
@@ -147,9 +153,9 @@ class scanDimKernel {
     }
 
    protected:
-    sycl::accessor<To> out_acc_;
-    sycl::accessor<To> tmp_acc_;
-    sycl::accessor<Ti> in_acc_;
+    write_accessor<To> out_acc_;
+    write_accessor<To> tmp_acc_;
+    read_accessor<Ti> in_acc_;
     KParam oInfo_, tInfo_, iInfo_;
     const uint groups_x_, groups_y_, blocks_dim_, lim_, DIMY_;
     const bool isFinalPass_, inclusive_scan_;
@@ -161,8 +167,8 @@ class scanDimKernel {
 template<typename To, af_op_t op, int dim>
 class scanDimBcastKernel {
    public:
-    scanDimBcastKernel(sycl::accessor<To> out_acc, KParam oInfo,
-                       sycl::accessor<To> tmp_acc, KParam tInfo,
+    scanDimBcastKernel(write_accessor<To> out_acc, KParam oInfo,
+                       read_accessor<To> tmp_acc, KParam tInfo,
                        const uint groups_x, const uint groups_y,
                        const uint groups_dim, const uint lim,
                        const bool inclusive_scan, sycl::stream debug)
@@ -233,8 +239,8 @@ class scanDimBcastKernel {
     }
 
    protected:
-    sycl::accessor<To> out_acc_;
-    sycl::accessor<To> tmp_acc_;
+    write_accessor<To> out_acc_;
+    read_accessor<To> tmp_acc_;
     KParam oInfo_, tInfo_;
     const uint groups_x_, groups_y_, groups_dim_, lim_;
     const bool inclusive_scan_;
@@ -253,9 +259,9 @@ static void scan_dim_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
 
     getQueue().submit([&](sycl::handler &h) {
         // TODO: specify access modes in all kernels
-        auto out_acc = out.data->get_access(h);
-        auto tmp_acc = tmp.data->get_access(h);
-        auto in_acc  = in.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
+        write_accessor<To> tmp_acc{*tmp.data, h};
+        read_accessor<Ti> in_acc{*in.data, h};
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
@@ -284,8 +290,8 @@ static void bcast_dim_launcher(Param<To> out, Param<To> tmp,
     uint lim = divup(out.info.dims[dim], (threads_y * blocks_all[dim]));
 
     getQueue().submit([&](sycl::handler &h) {
-        auto out_acc = out.data->get_access(h);
-        auto tmp_acc = tmp.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
+        read_accessor<To> tmp_acc{*tmp.data, h};
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index 0efed3de48..9a377ca2d9 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -25,16 +25,22 @@ using local_accessor =
     sycl::accessor<T, dimensions, sycl::access::mode::read_write,
                    sycl::access::target::local>;
 
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
 template<typename Ti, typename To, af_op_t op>
 class scanFirstKernel {
    public:
-    scanFirstKernel(sycl::accessor<To> out_acc, KParam oInfo,
-                    sycl::accessor<To> tmp_acc, KParam tInfo,
-                    sycl::accessor<Ti> in_acc, KParam iInfo,
-                    const uint groups_x, const uint groups_y, const uint lim,
-                    const bool isFinalPass, const uint DIMX,
-                    const bool inclusive_scan, local_accessor<To, 1> s_val,
-                    local_accessor<To, 1> s_tmp, sycl::stream debug_stream)
+    scanFirstKernel(write_accessor<To> out_acc, KParam oInfo,
+                    write_accessor<To> tmp_acc, KParam tInfo,
+                    read_accessor<Ti> in_acc, KParam iInfo, const uint groups_x,
+                    const uint groups_y, const uint lim, const bool isFinalPass,
+                    const uint DIMX, const bool inclusive_scan,
+                    local_accessor<To, 1> s_val, local_accessor<To, 1> s_tmp,
+                    sycl::stream debug_stream)
         : out_acc_(out_acc)
         , oInfo_(oInfo)
         , tmp_acc_(tmp_acc)
@@ -131,9 +137,9 @@ class scanFirstKernel {
     }
 
    protected:
-    sycl::accessor<To> out_acc_;
-    sycl::accessor<To> tmp_acc_;
-    sycl::accessor<Ti> in_acc_;
+    write_accessor<To> out_acc_;
+    write_accessor<To> tmp_acc_;
+    read_accessor<Ti> in_acc_;
     KParam oInfo_, tInfo_, iInfo_;
     const uint groups_x_, groups_y_, lim_, DIMX_;
     const bool isFinalPass_, inclusive_scan_;
@@ -145,8 +151,8 @@ class scanFirstKernel {
 template<typename To, af_op_t op>
 class scanFirstBcastKernel {
    public:
-    scanFirstBcastKernel(sycl::accessor<To> out_acc, KParam oInfo,
-                         sycl::accessor<To> tmp_acc, KParam tInfo,
+    scanFirstBcastKernel(write_accessor<To> out_acc, KParam oInfo,
+                         read_accessor<To> tmp_acc, KParam tInfo,
                          const uint groups_x, const uint groups_y,
                          const uint lim, const bool inclusive_scan,
                          sycl::stream debug_stream)
@@ -199,8 +205,8 @@ class scanFirstBcastKernel {
     }
 
    protected:
-    sycl::accessor<To> out_acc_;
-    sycl::accessor<To> tmp_acc_;
+    write_accessor<To> out_acc_;
+    read_accessor<To> tmp_acc_;
     KParam oInfo_, tInfo_;
     const uint groups_x_, groups_y_, lim_;
     const bool inclusive_scan_;
@@ -218,9 +224,9 @@ static void scan_first_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
     uint lim = divup(out.info.dims[0], (threads_x * groups_x));
 
     getQueue().submit([&](sycl::handler &h) {
-        auto out_acc = out.data->get_access(h);
-        auto tmp_acc = tmp.data->get_access(h);
-        auto in_acc  = in.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
+        write_accessor<To> tmp_acc{*tmp.data, h};
+        read_accessor<Ti> in_acc{*in.data, h};
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
@@ -250,8 +256,8 @@ static void bcast_first_launcher(Param<To> out, Param<To> tmp,
     uint lim = divup(out.info.dims[0], (threads_x * groups_x));
 
     getQueue().submit([&](sycl::handler &h) {
-        auto out_acc = out.data->get_access(h);
-        auto tmp_acc = tmp.data->get_access(h);
+        write_accessor<To> out_acc{*out.data, h};
+        read_accessor<To> tmp_acc{*tmp.data, h};
 
         sycl::stream debug_stream(2048 * 256, 128, h);
 
@@ -306,4 +312,4 @@ static void scan_first(Param<To> out, Param<Ti> in, bool inclusive_scan) {
 }
 
 }  // namespace kernel
-}  // namespace oneapi
\ No newline at end of file
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index c927c995da..e55c72b367 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -23,13 +23,19 @@
 namespace oneapi {
 namespace kernel {
 
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
 template<typename T>
 class whereKernel {
    public:
-    whereKernel(sycl::accessor<uint> out_acc, KParam oInfo,
-                sycl::accessor<uint> otmp_acc, KParam otInfo,
-                sycl::accessor<uint> rtmp_acc, KParam rtInfo,
-                sycl::accessor<T> in_acc, KParam iInfo, uint groups_x,
+    whereKernel(write_accessor<uint> out_acc, KParam oInfo,
+                read_accessor<uint> otmp_acc, KParam otInfo,
+                read_accessor<uint> rtmp_acc, KParam rtInfo,
+                read_accessor<T> in_acc, KParam iInfo, uint groups_x,
                 uint groups_y, uint lim, sycl::stream debug)
         : out_acc_(out_acc)
         , oInfo_(oInfo)
@@ -87,10 +93,10 @@ class whereKernel {
     }
 
    protected:
-    sycl::accessor<uint> out_acc_;
-    sycl::accessor<uint> otmp_acc_;
-    sycl::accessor<uint> rtmp_acc_;
-    sycl::accessor<T> in_acc_;
+    write_accessor<uint> out_acc_;
+    read_accessor<uint> otmp_acc_;
+    read_accessor<uint> rtmp_acc_;
+    read_accessor<T> in_acc_;
     KParam oInfo_, otInfo_, rtInfo_, iInfo_;
     uint groups_x_, groups_y_, lim_;
     sycl::stream debug_;
@@ -170,10 +176,10 @@ static void where(Param<uint> &out, Param<T> in) {
     uint lim = divup(otmp.info.dims[0], (threads_x * groups_x));
 
     getQueue().submit([&](sycl::handler &h) {
-        auto out_acc  = out.data->get_access(h);
-        auto otmp_acc = otmp.data->get_access(h);
-        auto rtmp_acc = rtmp.data->get_access(h);
-        auto in_acc   = in.data->get_access(h);
+        write_accessor<uint> out_acc{*out.data, h};
+        read_accessor<uint> otmp_acc{*otmp.data, h};
+        read_accessor<uint> rtmp_acc{*rtmp.data, h};
+        read_accessor<T> in_acc{*in.data, h};
 
         sycl::stream debug_stream(2048 * 256, 128, h);
         h.parallel_for(sycl::nd_range<2>(global, local),

From 013b196e22df9eb872e10f0228ae7d484ec4269d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 7 Nov 2022 16:57:37 -0500
Subject: [PATCH 122/473] Fix errors on Linux builds for reduce, scan, and
 where

---
 src/api/c/optypes.hpp                    |  2 +-
 src/backend/oneapi/copy.cpp              |  5 +++--
 src/backend/oneapi/kernel/mean.hpp       |  8 +++----
 src/backend/oneapi/kernel/reduce.hpp     | 28 ++++++++++++------------
 src/backend/oneapi/kernel/scan_dim.hpp   | 12 +++++-----
 src/backend/oneapi/kernel/scan_first.hpp | 10 ++++-----
 src/backend/oneapi/kernel/where.hpp      | 13 +++++------
 7 files changed, 38 insertions(+), 40 deletions(-)

diff --git a/src/api/c/optypes.hpp b/src/api/c/optypes.hpp
index 696ae07668..44f1fd68d6 100644
--- a/src/api/c/optypes.hpp
+++ b/src/api/c/optypes.hpp
@@ -9,7 +9,7 @@
 
 #pragma once
 
-typedef enum af_op_t : int {
+enum af_op_t : int {
     af_none_t = -1,
     af_add_t  = 0,
     af_sub_t,
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index f24db5650c..f49689a423 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -218,8 +218,9 @@ T getScalar(const Array<T> &in) {
 
     getQueue()
         .submit([&](sycl::handler &h) {
-            auto acc_in  = in.getData()->get_access(h, sycl::range{1},
-                                                    sycl::id{in.getOffset()});
+            auto acc_in = in.getData()->get_access(
+                h, sycl::range{1},
+                sycl::id{static_cast<uintl>(in.getOffset())});
             auto acc_out = retBuffer.get_access();
             h.copy(acc_in, acc_out);
         })
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index f64fc94b96..8a2e07d93c 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -583,7 +583,7 @@ void mean(Param<To> out, Param<Ti> in, int dim) {
 
 template<typename T, typename Tw>
 T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
-    int in_elements =
+    uintl in_elements =
         in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
     // FIXME: Use better heuristics to get to the optimum number
     if (in_elements > 4096) {
@@ -621,7 +621,7 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         Array<Tw> tmpWt = createEmptyArray<Tw>(
             {blocks_x, in.info.dims[1], in.info.dims[2], in.info.dims[3]});
 
-        int tmp_elements = tmpOut.elements();
+        uintl tmp_elements = tmpOut.elements();
 
         mean_first_launcher<T, Tw, T>(tmpOut, tmpWt, in, iwt, blocks_x,
                                       blocks_y, threads_x);
@@ -693,7 +693,7 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
 template<typename Ti, typename Tw, typename To>
 To mean_all(Param<Ti> in) {
     using std::unique_ptr;
-    int in_elements =
+    uintl in_elements =
         in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
     bool is_linear = (in.info.strides[0] == 1);
     for (int k = 1; k < 4; k++) {
@@ -728,7 +728,7 @@ To mean_all(Param<Ti> in) {
         mean_first_launcher<Ti, Tw, To>(tmpOut, tmpCt, in, iwt, blocks_x,
                                         blocks_y, threads_x);
 
-        int tmp_elements = tmpOut.elements();
+        uintl tmp_elements = tmpOut.elements();
         std::vector<To> h_ptr(tmp_elements);
         std::vector<Tw> h_cptr(tmp_elements);
 
diff --git a/src/backend/oneapi/kernel/reduce.hpp b/src/backend/oneapi/kernel/reduce.hpp
index cae5c9854c..6fa38e0269 100644
--- a/src/backend/oneapi/kernel/reduce.hpp
+++ b/src/backend/oneapi/kernel/reduce.hpp
@@ -31,20 +31,6 @@
 namespace oneapi {
 namespace kernel {
 
-template<typename Ti, typename To, af_op_t op>
-void reduce_cpu_dispatch(Param<To> out, Param<Ti> in, int dim, bool change_nan,
-                         double nanval) {
-    // TODO: use kernels optimized for SIMD-based subgroup sizes
-    reduce_default_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
-}
-
-template<typename Ti, typename To, af_op_t op>
-void reduce_gpu_dispatch(Param<To> out, Param<Ti> in, int dim, bool change_nan,
-                         double nanval) {
-    // TODO: use kernels optimized for gpu subgroup sizes
-    reduce_default_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
-}
-
 template<typename Ti, typename To, af_op_t op>
 void reduce_default_dispatch(Param<To> out, Param<Ti> in, int dim,
                              bool change_nan, double nanval) {
@@ -64,6 +50,20 @@ void reduce_default_dispatch(Param<To> out, Param<Ti> in, int dim,
     }
 }
 
+template<typename Ti, typename To, af_op_t op>
+void reduce_cpu_dispatch(Param<To> out, Param<Ti> in, int dim, bool change_nan,
+                         double nanval) {
+    // TODO: use kernels optimized for SIMD-based subgroup sizes
+    reduce_default_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
+}
+
+template<typename Ti, typename To, af_op_t op>
+void reduce_gpu_dispatch(Param<To> out, Param<Ti> in, int dim, bool change_nan,
+                         double nanval) {
+    // TODO: use kernels optimized for gpu subgroup sizes
+    reduce_default_dispatch<Ti, To, op>(out, in, dim, change_nan, nanval);
+}
+
 template<typename Ti, typename To, af_op_t op>
 void reduce(Param<To> out, Param<Ti> in, int dim, bool change_nan,
             double nanval) {
diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp
index f617c782dc..8c1a6e9140 100644
--- a/src/backend/oneapi/kernel/scan_dim.hpp
+++ b/src/backend/oneapi/kernel/scan_dim.hpp
@@ -42,17 +42,17 @@ class scanDimKernel {
                   const bool inclusive_scan, local_accessor<To, 1> s_val,
                   local_accessor<To, 1> s_tmp, sycl::stream debug)
         : out_acc_(out_acc)
-        , oInfo_(oInfo)
         , tmp_acc_(tmp_acc)
-        , tInfo_(tInfo)
         , in_acc_(in_acc)
+        , oInfo_(oInfo)
+        , tInfo_(tInfo)
         , iInfo_(iInfo)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , blocks_dim_(blocks_dim)
         , lim_(lim)
-        , isFinalPass_(isFinalPass)
         , DIMY_(DIMY)
+        , isFinalPass_(isFinalPass)
         , inclusive_scan_(inclusive_scan)
         , s_val_(s_val)
         , s_tmp_(s_tmp)
@@ -71,7 +71,7 @@ class scanDimKernel {
         const uint xid       = groupId_x * g.get_local_range(0) + lidx;
         const uint yid       = groupId_y;
 
-        int ids[4] = {xid, yid, zid, wid};
+        uint ids[4] = {xid, yid, zid, wid};
 
         const Ti *iptr = in_acc_.get_pointer();
         To *optr       = out_acc_.get_pointer();
@@ -173,8 +173,8 @@ class scanDimBcastKernel {
                        const uint groups_dim, const uint lim,
                        const bool inclusive_scan, sycl::stream debug)
         : out_acc_(out_acc)
-        , oInfo_(oInfo)
         , tmp_acc_(tmp_acc)
+        , oInfo_(oInfo)
         , tInfo_(tInfo)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
@@ -195,7 +195,7 @@ class scanDimBcastKernel {
         const uint xid       = groupId_x * g.get_local_range(0) + lidx;
         const uint yid       = groupId_y;
 
-        int ids[4] = {xid, yid, zid, wid};
+        uint ids[4] = {xid, yid, zid, wid};
 
         const To *tptr = tmp_acc_.get_pointer();
         To *optr       = out_acc_.get_pointer();
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index 9a377ca2d9..a7fe567c75 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -42,16 +42,16 @@ class scanFirstKernel {
                     local_accessor<To, 1> s_val, local_accessor<To, 1> s_tmp,
                     sycl::stream debug_stream)
         : out_acc_(out_acc)
-        , oInfo_(oInfo)
         , tmp_acc_(tmp_acc)
-        , tInfo_(tInfo)
         , in_acc_(in_acc)
+        , oInfo_(oInfo)
+        , tInfo_(tInfo)
         , iInfo_(iInfo)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , lim_(lim)
-        , isFinalPass_(isFinalPass)
         , DIMX_(DIMX)
+        , isFinalPass_(isFinalPass)
         , inclusive_scan_(inclusive_scan)
         , s_val_(s_val)
         , s_tmp_(s_tmp)
@@ -61,7 +61,6 @@ class scanFirstKernel {
         sycl::group g   = it.get_group();
         const uint lidx = it.get_local_id(0);
         const uint lidy = it.get_local_id(1);
-        const uint lid  = lidy * g.get_local_range(0) + lidx;
 
         const uint zid       = g.get_group_id(0) / groups_x_;
         const uint wid       = g.get_group_id(1) / groups_y_;
@@ -157,8 +156,8 @@ class scanFirstBcastKernel {
                          const uint lim, const bool inclusive_scan,
                          sycl::stream debug_stream)
         : out_acc_(out_acc)
-        , oInfo_(oInfo)
         , tmp_acc_(tmp_acc)
+        , oInfo_(oInfo)
         , tInfo_(tInfo)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
@@ -170,7 +169,6 @@ class scanFirstBcastKernel {
         sycl::group g   = it.get_group();
         const uint lidx = it.get_local_id(0);
         const uint lidy = it.get_local_id(1);
-        const uint lid  = lidy * g.get_local_range(0) + lidx;
 
         const uint zid       = g.get_group_id(0) / groups_x_;
         const uint wid       = g.get_group_id(1) / groups_y_;
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index e55c72b367..cb8887fb84 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -54,7 +54,6 @@ class whereKernel {
         sycl::group g   = it.get_group();
         const uint lidx = it.get_local_id(0);
         const uint lidy = it.get_local_id(1);
-        const uint lid  = lidy * g.get_local_range(0) + lidx;
 
         const uint zid       = g.get_group_id(0) / groups_x_;
         const uint wid       = g.get_group_id(1) / groups_y_;
@@ -126,12 +125,12 @@ static void where(Param<uint> &out, Param<T> in) {
         otmp.info.strides[k] = otmp.info.strides[k - 1] * otmp.info.dims[k - 1];
     }
 
-    int rtmp_elements = rtmp.info.strides[3] * rtmp.info.dims[3];
-    int otmp_elements = otmp.info.strides[3] * otmp.info.dims[3];
-    auto rtmp_alloc   = memAlloc<uint>(rtmp_elements);
-    auto otmp_alloc   = memAlloc<uint>(otmp_elements);
-    rtmp.data         = rtmp_alloc.get();
-    otmp.data         = otmp_alloc.get();
+    uintl rtmp_elements = rtmp.info.strides[3] * rtmp.info.dims[3];
+    uintl otmp_elements = otmp.info.strides[3] * otmp.info.dims[3];
+    auto rtmp_alloc     = memAlloc<uint>(rtmp_elements);
+    auto otmp_alloc     = memAlloc<uint>(otmp_elements);
+    rtmp.data           = rtmp_alloc.get();
+    otmp.data           = otmp_alloc.get();
 
     scan_first_launcher<T, uint, af_notzero_t>(
         otmp, rtmp, in, groups_x, groups_y, threads_x, false, true);

From 3775fd7390da1a552be6f1b22be572d30e146295 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Sat, 8 Oct 2022 09:06:43 -0400
Subject: [PATCH 123/473] approx1 port to oneapi. tests out aside from Subs,
 JIT, Memory

---
 src/backend/oneapi/CMakeLists.txt    |   1 +
 src/backend/oneapi/approx.cpp        |  15 +-
 src/backend/oneapi/kernel/approx.hpp | 309 +++++++++++++++++++++++++++
 3 files changed, 317 insertions(+), 8 deletions(-)
 create mode 100755 src/backend/oneapi/kernel/approx.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 4b0f9f0a29..c2c78dc9c6 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -205,6 +205,7 @@ add_library(afoneapi
 target_sources(afoneapi
   PRIVATE
     kernel/KParam.hpp
+    kernel/approx.hpp
     kernel/assign.hpp
     kernel/diagonal.hpp
     kernel/diff.hpp
diff --git a/src/backend/oneapi/approx.cpp b/src/backend/oneapi/approx.cpp
index e11216d00c..f25132f073 100644
--- a/src/backend/oneapi/approx.cpp
+++ b/src/backend/oneapi/approx.cpp
@@ -9,29 +9,28 @@
 
 #include <approx.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/approx.hpp>
 
 namespace oneapi {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
              const int xdim, const Tp &xi_beg, const Tp &xi_step,
              const af_interp_type method, const float offGrid) {
-    ONEAPI_NOT_SUPPORTED("");
-    return;
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            // kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step,
-            // offGrid, method, 1);
+            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    method, 1);
             break;
         case AF_INTERP_LINEAR:
         case AF_INTERP_LINEAR_COSINE:
-            // kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step,
-            // offGrid, method, 2);
+            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    method, 2);
             break;
         case AF_INTERP_CUBIC:
         case AF_INTERP_CUBIC_SPLINE:
-            // kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step,
-            // offGrid, method, 3);
+            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    method, 3);
             break;
         default: break;
     }
diff --git a/src/backend/oneapi/kernel/approx.hpp b/src/backend/oneapi/kernel/approx.hpp
new file mode 100755
index 0000000000..de96866f99
--- /dev/null
+++ b/src/backend/oneapi/kernel/approx.hpp
@@ -0,0 +1,309 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+constexpr int TILE_DIM  = 32;
+constexpr int THREADS_X = TILE_DIM;
+constexpr int THREADS_Y = 256 / TILE_DIM;
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename Ty, typename Tp>
+class approx1Kernel {
+   public:
+    approx1Kernel(sycl::accessor<Ty> d_yo, const KParam yo,
+                  sycl::accessor<Ty> d_yi, const KParam yi,
+                  sycl::accessor<Tp> d_xo, const KParam xo, const Tp xi_beg,
+                  const Tp xi_step_reproc, const Ty offGrid,
+                  const int blocksMatX, const int batch, const int method,
+                  const int XDIM, const int INTERP_ORDER)
+        : d_yo_(d_yo)
+        , yo_(yo)
+        , d_yi_(d_yi)
+        , yi_(yi)
+        , d_xo_(d_xo)
+        , xo_(xo)
+        , xi_beg_(xi_beg)
+        , xi_step_reproc_(xi_step_reproc)
+        , offGrid_(offGrid)
+        , blocksMatX_(blocksMatX)
+        , batch_(batch)
+        , method_(method)
+        , XDIM_(XDIM)
+        , INTERP_ORDER_(INTERP_ORDER) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const int idw = g.get_group_id(1) / yo_.dims[2];
+        const int idz = g.get_group_id(1) - idw * yo_.dims[2];
+
+        const int idy        = g.get_group_id(0) / blocksMatX_;
+        const int blockIdx_x = g.get_group_id(0) - idy * blocksMatX_;
+        const int idx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+
+        if (idx >= yo_.dims[0] || idy >= yo_.dims[1] || idz >= yo_.dims[2] ||
+            idw >= yo_.dims[3])
+            return;
+
+        // FIXME: Only cubic interpolation is doing clamping
+        // We need to make it consistent across all methods
+        // Not changing the behavior because tests will fail
+        const bool doclamp = INTERP_ORDER_ == 3;
+
+        bool is_off[] = {xo_.dims[0] > 1, xo_.dims[1] > 1, xo_.dims[2] > 1,
+                         xo_.dims[3] > 1};
+
+        const int yo_idx = idw * yo_.strides[3] + idz * yo_.strides[2] +
+                           idy * yo_.strides[1] + idx + yo_.offset;
+
+        int xo_idx = idx * is_off[0] + xo_.offset;
+        if (batch_) {
+            xo_idx += idw * xo_.strides[3] * is_off[3];
+            xo_idx += idz * xo_.strides[2] * is_off[2];
+            xo_idx += idy * xo_.strides[1] * is_off[1];
+        }
+
+        const Tp x = (d_xo_[xo_idx] - xi_beg_) * xi_step_reproc_;
+
+#pragma unroll
+        for (int flagIdx = 0; flagIdx < 4; ++flagIdx) {
+            is_off[flagIdx] = true;
+        }
+        is_off[XDIM_] = false;
+
+        if (x < 0 || yi_.dims[XDIM_] < x + 1) {
+            d_yo_[yo_idx] = offGrid_;
+            return;
+        }
+
+        int yi_idx = idx * is_off[0] + yi_.offset;
+        yi_idx += idw * yi_.strides[3] * is_off[3];
+        yi_idx += idz * yi_.strides[2] * is_off[2];
+        yi_idx += idy * yi_.strides[1] * is_off[1];
+
+        if (INTERP_ORDER_ == 1)
+            interp1o1(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
+                      doclamp, 1);
+        if (INTERP_ORDER_ == 2)
+            interp1o2(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
+                      doclamp, 1);
+        if (INTERP_ORDER_ == 3)
+            interp1o3(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
+                      doclamp, 1);
+    }
+
+    void interp1o1(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        Ty zero = (Ty)0;
+
+        const int x_lim    = in.dims[XDIM_];
+        const int x_stride = in.strides[XDIM_];
+
+        int xid   = (method == AF_INTERP_LOWER ? floor(x) : round(x));
+        bool cond = xid >= 0 && xid < x_lim;
+        if (doclamp) xid = fmax(0, fmin(xid, x_lim));
+
+        const int idx = ioff + xid * x_stride;
+
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * in.strides[batch_dim];
+            d_out[ooff + n * out.strides[batch_dim]] =
+                (doclamp || cond) ? d_in[idx_n] : zero;
+        }
+    }
+
+#if IS_CPLX
+#if USE_DOUBLE
+    typedef double ScalarTy;
+#else
+    typedef float ScalarTy;
+#endif
+    Ty __mulrc(ScalarTy s, Ty v) {
+        InterpInTy out = {s * v.x, s * v.y};
+        return out;
+    }
+#define MULRC(a, b) __mulrc(a, b)
+#define MULCR(a, b) __mulrc(b, a)
+#else
+#define MULRC(a, b) (a) * (b)
+#define MULCR(a, b) (a) * (b)
+#endif
+
+    Ty linearInterpFunc(Ty val[2], Tp ratio) const {
+        return MULRC((1 - ratio), val[0]) + MULRC(ratio, val[1]);
+    }
+
+    Ty bilinearInterpFunc(Ty val[2][2], Tp xratio, Tp yratio) const {
+        Ty res[2];
+        res[0] = linearInterpFunc(val[0], xratio);
+        res[1] = linearInterpFunc(val[1], xratio);
+        return linearInterpFunc(res, yratio);
+    }
+
+    Ty cubicInterpFunc(Ty val[4], Tp xratio, bool spline) const {
+        Ty a0, a1, a2, a3;
+        if (spline) {
+            a0 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)1.5, val[1]) +
+                 MULRC((Tp)-1.5, val[2]) + MULRC((Tp)0.5, val[3]);
+
+            a1 = MULRC((Tp)1.0, val[0]) + MULRC((Tp)-2.5, val[1]) +
+                 MULRC((Tp)2.0, val[2]) + MULRC((Tp)-0.5, val[3]);
+
+            a2 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)0.5, val[2]);
+
+            a3 = val[1];
+        } else {
+            a0 = val[3] - val[2] - val[0] + val[1];
+            a1 = val[0] - val[1] - a0;
+            a2 = val[2] - val[0];
+            a3 = val[1];
+        }
+
+        Tp xratio2 = xratio * xratio;
+        Tp xratio3 = xratio2 * xratio;
+
+        return MULCR(a0, xratio3) + MULCR(a1, xratio2) + MULCR(a2, xratio) + a3;
+    }
+
+    Ty bicubicInterpFunc(Ty val[4][4], Tp xratio, Tp yratio,
+                         bool spline) const {
+        Ty res[4];
+        res[0] = cubicInterpFunc(val[0], xratio, spline);
+        res[1] = cubicInterpFunc(val[1], xratio, spline);
+        res[2] = cubicInterpFunc(val[2], xratio, spline);
+        res[3] = cubicInterpFunc(val[3], xratio, spline);
+        return cubicInterpFunc(res, yratio, spline);
+    }
+
+    void interp1o2(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        const int grid_x = floor(x);    // nearest grid
+        const Tp off_x   = x - grid_x;  // fractional offset
+
+        const int x_lim    = in.dims[XDIM_];
+        const int x_stride = in.strides[XDIM_];
+        const int idx      = ioff + grid_x * x_stride;
+
+        Ty zero      = (Ty)0;
+        bool cond[2] = {true, grid_x + 1 < x_lim};
+        int offx[2]  = {0, cond[1] ? 1 : 0};
+        Tp ratio     = off_x;
+        if (method == AF_INTERP_LINEAR_COSINE) {
+            ratio = (1 - cos(ratio * (Tp)M_PI)) / 2;
+        }
+
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * in.strides[batch_dim];
+            Ty val[2] = {
+                (doclamp || cond[0]) ? d_in[idx_n + offx[0] * x_stride] : zero,
+                (doclamp || cond[1]) ? d_in[idx_n + offx[1] * x_stride] : zero};
+
+            d_out[ooff + n * out.strides[batch_dim]] =
+                linearInterpFunc(val, ratio);
+        }
+    }
+
+    void interp1o3(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        const int grid_x = floor(x);    // nearest grid
+        const Tp off_x   = x - grid_x;  // fractional offset
+
+        const int x_lim    = in.dims[XDIM_];
+        const int x_stride = in.strides[XDIM_];
+        const int idx      = ioff + grid_x * x_stride;
+
+        bool cond[4] = {grid_x - 1 >= 0, true, grid_x + 1 < x_lim,
+                        grid_x + 2 < x_lim};
+        int off[4]   = {cond[0] ? -1 : 0, 0, cond[2] ? 1 : 0,
+                      cond[3] ? 2 : (cond[2] ? 1 : 0)};
+
+        Ty zero = (Ty)0;
+
+        for (int n = 0; n < batch; n++) {
+            Ty val[4];
+            int idx_n = idx + n * in.strides[batch_dim];
+            for (int i = 0; i < 4; i++) {
+                val[i] = (doclamp || cond[i]) ? d_in[idx_n + off[i] * x_stride]
+                                              : zero;
+            }
+            bool spline = method == AF_INTERP_CUBIC_SPLINE;
+            d_out[ooff + n * out.strides[batch_dim]] =
+                cubicInterpFunc(val, off_x, spline);
+        }
+    }
+
+   private:
+    sycl::accessor<Ty> d_yo_;
+    const KParam yo_;
+    sycl::accessor<Ty> d_yi_;
+    const KParam yi_;
+    sycl::accessor<Tp> d_xo_;
+    const KParam xo_;
+    const Tp xi_beg_;
+    const Tp xi_step_reproc_;
+    const Ty offGrid_;
+    const int blocksMatX_;
+    const int batch_;
+    const int method_;
+    const int XDIM_;
+    const int INTERP_ORDER_;
+};
+
+template<typename Ty, typename Tp>
+void approx1(Param<Ty> yo, const Param<Ty> yi, const Param<Tp> xo,
+             const int xdim, const Tp xi_beg, const Tp xi_step,
+             const float offGrid, const af_interp_type method,
+             const int order) {
+    constexpr int THREADS = 256;
+
+    auto local         = sycl::range{THREADS, 1};
+    dim_t blocksPerMat = divup(yo.info.dims[0], local[0]);
+    auto global        = sycl::range{blocksPerMat * local[0] * yo.info.dims[1],
+                              yo.info.dims[2] * yo.info.dims[3] * local[1]};
+
+    // Passing bools to opencl kernels is not allowed
+    bool batch =
+        !(xo.info.dims[1] == 1 && xo.info.dims[2] == 1 && xo.info.dims[3] == 1);
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto yoAcc = yo.data->get_access(h);
+        auto yiAcc = yi.data->get_access(h);
+        auto xoAcc = xo.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            approx1Kernel<Ty, Tp>(yoAcc, yo.info, yiAcc, yi.info, xoAcc,
+                                  xo.info, xi_beg, Tp(1) / xi_step, (Ty)offGrid,
+                                  (int)blocksPerMat, (int)batch, (int)method,
+                                  xdim, order));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi

From 55583412363c22210ed40601df5ba84e5f18c8de Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Mon, 10 Oct 2022 13:45:31 -0400
Subject: [PATCH 124/473] approx2 port to oneapi. tests out aside from Subs,
 JIT, Memory

---
 src/backend/oneapi/approx.cpp        |  14 +-
 src/backend/oneapi/arith.hpp         |   3 +
 src/backend/oneapi/kernel/approx.hpp | 344 +++++++++++++++++++++++++--
 3 files changed, 336 insertions(+), 25 deletions(-)

diff --git a/src/backend/oneapi/approx.cpp b/src/backend/oneapi/approx.cpp
index f25132f073..43ff5f7dcf 100644
--- a/src/backend/oneapi/approx.cpp
+++ b/src/backend/oneapi/approx.cpp
@@ -42,27 +42,25 @@ void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
              const Array<Tp> &yo, const int ydim, const Tp &yi_beg,
              const Tp &yi_step, const af_interp_type method,
              const float offGrid) {
-    ONEAPI_NOT_SUPPORTED("");
-    return;
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            // kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
-            // ydim, yi_beg, yi_step, offGrid, method, 1);
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+                                    ydim, yi_beg, yi_step, offGrid, method, 1);
             break;
         case AF_INTERP_LINEAR:
         case AF_INTERP_BILINEAR:
         case AF_INTERP_LINEAR_COSINE:
         case AF_INTERP_BILINEAR_COSINE:
-            // kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
-            // ydim, yi_beg, yi_step, offGrid, method, 2);
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+                                    ydim, yi_beg, yi_step, offGrid, method, 2);
             break;
         case AF_INTERP_CUBIC:
         case AF_INTERP_BICUBIC:
         case AF_INTERP_CUBIC_SPLINE:
         case AF_INTERP_BICUBIC_SPLINE:
-            // kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
-            // ydim, yi_beg, yi_step, offGrid, method, 3);
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+                                    ydim, yi_beg, yi_step, offGrid, method, 3);
             break;
         default: break;
     }
diff --git a/src/backend/oneapi/arith.hpp b/src/backend/oneapi/arith.hpp
index 2a004b5766..1311d4d607 100644
--- a/src/backend/oneapi/arith.hpp
+++ b/src/backend/oneapi/arith.hpp
@@ -11,6 +11,7 @@
 
 #include <Array.hpp>
 #include <common/jit/BinaryNode.hpp>
+#include <err_oneapi.hpp>
 #include <optypes.hpp>
 #include <af/dim4.hpp>
 
@@ -19,12 +20,14 @@ namespace oneapi {
 template<typename T, af_op_t op>
 Array<T> arithOp(const Array<T> &&lhs, const Array<T> &&rhs,
                  const af::dim4 &odims) {
+    ONEAPI_NOT_SUPPORTED(__FUNCTION__);
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 
 template<typename T, af_op_t op>
 Array<T> arithOp(const Array<T> &lhs, const Array<T> &rhs,
                  const af::dim4 &odims) {
+    ONEAPI_NOT_SUPPORTED(__FUNCTION__);
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/approx.hpp b/src/backend/oneapi/kernel/approx.hpp
index de96866f99..15033317dd 100755
--- a/src/backend/oneapi/kernel/approx.hpp
+++ b/src/backend/oneapi/kernel/approx.hpp
@@ -155,13 +155,6 @@ class approx1Kernel {
         return MULRC((1 - ratio), val[0]) + MULRC(ratio, val[1]);
     }
 
-    Ty bilinearInterpFunc(Ty val[2][2], Tp xratio, Tp yratio) const {
-        Ty res[2];
-        res[0] = linearInterpFunc(val[0], xratio);
-        res[1] = linearInterpFunc(val[1], xratio);
-        return linearInterpFunc(res, yratio);
-    }
-
     Ty cubicInterpFunc(Ty val[4], Tp xratio, bool spline) const {
         Ty a0, a1, a2, a3;
         if (spline) {
@@ -187,16 +180,6 @@ class approx1Kernel {
         return MULCR(a0, xratio3) + MULCR(a1, xratio2) + MULCR(a2, xratio) + a3;
     }
 
-    Ty bicubicInterpFunc(Ty val[4][4], Tp xratio, Tp yratio,
-                         bool spline) const {
-        Ty res[4];
-        res[0] = cubicInterpFunc(val[0], xratio, spline);
-        res[1] = cubicInterpFunc(val[1], xratio, spline);
-        res[2] = cubicInterpFunc(val[2], xratio, spline);
-        res[3] = cubicInterpFunc(val[3], xratio, spline);
-        return cubicInterpFunc(res, yratio, spline);
-    }
-
     void interp1o2(sycl::accessor<Ty> d_out, KParam out, int ooff,
                    sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
                    int method, int batch, bool doclamp, int batch_dim) const {
@@ -305,5 +288,332 @@ void approx1(Param<Ty> yo, const Param<Ty> yi, const Param<Tp> xo,
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 
+template<typename Ty, typename Tp>
+class approx2Kernel {
+   public:
+    approx2Kernel(sycl::accessor<Ty> d_zo, const KParam zo,
+                  sycl::accessor<Ty> d_zi, const KParam zi,
+                  sycl::accessor<Tp> d_xo, const KParam xo,
+                  sycl::accessor<Tp> d_yo, const KParam yo, const Tp xi_beg,
+                  const Tp xi_step_reproc, const Tp yi_beg,
+                  const Tp yi_step_reproc, const Ty offGrid,
+                  const int blocksMatX, const int blocksMatY, const int batch,
+                  int method, const int XDIM, const int YDIM,
+                  const int INTERP_ORDER)
+        : d_zo_(d_zo)
+        , zo_(zo)
+        , d_zi_(d_zi)
+        , zi_(zi)
+        , d_xo_(d_xo)
+        , xo_(xo)
+        , d_yo_(d_yo)
+        , yo_(yo)
+        , xi_beg_(xi_beg)
+        , xi_step_reproc_(xi_step_reproc)
+        , yi_beg_(yi_beg)
+        , yi_step_reproc_(yi_step_reproc)
+        , offGrid_(offGrid)
+        , blocksMatX_(blocksMatX)
+        , blocksMatY_(blocksMatY)
+        , batch_(batch)
+        , method_(method)
+        , XDIM_(XDIM)
+        , YDIM_(YDIM)
+        , INTERP_ORDER_(INTERP_ORDER) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const int idz = g.get_group_id(0) / blocksMatX_;
+        const int idw = g.get_group_id(1) / blocksMatY_;
+
+        const int blockIdx_x = g.get_group_id(0) - idz * blocksMatX_;
+        const int blockIdx_y = g.get_group_id(1) - idw * blocksMatY_;
+
+        const int idx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+        const int idy = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+        if (idx >= zo_.dims[0] || idy >= zo_.dims[1] || idz >= zo_.dims[2] ||
+            idw >= zo_.dims[3])
+            return;
+
+        // FIXME: Only cubic interpolation is doing clamping
+        // We need to make it consistent across all methods
+        // Not changing the behavior because tests will fail
+        const bool doclamp = INTERP_ORDER_ == 3;
+
+        bool is_off[] = {xo_.dims[0] > 1, xo_.dims[1] > 1, xo_.dims[2] > 1,
+                         xo_.dims[3] > 1};
+
+        const int zo_idx = idw * zo_.strides[3] + idz * zo_.strides[2] +
+                           idy * zo_.strides[1] + idx + zo_.offset;
+        int xo_idx =
+            idy * xo_.strides[1] * is_off[1] + idx * is_off[0] + xo_.offset;
+
+        int yo_idx =
+            idy * yo_.strides[1] * is_off[1] + idx * is_off[0] + yo_.offset;
+        if (batch_) {
+            xo_idx += idw * xo_.strides[3] * is_off[3] +
+                      idz * xo_.strides[2] * is_off[2];
+            yo_idx += idw * yo_.strides[3] * is_off[3] +
+                      idz * yo_.strides[2] * is_off[2];
+        }
+
+#pragma unroll
+        for (int flagIdx = 0; flagIdx < 4; ++flagIdx) {
+            is_off[flagIdx] = true;
+        }
+        is_off[XDIM_] = false;
+        is_off[YDIM_] = false;
+
+        const Tp x = (d_xo_[xo_idx] - xi_beg_) * xi_step_reproc_;
+        const Tp y = (d_yo_[yo_idx] - yi_beg_) * yi_step_reproc_;
+
+        if (x < 0 || y < 0 || zi_.dims[XDIM_] < x + 1 ||
+            zi_.dims[YDIM_] < y + 1) {
+            d_zo_[zo_idx] = offGrid_;
+            return;
+        }
+
+        int zi_idx =
+            idy * zi_.strides[1] * is_off[1] + idx * is_off[0] + zi_.offset;
+        zi_idx +=
+            idw * zi_.strides[3] * is_off[3] + idz * zi_.strides[2] * is_off[2];
+
+        if (INTERP_ORDER_ == 1)
+            interp2o1(d_zo_, zo_, zo_idx, d_zi_, zi_, zi_idx, x, y, method_, 1,
+                      doclamp, 2);
+        if (INTERP_ORDER_ == 2)
+            interp2o2(d_zo_, zo_, zo_idx, d_zi_, zi_, zi_idx, x, y, method_, 1,
+                      doclamp, 2);
+        if (INTERP_ORDER_ == 3)
+            interp2o3(d_zo_, zo_, zo_idx, d_zi_, zi_, zi_idx, x, y, method_, 1,
+                      doclamp, 2);
+    }
+
+    Ty linearInterpFunc(Ty val[2], Tp ratio) const {
+        return MULRC((1 - ratio), val[0]) + MULRC(ratio, val[1]);
+    }
+
+    Ty bilinearInterpFunc(Ty val[2][2], Tp xratio, Tp yratio) const {
+        Ty res[2];
+        res[0] = linearInterpFunc(val[0], xratio);
+        res[1] = linearInterpFunc(val[1], xratio);
+        return linearInterpFunc(res, yratio);
+    }
+
+    Ty cubicInterpFunc(Ty val[4], Tp xratio, bool spline) const {
+        Ty a0, a1, a2, a3;
+        if (spline) {
+            a0 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)1.5, val[1]) +
+                 MULRC((Tp)-1.5, val[2]) + MULRC((Tp)0.5, val[3]);
+
+            a1 = MULRC((Tp)1.0, val[0]) + MULRC((Tp)-2.5, val[1]) +
+                 MULRC((Tp)2.0, val[2]) + MULRC((Tp)-0.5, val[3]);
+
+            a2 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)0.5, val[2]);
+
+            a3 = val[1];
+        } else {
+            a0 = val[3] - val[2] - val[0] + val[1];
+            a1 = val[0] - val[1] - a0;
+            a2 = val[2] - val[0];
+            a3 = val[1];
+        }
+
+        Tp xratio2 = xratio * xratio;
+        Tp xratio3 = xratio2 * xratio;
+
+        return MULCR(a0, xratio3) + MULCR(a1, xratio2) + MULCR(a2, xratio) + a3;
+    }
+
+    Ty bicubicInterpFunc(Ty val[4][4], Tp xratio, Tp yratio,
+                         bool spline) const {
+        Ty res[4];
+        res[0] = cubicInterpFunc(val[0], xratio, spline);
+        res[1] = cubicInterpFunc(val[1], xratio, spline);
+        res[2] = cubicInterpFunc(val[2], xratio, spline);
+        res[3] = cubicInterpFunc(val[3], xratio, spline);
+        return cubicInterpFunc(res, yratio, spline);
+    }
+
+    void interp2o1(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x, Tp y,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        int xid = (method == AF_INTERP_LOWER ? floor(x) : round(x));
+        int yid = (method == AF_INTERP_LOWER ? floor(y) : round(y));
+
+        const int x_lim    = in.dims[XDIM_];
+        const int y_lim    = in.dims[YDIM_];
+        const int x_stride = in.strides[XDIM_];
+        const int y_stride = in.strides[YDIM_];
+
+        if (doclamp) {
+            xid = fmax(0, fmin(xid, x_lim));
+            yid = fmax(0, fmin(yid, y_lim));
+        }
+        const int idx = ioff + yid * y_stride + xid * x_stride;
+
+        bool condX = xid >= 0 && xid < x_lim;
+        bool condY = yid >= 0 && yid < y_lim;
+
+        Ty zero = (Ty)0;
+        ;
+        bool cond = condX && condY;
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * in.strides[batch_dim];
+            d_out[ooff + n * out.strides[batch_dim]] =
+                (doclamp || cond) ? d_in[idx_n] : zero;
+        }
+    }
+
+    void interp2o2(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x, Tp y,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        const int grid_x = floor(x);
+        const Tp off_x   = x - grid_x;
+
+        const int grid_y = floor(y);
+        const Tp off_y   = y - grid_y;
+
+        const int x_lim    = in.dims[XDIM_];
+        const int y_lim    = in.dims[YDIM_];
+        const int x_stride = in.strides[XDIM_];
+        const int y_stride = in.strides[YDIM_];
+        const int idx      = ioff + grid_y * y_stride + grid_x * x_stride;
+
+        bool condX[2] = {true, x + 1 < x_lim};
+        bool condY[2] = {true, y + 1 < y_lim};
+        int offx[2]   = {0, condX[1] ? 1 : 0};
+        int offy[2]   = {0, condY[1] ? 1 : 0};
+
+        Tp xratio = off_x, yratio = off_y;
+        if (method == AF_INTERP_LINEAR_COSINE) {
+            xratio = (1 - cos(xratio * (Tp)M_PI)) / 2;
+            yratio = (1 - cos(yratio * (Tp)M_PI)) / 2;
+        }
+
+        Ty zero = (Ty)0;
+        ;
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * in.strides[batch_dim];
+            Ty val[2][2];
+            for (int j = 0; j < 2; j++) {
+                int off_y = idx_n + offy[j] * y_stride;
+                for (int i = 0; i < 2; i++) {
+                    bool cond = (doclamp || (condX[i] && condY[j]));
+                    val[j][i] = cond ? d_in[off_y + offx[i] * x_stride] : zero;
+                }
+            }
+            d_out[ooff + n * out.strides[batch_dim]] =
+                bilinearInterpFunc(val, xratio, yratio);
+        }
+    }
+
+    void interp2o3(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x, Tp y,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        const int grid_x = floor(x);
+        const Tp off_x   = x - grid_x;
+
+        const int grid_y = floor(y);
+        const Tp off_y   = y - grid_y;
+
+        const int x_lim    = in.dims[XDIM_];
+        const int y_lim    = in.dims[YDIM_];
+        const int x_stride = in.strides[XDIM_];
+        const int y_stride = in.strides[YDIM_];
+        const int idx      = ioff + grid_y * y_stride + grid_x * x_stride;
+
+        // used for setting values at boundaries
+        bool condX[4] = {grid_x - 1 >= 0, true, grid_x + 1 < x_lim,
+                         grid_x + 2 < x_lim};
+        bool condY[4] = {grid_y - 1 >= 0, true, grid_y + 1 < y_lim,
+                         grid_y + 2 < y_lim};
+        int offX[4]   = {condX[0] ? -1 : 0, 0, condX[2] ? 1 : 0,
+                       condX[3] ? 2 : (condX[2] ? 1 : 0)};
+        int offY[4]   = {condY[0] ? -1 : 0, 0, condY[2] ? 1 : 0,
+                       condY[3] ? 2 : (condY[2] ? 1 : 0)};
+
+        Ty zero = (Ty)0;
+        ;
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * in.strides[batch_dim];
+            // for bicubic interpolation, work with 4x4 val at a time
+            Ty val[4][4];
+#pragma unroll
+            for (int j = 0; j < 4; j++) {
+                int ioff_j = idx_n + offY[j] * y_stride;
+#pragma unroll
+                for (int i = 0; i < 4; i++) {
+                    bool cond = (doclamp || (condX[i] && condY[j]));
+                    val[j][i] = cond ? d_in[ioff_j + offX[i] * x_stride] : zero;
+                }
+            }
+            bool spline = method == AF_INTERP_CUBIC_SPLINE ||
+                          method == AF_INTERP_BICUBIC_SPLINE;
+            d_out[ooff + n * out.strides[batch_dim]] =
+                bicubicInterpFunc(val, off_x, off_y, spline);
+        }
+    }
+
+   private:
+    sycl::accessor<Ty> d_zo_;
+    const KParam zo_;
+    sycl::accessor<Ty> d_zi_;
+    const KParam zi_;
+    sycl::accessor<Tp> d_xo_;
+    const KParam xo_;
+    sycl::accessor<Tp> d_yo_;
+    const KParam yo_;
+    const Tp xi_beg_;
+    const Tp xi_step_reproc_;
+    const Tp yi_beg_;
+    const Tp yi_step_reproc_;
+    const Ty offGrid_;
+    const int blocksMatX_;
+    const int blocksMatY_;
+    const int batch_;
+    int method_;
+    const int XDIM_;
+    const int YDIM_;
+    const int INTERP_ORDER_;
+};
+
+template<typename Ty, typename Tp>
+void approx2(Param<Ty> zo, const Param<Ty> zi, const Param<Tp> xo,
+             const int xdim, const Tp &xi_beg, const Tp &xi_step,
+             const Param<Tp> yo, const int ydim, const Tp &yi_beg,
+             const Tp &yi_step, const float offGrid,
+             const af_interp_type method, const int order) {
+    constexpr int TX = 16;
+    constexpr int TY = 16;
+
+    auto local          = sycl::range{TX, TY};
+    dim_t blocksPerMatX = divup(zo.info.dims[0], local[0]);
+    dim_t blocksPerMatY = divup(zo.info.dims[1], local[1]);
+    auto global = sycl::range{blocksPerMatX * local[0] * zo.info.dims[2],
+                              blocksPerMatY * local[1] * zo.info.dims[3]};
+
+    // Passing bools to opencl kernels is not allowed
+    bool batch = !(xo.info.dims[2] == 1 && xo.info.dims[3] == 1);
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto zoAcc = zo.data->get_access(h);
+        auto ziAcc = zi.data->get_access(h);
+        auto xoAcc = xo.data->get_access(h);
+        auto yoAcc = yo.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            approx2Kernel<Ty, Tp>(
+                zoAcc, zo.info, ziAcc, zi.info, xoAcc, xo.info, yoAcc, yo.info,
+                xi_beg, Tp(1) / xi_step, yi_beg, Tp(1) / yi_step, (Ty)offGrid,
+                static_cast<int>(blocksPerMatX),
+                static_cast<int>(blocksPerMatY), static_cast<int>(batch),
+                static_cast<int>(method), xdim, ydim, order));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
 }  // namespace kernel
 }  // namespace oneapi

From 1157453a2749f55ca20f84d29fe87ff841338296 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Mon, 10 Oct 2022 14:27:01 -0400
Subject: [PATCH 125/473] format

---
 src/backend/oneapi/approx.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/backend/oneapi/approx.cpp b/src/backend/oneapi/approx.cpp
index 43ff5f7dcf..c96764194c 100644
--- a/src/backend/oneapi/approx.cpp
+++ b/src/backend/oneapi/approx.cpp
@@ -45,22 +45,22 @@ void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
-                                    ydim, yi_beg, yi_step, offGrid, method, 1);
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    yi_beg, yi_step, offGrid, method, 1);
             break;
         case AF_INTERP_LINEAR:
         case AF_INTERP_BILINEAR:
         case AF_INTERP_LINEAR_COSINE:
         case AF_INTERP_BILINEAR_COSINE:
-            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
-                                    ydim, yi_beg, yi_step, offGrid, method, 2);
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    yi_beg, yi_step, offGrid, method, 2);
             break;
         case AF_INTERP_CUBIC:
         case AF_INTERP_BICUBIC:
         case AF_INTERP_CUBIC_SPLINE:
         case AF_INTERP_BICUBIC_SPLINE:
-            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
-                                    ydim, yi_beg, yi_step, offGrid, method, 3);
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    yi_beg, yi_step, offGrid, method, 3);
             break;
         default: break;
     }

From 3fc5bd93d2f5cf87b3b5442e678e85a504e47c64 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Tue, 11 Oct 2022 10:13:29 -0400
Subject: [PATCH 126/473] split oneapi approx into approx1 and approx2 separate
 sources

---
 src/backend/oneapi/CMakeLists.txt             |   6 +-
 src/backend/oneapi/approx.cpp                 |   3 +-
 src/backend/oneapi/approx1.cpp                |  49 +++
 src/backend/oneapi/approx2.cpp                |  56 ++++
 src/backend/oneapi/kernel/approx1.hpp         | 278 ++++++++++++++++++
 .../oneapi/kernel/{approx.hpp => approx2.hpp} | 261 +---------------
 6 files changed, 392 insertions(+), 261 deletions(-)
 create mode 100644 src/backend/oneapi/approx1.cpp
 create mode 100644 src/backend/oneapi/approx2.cpp
 create mode 100644 src/backend/oneapi/kernel/approx1.hpp
 rename src/backend/oneapi/kernel/{approx.hpp => approx2.hpp} (59%)
 mode change 100755 => 100644

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index c2c78dc9c6..e70af234c3 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -22,7 +22,8 @@ add_library(afoneapi
   anisotropic_diffusion.cpp
   anisotropic_diffusion.hpp
   any.cpp
-  approx.cpp
+  approx1.cpp
+  approx2.cpp
   approx.hpp
   arith.hpp
   assign.cpp
@@ -205,7 +206,8 @@ add_library(afoneapi
 target_sources(afoneapi
   PRIVATE
     kernel/KParam.hpp
-    kernel/approx.hpp
+    kernel/approx1.hpp
+    kernel/approx2.hpp
     kernel/assign.hpp
     kernel/diagonal.hpp
     kernel/diff.hpp
diff --git a/src/backend/oneapi/approx.cpp b/src/backend/oneapi/approx.cpp
index c96764194c..da153301d2 100644
--- a/src/backend/oneapi/approx.cpp
+++ b/src/backend/oneapi/approx.cpp
@@ -9,7 +9,8 @@
 
 #include <approx.hpp>
 #include <err_oneapi.hpp>
-#include <kernel/approx.hpp>
+#include <kernel/approx1.hpp>
+#include <kernel/approx2.hpp>
 
 namespace oneapi {
 template<typename Ty, typename Tp>
diff --git a/src/backend/oneapi/approx1.cpp b/src/backend/oneapi/approx1.cpp
new file mode 100644
index 0000000000..cee2aa9b15
--- /dev/null
+++ b/src/backend/oneapi/approx1.cpp
@@ -0,0 +1,49 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#include <approx.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/approx1.hpp>
+
+namespace oneapi {
+template<typename Ty, typename Tp>
+void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
+             const int xdim, const Tp &xi_beg, const Tp &xi_step,
+             const af_interp_type method, const float offGrid) {
+    switch (method) {
+        case AF_INTERP_NEAREST:
+        case AF_INTERP_LOWER:
+            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    method, 1);
+            break;
+        case AF_INTERP_LINEAR:
+        case AF_INTERP_LINEAR_COSINE:
+            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    method, 2);
+            break;
+        case AF_INTERP_CUBIC:
+        case AF_INTERP_CUBIC_SPLINE:
+            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
+                                    method, 3);
+            break;
+        default: break;
+    }
+}
+
+#define INSTANTIATE(Ty, Tp)                                       \
+    template void approx1<Ty, Tp>(                                \
+        Array<Ty> & yo, const Array<Ty> &yi, const Array<Tp> &xo, \
+        const int xdim, const Tp &xi_beg, const Tp &xi_step,      \
+        const af_interp_type method, const float offGrid);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+INSTANTIATE(cfloat, float)
+INSTANTIATE(cdouble, double)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/approx2.cpp b/src/backend/oneapi/approx2.cpp
new file mode 100644
index 0000000000..e22d5406ee
--- /dev/null
+++ b/src/backend/oneapi/approx2.cpp
@@ -0,0 +1,56 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#include <approx.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/approx2.hpp>
+
+namespace oneapi {
+template<typename Ty, typename Tp>
+void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
+             const int xdim, const Tp &xi_beg, const Tp &xi_step,
+             const Array<Tp> &yo, const int ydim, const Tp &yi_beg,
+             const Tp &yi_step, const af_interp_type method,
+             const float offGrid) {
+    switch (method) {
+        case AF_INTERP_NEAREST:
+        case AF_INTERP_LOWER:
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    yi_beg, yi_step, offGrid, method, 1);
+            break;
+        case AF_INTERP_LINEAR:
+        case AF_INTERP_BILINEAR:
+        case AF_INTERP_LINEAR_COSINE:
+        case AF_INTERP_BILINEAR_COSINE:
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    yi_beg, yi_step, offGrid, method, 2);
+            break;
+        case AF_INTERP_CUBIC:
+        case AF_INTERP_BICUBIC:
+        case AF_INTERP_CUBIC_SPLINE:
+        case AF_INTERP_BICUBIC_SPLINE:
+            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
+                                    yi_beg, yi_step, offGrid, method, 3);
+            break;
+        default: break;
+    }
+}
+
+#define INSTANTIATE(Ty, Tp)                                       \
+    template void approx2<Ty, Tp>(                                \
+        Array<Ty> & zo, const Array<Ty> &zi, const Array<Tp> &xo, \
+        const int xdim, const Tp &xi_beg, const Tp &xi_step,      \
+        const Array<Tp> &yo, const int ydim, const Tp &yi_beg,    \
+        const Tp &yi_step, const af_interp_type method, const float offGrid);
+
+INSTANTIATE(float, float)
+INSTANTIATE(double, double)
+INSTANTIATE(cfloat, float)
+INSTANTIATE(cdouble, double)
+
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/approx1.hpp b/src/backend/oneapi/kernel/approx1.hpp
new file mode 100644
index 0000000000..ad2ac257ea
--- /dev/null
+++ b/src/backend/oneapi/kernel/approx1.hpp
@@ -0,0 +1,278 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+#define MULRC(a, b) (a) * (b)
+#define MULCR(a, b) (a) * (b)
+
+namespace oneapi {
+namespace kernel {
+
+constexpr int TILE_DIM  = 32;
+constexpr int THREADS_X = TILE_DIM;
+constexpr int THREADS_Y = 256 / TILE_DIM;
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename Ty, typename Tp>
+class approx1Kernel {
+   public:
+    approx1Kernel(sycl::accessor<Ty> d_yo, const KParam yo,
+                  sycl::accessor<Ty> d_yi, const KParam yi,
+                  sycl::accessor<Tp> d_xo, const KParam xo, const Tp xi_beg,
+                  const Tp xi_step_reproc, const Ty offGrid,
+                  const int blocksMatX, const int batch, const int method,
+                  const int XDIM, const int INTERP_ORDER)
+        : d_yo_(d_yo)
+        , yo_(yo)
+        , d_yi_(d_yi)
+        , yi_(yi)
+        , d_xo_(d_xo)
+        , xo_(xo)
+        , xi_beg_(xi_beg)
+        , xi_step_reproc_(xi_step_reproc)
+        , offGrid_(offGrid)
+        , blocksMatX_(blocksMatX)
+        , batch_(batch)
+        , method_(method)
+        , XDIM_(XDIM)
+        , INTERP_ORDER_(INTERP_ORDER) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const int idw = g.get_group_id(1) / yo_.dims[2];
+        const int idz = g.get_group_id(1) - idw * yo_.dims[2];
+
+        const int idy        = g.get_group_id(0) / blocksMatX_;
+        const int blockIdx_x = g.get_group_id(0) - idy * blocksMatX_;
+        const int idx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+
+        if (idx >= yo_.dims[0] || idy >= yo_.dims[1] || idz >= yo_.dims[2] ||
+            idw >= yo_.dims[3])
+            return;
+
+        // FIXME: Only cubic interpolation is doing clamping
+        // We need to make it consistent across all methods
+        // Not changing the behavior because tests will fail
+        const bool doclamp = INTERP_ORDER_ == 3;
+
+        bool is_off[] = {xo_.dims[0] > 1, xo_.dims[1] > 1, xo_.dims[2] > 1,
+                         xo_.dims[3] > 1};
+
+        const int yo_idx = idw * yo_.strides[3] + idz * yo_.strides[2] +
+                           idy * yo_.strides[1] + idx + yo_.offset;
+
+        int xo_idx = idx * is_off[0] + xo_.offset;
+        if (batch_) {
+            xo_idx += idw * xo_.strides[3] * is_off[3];
+            xo_idx += idz * xo_.strides[2] * is_off[2];
+            xo_idx += idy * xo_.strides[1] * is_off[1];
+        }
+
+        const Tp x = (d_xo_[xo_idx] - xi_beg_) * xi_step_reproc_;
+
+#pragma unroll
+        for (int flagIdx = 0; flagIdx < 4; ++flagIdx) {
+            is_off[flagIdx] = true;
+        }
+        is_off[XDIM_] = false;
+
+        if (x < 0 || yi_.dims[XDIM_] < x + 1) {
+            d_yo_[yo_idx] = offGrid_;
+            return;
+        }
+
+        int yi_idx = idx * is_off[0] + yi_.offset;
+        yi_idx += idw * yi_.strides[3] * is_off[3];
+        yi_idx += idz * yi_.strides[2] * is_off[2];
+        yi_idx += idy * yi_.strides[1] * is_off[1];
+
+        if (INTERP_ORDER_ == 1)
+            interp1o1(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
+                      doclamp, 1);
+        if (INTERP_ORDER_ == 2)
+            interp1o2(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
+                      doclamp, 1);
+        if (INTERP_ORDER_ == 3)
+            interp1o3(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
+                      doclamp, 1);
+    }
+
+    void interp1o1(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        Ty zero = (Ty)0;
+
+        const int x_lim    = in.dims[XDIM_];
+        const int x_stride = in.strides[XDIM_];
+
+        int xid   = (method == AF_INTERP_LOWER ? floor(x) : round(x));
+        bool cond = xid >= 0 && xid < x_lim;
+        if (doclamp) xid = fmax(0, fmin(xid, x_lim));
+
+        const int idx = ioff + xid * x_stride;
+
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * in.strides[batch_dim];
+            d_out[ooff + n * out.strides[batch_dim]] =
+                (doclamp || cond) ? d_in[idx_n] : zero;
+        }
+    }
+
+    Ty linearInterpFunc(Ty val[2], Tp ratio) const {
+        return MULRC((1 - ratio), val[0]) + MULRC(ratio, val[1]);
+    }
+
+    Ty cubicInterpFunc(Ty val[4], Tp xratio, bool spline) const {
+        Ty a0, a1, a2, a3;
+        if (spline) {
+            a0 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)1.5, val[1]) +
+                 MULRC((Tp)-1.5, val[2]) + MULRC((Tp)0.5, val[3]);
+
+            a1 = MULRC((Tp)1.0, val[0]) + MULRC((Tp)-2.5, val[1]) +
+                 MULRC((Tp)2.0, val[2]) + MULRC((Tp)-0.5, val[3]);
+
+            a2 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)0.5, val[2]);
+
+            a3 = val[1];
+        } else {
+            a0 = val[3] - val[2] - val[0] + val[1];
+            a1 = val[0] - val[1] - a0;
+            a2 = val[2] - val[0];
+            a3 = val[1];
+        }
+
+        Tp xratio2 = xratio * xratio;
+        Tp xratio3 = xratio2 * xratio;
+
+        return MULCR(a0, xratio3) + MULCR(a1, xratio2) + MULCR(a2, xratio) + a3;
+    }
+
+    void interp1o2(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        const int grid_x = floor(x);    // nearest grid
+        const Tp off_x   = x - grid_x;  // fractional offset
+
+        const int x_lim    = in.dims[XDIM_];
+        const int x_stride = in.strides[XDIM_];
+        const int idx      = ioff + grid_x * x_stride;
+
+        Ty zero      = (Ty)0;
+        bool cond[2] = {true, grid_x + 1 < x_lim};
+        int offx[2]  = {0, cond[1] ? 1 : 0};
+        Tp ratio     = off_x;
+        if (method == AF_INTERP_LINEAR_COSINE) {
+            ratio = (1 - cos(ratio * (Tp)M_PI)) / 2;
+        }
+
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * in.strides[batch_dim];
+            Ty val[2] = {
+                (doclamp || cond[0]) ? d_in[idx_n + offx[0] * x_stride] : zero,
+                (doclamp || cond[1]) ? d_in[idx_n + offx[1] * x_stride] : zero};
+
+            d_out[ooff + n * out.strides[batch_dim]] =
+                linearInterpFunc(val, ratio);
+        }
+    }
+
+    void interp1o3(sycl::accessor<Ty> d_out, KParam out, int ooff,
+                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
+                   int method, int batch, bool doclamp, int batch_dim) const {
+        const int grid_x = floor(x);    // nearest grid
+        const Tp off_x   = x - grid_x;  // fractional offset
+
+        const int x_lim    = in.dims[XDIM_];
+        const int x_stride = in.strides[XDIM_];
+        const int idx      = ioff + grid_x * x_stride;
+
+        bool cond[4] = {grid_x - 1 >= 0, true, grid_x + 1 < x_lim,
+                        grid_x + 2 < x_lim};
+        int off[4]   = {cond[0] ? -1 : 0, 0, cond[2] ? 1 : 0,
+                      cond[3] ? 2 : (cond[2] ? 1 : 0)};
+
+        Ty zero = (Ty)0;
+
+        for (int n = 0; n < batch; n++) {
+            Ty val[4];
+            int idx_n = idx + n * in.strides[batch_dim];
+            for (int i = 0; i < 4; i++) {
+                val[i] = (doclamp || cond[i]) ? d_in[idx_n + off[i] * x_stride]
+                                              : zero;
+            }
+            bool spline = method == AF_INTERP_CUBIC_SPLINE;
+            d_out[ooff + n * out.strides[batch_dim]] =
+                cubicInterpFunc(val, off_x, spline);
+        }
+    }
+
+   private:
+    sycl::accessor<Ty> d_yo_;
+    const KParam yo_;
+    sycl::accessor<Ty> d_yi_;
+    const KParam yi_;
+    sycl::accessor<Tp> d_xo_;
+    const KParam xo_;
+    const Tp xi_beg_;
+    const Tp xi_step_reproc_;
+    const Ty offGrid_;
+    const int blocksMatX_;
+    const int batch_;
+    const int method_;
+    const int XDIM_;
+    const int INTERP_ORDER_;
+};
+
+template<typename Ty, typename Tp>
+void approx1(Param<Ty> yo, const Param<Ty> yi, const Param<Tp> xo,
+             const int xdim, const Tp xi_beg, const Tp xi_step,
+             const float offGrid, const af_interp_type method,
+             const int order) {
+    constexpr int THREADS = 256;
+
+    auto local         = sycl::range{THREADS, 1};
+    dim_t blocksPerMat = divup(yo.info.dims[0], local[0]);
+    auto global        = sycl::range{blocksPerMat * local[0] * yo.info.dims[1],
+                              yo.info.dims[2] * yo.info.dims[3] * local[1]};
+
+    // Passing bools to opencl kernels is not allowed
+    bool batch =
+        !(xo.info.dims[1] == 1 && xo.info.dims[2] == 1 && xo.info.dims[3] == 1);
+
+    getQueue().submit([&](sycl::handler &h) {
+        auto yoAcc = yo.data->get_access(h);
+        auto yiAcc = yi.data->get_access(h);
+        auto xoAcc = xo.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            approx1Kernel<Ty, Tp>(yoAcc, yo.info, yiAcc, yi.info, xoAcc,
+                                  xo.info, xi_beg, Tp(1) / xi_step, (Ty)offGrid,
+                                  (int)blocksPerMat, (int)batch, (int)method,
+                                  xdim, order));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/approx.hpp b/src/backend/oneapi/kernel/approx2.hpp
old mode 100755
new mode 100644
similarity index 59%
rename from src/backend/oneapi/kernel/approx.hpp
rename to src/backend/oneapi/kernel/approx2.hpp
index 15033317dd..5df6111d6a
--- a/src/backend/oneapi/kernel/approx.hpp
+++ b/src/backend/oneapi/kernel/approx2.hpp
@@ -18,6 +18,9 @@
 #include <string>
 #include <vector>
 
+#define MULRC(a, b) (a) * (b)
+#define MULCR(a, b) (a) * (b)
+
 namespace oneapi {
 namespace kernel {
 
@@ -30,264 +33,6 @@ using local_accessor =
     sycl::accessor<T, dimensions, sycl::access::mode::read_write,
                    sycl::access::target::local>;
 
-template<typename Ty, typename Tp>
-class approx1Kernel {
-   public:
-    approx1Kernel(sycl::accessor<Ty> d_yo, const KParam yo,
-                  sycl::accessor<Ty> d_yi, const KParam yi,
-                  sycl::accessor<Tp> d_xo, const KParam xo, const Tp xi_beg,
-                  const Tp xi_step_reproc, const Ty offGrid,
-                  const int blocksMatX, const int batch, const int method,
-                  const int XDIM, const int INTERP_ORDER)
-        : d_yo_(d_yo)
-        , yo_(yo)
-        , d_yi_(d_yi)
-        , yi_(yi)
-        , d_xo_(d_xo)
-        , xo_(xo)
-        , xi_beg_(xi_beg)
-        , xi_step_reproc_(xi_step_reproc)
-        , offGrid_(offGrid)
-        , blocksMatX_(blocksMatX)
-        , batch_(batch)
-        , method_(method)
-        , XDIM_(XDIM)
-        , INTERP_ORDER_(INTERP_ORDER) {}
-    void operator()(sycl::nd_item<2> it) const {
-        sycl::group g = it.get_group();
-        const int idw = g.get_group_id(1) / yo_.dims[2];
-        const int idz = g.get_group_id(1) - idw * yo_.dims[2];
-
-        const int idy        = g.get_group_id(0) / blocksMatX_;
-        const int blockIdx_x = g.get_group_id(0) - idy * blocksMatX_;
-        const int idx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
-
-        if (idx >= yo_.dims[0] || idy >= yo_.dims[1] || idz >= yo_.dims[2] ||
-            idw >= yo_.dims[3])
-            return;
-
-        // FIXME: Only cubic interpolation is doing clamping
-        // We need to make it consistent across all methods
-        // Not changing the behavior because tests will fail
-        const bool doclamp = INTERP_ORDER_ == 3;
-
-        bool is_off[] = {xo_.dims[0] > 1, xo_.dims[1] > 1, xo_.dims[2] > 1,
-                         xo_.dims[3] > 1};
-
-        const int yo_idx = idw * yo_.strides[3] + idz * yo_.strides[2] +
-                           idy * yo_.strides[1] + idx + yo_.offset;
-
-        int xo_idx = idx * is_off[0] + xo_.offset;
-        if (batch_) {
-            xo_idx += idw * xo_.strides[3] * is_off[3];
-            xo_idx += idz * xo_.strides[2] * is_off[2];
-            xo_idx += idy * xo_.strides[1] * is_off[1];
-        }
-
-        const Tp x = (d_xo_[xo_idx] - xi_beg_) * xi_step_reproc_;
-
-#pragma unroll
-        for (int flagIdx = 0; flagIdx < 4; ++flagIdx) {
-            is_off[flagIdx] = true;
-        }
-        is_off[XDIM_] = false;
-
-        if (x < 0 || yi_.dims[XDIM_] < x + 1) {
-            d_yo_[yo_idx] = offGrid_;
-            return;
-        }
-
-        int yi_idx = idx * is_off[0] + yi_.offset;
-        yi_idx += idw * yi_.strides[3] * is_off[3];
-        yi_idx += idz * yi_.strides[2] * is_off[2];
-        yi_idx += idy * yi_.strides[1] * is_off[1];
-
-        if (INTERP_ORDER_ == 1)
-            interp1o1(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
-                      doclamp, 1);
-        if (INTERP_ORDER_ == 2)
-            interp1o2(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
-                      doclamp, 1);
-        if (INTERP_ORDER_ == 3)
-            interp1o3(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
-                      doclamp, 1);
-    }
-
-    void interp1o1(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        Ty zero = (Ty)0;
-
-        const int x_lim    = in.dims[XDIM_];
-        const int x_stride = in.strides[XDIM_];
-
-        int xid   = (method == AF_INTERP_LOWER ? floor(x) : round(x));
-        bool cond = xid >= 0 && xid < x_lim;
-        if (doclamp) xid = fmax(0, fmin(xid, x_lim));
-
-        const int idx = ioff + xid * x_stride;
-
-        for (int n = 0; n < batch; n++) {
-            int idx_n = idx + n * in.strides[batch_dim];
-            d_out[ooff + n * out.strides[batch_dim]] =
-                (doclamp || cond) ? d_in[idx_n] : zero;
-        }
-    }
-
-#if IS_CPLX
-#if USE_DOUBLE
-    typedef double ScalarTy;
-#else
-    typedef float ScalarTy;
-#endif
-    Ty __mulrc(ScalarTy s, Ty v) {
-        InterpInTy out = {s * v.x, s * v.y};
-        return out;
-    }
-#define MULRC(a, b) __mulrc(a, b)
-#define MULCR(a, b) __mulrc(b, a)
-#else
-#define MULRC(a, b) (a) * (b)
-#define MULCR(a, b) (a) * (b)
-#endif
-
-    Ty linearInterpFunc(Ty val[2], Tp ratio) const {
-        return MULRC((1 - ratio), val[0]) + MULRC(ratio, val[1]);
-    }
-
-    Ty cubicInterpFunc(Ty val[4], Tp xratio, bool spline) const {
-        Ty a0, a1, a2, a3;
-        if (spline) {
-            a0 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)1.5, val[1]) +
-                 MULRC((Tp)-1.5, val[2]) + MULRC((Tp)0.5, val[3]);
-
-            a1 = MULRC((Tp)1.0, val[0]) + MULRC((Tp)-2.5, val[1]) +
-                 MULRC((Tp)2.0, val[2]) + MULRC((Tp)-0.5, val[3]);
-
-            a2 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)0.5, val[2]);
-
-            a3 = val[1];
-        } else {
-            a0 = val[3] - val[2] - val[0] + val[1];
-            a1 = val[0] - val[1] - a0;
-            a2 = val[2] - val[0];
-            a3 = val[1];
-        }
-
-        Tp xratio2 = xratio * xratio;
-        Tp xratio3 = xratio2 * xratio;
-
-        return MULCR(a0, xratio3) + MULCR(a1, xratio2) + MULCR(a2, xratio) + a3;
-    }
-
-    void interp1o2(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        const int grid_x = floor(x);    // nearest grid
-        const Tp off_x   = x - grid_x;  // fractional offset
-
-        const int x_lim    = in.dims[XDIM_];
-        const int x_stride = in.strides[XDIM_];
-        const int idx      = ioff + grid_x * x_stride;
-
-        Ty zero      = (Ty)0;
-        bool cond[2] = {true, grid_x + 1 < x_lim};
-        int offx[2]  = {0, cond[1] ? 1 : 0};
-        Tp ratio     = off_x;
-        if (method == AF_INTERP_LINEAR_COSINE) {
-            ratio = (1 - cos(ratio * (Tp)M_PI)) / 2;
-        }
-
-        for (int n = 0; n < batch; n++) {
-            int idx_n = idx + n * in.strides[batch_dim];
-            Ty val[2] = {
-                (doclamp || cond[0]) ? d_in[idx_n + offx[0] * x_stride] : zero,
-                (doclamp || cond[1]) ? d_in[idx_n + offx[1] * x_stride] : zero};
-
-            d_out[ooff + n * out.strides[batch_dim]] =
-                linearInterpFunc(val, ratio);
-        }
-    }
-
-    void interp1o3(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        const int grid_x = floor(x);    // nearest grid
-        const Tp off_x   = x - grid_x;  // fractional offset
-
-        const int x_lim    = in.dims[XDIM_];
-        const int x_stride = in.strides[XDIM_];
-        const int idx      = ioff + grid_x * x_stride;
-
-        bool cond[4] = {grid_x - 1 >= 0, true, grid_x + 1 < x_lim,
-                        grid_x + 2 < x_lim};
-        int off[4]   = {cond[0] ? -1 : 0, 0, cond[2] ? 1 : 0,
-                      cond[3] ? 2 : (cond[2] ? 1 : 0)};
-
-        Ty zero = (Ty)0;
-
-        for (int n = 0; n < batch; n++) {
-            Ty val[4];
-            int idx_n = idx + n * in.strides[batch_dim];
-            for (int i = 0; i < 4; i++) {
-                val[i] = (doclamp || cond[i]) ? d_in[idx_n + off[i] * x_stride]
-                                              : zero;
-            }
-            bool spline = method == AF_INTERP_CUBIC_SPLINE;
-            d_out[ooff + n * out.strides[batch_dim]] =
-                cubicInterpFunc(val, off_x, spline);
-        }
-    }
-
-   private:
-    sycl::accessor<Ty> d_yo_;
-    const KParam yo_;
-    sycl::accessor<Ty> d_yi_;
-    const KParam yi_;
-    sycl::accessor<Tp> d_xo_;
-    const KParam xo_;
-    const Tp xi_beg_;
-    const Tp xi_step_reproc_;
-    const Ty offGrid_;
-    const int blocksMatX_;
-    const int batch_;
-    const int method_;
-    const int XDIM_;
-    const int INTERP_ORDER_;
-};
-
-template<typename Ty, typename Tp>
-void approx1(Param<Ty> yo, const Param<Ty> yi, const Param<Tp> xo,
-             const int xdim, const Tp xi_beg, const Tp xi_step,
-             const float offGrid, const af_interp_type method,
-             const int order) {
-    constexpr int THREADS = 256;
-
-    auto local         = sycl::range{THREADS, 1};
-    dim_t blocksPerMat = divup(yo.info.dims[0], local[0]);
-    auto global        = sycl::range{blocksPerMat * local[0] * yo.info.dims[1],
-                              yo.info.dims[2] * yo.info.dims[3] * local[1]};
-
-    // Passing bools to opencl kernels is not allowed
-    bool batch =
-        !(xo.info.dims[1] == 1 && xo.info.dims[2] == 1 && xo.info.dims[3] == 1);
-
-    getQueue().submit([&](sycl::handler &h) {
-        auto yoAcc = yo.data->get_access(h);
-        auto yiAcc = yi.data->get_access(h);
-        auto xoAcc = xo.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
-
-        h.parallel_for(
-            sycl::nd_range{global, local},
-            approx1Kernel<Ty, Tp>(yoAcc, yo.info, yiAcc, yi.info, xoAcc,
-                                  xo.info, xi_beg, Tp(1) / xi_step, (Ty)offGrid,
-                                  (int)blocksPerMat, (int)batch, (int)method,
-                                  xdim, order));
-    });
-    ONEAPI_DEBUG_FINISH(getQueue());
-}
-
 template<typename Ty, typename Tp>
 class approx2Kernel {
    public:

From af3ef613ec43bbc66bc6f66283746bb789ac7450 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 7 Nov 2022 21:20:33 -0500
Subject: [PATCH 127/473] extract interp functor from approx1/approx2 for reuse
 in other kernels

---
 src/backend/oneapi/CMakeLists.txt     |   1 +
 src/backend/oneapi/approx.cpp         |  12 +-
 src/backend/oneapi/approx1.cpp        |  12 +-
 src/backend/oneapi/approx2.cpp        |  12 +-
 src/backend/oneapi/kernel/approx1.hpp | 241 +++++-------------
 src/backend/oneapi/kernel/approx2.hpp | 298 +++++-----------------
 src/backend/oneapi/kernel/interp.hpp  | 342 ++++++++++++++++++++++++++
 7 files changed, 489 insertions(+), 429 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/interp.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index e70af234c3..d6bfaae598 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -211,6 +211,7 @@ target_sources(afoneapi
     kernel/assign.hpp
     kernel/diagonal.hpp
     kernel/diff.hpp
+    kernel/interp.hpp
     kernel/iota.hpp
     kernel/histogram.hpp
     kernel/memcopy.hpp
diff --git a/src/backend/oneapi/approx.cpp b/src/backend/oneapi/approx.cpp
index da153301d2..4ad0c27d9b 100644
--- a/src/backend/oneapi/approx.cpp
+++ b/src/backend/oneapi/approx.cpp
@@ -20,18 +20,18 @@ void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    method, 1);
+            kernel::approx1<Ty, Tp, 1>(yo, yi, xo, xdim, xi_beg, xi_step,
+                                       offGrid, method);
             break;
         case AF_INTERP_LINEAR:
         case AF_INTERP_LINEAR_COSINE:
-            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    method, 2);
+            kernel::approx1<Ty, Tp, 2>(yo, yi, xo, xdim, xi_beg, xi_step,
+                                       offGrid, method);
             break;
         case AF_INTERP_CUBIC:
         case AF_INTERP_CUBIC_SPLINE:
-            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    method, 3);
+            kernel::approx1<Ty, Tp, 3>(yo, yi, xo, xdim, xi_beg, xi_step,
+                                       offGrid, method);
             break;
         default: break;
     }
diff --git a/src/backend/oneapi/approx1.cpp b/src/backend/oneapi/approx1.cpp
index cee2aa9b15..8906f57016 100644
--- a/src/backend/oneapi/approx1.cpp
+++ b/src/backend/oneapi/approx1.cpp
@@ -18,18 +18,18 @@ void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    method, 1);
+            kernel::approx1<Ty, Tp, 1>(yo, yi, xo, xdim, xi_beg, xi_step,
+                                       offGrid, method);
             break;
         case AF_INTERP_LINEAR:
         case AF_INTERP_LINEAR_COSINE:
-            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    method, 2);
+            kernel::approx1<Ty, Tp, 2>(yo, yi, xo, xdim, xi_beg, xi_step,
+                                       offGrid, method);
             break;
         case AF_INTERP_CUBIC:
         case AF_INTERP_CUBIC_SPLINE:
-            kernel::approx1<Ty, Tp>(yo, yi, xo, xdim, xi_beg, xi_step, offGrid,
-                                    method, 3);
+            kernel::approx1<Ty, Tp, 3>(yo, yi, xo, xdim, xi_beg, xi_step,
+                                       offGrid, method);
             break;
         default: break;
     }
diff --git a/src/backend/oneapi/approx2.cpp b/src/backend/oneapi/approx2.cpp
index e22d5406ee..3330aaa42f 100644
--- a/src/backend/oneapi/approx2.cpp
+++ b/src/backend/oneapi/approx2.cpp
@@ -20,22 +20,22 @@ void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
-                                    yi_beg, yi_step, offGrid, method, 1);
+            kernel::approx2<Ty, Tp, 1>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+                                       ydim, yi_beg, yi_step, offGrid, method);
             break;
         case AF_INTERP_LINEAR:
         case AF_INTERP_BILINEAR:
         case AF_INTERP_LINEAR_COSINE:
         case AF_INTERP_BILINEAR_COSINE:
-            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
-                                    yi_beg, yi_step, offGrid, method, 2);
+            kernel::approx2<Ty, Tp, 2>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+                                       ydim, yi_beg, yi_step, offGrid, method);
             break;
         case AF_INTERP_CUBIC:
         case AF_INTERP_BICUBIC:
         case AF_INTERP_CUBIC_SPLINE:
         case AF_INTERP_BICUBIC_SPLINE:
-            kernel::approx2<Ty, Tp>(zo, zi, xo, xdim, xi_beg, xi_step, yo, ydim,
-                                    yi_beg, yi_step, offGrid, method, 3);
+            kernel::approx2<Ty, Tp, 3>(zo, zi, xo, xdim, xi_beg, xi_step, yo,
+                                       ydim, yi_beg, yi_step, offGrid, method);
             break;
         default: break;
     }
diff --git a/src/backend/oneapi/kernel/approx1.hpp b/src/backend/oneapi/kernel/approx1.hpp
index ad2ac257ea..95b4ceb65c 100644
--- a/src/backend/oneapi/kernel/approx1.hpp
+++ b/src/backend/oneapi/kernel/approx1.hpp
@@ -13,14 +13,14 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/interp.hpp>
+#include <af/constants.h>
+// #include <kernel/default_config.hpp>
 #include <traits.hpp>
 
 #include <string>
 #include <vector>
 
-#define MULRC(a, b) (a) * (b)
-#define MULCR(a, b) (a) * (b)
-
 namespace oneapi {
 namespace kernel {
 
@@ -33,58 +33,64 @@ using local_accessor =
     sycl::accessor<T, dimensions, sycl::access::mode::read_write,
                    sycl::access::target::local>;
 
-template<typename Ty, typename Tp>
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename Ty, typename Tp, int order>
 class approx1Kernel {
    public:
-    approx1Kernel(sycl::accessor<Ty> d_yo, const KParam yo,
-                  sycl::accessor<Ty> d_yi, const KParam yi,
-                  sycl::accessor<Tp> d_xo, const KParam xo, const Tp xi_beg,
+    approx1Kernel(write_accessor<Ty> d_yo, const KParam yoInfo,
+                  read_accessor<Ty> d_yi, const KParam yiInfo,
+                  read_accessor<Tp> d_xo, const KParam xoInfo, const Tp xi_beg,
                   const Tp xi_step_reproc, const Ty offGrid,
-                  const int blocksMatX, const int batch, const int method,
-                  const int XDIM, const int INTERP_ORDER)
+                  const int blocksMatX, const af_interp_type method,
+                  const bool batch, const int XDIM)
         : d_yo_(d_yo)
-        , yo_(yo)
+        , yoInfo_(yoInfo)
         , d_yi_(d_yi)
-        , yi_(yi)
+        , yiInfo_(yiInfo)
         , d_xo_(d_xo)
-        , xo_(xo)
+        , xoInfo_(xoInfo)
         , xi_beg_(xi_beg)
         , xi_step_reproc_(xi_step_reproc)
         , offGrid_(offGrid)
         , blocksMatX_(blocksMatX)
-        , batch_(batch)
         , method_(method)
-        , XDIM_(XDIM)
-        , INTERP_ORDER_(INTERP_ORDER) {}
+        , batch_(batch)
+        , XDIM_(XDIM) {}
+
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g = it.get_group();
-        const int idw = g.get_group_id(1) / yo_.dims[2];
-        const int idz = g.get_group_id(1) - idw * yo_.dims[2];
+        const int idw = g.get_group_id(1) / yoInfo_.dims[2];
+        const int idz = g.get_group_id(1) - idw * yoInfo_.dims[2];
 
         const int idy        = g.get_group_id(0) / blocksMatX_;
         const int blockIdx_x = g.get_group_id(0) - idy * blocksMatX_;
         const int idx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
 
-        if (idx >= yo_.dims[0] || idy >= yo_.dims[1] || idz >= yo_.dims[2] ||
-            idw >= yo_.dims[3])
+        if (idx >= yoInfo_.dims[0] || idy >= yoInfo_.dims[1] ||
+            idz >= yoInfo_.dims[2] || idw >= yoInfo_.dims[3])
             return;
 
         // FIXME: Only cubic interpolation is doing clamping
         // We need to make it consistent across all methods
         // Not changing the behavior because tests will fail
-        const bool doclamp = INTERP_ORDER_ == 3;
+        const bool doclamp = order == 3;
 
-        bool is_off[] = {xo_.dims[0] > 1, xo_.dims[1] > 1, xo_.dims[2] > 1,
-                         xo_.dims[3] > 1};
+        bool is_off[] = {xoInfo_.dims[0] > 1, xoInfo_.dims[1] > 1,
+                         xoInfo_.dims[2] > 1, xoInfo_.dims[3] > 1};
 
-        const int yo_idx = idw * yo_.strides[3] + idz * yo_.strides[2] +
-                           idy * yo_.strides[1] + idx + yo_.offset;
+        const int yo_idx = idw * yoInfo_.strides[3] + idz * yoInfo_.strides[2] +
+                           idy * yoInfo_.strides[1] + idx + yoInfo_.offset;
 
-        int xo_idx = idx * is_off[0] + xo_.offset;
+        int xo_idx = idx * is_off[0] + xoInfo_.offset;
         if (batch_) {
-            xo_idx += idw * xo_.strides[3] * is_off[3];
-            xo_idx += idz * xo_.strides[2] * is_off[2];
-            xo_idx += idy * xo_.strides[1] * is_off[1];
+            xo_idx += idw * xoInfo_.strides[3] * is_off[3];
+            xo_idx += idz * xoInfo_.strides[2] * is_off[2];
+            xo_idx += idy * xoInfo_.strides[1] * is_off[1];
         }
 
         const Tp x = (d_xo_[xo_idx] - xi_beg_) * xi_step_reproc_;
@@ -95,181 +101,62 @@ class approx1Kernel {
         }
         is_off[XDIM_] = false;
 
-        if (x < 0 || yi_.dims[XDIM_] < x + 1) {
+        if (x < 0 || yiInfo_.dims[XDIM_] < x + 1) {
             d_yo_[yo_idx] = offGrid_;
             return;
         }
 
-        int yi_idx = idx * is_off[0] + yi_.offset;
-        yi_idx += idw * yi_.strides[3] * is_off[3];
-        yi_idx += idz * yi_.strides[2] * is_off[2];
-        yi_idx += idy * yi_.strides[1] * is_off[1];
-
-        if (INTERP_ORDER_ == 1)
-            interp1o1(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
-                      doclamp, 1);
-        if (INTERP_ORDER_ == 2)
-            interp1o2(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
-                      doclamp, 1);
-        if (INTERP_ORDER_ == 3)
-            interp1o3(d_yo_, yo_, yo_idx, d_yi_, yi_, yi_idx, x, method_, 1,
-                      doclamp, 1);
-    }
-
-    void interp1o1(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        Ty zero = (Ty)0;
-
-        const int x_lim    = in.dims[XDIM_];
-        const int x_stride = in.strides[XDIM_];
-
-        int xid   = (method == AF_INTERP_LOWER ? floor(x) : round(x));
-        bool cond = xid >= 0 && xid < x_lim;
-        if (doclamp) xid = fmax(0, fmin(xid, x_lim));
-
-        const int idx = ioff + xid * x_stride;
-
-        for (int n = 0; n < batch; n++) {
-            int idx_n = idx + n * in.strides[batch_dim];
-            d_out[ooff + n * out.strides[batch_dim]] =
-                (doclamp || cond) ? d_in[idx_n] : zero;
-        }
-    }
-
-    Ty linearInterpFunc(Ty val[2], Tp ratio) const {
-        return MULRC((1 - ratio), val[0]) + MULRC(ratio, val[1]);
-    }
-
-    Ty cubicInterpFunc(Ty val[4], Tp xratio, bool spline) const {
-        Ty a0, a1, a2, a3;
-        if (spline) {
-            a0 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)1.5, val[1]) +
-                 MULRC((Tp)-1.5, val[2]) + MULRC((Tp)0.5, val[3]);
-
-            a1 = MULRC((Tp)1.0, val[0]) + MULRC((Tp)-2.5, val[1]) +
-                 MULRC((Tp)2.0, val[2]) + MULRC((Tp)-0.5, val[3]);
-
-            a2 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)0.5, val[2]);
-
-            a3 = val[1];
-        } else {
-            a0 = val[3] - val[2] - val[0] + val[1];
-            a1 = val[0] - val[1] - a0;
-            a2 = val[2] - val[0];
-            a3 = val[1];
-        }
-
-        Tp xratio2 = xratio * xratio;
-        Tp xratio3 = xratio2 * xratio;
-
-        return MULCR(a0, xratio3) + MULCR(a1, xratio2) + MULCR(a2, xratio) + a3;
-    }
-
-    void interp1o2(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        const int grid_x = floor(x);    // nearest grid
-        const Tp off_x   = x - grid_x;  // fractional offset
-
-        const int x_lim    = in.dims[XDIM_];
-        const int x_stride = in.strides[XDIM_];
-        const int idx      = ioff + grid_x * x_stride;
-
-        Ty zero      = (Ty)0;
-        bool cond[2] = {true, grid_x + 1 < x_lim};
-        int offx[2]  = {0, cond[1] ? 1 : 0};
-        Tp ratio     = off_x;
-        if (method == AF_INTERP_LINEAR_COSINE) {
-            ratio = (1 - cos(ratio * (Tp)M_PI)) / 2;
-        }
+        int yi_idx = idx * is_off[0] + yiInfo_.offset;
+        yi_idx += idw * yiInfo_.strides[3] * is_off[3];
+        yi_idx += idz * yiInfo_.strides[2] * is_off[2];
+        yi_idx += idy * yiInfo_.strides[1] * is_off[1];
 
-        for (int n = 0; n < batch; n++) {
-            int idx_n = idx + n * in.strides[batch_dim];
-            Ty val[2] = {
-                (doclamp || cond[0]) ? d_in[idx_n + offx[0] * x_stride] : zero,
-                (doclamp || cond[1]) ? d_in[idx_n + offx[1] * x_stride] : zero};
-
-            d_out[ooff + n * out.strides[batch_dim]] =
-                linearInterpFunc(val, ratio);
-        }
-    }
-
-    void interp1o3(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        const int grid_x = floor(x);    // nearest grid
-        const Tp off_x   = x - grid_x;  // fractional offset
-
-        const int x_lim    = in.dims[XDIM_];
-        const int x_stride = in.strides[XDIM_];
-        const int idx      = ioff + grid_x * x_stride;
-
-        bool cond[4] = {grid_x - 1 >= 0, true, grid_x + 1 < x_lim,
-                        grid_x + 2 < x_lim};
-        int off[4]   = {cond[0] ? -1 : 0, 0, cond[2] ? 1 : 0,
-                      cond[3] ? 2 : (cond[2] ? 1 : 0)};
-
-        Ty zero = (Ty)0;
-
-        for (int n = 0; n < batch; n++) {
-            Ty val[4];
-            int idx_n = idx + n * in.strides[batch_dim];
-            for (int i = 0; i < 4; i++) {
-                val[i] = (doclamp || cond[i]) ? d_in[idx_n + off[i] * x_stride]
-                                              : zero;
-            }
-            bool spline = method == AF_INTERP_CUBIC_SPLINE;
-            d_out[ooff + n * out.strides[batch_dim]] =
-                cubicInterpFunc(val, off_x, spline);
-        }
+        Interp1<Ty, Tp, order> interp;
+        interp(d_yo_, yoInfo_, yo_idx, d_yi_, yiInfo_, yi_idx, x, XDIM_,
+               method_, 1, doclamp);
     }
 
-   private:
-    sycl::accessor<Ty> d_yo_;
-    const KParam yo_;
-    sycl::accessor<Ty> d_yi_;
-    const KParam yi_;
-    sycl::accessor<Tp> d_xo_;
-    const KParam xo_;
+   protected:
+    write_accessor<Ty> d_yo_;
+    const KParam yoInfo_;
+    read_accessor<Ty> d_yi_;
+    const KParam yiInfo_;
+    read_accessor<Tp> d_xo_;
+    const KParam xoInfo_;
     const Tp xi_beg_;
     const Tp xi_step_reproc_;
     const Ty offGrid_;
     const int blocksMatX_;
-    const int batch_;
-    const int method_;
+    const af_interp_type method_;
+    const bool batch_;
     const int XDIM_;
-    const int INTERP_ORDER_;
 };
 
-template<typename Ty, typename Tp>
+template<typename Ty, typename Tp, int order>
 void approx1(Param<Ty> yo, const Param<Ty> yi, const Param<Tp> xo,
              const int xdim, const Tp xi_beg, const Tp xi_step,
-             const float offGrid, const af_interp_type method,
-             const int order) {
+             const float offGrid, const af_interp_type method) {
     constexpr int THREADS = 256;
 
-    auto local         = sycl::range{THREADS, 1};
-    dim_t blocksPerMat = divup(yo.info.dims[0], local[0]);
-    auto global        = sycl::range{blocksPerMat * local[0] * yo.info.dims[1],
+    auto local        = sycl::range{THREADS, 1};
+    uint blocksPerMat = divup(yo.info.dims[0], local[0]);
+    auto global       = sycl::range{blocksPerMat * local[0] * yo.info.dims[1],
                               yo.info.dims[2] * yo.info.dims[3] * local[1]};
 
-    // Passing bools to opencl kernels is not allowed
     bool batch =
         !(xo.info.dims[1] == 1 && xo.info.dims[2] == 1 && xo.info.dims[3] == 1);
 
     getQueue().submit([&](sycl::handler &h) {
-        auto yoAcc = yo.data->get_access(h);
-        auto yiAcc = yi.data->get_access(h);
-        auto xoAcc = xo.data->get_access(h);
+        write_accessor<Ty> yoAcc{*yo.data, h};
+        read_accessor<Ty> yiAcc{*yi.data, h};
+        read_accessor<Tp> xoAcc{*xo.data, h};
         sycl::stream debugStream(128, 128, h);
 
-        h.parallel_for(
-            sycl::nd_range{global, local},
-            approx1Kernel<Ty, Tp>(yoAcc, yo.info, yiAcc, yi.info, xoAcc,
-                                  xo.info, xi_beg, Tp(1) / xi_step, (Ty)offGrid,
-                                  (int)blocksPerMat, (int)batch, (int)method,
-                                  xdim, order));
+        h.parallel_for(sycl::nd_range{global, local},
+                       approx1Kernel<Ty, Tp, order>(
+                           yoAcc, yo.info, yiAcc, yi.info, xoAcc, xo.info,
+                           xi_beg, Tp(1) / xi_step, (Ty)offGrid,
+                           (uint)blocksPerMat, method, batch, xdim));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/approx2.hpp b/src/backend/oneapi/kernel/approx2.hpp
index 5df6111d6a..94b2f7060c 100644
--- a/src/backend/oneapi/kernel/approx2.hpp
+++ b/src/backend/oneapi/kernel/approx2.hpp
@@ -13,14 +13,14 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/interp.hpp>
+#include <af/constants.h>
+// #include <kernel/default_config.hpp>
 #include <traits.hpp>
 
 #include <string>
 #include <vector>
 
-#define MULRC(a, b) (a) * (b)
-#define MULCR(a, b) (a) * (b)
-
 namespace oneapi {
 namespace kernel {
 
@@ -33,26 +33,31 @@ using local_accessor =
     sycl::accessor<T, dimensions, sycl::access::mode::read_write,
                    sycl::access::target::local>;
 
-template<typename Ty, typename Tp>
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename Ty, typename Tp, int order>
 class approx2Kernel {
    public:
-    approx2Kernel(sycl::accessor<Ty> d_zo, const KParam zo,
-                  sycl::accessor<Ty> d_zi, const KParam zi,
-                  sycl::accessor<Tp> d_xo, const KParam xo,
-                  sycl::accessor<Tp> d_yo, const KParam yo, const Tp xi_beg,
+    approx2Kernel(write_accessor<Ty> d_zo, const KParam zo,
+                  read_accessor<Ty> d_zi, const KParam zi,
+                  read_accessor<Tp> d_xo, const KParam xo,
+                  read_accessor<Tp> d_yo, const KParam yo, const Tp xi_beg,
                   const Tp xi_step_reproc, const Tp yi_beg,
                   const Tp yi_step_reproc, const Ty offGrid,
-                  const int blocksMatX, const int blocksMatY, const int batch,
-                  int method, const int XDIM, const int YDIM,
-                  const int INTERP_ORDER)
+                  const int blocksMatX, const int blocksMatY, const bool batch,
+                  const af_interp_type method, const int XDIM, const int YDIM)
         : d_zo_(d_zo)
-        , zo_(zo)
+        , zoInfo_(zo)
         , d_zi_(d_zi)
-        , zi_(zi)
+        , ziInfo_(zi)
         , d_xo_(d_xo)
-        , xo_(xo)
+        , xoInfo_(xo)
         , d_yo_(d_yo)
-        , yo_(yo)
+        , yoInfo_(yo)
         , xi_beg_(xi_beg)
         , xi_step_reproc_(xi_step_reproc)
         , yi_beg_(yi_beg)
@@ -63,8 +68,8 @@ class approx2Kernel {
         , batch_(batch)
         , method_(method)
         , XDIM_(XDIM)
-        , YDIM_(YDIM)
-        , INTERP_ORDER_(INTERP_ORDER) {}
+        , YDIM_(YDIM) {}
+
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g = it.get_group();
         const int idz = g.get_group_id(0) / blocksMatX_;
@@ -76,30 +81,30 @@ class approx2Kernel {
         const int idx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
         const int idy = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
 
-        if (idx >= zo_.dims[0] || idy >= zo_.dims[1] || idz >= zo_.dims[2] ||
-            idw >= zo_.dims[3])
+        if (idx >= zoInfo_.dims[0] || idy >= zoInfo_.dims[1] ||
+            idz >= zoInfo_.dims[2] || idw >= zoInfo_.dims[3])
             return;
 
         // FIXME: Only cubic interpolation is doing clamping
         // We need to make it consistent across all methods
         // Not changing the behavior because tests will fail
-        const bool doclamp = INTERP_ORDER_ == 3;
+        const bool doclamp = order == 3;
 
-        bool is_off[] = {xo_.dims[0] > 1, xo_.dims[1] > 1, xo_.dims[2] > 1,
-                         xo_.dims[3] > 1};
+        bool is_off[] = {xoInfo_.dims[0] > 1, xoInfo_.dims[1] > 1,
+                         xoInfo_.dims[2] > 1, xoInfo_.dims[3] > 1};
 
-        const int zo_idx = idw * zo_.strides[3] + idz * zo_.strides[2] +
-                           idy * zo_.strides[1] + idx + zo_.offset;
-        int xo_idx =
-            idy * xo_.strides[1] * is_off[1] + idx * is_off[0] + xo_.offset;
+        const int zo_idx = idw * zoInfo_.strides[3] + idz * zoInfo_.strides[2] +
+                           idy * zoInfo_.strides[1] + idx + zoInfo_.offset;
+        int xo_idx = idy * xoInfo_.strides[1] * is_off[1] + idx * is_off[0] +
+                     xoInfo_.offset;
 
-        int yo_idx =
-            idy * yo_.strides[1] * is_off[1] + idx * is_off[0] + yo_.offset;
+        int yo_idx = idy * yoInfo_.strides[1] * is_off[1] + idx * is_off[0] +
+                     yoInfo_.offset;
         if (batch_) {
-            xo_idx += idw * xo_.strides[3] * is_off[3] +
-                      idz * xo_.strides[2] * is_off[2];
-            yo_idx += idw * yo_.strides[3] * is_off[3] +
-                      idz * yo_.strides[2] * is_off[2];
+            xo_idx += idw * xoInfo_.strides[3] * is_off[3] +
+                      idz * xoInfo_.strides[2] * is_off[2];
+            yo_idx += idw * yoInfo_.strides[3] * is_off[3] +
+                      idz * yoInfo_.strides[2] * is_off[2];
         }
 
 #pragma unroll
@@ -112,203 +117,31 @@ class approx2Kernel {
         const Tp x = (d_xo_[xo_idx] - xi_beg_) * xi_step_reproc_;
         const Tp y = (d_yo_[yo_idx] - yi_beg_) * yi_step_reproc_;
 
-        if (x < 0 || y < 0 || zi_.dims[XDIM_] < x + 1 ||
-            zi_.dims[YDIM_] < y + 1) {
+        if (x < 0 || y < 0 || ziInfo_.dims[XDIM_] < x + 1 ||
+            ziInfo_.dims[YDIM_] < y + 1) {
             d_zo_[zo_idx] = offGrid_;
             return;
         }
 
-        int zi_idx =
-            idy * zi_.strides[1] * is_off[1] + idx * is_off[0] + zi_.offset;
-        zi_idx +=
-            idw * zi_.strides[3] * is_off[3] + idz * zi_.strides[2] * is_off[2];
-
-        if (INTERP_ORDER_ == 1)
-            interp2o1(d_zo_, zo_, zo_idx, d_zi_, zi_, zi_idx, x, y, method_, 1,
-                      doclamp, 2);
-        if (INTERP_ORDER_ == 2)
-            interp2o2(d_zo_, zo_, zo_idx, d_zi_, zi_, zi_idx, x, y, method_, 1,
-                      doclamp, 2);
-        if (INTERP_ORDER_ == 3)
-            interp2o3(d_zo_, zo_, zo_idx, d_zi_, zi_, zi_idx, x, y, method_, 1,
-                      doclamp, 2);
-    }
-
-    Ty linearInterpFunc(Ty val[2], Tp ratio) const {
-        return MULRC((1 - ratio), val[0]) + MULRC(ratio, val[1]);
-    }
-
-    Ty bilinearInterpFunc(Ty val[2][2], Tp xratio, Tp yratio) const {
-        Ty res[2];
-        res[0] = linearInterpFunc(val[0], xratio);
-        res[1] = linearInterpFunc(val[1], xratio);
-        return linearInterpFunc(res, yratio);
-    }
-
-    Ty cubicInterpFunc(Ty val[4], Tp xratio, bool spline) const {
-        Ty a0, a1, a2, a3;
-        if (spline) {
-            a0 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)1.5, val[1]) +
-                 MULRC((Tp)-1.5, val[2]) + MULRC((Tp)0.5, val[3]);
-
-            a1 = MULRC((Tp)1.0, val[0]) + MULRC((Tp)-2.5, val[1]) +
-                 MULRC((Tp)2.0, val[2]) + MULRC((Tp)-0.5, val[3]);
-
-            a2 = MULRC((Tp)-0.5, val[0]) + MULRC((Tp)0.5, val[2]);
-
-            a3 = val[1];
-        } else {
-            a0 = val[3] - val[2] - val[0] + val[1];
-            a1 = val[0] - val[1] - a0;
-            a2 = val[2] - val[0];
-            a3 = val[1];
-        }
-
-        Tp xratio2 = xratio * xratio;
-        Tp xratio3 = xratio2 * xratio;
-
-        return MULCR(a0, xratio3) + MULCR(a1, xratio2) + MULCR(a2, xratio) + a3;
-    }
-
-    Ty bicubicInterpFunc(Ty val[4][4], Tp xratio, Tp yratio,
-                         bool spline) const {
-        Ty res[4];
-        res[0] = cubicInterpFunc(val[0], xratio, spline);
-        res[1] = cubicInterpFunc(val[1], xratio, spline);
-        res[2] = cubicInterpFunc(val[2], xratio, spline);
-        res[3] = cubicInterpFunc(val[3], xratio, spline);
-        return cubicInterpFunc(res, yratio, spline);
-    }
-
-    void interp2o1(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x, Tp y,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        int xid = (method == AF_INTERP_LOWER ? floor(x) : round(x));
-        int yid = (method == AF_INTERP_LOWER ? floor(y) : round(y));
-
-        const int x_lim    = in.dims[XDIM_];
-        const int y_lim    = in.dims[YDIM_];
-        const int x_stride = in.strides[XDIM_];
-        const int y_stride = in.strides[YDIM_];
-
-        if (doclamp) {
-            xid = fmax(0, fmin(xid, x_lim));
-            yid = fmax(0, fmin(yid, y_lim));
-        }
-        const int idx = ioff + yid * y_stride + xid * x_stride;
-
-        bool condX = xid >= 0 && xid < x_lim;
-        bool condY = yid >= 0 && yid < y_lim;
-
-        Ty zero = (Ty)0;
-        ;
-        bool cond = condX && condY;
-        for (int n = 0; n < batch; n++) {
-            int idx_n = idx + n * in.strides[batch_dim];
-            d_out[ooff + n * out.strides[batch_dim]] =
-                (doclamp || cond) ? d_in[idx_n] : zero;
-        }
-    }
-
-    void interp2o2(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x, Tp y,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        const int grid_x = floor(x);
-        const Tp off_x   = x - grid_x;
-
-        const int grid_y = floor(y);
-        const Tp off_y   = y - grid_y;
-
-        const int x_lim    = in.dims[XDIM_];
-        const int y_lim    = in.dims[YDIM_];
-        const int x_stride = in.strides[XDIM_];
-        const int y_stride = in.strides[YDIM_];
-        const int idx      = ioff + grid_y * y_stride + grid_x * x_stride;
-
-        bool condX[2] = {true, x + 1 < x_lim};
-        bool condY[2] = {true, y + 1 < y_lim};
-        int offx[2]   = {0, condX[1] ? 1 : 0};
-        int offy[2]   = {0, condY[1] ? 1 : 0};
-
-        Tp xratio = off_x, yratio = off_y;
-        if (method == AF_INTERP_LINEAR_COSINE) {
-            xratio = (1 - cos(xratio * (Tp)M_PI)) / 2;
-            yratio = (1 - cos(yratio * (Tp)M_PI)) / 2;
-        }
-
-        Ty zero = (Ty)0;
-        ;
-        for (int n = 0; n < batch; n++) {
-            int idx_n = idx + n * in.strides[batch_dim];
-            Ty val[2][2];
-            for (int j = 0; j < 2; j++) {
-                int off_y = idx_n + offy[j] * y_stride;
-                for (int i = 0; i < 2; i++) {
-                    bool cond = (doclamp || (condX[i] && condY[j]));
-                    val[j][i] = cond ? d_in[off_y + offx[i] * x_stride] : zero;
-                }
-            }
-            d_out[ooff + n * out.strides[batch_dim]] =
-                bilinearInterpFunc(val, xratio, yratio);
-        }
-    }
-
-    void interp2o3(sycl::accessor<Ty> d_out, KParam out, int ooff,
-                   sycl::accessor<Ty> d_in, KParam in, int ioff, Tp x, Tp y,
-                   int method, int batch, bool doclamp, int batch_dim) const {
-        const int grid_x = floor(x);
-        const Tp off_x   = x - grid_x;
-
-        const int grid_y = floor(y);
-        const Tp off_y   = y - grid_y;
+        int zi_idx = idy * ziInfo_.strides[1] * is_off[1] + idx * is_off[0] +
+                     ziInfo_.offset;
+        zi_idx += idw * ziInfo_.strides[3] * is_off[3] +
+                  idz * ziInfo_.strides[2] * is_off[2];
 
-        const int x_lim    = in.dims[XDIM_];
-        const int y_lim    = in.dims[YDIM_];
-        const int x_stride = in.strides[XDIM_];
-        const int y_stride = in.strides[YDIM_];
-        const int idx      = ioff + grid_y * y_stride + grid_x * x_stride;
-
-        // used for setting values at boundaries
-        bool condX[4] = {grid_x - 1 >= 0, true, grid_x + 1 < x_lim,
-                         grid_x + 2 < x_lim};
-        bool condY[4] = {grid_y - 1 >= 0, true, grid_y + 1 < y_lim,
-                         grid_y + 2 < y_lim};
-        int offX[4]   = {condX[0] ? -1 : 0, 0, condX[2] ? 1 : 0,
-                       condX[3] ? 2 : (condX[2] ? 1 : 0)};
-        int offY[4]   = {condY[0] ? -1 : 0, 0, condY[2] ? 1 : 0,
-                       condY[3] ? 2 : (condY[2] ? 1 : 0)};
-
-        Ty zero = (Ty)0;
-        ;
-        for (int n = 0; n < batch; n++) {
-            int idx_n = idx + n * in.strides[batch_dim];
-            // for bicubic interpolation, work with 4x4 val at a time
-            Ty val[4][4];
-#pragma unroll
-            for (int j = 0; j < 4; j++) {
-                int ioff_j = idx_n + offY[j] * y_stride;
-#pragma unroll
-                for (int i = 0; i < 4; i++) {
-                    bool cond = (doclamp || (condX[i] && condY[j]));
-                    val[j][i] = cond ? d_in[ioff_j + offX[i] * x_stride] : zero;
-                }
-            }
-            bool spline = method == AF_INTERP_CUBIC_SPLINE ||
-                          method == AF_INTERP_BICUBIC_SPLINE;
-            d_out[ooff + n * out.strides[batch_dim]] =
-                bicubicInterpFunc(val, off_x, off_y, spline);
-        }
+        Interp2<Ty, Tp, order> interp;
+        interp(d_zo_, zoInfo_, zo_idx, d_zi_, ziInfo_, zi_idx, x, y, XDIM_,
+               YDIM_, method_, 1, doclamp);
     }
 
-   private:
-    sycl::accessor<Ty> d_zo_;
-    const KParam zo_;
-    sycl::accessor<Ty> d_zi_;
-    const KParam zi_;
-    sycl::accessor<Tp> d_xo_;
-    const KParam xo_;
-    sycl::accessor<Tp> d_yo_;
-    const KParam yo_;
+   protected:
+    write_accessor<Ty> d_zo_;
+    const KParam zoInfo_;
+    read_accessor<Ty> d_zi_;
+    const KParam ziInfo_;
+    read_accessor<Tp> d_xo_;
+    const KParam xoInfo_;
+    read_accessor<Tp> d_yo_;
+    const KParam yoInfo_;
     const Tp xi_beg_;
     const Tp xi_step_reproc_;
     const Tp yi_beg_;
@@ -317,18 +150,17 @@ class approx2Kernel {
     const int blocksMatX_;
     const int blocksMatY_;
     const int batch_;
-    int method_;
+    af::interpType method_;
     const int XDIM_;
     const int YDIM_;
-    const int INTERP_ORDER_;
 };
 
-template<typename Ty, typename Tp>
+template<typename Ty, typename Tp, int order>
 void approx2(Param<Ty> zo, const Param<Ty> zi, const Param<Tp> xo,
              const int xdim, const Tp &xi_beg, const Tp &xi_step,
              const Param<Tp> yo, const int ydim, const Tp &yi_beg,
              const Tp &yi_step, const float offGrid,
-             const af_interp_type method, const int order) {
+             const af_interp_type method) {
     constexpr int TX = 16;
     constexpr int TY = 16;
 
@@ -342,20 +174,18 @@ void approx2(Param<Ty> zo, const Param<Ty> zi, const Param<Tp> xo,
     bool batch = !(xo.info.dims[2] == 1 && xo.info.dims[3] == 1);
 
     getQueue().submit([&](sycl::handler &h) {
-        auto zoAcc = zo.data->get_access(h);
-        auto ziAcc = zi.data->get_access(h);
-        auto xoAcc = xo.data->get_access(h);
-        auto yoAcc = yo.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
+        write_accessor<Ty> zoAcc{*zo.data, h};
+        read_accessor<Ty> ziAcc{*zi.data, h};
+        read_accessor<Tp> xoAcc{*xo.data, h};
+        read_accessor<Tp> yoAcc{*yo.data, h};
 
         h.parallel_for(
             sycl::nd_range{global, local},
-            approx2Kernel<Ty, Tp>(
+            approx2Kernel<Ty, Tp, order>(
                 zoAcc, zo.info, ziAcc, zi.info, xoAcc, xo.info, yoAcc, yo.info,
                 xi_beg, Tp(1) / xi_step, yi_beg, Tp(1) / yi_step, (Ty)offGrid,
                 static_cast<int>(blocksPerMatX),
-                static_cast<int>(blocksPerMatY), static_cast<int>(batch),
-                static_cast<int>(method), xdim, ydim, order));
+                static_cast<int>(blocksPerMatY), batch, method, xdim, ydim));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
new file mode 100644
index 0000000000..1e3ac19287
--- /dev/null
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -0,0 +1,342 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Param.hpp>
+#include <math.hpp>
+#include <types.hpp>
+#include <algorithm>
+
+namespace oneapi {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T>
+struct itype_t {
+    typedef float wtype;
+    typedef float vtype;
+};
+
+template<>
+struct itype_t<double> {
+    typedef double wtype;
+    typedef double vtype;
+};
+
+template<>
+struct itype_t<cfloat> {
+    typedef float wtype;
+    typedef cfloat vtype;
+};
+
+template<>
+struct itype_t<cdouble> {
+    typedef double wtype;
+    typedef cdouble vtype;
+};
+
+template<typename Ty, typename Tp>
+Ty linearInterpFunc(Ty val[2], Tp ratio) {
+    return (1 - ratio) * val[0] + ratio * val[1];
+}
+
+template<typename Ty, typename Tp>
+Ty bilinearInterpFunc(Ty val[2][2], Tp xratio, Tp yratio) {
+    Ty res[2];
+    res[0] = linearInterpFunc(val[0], xratio);
+    res[1] = linearInterpFunc(val[1], xratio);
+    return linearInterpFunc(res, yratio);
+}
+
+template<typename Ty, typename Tp>
+inline static Ty cubicInterpFunc(Ty val[4], Tp xratio, bool spline) {
+    Ty a0, a1, a2, a3;
+    if (spline) {
+        a0 = scalar<Ty>(-0.5) * val[0] + scalar<Ty>(1.5) * val[1] +
+             scalar<Ty>(-1.5) * val[2] + scalar<Ty>(0.5) * val[3];
+
+        a1 = scalar<Ty>(1.0) * val[0] + scalar<Ty>(-2.5) * val[1] +
+             scalar<Ty>(2.0) * val[2] + scalar<Ty>(-0.5) * val[3];
+
+        a2 = scalar<Ty>(-0.5) * val[0] + scalar<Ty>(0.5) * val[2];
+
+        a3 = val[1];
+    } else {
+        a0 = val[3] - val[2] - val[0] + val[1];
+        a1 = val[0] - val[1] - a0;
+        a2 = val[2] - val[0];
+        a3 = val[1];
+    }
+
+    Tp xratio2 = xratio * xratio;
+    Tp xratio3 = xratio2 * xratio;
+
+    return a0 * xratio3 + a1 * xratio2 + a2 * xratio + a3;
+}
+
+template<typename Ty, typename Tp>
+inline static Ty bicubicInterpFunc(Ty val[4][4], Tp xratio, Tp yratio,
+                                   bool spline) {
+    Ty res[4];
+    res[0] = cubicInterpFunc(val[0], xratio, spline);
+    res[1] = cubicInterpFunc(val[1], xratio, spline);
+    res[2] = cubicInterpFunc(val[2], xratio, spline);
+    res[3] = cubicInterpFunc(val[3], xratio, spline);
+    return cubicInterpFunc(res, yratio, spline);
+}
+
+template<typename Ty, typename Tp, int order>
+struct Interp1 {};
+
+template<typename Ty, typename Tp>
+struct Interp1<Ty, Tp, 1> {
+    void operator()(write_accessor<Ty> out, KParam oInfo, int ooff,
+                    read_accessor<Ty> in, KParam iInfo, int ioff, Tp x,
+                    int xdim, af::interpType method, int batch, bool clamp,
+                    int batch_dim = 1) {
+        Ty zero = scalar<Ty>(0);
+
+        const int x_lim    = iInfo.dims[xdim];
+        const int x_stride = iInfo.strides[xdim];
+
+        int xid   = (method == AF_INTERP_LOWER ? floor(x) : round(x));
+        bool cond = xid >= 0 && xid < x_lim;
+        if (clamp) xid = std::max((int)0, std::min(xid, x_lim));
+
+        const int idx = ioff + xid * x_stride;
+
+        for (int n = 0; n < batch; n++) {
+            Ty outval =
+                (cond || clamp) ? in[idx + n * iInfo.strides[batch_dim]] : zero;
+            out[ooff + n * oInfo.strides[batch_dim]] = outval;
+        }
+    }
+};
+
+template<typename Ty, typename Tp>
+struct Interp1<Ty, Tp, 2> {
+    void operator()(write_accessor<Ty> out, KParam oInfo, int ooff,
+                    read_accessor<Ty> in, KParam iInfo, int ioff, Tp x,
+                    int xdim, af::interpType method, int batch, bool clamp,
+                    int batch_dim = 1) {
+        typedef typename itype_t<Tp>::wtype WT;
+        typedef typename itype_t<Ty>::vtype VT;
+
+        const int grid_x = floor(x);    // nearest grid
+        const WT off_x   = x - grid_x;  // fractional offset
+
+        const int x_lim    = iInfo.dims[xdim];
+        const int x_stride = iInfo.strides[xdim];
+        const int idx      = ioff + grid_x * x_stride;
+
+        bool cond[2] = {true, grid_x + 1 < x_lim};
+        int offx[2]  = {0, cond[1] ? 1 : 0};
+        WT ratio     = off_x;
+        if (method == AF_INTERP_LINEAR_COSINE) {
+            // Smooth the factional part with cosine
+            ratio = (1 - cos(ratio * af::Pi)) / 2;
+        }
+
+        Ty zero = scalar<Ty>(0);
+
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * iInfo.strides[batch_dim];
+            VT val[2] = {
+                (clamp || cond[0]) ? in[idx_n + offx[0] * x_stride] : zero,
+                (clamp || cond[1]) ? in[idx_n + offx[1] * x_stride] : zero};
+            out[ooff + n * oInfo.strides[batch_dim]] =
+                linearInterpFunc(val, ratio);
+        }
+    }
+};
+
+template<typename Ty, typename Tp>
+struct Interp1<Ty, Tp, 3> {
+    void operator()(write_accessor<Ty> out, KParam oInfo, int ooff,
+                    read_accessor<Ty> in, KParam iInfo, int ioff, Tp x,
+                    int xdim, af::interpType method, int batch, bool clamp,
+                    int batch_dim = 1) {
+        typedef typename itype_t<Tp>::wtype WT;
+        typedef typename itype_t<Ty>::vtype VT;
+
+        const int grid_x = floor(x);    // nearest grid
+        const WT off_x   = x - grid_x;  // fractional offset
+
+        const int x_lim    = iInfo.dims[xdim];
+        const int x_stride = iInfo.strides[xdim];
+        const int idx      = ioff + grid_x * x_stride;
+
+        bool cond[4] = {grid_x - 1 >= 0, true, grid_x + 1 < x_lim,
+                        grid_x + 2 < x_lim};
+        int offx[4]  = {cond[0] ? -1 : 0, 0, cond[2] ? 1 : 0,
+                       cond[3] ? 2 : (cond[2] ? 1 : 0)};
+
+        bool spline = method == AF_INTERP_CUBIC_SPLINE;
+        Ty zero     = scalar<Ty>(0);
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * iInfo.strides[batch_dim];
+            VT val[4];
+            for (int i = 0; i < 4; i++) {
+                val[i] =
+                    (clamp || cond[i]) ? in[idx_n + offx[i] * x_stride] : zero;
+            }
+            out[ooff + n * oInfo.strides[batch_dim]] =
+                cubicInterpFunc(val, off_x, spline);
+        }
+    }
+};
+
+template<typename Ty, typename Tp, int order>
+struct Interp2 {};
+
+template<typename Ty, typename Tp>
+struct Interp2<Ty, Tp, 1> {
+    void operator()(write_accessor<Ty> out, KParam oInfo, int ooff,
+                    read_accessor<Ty> in, KParam iInfo, int ioff, Tp x, Tp y,
+                    int xdim, int ydim, af::interpType method, int batch,
+                    bool clamp, int batch_dim = 2) {
+        int xid = (method == AF_INTERP_LOWER ? floor(x) : round(x));
+        int yid = (method == AF_INTERP_LOWER ? floor(y) : round(y));
+
+        const int x_lim    = iInfo.dims[xdim];
+        const int y_lim    = iInfo.dims[ydim];
+        const int x_stride = iInfo.strides[xdim];
+        const int y_stride = iInfo.strides[ydim];
+
+        if (clamp) {
+            xid = std::max(0, std::min(xid, (int)iInfo.dims[xdim]));
+            yid = std::max(0, std::min(yid, (int)iInfo.dims[ydim]));
+        }
+
+        const int idx = ioff + yid * y_stride + xid * x_stride;
+
+        bool condX = xid >= 0 && xid < x_lim;
+        bool condY = yid >= 0 && yid < y_lim;
+
+        Ty zero   = scalar<Ty>(0);
+        bool cond = condX && condY;
+
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * iInfo.strides[batch_dim];
+            Ty val    = (clamp || cond) ? in[idx_n] : zero;
+            out[ooff + n * oInfo.strides[batch_dim]] = val;
+        }
+    }
+};
+
+template<typename Ty, typename Tp>
+struct Interp2<Ty, Tp, 2> {
+    void operator()(write_accessor<Ty> out, KParam oInfo, int ooff,
+                    read_accessor<Ty> in, KParam iInfo, int ioff, Tp x, Tp y,
+                    int xdim, int ydim, af::interpType method, int batch,
+                    bool clamp, int batch_dim = 2) {
+        typedef typename itype_t<Tp>::wtype WT;
+        typedef typename itype_t<Ty>::vtype VT;
+
+        const int grid_x = floor(x);
+        const WT off_x   = x - grid_x;
+
+        const int grid_y = floor(y);
+        const WT off_y   = y - grid_y;
+
+        const int x_lim    = iInfo.dims[xdim];
+        const int y_lim    = iInfo.dims[ydim];
+        const int x_stride = iInfo.strides[xdim];
+        const int y_stride = iInfo.strides[ydim];
+        const int idx      = ioff + grid_y * y_stride + grid_x * x_stride;
+
+        bool condX[2] = {true, x + 1 < x_lim};
+        bool condY[2] = {true, y + 1 < y_lim};
+        int offx[2]   = {0, condX[1] ? 1 : 0};
+        int offy[2]   = {0, condY[1] ? 1 : 0};
+
+        WT xratio = off_x, yratio = off_y;
+        if (method == AF_INTERP_LINEAR_COSINE ||
+            method == AF_INTERP_BILINEAR_COSINE) {
+            // Smooth the factional part with cosine
+            xratio = (1 - cos(xratio * af::Pi)) / 2;
+            yratio = (1 - cos(yratio * af::Pi)) / 2;
+        }
+
+        Ty zero = scalar<Ty>(0);
+
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * iInfo.strides[batch_dim];
+            VT val[2][2];
+            for (int j = 0; j < 2; j++) {
+                int ioff_j = idx_n + offy[j] * y_stride;
+                for (int i = 0; i < 2; i++) {
+                    bool cond = clamp || (condX[i] && condY[j]);
+                    val[j][i] = (cond) ? in[ioff_j + offx[i] * x_stride] : zero;
+                }
+            }
+            out[ooff + n * oInfo.strides[batch_dim]] =
+                bilinearInterpFunc(val, xratio, yratio);
+        }
+    }
+};
+
+template<typename Ty, typename Tp>
+struct Interp2<Ty, Tp, 3> {
+    void operator()(write_accessor<Ty> out, KParam oInfo, int ooff,
+                    read_accessor<Ty> in, KParam iInfo, int ioff, Tp x, Tp y,
+                    int xdim, int ydim, af::interpType method, int batch,
+                    bool clamp, int batch_dim = 2) {
+        typedef typename itype_t<Tp>::wtype WT;
+        typedef typename itype_t<Ty>::vtype VT;
+
+        const int grid_x = floor(x);
+        const WT off_x   = x - grid_x;
+
+        const int grid_y = floor(y);
+        const WT off_y   = y - grid_y;
+
+        const int x_lim    = iInfo.dims[xdim];
+        const int y_lim    = iInfo.dims[ydim];
+        const int x_stride = iInfo.strides[xdim];
+        const int y_stride = iInfo.strides[ydim];
+        const int idx      = ioff + grid_y * y_stride + grid_x * x_stride;
+
+        // used for setting values at boundaries
+        bool condX[4] = {grid_x - 1 >= 0, true, grid_x + 1 < x_lim,
+                         grid_x + 2 < x_lim};
+        bool condY[4] = {grid_y - 1 >= 0, true, grid_y + 1 < y_lim,
+                         grid_y + 2 < y_lim};
+        int offX[4]   = {condX[0] ? -1 : 0, 0, condX[2] ? 1 : 0,
+                       condX[3] ? 2 : (condX[2] ? 1 : 0)};
+        int offY[4]   = {condY[0] ? -1 : 0, 0, condY[2] ? 1 : 0,
+                       condY[3] ? 2 : (condY[2] ? 1 : 0)};
+
+        // for bicubic interpolation, work with 4x4 val at a time
+        Ty zero     = scalar<Ty>(0);
+        bool spline = (method == AF_INTERP_CUBIC_SPLINE ||
+                       method == AF_INTERP_BICUBIC_SPLINE);
+        for (int n = 0; n < batch; n++) {
+            int idx_n = idx + n * iInfo.strides[batch_dim];
+            VT val[4][4];
+#pragma unroll
+            for (int j = 0; j < 4; j++) {
+                int ioff_j = idx_n + offY[j] * y_stride;
+#pragma unroll
+                for (int i = 0; i < 4; i++) {
+                    bool cond = clamp || (condX[i] && condY[j]);
+                    val[j][i] = (cond) ? in[ioff_j + offX[i] * x_stride] : zero;
+                }
+            }
+
+            out[ooff + n * oInfo.strides[batch_dim]] =
+                bicubicInterpFunc(val, off_x, off_y, spline);
+        }
+    }
+};
+
+}  // namespace oneapi

From 51a4f6936a1ef0ecb93914ff1d464a72d111e3b1 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Tue, 11 Oct 2022 13:32:39 -0400
Subject: [PATCH 128/473] bilateral port to oneapi. tests pass except GFOR b/c
 of missing JIT

---
 src/backend/oneapi/CMakeLists.txt       |   1 +
 src/backend/oneapi/bilateral.cpp        |   3 +-
 src/backend/oneapi/kernel/bilateral.hpp | 217 ++++++++++++++++++++++++
 3 files changed, 220 insertions(+), 1 deletion(-)
 create mode 100755 src/backend/oneapi/kernel/bilateral.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index d6bfaae598..0561573a44 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -209,6 +209,7 @@ target_sources(afoneapi
     kernel/approx1.hpp
     kernel/approx2.hpp
     kernel/assign.hpp
+    kernel/bilateral.hpp
     kernel/diagonal.hpp
     kernel/diff.hpp
     kernel/interp.hpp
diff --git a/src/backend/oneapi/bilateral.cpp b/src/backend/oneapi/bilateral.cpp
index 59b050d2bf..75b97d5509 100644
--- a/src/backend/oneapi/bilateral.cpp
+++ b/src/backend/oneapi/bilateral.cpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <bilateral.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/bilateral.hpp>
 #include <af/dim4.hpp>
 
 using af::dim4;
@@ -19,8 +20,8 @@ namespace oneapi {
 template<typename inType, typename outType>
 Array<outType> bilateral(const Array<inType> &in, const float &sSigma,
                          const float &cSigma) {
-    ONEAPI_NOT_SUPPORTED("");
     Array<outType> out = createEmptyArray<outType>(in.dims());
+    kernel::bilateral<inType, outType>(out, in, sSigma, cSigma);
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/bilateral.hpp b/src/backend/oneapi/kernel/bilateral.hpp
new file mode 100755
index 0000000000..aba8b93d87
--- /dev/null
+++ b/src/backend/oneapi/kernel/bilateral.hpp
@@ -0,0 +1,217 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename outType, bool USE_NATIVE_EXP>
+auto exp_native_nonnative(float in) {
+    if constexpr (USE_NATIVE_EXP)
+        return sycl::native::exp(in);
+    else
+        return exp(in);
+}
+
+template<typename outType, typename inType, bool USE_NATIVE_EXP>
+class bilateralKernel {
+   public:
+    bilateralKernel(sycl::accessor<outType> d_dst, KParam oInfo,
+                    sycl::accessor<inType> d_src, KParam iInfo,
+                    local_accessor<outType, 1> localMem,
+                    local_accessor<outType, 1> gauss2d, float sigma_space,
+                    float sigma_color, int gaussOff, int nBBS0, int nBBS1)
+        : d_dst_(d_dst)
+        , oInfo_(oInfo)
+        , d_src_(d_src)
+        , iInfo_(iInfo)
+        , localMem_(localMem)
+        , gauss2d_(gauss2d)
+        , sigma_space_(sigma_space)
+        , sigma_color_(sigma_color)
+        , gaussOff_(gaussOff)
+        , nBBS0_(nBBS0)
+        , nBBS1_(nBBS1) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g                   = it.get_group();
+        const int radius                = fmax((int)(sigma_space_ * 1.5f), 1);
+        const int padding               = 2 * radius;
+        const int window_size           = padding + 1;
+        const int shrdLen               = g.get_local_range(0) + padding;
+        const float variance_range      = sigma_color_ * sigma_color_;
+        const float variance_space      = sigma_space_ * sigma_space_;
+        const float variance_space_neg2 = -2.0 * variance_space;
+        const float inv_variance_range_neg2 = -0.5 / (variance_range);
+
+        // gfor batch offsets
+        unsigned b2 = g.get_group_id(0) / nBBS0_;
+        unsigned b3 = g.get_group_id(1) / nBBS1_;
+
+        const inType* in =
+            d_src_.get_pointer() +
+            (b2 * iInfo_.strides[2] + b3 * iInfo_.strides[3] + iInfo_.offset);
+        outType* out = d_dst_.get_pointer() +
+                       (b2 * oInfo_.strides[2] + b3 * oInfo_.strides[3]);
+
+        int lx = it.get_local_id(0);
+        int ly = it.get_local_id(1);
+
+        const int gx =
+            g.get_local_range(0) * (g.get_group_id(0) - b2 * nBBS0_) + lx;
+        const int gy =
+            g.get_local_range(1) * (g.get_group_id(1) - b3 * nBBS1_) + ly;
+
+        // generate gauss2d_ spatial variance values for block
+        if (lx < window_size && ly < window_size) {
+            int x = lx - radius;
+            int y = ly - radius;
+            gauss2d_[ly * window_size + lx] =
+                exp_native_nonnative<outType, USE_NATIVE_EXP>(
+                    ((x * x) + (y * y)) / variance_space_neg2);
+        }
+
+        int s0 = iInfo_.strides[0];
+        int s1 = iInfo_.strides[1];
+        int d0 = iInfo_.dims[0];
+        int d1 = iInfo_.dims[1];
+        // pull image to local memory
+        for (int b = ly, gy2 = gy; b < shrdLen;
+             b += g.get_local_range(1), gy2 += g.get_local_range(1)) {
+            // move row_set g.get_local_range(1) along coloumns
+            for (int a = lx, gx2 = gx; a < shrdLen;
+                 a += g.get_local_range(0), gx2 += g.get_local_range(0)) {
+                load2LocalMem(localMem_, in, a, b, shrdLen, d0, d1,
+                              gx2 - radius, gy2 - radius, s1, s0);
+            }
+        }
+
+        it.barrier();
+
+        if (gx < iInfo_.dims[0] && gy < iInfo_.dims[1]) {
+            lx += radius;
+            ly += radius;
+            outType center_color = localMem_[ly * shrdLen + lx];
+            outType res          = 0;
+            outType norm         = 0;
+
+            int joff = (ly - radius) * shrdLen + (lx - radius);
+            int goff = 0;
+
+#pragma unroll
+            for (int wj = 0; wj < window_size; ++wj) {
+#pragma unroll
+                for (int wi = 0; wi < window_size; ++wi) {
+                    outType tmp_color = localMem_[joff + wi];
+                    const outType c   = center_color - tmp_color;
+                    outType gauss_range =
+                        exp_native_nonnative<outType, USE_NATIVE_EXP>(
+                            c * c * inv_variance_range_neg2);
+                    outType weight = gauss2d_[goff + wi] * gauss_range;
+                    norm += weight;
+                    res += tmp_color * weight;
+                }
+                joff += shrdLen;
+                goff += window_size;
+            }
+            out[gy * oInfo_.strides[1] + gx] = res / norm;
+        }
+    }
+
+    int lIdx(int x, int y, int stride1, int stride0) const {
+        return (y * stride1 + x * stride0);
+    }
+
+    void load2LocalMem(local_accessor<outType, 1> shrd, const inType* in,
+                       int lx, int ly, int shrdStride, int dim0, int dim1,
+                       int gx, int gy, int inStride1, int inStride0) const {
+        int gx_ = std::clamp(gx, 0, dim0 - 1);
+        int gy_ = std::clamp(gy, 0, dim1 - 1);
+        shrd[lIdx(lx, ly, shrdStride, 1)] =
+            (outType)in[lIdx(gx_, gy_, inStride1, inStride0)];
+    }
+
+   private:
+    sycl::accessor<outType> d_dst_;
+    KParam oInfo_;
+    sycl::accessor<inType> d_src_;
+    KParam iInfo_;
+    local_accessor<outType, 1> localMem_;
+    local_accessor<outType, 1> gauss2d_;
+    float sigma_space_;
+    float sigma_color_;
+    int gaussOff_;
+    int nBBS0_;
+    int nBBS1_;
+};
+
+template<typename inType, typename outType>
+void bilateral(Param<outType> out, const Param<inType> in, const float s_sigma,
+               const float c_sigma) {
+    constexpr int THREADS_X     = 16;
+    constexpr int THREADS_Y     = 16;
+    constexpr bool UseNativeExp = !std::is_same<inType, double>::value ||
+                                  std::is_same<inType, cdouble>::value;
+
+    auto local = sycl::range{THREADS_X, THREADS_Y};
+
+    const int blk_x = divup(in.info.dims[0], THREADS_X);
+    const int blk_y = divup(in.info.dims[1], THREADS_Y);
+
+    auto global = sycl::range{(size_t)(blk_x * in.info.dims[2] * THREADS_X),
+                              (size_t)(blk_y * in.info.dims[3] * THREADS_Y)};
+
+    // calculate local memory size
+    int radius          = (int)std::max(s_sigma * 1.5f, 1.f);
+    int num_shrd_elems  = (THREADS_X + 2 * radius) * (THREADS_Y + 2 * radius);
+    int num_gauss_elems = (2 * radius + 1) * (2 * radius + 1);
+    size_t localMemSize = (num_shrd_elems + num_gauss_elems) * sizeof(outType);
+    size_t MaxLocalSize =
+        getQueue().get_device().get_info<sycl::info::device::local_mem_size>();
+    if (localMemSize > MaxLocalSize) {
+        char errMessage[256];
+        snprintf(errMessage, sizeof(errMessage),
+                 "\nOneAPI Bilateral filter doesn't support %f spatial sigma\n",
+                 s_sigma);
+        ONEAPI_NOT_SUPPORTED(errMessage);
+    }
+
+    getQueue().submit([&](sycl::handler& h) {
+        auto inAcc  = in.data->get_access(h);
+        auto outAcc = out.data->get_access(h);
+        sycl::stream debugStream(128, 128, h);
+
+        auto localMem = local_accessor<outType, 1>(num_shrd_elems, h);
+        auto gauss2d  = local_accessor<outType, 1>(num_shrd_elems, h);
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       bilateralKernel<outType, inType, UseNativeExp>(
+                           outAcc, out.info, inAcc, in.info, localMem, gauss2d,
+                           s_sigma, c_sigma, num_shrd_elems, blk_x, blk_y));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi

From 46735cd7cd5ccb5e983338fc6d8527c8ba0eb892 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Fri, 11 Nov 2022 13:09:16 -0500
Subject: [PATCH 129/473] fix: interp.hpp missing af/constants.h

---
 src/backend/oneapi/kernel/interp.hpp | 1 +
 1 file changed, 1 insertion(+)
 mode change 100644 => 100755 src/backend/oneapi/kernel/interp.hpp

diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
old mode 100644
new mode 100755
index 1e3ac19287..778aff8202
--- a/src/backend/oneapi/kernel/interp.hpp
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -11,6 +11,7 @@
 #include <math.hpp>
 #include <types.hpp>
 #include <algorithm>
+#include <af/constants.h>
 
 namespace oneapi {
 

From 8d4f680e6b5e79531dc5596b7e6382ee2e0d857e Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Fri, 11 Nov 2022 13:13:42 -0500
Subject: [PATCH 130/473] formatting

---
 src/backend/oneapi/kernel/interp.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
index 778aff8202..6f43fb52f2 100755
--- a/src/backend/oneapi/kernel/interp.hpp
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -10,8 +10,8 @@
 #include <Param.hpp>
 #include <math.hpp>
 #include <types.hpp>
-#include <algorithm>
 #include <af/constants.h>
+#include <algorithm>
 
 namespace oneapi {
 

From dbc33fc7065d1a2dfae177dec0a204acbb722146 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 14 Nov 2022 16:01:26 -0500
Subject: [PATCH 131/473] Fix documentation for af_clamp

---
 docs/details/arith.dox | 7 ++++++-
 include/af/arith.h     | 6 +++---
 test/clamp.cpp         | 8 ++++----
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index f53de09a87..8461ecd100 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -190,7 +190,6 @@ Bitwise xor operation of two inputs
 Minimum of two inputs.
 
 
-
 \defgroup arith_func_max max
 
 \ingroup numeric_mat
@@ -198,6 +197,12 @@ Minimum of two inputs.
 Maximum of two inputs.
 
 
+\defgroup arith_func_clamp clamp
+
+\ingroup numeric_mat
+
+Limits the range of the in array to the values between lo and hi
+
 
 \defgroup arith_func_rem rem
 
diff --git a/include/af/arith.h b/include/af/arith.h
index 319bda674b..89bd39bd64 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -888,16 +888,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for max of two arrays
+       C Interface for clamp
 
-       \param[out] out will contain the values from \p clamped between \p lo and \p hi
+       \param[out] out will contain the values from \p in clamped between \p lo and \p hi
        \param[in] in Input array
        \param[in] lo Value for lower limit
        \param[in] hi Value for upper limit
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_max
+       \ingroup arith_func_clamp
     */
     AFAPI af_err af_clamp(af_array *out, const af_array in,
                           const af_array lo, const af_array hi, const bool batch);
diff --git a/test/clamp.cpp b/test/clamp.cpp
index 7f888a56ac..d27ad3a16d 100644
--- a/test/clamp.cpp
+++ b/test/clamp.cpp
@@ -144,7 +144,7 @@ TEST_P(ClampFloatingPoint, Basic) {
     ASSERT_ARRAYS_NEAR(gold_, out, 1e-5);
 }
 
-TEST(ClampTests, FloatArrayArray) {
+TEST(Clamp, FloatArrayArray) {
     array in = randu(num, f32);
     array lo = randu(num, f32) / 10;        // Ensure lo <= 0.1
     array hi = 1.0 - randu(num, f32) / 10;  // Ensure hi >= 0.9
@@ -165,7 +165,7 @@ TEST(ClampTests, FloatArrayArray) {
     }
 }
 
-TEST(ClampTests, FloatArrayScalar) {
+TEST(Clamp, FloatArrayScalar) {
     array in = randu(num, f32);
     array lo = randu(num, f32) / 10;  // Ensure lo <= 0.1
     float hi = 0.9;
@@ -185,7 +185,7 @@ TEST(ClampTests, FloatArrayScalar) {
     }
 }
 
-TEST(ClampTests, FloatScalarArray) {
+TEST(Clamp, FloatScalarArray) {
     array in = randu(num, f32);
     float lo = 0.1;
     array hi = 1.0 - randu(num, f32) / 10;  // Ensure hi >= 0.9
@@ -205,7 +205,7 @@ TEST(ClampTests, FloatScalarArray) {
     }
 }
 
-TEST(ClampTests, FloatScalarScalar) {
+TEST(Clamp, FloatScalarScalar) {
     array in = randu(num, f32);
     float lo = 0.1;
     float hi = 0.9;

From af95a357f6f6ff584b3e2dc4da9fca7a25a75ba7 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 14 Nov 2022 16:44:37 -0500
Subject: [PATCH 132/473] Avoid installing system forge when
 AF_INSTALL_STANDALONE not set

The install target was copying the forge library installed on the system. This
is not expected because the install command only copies the artifacts generated
by the project and not libraries installed on the system. We do want system
libraries to be installed when AF_INSTALL_STANDALONE is enabled. This commit
addresses both of these issues.
---
 CMakeModules/AFconfigure_forge_dep.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CMakeModules/AFconfigure_forge_dep.cmake b/CMakeModules/AFconfigure_forge_dep.cmake
index 6944d9e9f1..8bf27d3a9e 100644
--- a/CMakeModules/AFconfigure_forge_dep.cmake
+++ b/CMakeModules/AFconfigure_forge_dep.cmake
@@ -75,7 +75,8 @@ else(AF_BUILD_FORGE)
 
     if(TARGET Forge::forge)
         get_target_property(fg_lib_type Forge::forge TYPE)
-        if(NOT ${fg_lib_type} STREQUAL "STATIC_LIBRARY")
+        if(NOT ${fg_lib_type} STREQUAL "STATIC_LIBRARY" AND
+           AF_INSTALL_STANDALONE)
             install(FILES
                     $<TARGET_FILE:Forge::forge>
                     $<$<PLATFORM_ID:Linux>:$<TARGET_SONAME_FILE:Forge::forge>>

From 35a88d9992d4698a0a77dc79e5724babd04d4023 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 18 Nov 2022 16:58:22 -0500
Subject: [PATCH 133/473] Fix ireduce failure in clang 14 due to b8 RNG
 optimization

The random number generator for b8 was producing incorrect results on clang 14
due to loop unrolling. This commit addresses the underlying issue caused by
ineffective indexing on the b8 RNG and updates one ireduce test to use the
ASSERT_VEC_ARRAYS_EQ function
---
 src/backend/cpu/kernel/random_engine.hpp |   4 +-
 test/ireduce.cpp                         | 125 ++++++++++++-----------
 2 files changed, 66 insertions(+), 63 deletions(-)

diff --git a/src/backend/cpu/kernel/random_engine.hpp b/src/backend/cpu/kernel/random_engine.hpp
index 29484e26da..6eaa862031 100644
--- a/src/backend/cpu/kernel/random_engine.hpp
+++ b/src/backend/cpu/kernel/random_engine.hpp
@@ -99,8 +99,8 @@ double getDouble01(uint *val, uint index) {
 
 template<>
 char transform<char>(uint *val, uint index) {
-    char v = val[index >> 2] >> (8 << (index & 3));
-    v      = (v & 0x1) ? 1 : 0;
+    char v = val[index >> 2] >> (index & 3);
+    v = v & 0x1;
     return v;
 }
 
diff --git a/test/ireduce.cpp b/test/ireduce.cpp
index 92596528d4..1e55b9ac23 100644
--- a/test/ireduce.cpp
+++ b/test/ireduce.cpp
@@ -32,67 +32,70 @@ using af::span;
 using std::complex;
 using std::vector;
 
-#define MINMAXOP(fn, ty)                                        \
-    TEST(IndexedReduce, fn##_##ty##_0) {                        \
-        SUPPORTED_TYPE_CHECK(ty);                               \
-        dtype dty    = (dtype)dtype_traits<ty>::af_type;        \
-        const int nx = 10000;                                   \
-        const int ny = 100;                                     \
-        array in     = randu(nx, ny, dty);                      \
-        array val, idx;                                         \
-        fn(val, idx, in, 0);                                    \
-                                                                \
-        ty *h_in    = in.host<ty>();                            \
-        ty *h_in_st = h_in;                                     \
-        ty *h_val   = val.host<ty>();                           \
-        uint *h_idx = idx.host<uint>();                         \
-        for (int i = 0; i < ny; i++) {                          \
-            ty tmp = *std::fn##_element(h_in, h_in + nx);       \
-            ASSERT_EQ(tmp, h_val[i]) << "for index" << i;       \
-            ASSERT_EQ(h_in[h_idx[i]], tmp) << "for index" << i; \
-            h_in += nx;                                         \
-        }                                                       \
-        af_free_host(h_in_st);                                  \
-        af_free_host(h_val);                                    \
-        af_free_host(h_idx);                                    \
-    }                                                           \
-    TEST(IndexedReduce, fn##_##ty##_1) {                        \
-        SUPPORTED_TYPE_CHECK(ty);                               \
-        dtype dty    = (dtype)dtype_traits<ty>::af_type;        \
-        const int nx = 100;                                     \
-        const int ny = 100;                                     \
-        array in     = randu(nx, ny, dty);                      \
-        array val, idx;                                         \
-        fn(val, idx, in, 1);                                    \
-                                                                \
-        ty *h_in    = in.host<ty>();                            \
-        ty *h_val   = val.host<ty>();                           \
-        uint *h_idx = idx.host<uint>();                         \
-        for (int i = 0; i < nx; i++) {                          \
-            ty val = h_val[i];                                  \
-            for (int j = 0; j < ny; j++) {                      \
-                ty tmp = std::fn(val, h_in[j * nx + i]);        \
-                ASSERT_EQ(tmp, val);                            \
-            }                                                   \
-            ASSERT_EQ(val, h_in[h_idx[i] * nx + i]);            \
-        }                                                       \
-        af_free_host(h_in);                                     \
-        af_free_host(h_val);                                    \
-        af_free_host(h_idx);                                    \
-    }                                                           \
-    TEST(IndexedReduce, fn##_##ty##_all) {                      \
-        SUPPORTED_TYPE_CHECK(ty);                               \
-        dtype dty     = (dtype)dtype_traits<ty>::af_type;       \
-        const int num = 100000;                                 \
-        array in      = randu(num, dty);                        \
-        ty val;                                                 \
-        uint idx;                                               \
-        fn<ty>(&val, &idx, in);                                 \
-        ty *h_in = in.host<ty>();                               \
-        ty tmp   = *std::fn##_element(h_in, h_in + num);        \
-        ASSERT_EQ(tmp, val);                                    \
-        ASSERT_EQ(tmp, h_in[idx]);                              \
-        af_free_host(h_in);                                     \
+#define MINMAXOP(fn, ty)                                         \
+    TEST(IndexedReduce, fn##_##ty##_0) {                         \
+        SUPPORTED_TYPE_CHECK(ty);                                \
+        dtype dty    = (dtype)dtype_traits<ty>::af_type;         \
+        const int nx = 10;                                       \
+        const int ny = 100;                                      \
+        array in     = randu(nx, ny, dty);                       \
+        array val, idx;                                          \
+        fn(val, idx, in, 0);                                     \
+                                                                 \
+        ty *h_in    = in.host<ty>();                             \
+        ty *h_in_st = h_in;                                      \
+        uint *h_idx = idx.host<uint>();                          \
+        vector<ty> gold;                                         \
+        vector<ty> igold;                                        \
+        gold.reserve(ny);                                        \
+        igold.reserve(ny);                                       \
+        for (int i = 0; i < ny; i++) {                           \
+            gold.push_back(*std::fn##_element(h_in, h_in + nx)); \
+            igold.push_back(h_in[h_idx[i]]);                     \
+            h_in += nx;                                          \
+        }                                                        \
+        ASSERT_VEC_ARRAY_EQ(gold, af::dim4(1, ny), val);         \
+        ASSERT_VEC_ARRAY_EQ(igold, af::dim4(1, ny), val);        \
+        af_free_host(h_in_st);                                   \
+        af_free_host(h_idx);                                     \
+    }                                                            \
+    TEST(IndexedReduce, fn##_##ty##_1) {                         \
+        SUPPORTED_TYPE_CHECK(ty);                                \
+        dtype dty    = (dtype)dtype_traits<ty>::af_type;         \
+        const int nx = 100;                                      \
+        const int ny = 100;                                      \
+        array in     = randu(nx, ny, dty);                       \
+        array val, idx;                                          \
+        fn(val, idx, in, 1);                                     \
+                                                                 \
+        ty *h_in    = in.host<ty>();                             \
+        ty *h_val   = val.host<ty>();                            \
+        uint *h_idx = idx.host<uint>();                          \
+        for (int i = 0; i < nx; i++) {                           \
+            ty val = h_val[i];                                   \
+            for (int j = 0; j < ny; j++) {                       \
+                ty tmp = std::fn(val, h_in[j * nx + i]);         \
+                ASSERT_EQ(tmp, val);                             \
+            }                                                    \
+            ASSERT_EQ(val, h_in[h_idx[i] * nx + i]);             \
+        }                                                        \
+        af_free_host(h_in);                                      \
+        af_free_host(h_val);                                     \
+        af_free_host(h_idx);                                     \
+    }                                                            \
+    TEST(IndexedReduce, fn##_##ty##_all) {                       \
+        SUPPORTED_TYPE_CHECK(ty);                                \
+        dtype dty     = (dtype)dtype_traits<ty>::af_type;        \
+        const int num = 100000;                                  \
+        array in      = randu(num, dty);                         \
+        ty val;                                                  \
+        uint idx;                                                \
+        fn<ty>(&val, &idx, in);                                  \
+        ty *h_in = in.host<ty>();                                \
+        ty tmp   = *std::fn##_element(h_in, h_in + num);         \
+        ASSERT_EQ(tmp, val);                                     \
+        ASSERT_EQ(tmp, h_in[idx]);                               \
+        af_free_host(h_in);                                      \
     }
 
 MINMAXOP(min, float)

From d8900ea6b56ca8b442973067be1204c50c9a0aec Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 18 Nov 2022 17:22:17 -0500
Subject: [PATCH 134/473] Fix b8 RNG indexing so that the entire range of ctr
 is used

Previously the b8 RNG was only using the lest significant bits for the RNG
this is probably okay but it made the CPU indexing difficult. This commit
ensures that the LSB of each of the 4 integers are used instead of only
the first integer
---
 src/backend/cpu/kernel/random_engine.hpp      | 12 +++--
 src/backend/cuda/kernel/random_engine.hpp     | 48 +++++++++----------
 .../oneapi/kernel/random_engine_write.hpp     | 48 +++++++++----------
 .../opencl/kernel/random_engine_write.cl      | 48 +++++++++----------
 test/random.cpp                               |  2 +-
 5 files changed, 81 insertions(+), 77 deletions(-)

diff --git a/src/backend/cpu/kernel/random_engine.hpp b/src/backend/cpu/kernel/random_engine.hpp
index 6eaa862031..6f55f69719 100644
--- a/src/backend/cpu/kernel/random_engine.hpp
+++ b/src/backend/cpu/kernel/random_engine.hpp
@@ -99,14 +99,18 @@ double getDouble01(uint *val, uint index) {
 
 template<>
 char transform<char>(uint *val, uint index) {
-    char v = val[index >> 2] >> (index & 3);
-    v = v & 0x1;
+    char v = 0;
+    memcpy(&v, static_cast<char *>(static_cast<void *>(val)) + index,
+           sizeof(char));
+    v &= 0x1;
     return v;
 }
 
 template<>
 uchar transform<uchar>(uint *val, uint index) {
-    uchar v = val[index >> 2] >> (index << 3);
+    uchar v = 0;
+    memcpy(&v, static_cast<uchar *>(static_cast<void *>(val)) + index,
+           sizeof(uchar));
     return v;
 }
 
@@ -210,7 +214,7 @@ void philoxUniform(T *out, size_t elements, const uintl seed, uintl counter) {
 
                 // Use the same ctr array for each of the 4 locations,
                 // but each of the location gets a different ctr value
-                for (size_t buf_idx = 0; buf_idx < NUM_WRITES; ++buf_idx) {
+                for (uint buf_idx = 0; buf_idx < NUM_WRITES; ++buf_idx) {
                     size_t out_idx = iter + buf_idx * WRITE_STRIDE + i + j;
                     if (out_idx < elements) {
                         out[out_idx] = transform<T>(ctr, buf_idx);
diff --git a/src/backend/cuda/kernel/random_engine.hpp b/src/backend/cuda/kernel/random_engine.hpp
index e52e78d354..31f9a711ed 100644
--- a/src/backend/cuda/kernel/random_engine.hpp
+++ b/src/backend/cuda/kernel/random_engine.hpp
@@ -315,21 +315,21 @@ __device__ static void writeOut128Bytes(char *out, const uint &index,
                                         const uint &r1, const uint &r2,
                                         const uint &r3, const uint &r4) {
     out[index]                   = (r1)&0x1;
-    out[index + blockDim.x]      = (r1 >> 1) & 0x1;
-    out[index + 2 * blockDim.x]  = (r1 >> 2) & 0x1;
-    out[index + 3 * blockDim.x]  = (r1 >> 3) & 0x1;
+    out[index + blockDim.x]      = (r1 >> 8) & 0x1;
+    out[index + 2 * blockDim.x]  = (r1 >> 16) & 0x1;
+    out[index + 3 * blockDim.x]  = (r1 >> 24) & 0x1;
     out[index + 4 * blockDim.x]  = (r2)&0x1;
-    out[index + 5 * blockDim.x]  = (r2 >> 1) & 0x1;
-    out[index + 6 * blockDim.x]  = (r2 >> 2) & 0x1;
-    out[index + 7 * blockDim.x]  = (r2 >> 3) & 0x1;
+    out[index + 5 * blockDim.x]  = (r2 >> 8) & 0x1;
+    out[index + 6 * blockDim.x]  = (r2 >> 16) & 0x1;
+    out[index + 7 * blockDim.x]  = (r2 >> 24) & 0x1;
     out[index + 8 * blockDim.x]  = (r3)&0x1;
-    out[index + 9 * blockDim.x]  = (r3 >> 1) & 0x1;
-    out[index + 10 * blockDim.x] = (r3 >> 2) & 0x1;
-    out[index + 11 * blockDim.x] = (r3 >> 3) & 0x1;
+    out[index + 9 * blockDim.x]  = (r3 >> 8) & 0x1;
+    out[index + 10 * blockDim.x] = (r3 >> 16) & 0x1;
+    out[index + 11 * blockDim.x] = (r3 >> 24) & 0x1;
     out[index + 12 * blockDim.x] = (r4)&0x1;
-    out[index + 13 * blockDim.x] = (r4 >> 1) & 0x1;
-    out[index + 14 * blockDim.x] = (r4 >> 2) & 0x1;
-    out[index + 15 * blockDim.x] = (r4 >> 3) & 0x1;
+    out[index + 13 * blockDim.x] = (r4 >> 8) & 0x1;
+    out[index + 14 * blockDim.x] = (r4 >> 16) & 0x1;
+    out[index + 15 * blockDim.x] = (r4 >> 24) & 0x1;
 }
 
 __device__ static void writeOut128Bytes(short *out, const uint &index,
@@ -540,49 +540,49 @@ __device__ static void partialWriteOut128Bytes(char *out, const uint &index,
                                                const uint &elements) {
     if (index < elements) { out[index] = (r1)&0x1; }
     if (index + blockDim.x < elements) {
-        out[index + blockDim.x] = (r1 >> 1) & 0x1;
+        out[index + blockDim.x] = (r1 >> 8) & 0x1;
     }
     if (index + 2 * blockDim.x < elements) {
-        out[index + 2 * blockDim.x] = (r1 >> 2) & 0x1;
+        out[index + 2 * blockDim.x] = (r1 >> 16) & 0x1;
     }
     if (index + 3 * blockDim.x < elements) {
-        out[index + 3 * blockDim.x] = (r1 >> 3) & 0x1;
+        out[index + 3 * blockDim.x] = (r1 >> 24) & 0x1;
     }
     if (index + 4 * blockDim.x < elements) {
         out[index + 4 * blockDim.x] = (r2)&0x1;
     }
     if (index + 5 * blockDim.x < elements) {
-        out[index + 5 * blockDim.x] = (r2 >> 1) & 0x1;
+        out[index + 5 * blockDim.x] = (r2 >> 8) & 0x1;
     }
     if (index + 6 * blockDim.x < elements) {
-        out[index + 6 * blockDim.x] = (r2 >> 2) & 0x1;
+        out[index + 6 * blockDim.x] = (r2 >> 16) & 0x1;
     }
     if (index + 7 * blockDim.x < elements) {
-        out[index + 7 * blockDim.x] = (r2 >> 3) & 0x1;
+        out[index + 7 * blockDim.x] = (r2 >> 24) & 0x1;
     }
     if (index + 8 * blockDim.x < elements) {
         out[index + 8 * blockDim.x] = (r3)&0x1;
     }
     if (index + 9 * blockDim.x < elements) {
-        out[index + 9 * blockDim.x] = (r3 >> 1) & 0x1;
+        out[index + 9 * blockDim.x] = (r3 >> 8) & 0x1;
     }
     if (index + 10 * blockDim.x < elements) {
-        out[index + 10 * blockDim.x] = (r3 >> 2) & 0x1;
+        out[index + 10 * blockDim.x] = (r3 >> 16) & 0x1;
     }
     if (index + 11 * blockDim.x < elements) {
-        out[index + 11 * blockDim.x] = (r3 >> 3) & 0x1;
+        out[index + 11 * blockDim.x] = (r3 >> 24) & 0x1;
     }
     if (index + 12 * blockDim.x < elements) {
         out[index + 12 * blockDim.x] = (r4)&0x1;
     }
     if (index + 13 * blockDim.x < elements) {
-        out[index + 13 * blockDim.x] = (r4 >> 1) & 0x1;
+        out[index + 13 * blockDim.x] = (r4 >> 8) & 0x1;
     }
     if (index + 14 * blockDim.x < elements) {
-        out[index + 14 * blockDim.x] = (r4 >> 2) & 0x1;
+        out[index + 14 * blockDim.x] = (r4 >> 16) & 0x1;
     }
     if (index + 15 * blockDim.x < elements) {
-        out[index + 15 * blockDim.x] = (r4 >> 3) & 0x1;
+        out[index + 15 * blockDim.x] = (r4 >> 24) & 0x1;
     }
 }
 
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index 09f7a9c6e5..824feb95b8 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -310,21 +310,21 @@ static void writeOut128Bytes(char *out, const uint &index, const uint groupSz,
                              const uint &r1, const uint &r2, const uint &r3,
                              const uint &r4) {
     out[index]                = (r1)&0x1;
-    out[index + groupSz]      = (r1 >> 1) & 0x1;
-    out[index + 2 * groupSz]  = (r1 >> 2) & 0x1;
-    out[index + 3 * groupSz]  = (r1 >> 3) & 0x1;
+    out[index + groupSz]      = (r1 >> 8) & 0x1;
+    out[index + 2 * groupSz]  = (r1 >> 16) & 0x1;
+    out[index + 3 * groupSz]  = (r1 >> 24) & 0x1;
     out[index + 4 * groupSz]  = (r2)&0x1;
-    out[index + 5 * groupSz]  = (r2 >> 1) & 0x1;
-    out[index + 6 * groupSz]  = (r2 >> 2) & 0x1;
-    out[index + 7 * groupSz]  = (r2 >> 3) & 0x1;
+    out[index + 5 * groupSz]  = (r2 >> 8) & 0x1;
+    out[index + 6 * groupSz]  = (r2 >> 16) & 0x1;
+    out[index + 7 * groupSz]  = (r2 >> 24) & 0x1;
     out[index + 8 * groupSz]  = (r3)&0x1;
-    out[index + 9 * groupSz]  = (r3 >> 1) & 0x1;
-    out[index + 10 * groupSz] = (r3 >> 2) & 0x1;
-    out[index + 11 * groupSz] = (r3 >> 3) & 0x1;
+    out[index + 9 * groupSz]  = (r3 >> 8) & 0x1;
+    out[index + 10 * groupSz] = (r3 >> 16) & 0x1;
+    out[index + 11 * groupSz] = (r3 >> 24) & 0x1;
     out[index + 12 * groupSz] = (r4)&0x1;
-    out[index + 13 * groupSz] = (r4 >> 1) & 0x1;
-    out[index + 14 * groupSz] = (r4 >> 2) & 0x1;
-    out[index + 15 * groupSz] = (r4 >> 3) & 0x1;
+    out[index + 13 * groupSz] = (r4 >> 8) & 0x1;
+    out[index + 14 * groupSz] = (r4 >> 16) & 0x1;
+    out[index + 15 * groupSz] = (r4 >> 24) & 0x1;
 }
 
 static void writeOut128Bytes(short *out, const uint &index, const uint groupSz,
@@ -513,44 +513,44 @@ static void partialWriteOut128Bytes(char *out, const uint &index,
                                     const uint &r2, const uint &r3,
                                     const uint &r4, const uint &elements) {
     if (index < elements) { out[index] = (r1)&0x1; }
-    if (index + groupSz < elements) { out[index + groupSz] = (r1 >> 1) & 0x1; }
+    if (index + groupSz < elements) { out[index + groupSz] = (r1 >> 8) & 0x1; }
     if (index + 2 * groupSz < elements) {
-        out[index + 2 * groupSz] = (r1 >> 2) & 0x1;
+        out[index + 2 * groupSz] = (r1 >> 16) & 0x1;
     }
     if (index + 3 * groupSz < elements) {
-        out[index + 3 * groupSz] = (r1 >> 3) & 0x1;
+        out[index + 3 * groupSz] = (r1 >> 24) & 0x1;
     }
     if (index + 4 * groupSz < elements) { out[index + 4 * groupSz] = (r2)&0x1; }
     if (index + 5 * groupSz < elements) {
-        out[index + 5 * groupSz] = (r2 >> 1) & 0x1;
+        out[index + 5 * groupSz] = (r2 >> 8) & 0x1;
     }
     if (index + 6 * groupSz < elements) {
-        out[index + 6 * groupSz] = (r2 >> 2) & 0x1;
+        out[index + 6 * groupSz] = (r2 >> 16) & 0x1;
     }
     if (index + 7 * groupSz < elements) {
-        out[index + 7 * groupSz] = (r2 >> 3) & 0x1;
+        out[index + 7 * groupSz] = (r2 >> 24) & 0x1;
     }
     if (index + 8 * groupSz < elements) { out[index + 8 * groupSz] = (r3)&0x1; }
     if (index + 9 * groupSz < elements) {
-        out[index + 9 * groupSz] = (r3 >> 1) & 0x1;
+        out[index + 9 * groupSz] = (r3 >> 8) & 0x1;
     }
     if (index + 10 * groupSz < elements) {
-        out[index + 10 * groupSz] = (r3 >> 2) & 0x1;
+        out[index + 10 * groupSz] = (r3 >> 16) & 0x1;
     }
     if (index + 11 * groupSz < elements) {
-        out[index + 11 * groupSz] = (r3 >> 3) & 0x1;
+        out[index + 11 * groupSz] = (r3 >> 24) & 0x1;
     }
     if (index + 12 * groupSz < elements) {
         out[index + 12 * groupSz] = (r4)&0x1;
     }
     if (index + 13 * groupSz < elements) {
-        out[index + 13 * groupSz] = (r4 >> 1) & 0x1;
+        out[index + 13 * groupSz] = (r4 >> 8) & 0x1;
     }
     if (index + 14 * groupSz < elements) {
-        out[index + 14 * groupSz] = (r4 >> 2) & 0x1;
+        out[index + 14 * groupSz] = (r4 >> 16) & 0x1;
     }
     if (index + 15 * groupSz < elements) {
-        out[index + 15 * groupSz] = (r4 >> 3) & 0x1;
+        out[index + 15 * groupSz] = (r4 >> 24) & 0x1;
     }
 }
 
diff --git a/src/backend/opencl/kernel/random_engine_write.cl b/src/backend/opencl/kernel/random_engine_write.cl
index e61610b24a..8711987e44 100644
--- a/src/backend/opencl/kernel/random_engine_write.cl
+++ b/src/backend/opencl/kernel/random_engine_write.cl
@@ -50,21 +50,21 @@ void writeOut128Bytes_uchar(global uchar *out, uint index, uint r1, uint r2,
 void writeOut128Bytes_char(global char *out, uint index, uint r1, uint r2,
                            uint r3, uint r4) {
     out[index]                = (r1)&0x1;
-    out[index + THREADS]      = (r1 >> 1) & 0x1;
-    out[index + 2 * THREADS]  = (r1 >> 2) & 0x1;
-    out[index + 3 * THREADS]  = (r1 >> 3) & 0x1;
+    out[index + THREADS]      = (r1 >> 8) & 0x1;
+    out[index + 2 * THREADS]  = (r1 >> 16) & 0x1;
+    out[index + 3 * THREADS]  = (r1 >> 24) & 0x1;
     out[index + 4 * THREADS]  = (r2)&0x1;
-    out[index + 5 * THREADS]  = (r2 >> 1) & 0x1;
-    out[index + 6 * THREADS]  = (r2 >> 2) & 0x1;
-    out[index + 7 * THREADS]  = (r2 >> 3) & 0x1;
+    out[index + 5 * THREADS]  = (r2 >> 8) & 0x1;
+    out[index + 6 * THREADS]  = (r2 >> 16) & 0x1;
+    out[index + 7 * THREADS]  = (r2 >> 24) & 0x1;
     out[index + 8 * THREADS]  = (r3)&0x1;
-    out[index + 9 * THREADS]  = (r3 >> 1) & 0x1;
-    out[index + 10 * THREADS] = (r3 >> 2) & 0x1;
-    out[index + 11 * THREADS] = (r3 >> 3) & 0x1;
+    out[index + 9 * THREADS]  = (r3 >> 8) & 0x1;
+    out[index + 10 * THREADS] = (r3 >> 16) & 0x1;
+    out[index + 11 * THREADS] = (r3 >> 24) & 0x1;
     out[index + 12 * THREADS] = (r4)&0x1;
-    out[index + 13 * THREADS] = (r4 >> 1) & 0x1;
-    out[index + 14 * THREADS] = (r4 >> 2) & 0x1;
-    out[index + 15 * THREADS] = (r4 >> 3) & 0x1;
+    out[index + 13 * THREADS] = (r4 >> 8) & 0x1;
+    out[index + 14 * THREADS] = (r4 >> 16) & 0x1;
+    out[index + 15 * THREADS] = (r4 >> 24) & 0x1;
 }
 
 void writeOut128Bytes_short(global short *out, uint index, uint r1, uint r2,
@@ -187,44 +187,44 @@ void partialWriteOut128Bytes_uchar(global uchar *out, uint index, uint r1,
 void partialWriteOut128Bytes_char(global char *out, uint index, uint r1,
                                   uint r2, uint r3, uint r4, uint elements) {
     if (index < elements) { out[index] = (r1)&0x1; }
-    if (index + THREADS < elements) { out[index + THREADS] = (r1 >> 1) & 0x1; }
+    if (index + THREADS < elements) { out[index + THREADS] = (r1 >> 8) & 0x1; }
     if (index + 2 * THREADS < elements) {
-        out[index + 2 * THREADS] = (r1 >> 2) & 0x1;
+        out[index + 2 * THREADS] = (r1 >> 16) & 0x1;
     }
     if (index + 3 * THREADS < elements) {
-        out[index + 3 * THREADS] = (r1 >> 3) & 0x1;
+        out[index + 3 * THREADS] = (r1 >> 24) & 0x1;
     }
     if (index + 4 * THREADS < elements) { out[index + 4 * THREADS] = (r2)&0x1; }
     if (index + 5 * THREADS < elements) {
-        out[index + 5 * THREADS] = (r2 >> 1) & 0x1;
+        out[index + 5 * THREADS] = (r2 >> 8) & 0x1;
     }
     if (index + 6 * THREADS < elements) {
-        out[index + 6 * THREADS] = (r2 >> 2) & 0x1;
+        out[index + 6 * THREADS] = (r2 >> 16) & 0x1;
     }
     if (index + 7 * THREADS < elements) {
-        out[index + 7 * THREADS] = (r2 >> 3) & 0x1;
+        out[index + 7 * THREADS] = (r2 >> 24) & 0x1;
     }
     if (index + 8 * THREADS < elements) { out[index + 8 * THREADS] = (r3)&0x1; }
     if (index + 9 * THREADS < elements) {
-        out[index + 9 * THREADS] = (r3 >> 1) & 0x1;
+        out[index + 9 * THREADS] = (r3 >> 8) & 0x1;
     }
     if (index + 10 * THREADS < elements) {
-        out[index + 10 * THREADS] = (r3 >> 2) & 0x1;
+        out[index + 10 * THREADS] = (r3 >> 16) & 0x1;
     }
     if (index + 11 * THREADS < elements) {
-        out[index + 11 * THREADS] = (r3 >> 3) & 0x1;
+        out[index + 11 * THREADS] = (r3 >> 24) & 0x1;
     }
     if (index + 12 * THREADS < elements) {
         out[index + 12 * THREADS] = (r4)&0x1;
     }
     if (index + 13 * THREADS < elements) {
-        out[index + 13 * THREADS] = (r4 >> 1) & 0x1;
+        out[index + 13 * THREADS] = (r4 >> 8) & 0x1;
     }
     if (index + 14 * THREADS < elements) {
-        out[index + 14 * THREADS] = (r4 >> 2) & 0x1;
+        out[index + 14 * THREADS] = (r4 >> 16) & 0x1;
     }
     if (index + 15 * THREADS < elements) {
-        out[index + 15 * THREADS] = (r4 >> 3) & 0x1;
+        out[index + 15 * THREADS] = (r4 >> 24) & 0x1;
     }
 }
 
diff --git a/test/random.cpp b/test/random.cpp
index df65ac8006..d0860b70f2 100644
--- a/test/random.cpp
+++ b/test/random.cpp
@@ -36,7 +36,7 @@ class Random : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, intl,
-                         uintl, unsigned char, af_half>
+                         uintl, unsigned char, char, af_half>
     TestTypes;
 
 // register the type list

From 84046ca61a672ad50e6224f877b0f1df84fdbb11 Mon Sep 17 00:00:00 2001
From: ktdq <105746631+ktdq@users.noreply.github.com>
Date: Sun, 20 Nov 2022 00:52:49 -0500
Subject: [PATCH 135/473] Support 64bit hamming distance (#3314)

* support 64bit hamming distance on CUDA
* CPU support for 64 bit __popc in hamming distance
* adds hammingMatcher tests for uintll type

Co-authored-by: syurkevi <stefan@arrayfire.com>
---
 src/backend/cpu/kernel/nearest_neighbour.hpp  |  3 +-
 src/backend/cuda/kernel/nearest_neighbour.hpp |  2 +-
 test/hamming.cpp                              | 38 +++++++++++++++++++
 3 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/src/backend/cpu/kernel/nearest_neighbour.hpp b/src/backend/cpu/kernel/nearest_neighbour.hpp
index 599c04356b..39b005c4ed 100644
--- a/src/backend/cpu/kernel/nearest_neighbour.hpp
+++ b/src/backend/cpu/kernel/nearest_neighbour.hpp
@@ -17,6 +17,7 @@ namespace kernel {
 
 #include <intrin.h>
 #define __builtin_popcount __popcnt
+#define __builtin_popcountll __popcnt64
 
 #endif
 
@@ -44,7 +45,7 @@ struct dist_op<uint, To, AF_SHD> {
 
 template<typename To>
 struct dist_op<uintl, To, AF_SHD> {
-    To operator()(uintl v1, uintl v2) { return __builtin_popcount(v1 ^ v2); }
+    To operator()(uintl v1, uintl v2) { return __builtin_popcountll(v1 ^ v2); }
 };
 
 template<typename To>
diff --git a/src/backend/cuda/kernel/nearest_neighbour.hpp b/src/backend/cuda/kernel/nearest_neighbour.hpp
index f615a733db..170f81868a 100644
--- a/src/backend/cuda/kernel/nearest_neighbour.hpp
+++ b/src/backend/cuda/kernel/nearest_neighbour.hpp
@@ -52,7 +52,7 @@ struct dist_op<uint, To, AF_SHD> {
 
 template<typename To>
 struct dist_op<uintl, To, AF_SHD> {
-    __device__ To operator()(uintl v1, uintl v2) { return __popc(v1 ^ v2); }
+    __device__ To operator()(uintl v1, uintl v2) { return __popcll(v1 ^ v2); }
 };
 
 template<typename To>
diff --git a/test/hamming.cpp b/test/hamming.cpp
index 763e0f7774..b14a33db0a 100644
--- a/test/hamming.cpp
+++ b/test/hamming.cpp
@@ -153,3 +153,41 @@ TEST(HammingMatcher, CPP) {
     delete[] outIdx;
     delete[] outDist;
 }
+
+TEST(HammingMatcher64bit, CPP) {
+    using af::array;
+    using af::dim4;
+
+    vector<dim4> numDims;
+    vector<vector<unsigned long long>> in;
+    vector<vector<unsigned long long>> tests;
+
+    readTests<unsigned long long, unsigned long long, int>(
+        TEST_DIR "/hamming/hamming_500_5000_dim0_u32.test", numDims, in, tests);
+
+    dim4 qDims = numDims[0];
+    dim4 tDims = numDims[1];
+
+    array query(qDims, &(in[0].front()));
+    array train(tDims, &(in[1].front()));
+
+    array idx, dist;
+    hammingMatcher(idx, dist, query, train, 0, 1);
+
+    vector<unsigned long long> goldIdx  = tests[0];
+    vector<unsigned long long> goldDist = tests[1];
+    size_t nElems                       = goldIdx.size();
+    uint *outIdx                        = new uint[nElems];
+    uint *outDist                       = new uint[nElems];
+
+    idx.host(outIdx);
+    dist.host(outDist);
+
+    for (size_t elIter = 0; elIter < nElems; ++elIter) {
+        ASSERT_EQ(goldDist[elIter], outDist[elIter])
+            << "at: " << elIter << endl;
+    }
+
+    delete[] outIdx;
+    delete[] outDist;
+}

From 5a11efe8ca64ffbef367917da87a7400cccca7bb Mon Sep 17 00:00:00 2001
From: guillaume <schmidg109@gmail.com>
Date: Mon, 19 Sep 2022 08:13:12 +0200
Subject: [PATCH 136/473] Fixes local issue with to_string. Refactor out hash
 funcitons

The arguments provided to OpenCL uses the C++ standard library
function std::to_string(). This function uses the locale to render
it's argument to a string.

It is a problem when arrayfire is used in a
software initialised with non "C" locale.

For instance, on a French computer, to_string(1.0) will output the
string "1,0000000".
This string is provided to OpenCL kernels, generating a syntax error.

The most portable way to fix this problem is to use a local ostringstream
imbued withe "C" locale.

An Other way would be to use C++17 to_chars function, as it only renders it
argument with "C" locale, without impact from the application or
system locale.

The patch is pretty simple, it changes the toString() function to use
the stringstream in src/backend/common/TemplateArg.cpp and changed the
to_string calls to this toString function in types.cpp.
---
 CMakeLists.txt                           |   8 +-
 CMakeModules/bin2cpp.cpp                 |   4 +-
 src/api/c/device.cpp                     |   3 +
 src/api/c/type_util.hpp                  |   2 -
 src/api/unified/symbol_manager.cpp       |   1 +
 src/backend/common/CMakeLists.txt        |   4 +-
 src/backend/common/Source.hpp            |  17 ++
 src/backend/common/TemplateArg.cpp       | 295 ----------------------
 src/backend/common/TemplateArg.hpp       |  19 +-
 src/backend/common/deterministicHash.cpp |  47 ++++
 src/backend/common/deterministicHash.hpp |  36 +++
 src/backend/common/err_common.cpp        |  20 +-
 src/backend/common/graphics_common.cpp   |   1 +
 src/backend/common/half.cpp              |   6 +
 src/backend/common/half.hpp              |   1 +
 src/backend/common/jit/Node.cpp          |   1 +
 src/backend/common/jit/NodeIO.hpp        |   7 +-
 src/backend/common/kernel_cache.cpp      |   3 +-
 src/backend/common/kernel_cache.hpp      |   1 +
 src/backend/common/util.cpp              | 302 +++++++++++++++++++++--
 src/backend/common/util.hpp              |  39 +--
 src/backend/cpu/platform.cpp             |   2 +
 src/backend/cpu/queue.hpp                |   2 +-
 src/backend/cuda/compile_module.cpp      |   1 +
 src/backend/cuda/cudnnModule.cpp         |   1 +
 src/backend/cuda/device_manager.cpp      |   2 +
 src/backend/cuda/jit.cpp                 |   3 +
 src/backend/cuda/platform.cpp            |   2 +
 src/backend/opencl/compile_module.cpp    |   2 +
 src/backend/opencl/device_manager.cpp    |   1 +
 src/backend/opencl/jit.cpp               |   2 +
 src/backend/opencl/platform.cpp          |   2 +
 src/backend/opencl/types.cpp             |  27 +-
 33 files changed, 465 insertions(+), 399 deletions(-)
 create mode 100644 src/backend/common/Source.hpp
 delete mode 100644 src/backend/common/TemplateArg.cpp
 create mode 100644 src/backend/common/deterministicHash.cpp
 create mode 100644 src/backend/common/deterministicHash.hpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5689a8094b..099e9a72ae 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -317,15 +317,15 @@ if(CMAKE_CROSSCOMPILING)
                        "directory and build the bin2cpp target.")
   endif()
 else()
-  add_executable(bin2cpp ${ArrayFire_SOURCE_DIR}/CMakeModules/bin2cpp.cpp
-                         ${ArrayFire_SOURCE_DIR}/src/backend/common/util.cpp)
+  add_executable(bin2cpp CMakeModules/bin2cpp.cpp
+                         src/backend/common/deterministicHash.cpp
+                         src/backend/common/deterministicHash.hpp
+                         src/backend/common/Source.hpp)
   set_target_properties(bin2cpp
     PROPERTIES
       CXX_STANDARD 17)
   target_link_libraries(bin2cpp PRIVATE nonstd::span-lite)
 
-  # NOSPDLOG is used to remove the spdlog dependency from bin2cpp
-  target_compile_definitions(bin2cpp PRIVATE NOSPDLOG)
   if(WIN32)
     target_compile_definitions(bin2cpp PRIVATE OS_WIN)
   elseif(APPLE)
diff --git a/CMakeModules/bin2cpp.cpp b/CMakeModules/bin2cpp.cpp
index 217b3efe14..3426b1ebed 100644
--- a/CMakeModules/bin2cpp.cpp
+++ b/CMakeModules/bin2cpp.cpp
@@ -28,7 +28,7 @@
 #include <utility>
 #include <vector>
 
-#include <common/util.hpp>
+#include <common/deterministicHash.hpp>
 
 using namespace std;
 using std::cout;
@@ -275,7 +275,7 @@ int main(int argc, const char *const *const argv) {
 
     cout << "#pragma once\n";
     cout << "#include <cstddef>\n";          // defines size_t
-    cout << "#include <common/util.hpp>\n";  // defines common::Source
+    cout << "#include <common/Source.hpp>\n";  // defines common::Source
 
     int ns_cnt = 0;
     int level  = 0;
diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp
index cf65bfd81c..57c61be4c3 100644
--- a/src/api/c/device.cpp
+++ b/src/api/c/device.cpp
@@ -28,7 +28,10 @@
 #include <string>
 
 using af::dim4;
+using common::getCacheDirectory;
+using common::getEnvVar;
 using common::half;
+using common::JIT_KERNEL_CACHE_DIRECTORY_ENV_NAME;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/type_util.hpp b/src/api/c/type_util.hpp
index 1fa7dd7c87..4214882492 100644
--- a/src/api/c/type_util.hpp
+++ b/src/api/c/type_util.hpp
@@ -10,8 +10,6 @@
 #pragma once
 #include <af/defines.h>
 
-const char *getName(af_dtype type);
-
 // uchar to number converters
 template<typename T>
 struct ToNum {
diff --git a/src/api/unified/symbol_manager.cpp b/src/api/unified/symbol_manager.cpp
index ca11238773..a2efc6ee59 100644
--- a/src/api/unified/symbol_manager.cpp
+++ b/src/api/unified/symbol_manager.cpp
@@ -26,6 +26,7 @@
 #include <dlfcn.h>
 #endif
 
+using common::getEnvVar;
 using common::getErrorMessage;
 using common::getFunctionPointer;
 using common::loadLibrary;
diff --git a/src/backend/common/CMakeLists.txt b/src/backend/common/CMakeLists.txt
index 8f553814e7..1487d99c44 100644
--- a/src/backend/common/CMakeLists.txt
+++ b/src/backend/common/CMakeLists.txt
@@ -40,9 +40,9 @@ target_sources(afcommon_interface
     ${CMAKE_CURRENT_SOURCE_DIR}/MemoryManagerBase.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/MersenneTwister.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/ModuleInterface.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/Source.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/SparseArray.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/SparseArray.hpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/TemplateArg.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/TemplateArg.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/TemplateTypename.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/blas_headers.hpp
@@ -53,6 +53,8 @@ target_sources(afcommon_interface
     ${CMAKE_CURRENT_SOURCE_DIR}/complex.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/constants.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/defines.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/deterministicHash.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/deterministicHash.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dim4.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dispatch.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/dispatch.hpp
diff --git a/src/backend/common/Source.hpp b/src/backend/common/Source.hpp
new file mode 100644
index 0000000000..000c2809d2
--- /dev/null
+++ b/src/backend/common/Source.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+namespace common {
+struct Source {
+    const char* ptr;           // Pointer to the kernel source
+    const std::size_t length;  // Length of the kernel source
+    const std::size_t hash;    // hash value for the source *ptr;
+};
+}  // namespace common
diff --git a/src/backend/common/TemplateArg.cpp b/src/backend/common/TemplateArg.cpp
deleted file mode 100644
index 8cff5c4e24..0000000000
--- a/src/backend/common/TemplateArg.cpp
+++ /dev/null
@@ -1,295 +0,0 @@
-/*******************************************************
- * Copyright (c) 2020, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#include <common/TemplateArg.hpp>
-
-#include <common/internal_enums.hpp>
-#include <optypes.hpp>
-#include <af/defines.h>
-
-#include <cstdlib>
-#include <string>
-
-using std::string;
-
-template<typename T>
-string toString(T value) {
-    return std::to_string(value);
-}
-
-template string toString<int>(int);
-template string toString<long>(long);
-template string toString<long long>(long long);
-template string toString<unsigned>(unsigned);
-template string toString<unsigned long>(unsigned long);
-template string toString<unsigned long long>(unsigned long long);
-template string toString<float>(float);
-template string toString<double>(double);
-template string toString<long double>(long double);
-
-template<>
-string toString(TemplateArg arg) {
-    return arg._tparam;
-}
-
-template<>
-string toString(bool val) {
-    return string(val ? "true" : "false");
-}
-
-template<>
-string toString(const char* str) {
-    return string(str);
-}
-
-template<>
-string toString(const string str) {
-    return str;
-}
-
-template<>
-string toString(unsigned short val) {
-    return std::to_string((unsigned int)(val));
-}
-
-template<>
-string toString(short val) {
-    return std::to_string(int(val));
-}
-
-template<>
-string toString(unsigned char val) {
-    return std::to_string((unsigned int)(val));
-}
-
-template<>
-string toString(char val) {
-    return std::to_string(int(val));
-}
-
-string getOpEnumStr(af_op_t val) {
-    const char* retVal = NULL;
-#define CASE_STMT(v) \
-    case v: retVal = #v; break
-    switch (val) {
-        CASE_STMT(af_add_t);
-        CASE_STMT(af_sub_t);
-        CASE_STMT(af_mul_t);
-        CASE_STMT(af_div_t);
-
-        CASE_STMT(af_and_t);
-        CASE_STMT(af_or_t);
-        CASE_STMT(af_eq_t);
-        CASE_STMT(af_neq_t);
-        CASE_STMT(af_lt_t);
-        CASE_STMT(af_le_t);
-        CASE_STMT(af_gt_t);
-        CASE_STMT(af_ge_t);
-
-        CASE_STMT(af_bitnot_t);
-        CASE_STMT(af_bitor_t);
-        CASE_STMT(af_bitand_t);
-        CASE_STMT(af_bitxor_t);
-        CASE_STMT(af_bitshiftl_t);
-        CASE_STMT(af_bitshiftr_t);
-
-        CASE_STMT(af_min_t);
-        CASE_STMT(af_max_t);
-        CASE_STMT(af_cplx2_t);
-        CASE_STMT(af_atan2_t);
-        CASE_STMT(af_pow_t);
-        CASE_STMT(af_hypot_t);
-
-        CASE_STMT(af_sin_t);
-        CASE_STMT(af_cos_t);
-        CASE_STMT(af_tan_t);
-        CASE_STMT(af_asin_t);
-        CASE_STMT(af_acos_t);
-        CASE_STMT(af_atan_t);
-
-        CASE_STMT(af_sinh_t);
-        CASE_STMT(af_cosh_t);
-        CASE_STMT(af_tanh_t);
-        CASE_STMT(af_asinh_t);
-        CASE_STMT(af_acosh_t);
-        CASE_STMT(af_atanh_t);
-
-        CASE_STMT(af_exp_t);
-        CASE_STMT(af_expm1_t);
-        CASE_STMT(af_erf_t);
-        CASE_STMT(af_erfc_t);
-
-        CASE_STMT(af_log_t);
-        CASE_STMT(af_log10_t);
-        CASE_STMT(af_log1p_t);
-        CASE_STMT(af_log2_t);
-
-        CASE_STMT(af_sqrt_t);
-        CASE_STMT(af_cbrt_t);
-
-        CASE_STMT(af_abs_t);
-        CASE_STMT(af_cast_t);
-        CASE_STMT(af_cplx_t);
-        CASE_STMT(af_real_t);
-        CASE_STMT(af_imag_t);
-        CASE_STMT(af_conj_t);
-
-        CASE_STMT(af_floor_t);
-        CASE_STMT(af_ceil_t);
-        CASE_STMT(af_round_t);
-        CASE_STMT(af_trunc_t);
-        CASE_STMT(af_signbit_t);
-
-        CASE_STMT(af_rem_t);
-        CASE_STMT(af_mod_t);
-
-        CASE_STMT(af_tgamma_t);
-        CASE_STMT(af_lgamma_t);
-
-        CASE_STMT(af_notzero_t);
-
-        CASE_STMT(af_iszero_t);
-        CASE_STMT(af_isinf_t);
-        CASE_STMT(af_isnan_t);
-
-        CASE_STMT(af_sigmoid_t);
-
-        CASE_STMT(af_noop_t);
-
-        CASE_STMT(af_select_t);
-        CASE_STMT(af_not_select_t);
-        CASE_STMT(af_rsqrt_t);
-        CASE_STMT(af_moddims_t);
-
-        CASE_STMT(af_none_t);
-    }
-#undef CASE_STMT
-    return retVal;
-}
-
-template<>
-string toString(af_op_t val) {
-    return getOpEnumStr(val);
-}
-
-template<>
-string toString(af_interp_type p) {
-    const char* retVal = NULL;
-#define CASE_STMT(v) \
-    case v: retVal = #v; break
-    switch (p) {
-        CASE_STMT(AF_INTERP_NEAREST);
-        CASE_STMT(AF_INTERP_LINEAR);
-        CASE_STMT(AF_INTERP_BILINEAR);
-        CASE_STMT(AF_INTERP_CUBIC);
-        CASE_STMT(AF_INTERP_LOWER);
-        CASE_STMT(AF_INTERP_LINEAR_COSINE);
-        CASE_STMT(AF_INTERP_BILINEAR_COSINE);
-        CASE_STMT(AF_INTERP_BICUBIC);
-        CASE_STMT(AF_INTERP_CUBIC_SPLINE);
-        CASE_STMT(AF_INTERP_BICUBIC_SPLINE);
-    }
-#undef CASE_STMT
-    return retVal;
-}
-
-template<>
-string toString(af_border_type p) {
-    const char* retVal = NULL;
-#define CASE_STMT(v) \
-    case v: retVal = #v; break
-    switch (p) {
-        CASE_STMT(AF_PAD_ZERO);
-        CASE_STMT(AF_PAD_SYM);
-        CASE_STMT(AF_PAD_CLAMP_TO_EDGE);
-        CASE_STMT(AF_PAD_PERIODIC);
-    }
-#undef CASE_STMT
-    return retVal;
-}
-
-template<>
-string toString(af_moment_type p) {
-    const char* retVal = NULL;
-#define CASE_STMT(v) \
-    case v: retVal = #v; break
-    switch (p) {
-        CASE_STMT(AF_MOMENT_M00);
-        CASE_STMT(AF_MOMENT_M01);
-        CASE_STMT(AF_MOMENT_M10);
-        CASE_STMT(AF_MOMENT_M11);
-        CASE_STMT(AF_MOMENT_FIRST_ORDER);
-    }
-#undef CASE_STMT
-    return retVal;
-}
-
-template<>
-string toString(af_match_type p) {
-    const char* retVal = NULL;
-#define CASE_STMT(v) \
-    case v: retVal = #v; break
-    switch (p) {
-        CASE_STMT(AF_SAD);
-        CASE_STMT(AF_ZSAD);
-        CASE_STMT(AF_LSAD);
-        CASE_STMT(AF_SSD);
-        CASE_STMT(AF_ZSSD);
-        CASE_STMT(AF_LSSD);
-        CASE_STMT(AF_NCC);
-        CASE_STMT(AF_ZNCC);
-        CASE_STMT(AF_SHD);
-    }
-#undef CASE_STMT
-    return retVal;
-}
-
-template<>
-string toString(af_flux_function p) {
-    const char* retVal = NULL;
-#define CASE_STMT(v) \
-    case v: retVal = #v; break
-    switch (p) {
-        CASE_STMT(AF_FLUX_QUADRATIC);
-        CASE_STMT(AF_FLUX_EXPONENTIAL);
-        CASE_STMT(AF_FLUX_DEFAULT);
-    }
-#undef CASE_STMT
-    return retVal;
-}
-
-template<>
-string toString(AF_BATCH_KIND val) {
-    const char* retVal = NULL;
-#define CASE_STMT(v) \
-    case v: retVal = #v; break
-    switch (val) {
-        CASE_STMT(AF_BATCH_NONE);
-        CASE_STMT(AF_BATCH_LHS);
-        CASE_STMT(AF_BATCH_RHS);
-        CASE_STMT(AF_BATCH_SAME);
-        CASE_STMT(AF_BATCH_DIFF);
-        CASE_STMT(AF_BATCH_UNSUPPORTED);
-    }
-#undef CASE_STMT
-    return retVal;
-}
-
-template<>
-string toString(af_homography_type val) {
-    const char* retVal = NULL;
-#define CASE_STMT(v) \
-    case v: retVal = #v; break
-    switch (val) {
-        CASE_STMT(AF_HOMOGRAPHY_RANSAC);
-        CASE_STMT(AF_HOMOGRAPHY_LMEDS);
-    }
-#undef CASE_STMT
-    return retVal;
-}
diff --git a/src/backend/common/TemplateArg.hpp b/src/backend/common/TemplateArg.hpp
index a7dfbe4ceb..3a92bf643e 100644
--- a/src/backend/common/TemplateArg.hpp
+++ b/src/backend/common/TemplateArg.hpp
@@ -9,21 +9,15 @@
 
 #pragma once
 
+#include <common/util.hpp>
+
 #include <array>
 #include <string>
 #include <utility>
 
-#include <optypes.hpp>
-#include <traits.hpp>
-
 template<typename T>
 class TemplateTypename;
 
-template<typename T>
-std::string toString(T value);
-
-std::string getOpEnumStr(af_op_t val);
-
 struct TemplateArg {
     std::string _tparam;
 
@@ -33,7 +27,8 @@ struct TemplateArg {
     constexpr TemplateArg(TemplateTypename<T> arg) noexcept : _tparam(arg) {}
 
     template<typename T>
-    constexpr TemplateArg(T value) noexcept : _tparam(toString(value)) {}
+    constexpr TemplateArg(T value) noexcept
+        : _tparam(common::toString(value)) {}
 };
 
 template<typename... Targs>
@@ -43,6 +38,6 @@ std::array<TemplateArg, sizeof...(Targs)> TemplateArgs(Targs &&...args) {
 }
 
 #define DefineKey(arg) " -D " #arg
-#define DefineValue(arg) " -D " #arg "=" + toString(arg)
-#define DefineKeyValue(key, arg) " -D " #key "=" + toString(arg)
-#define DefineKeyFromStr(arg) toString(" -D " + std::string(arg))
+#define DefineValue(arg) " -D " #arg "=" + common::toString(arg)
+#define DefineKeyValue(key, arg) " -D " #key "=" + common::toString(arg)
+#define DefineKeyFromStr(arg) " -D " + std::string(arg)
diff --git a/src/backend/common/deterministicHash.cpp b/src/backend/common/deterministicHash.cpp
new file mode 100644
index 0000000000..0529f7c58b
--- /dev/null
+++ b/src/backend/common/deterministicHash.cpp
@@ -0,0 +1,47 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/deterministicHash.hpp>
+
+#include <nonstd/span.hpp>
+#include <numeric>
+#include <string>
+
+using nonstd::span;
+using std::accumulate;
+using std::string;
+
+size_t deterministicHash(const void* data, size_t byteSize, size_t prevHash) {
+    // Fowler-Noll-Vo "1a" 32 bit hash
+    // https://en.wikipedia.org/wiki/Fowler-Noll-Vo_hash_function
+    const auto* byteData = static_cast<const uint8_t*>(data);
+    return accumulate(
+        byteData, byteData + byteSize, prevHash,
+        [&](size_t hash, uint8_t data) { return (hash ^ data) * FNV1A_PRIME; });
+}
+
+size_t deterministicHash(const string& data, const size_t prevHash) {
+    return deterministicHash(data.data(), data.size(), prevHash);
+}
+
+size_t deterministicHash(span<const string> list, const size_t prevHash) {
+    size_t hash = prevHash;
+    for (auto s : list) { hash = deterministicHash(s.data(), s.size(), hash); }
+    return hash;
+}
+
+size_t deterministicHash(span<const common::Source> list) {
+    // Combine the different source codes, via their hashes
+    size_t hash = FNV1A_BASE_OFFSET;
+    for (auto s : list) {
+        size_t h = s.hash ? s.hash : deterministicHash(s.ptr, s.length);
+        hash     = deterministicHash(&h, sizeof(size_t), hash);
+    }
+    return hash;
+}
diff --git a/src/backend/common/deterministicHash.hpp b/src/backend/common/deterministicHash.hpp
new file mode 100644
index 0000000000..25b43a8893
--- /dev/null
+++ b/src/backend/common/deterministicHash.hpp
@@ -0,0 +1,36 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <nonstd/span.hpp>
+#include <string>
+
+#include <common/Source.hpp>
+
+/// Return the FNV-1a hash of the provided bata.
+///
+/// \param[in] data Binary data to hash
+/// \param[in] byteSize Size of the data in bytes
+/// \param[in] optional prevHash Hash of previous parts when string is split
+///
+/// \returns An unsigned integer representing the hash of the data
+constexpr std::size_t FNV1A_BASE_OFFSET = 0x811C9DC5;
+constexpr std::size_t FNV1A_PRIME       = 0x01000193;
+std::size_t deterministicHash(const void* data, std::size_t byteSize,
+                              const std::size_t prevHash = FNV1A_BASE_OFFSET);
+
+// This is just a wrapper around the above function.
+std::size_t deterministicHash(const std::string& data,
+                              const std::size_t prevHash = FNV1A_BASE_OFFSET);
+
+// This concatenates strings in the vector and computes hash
+std::size_t deterministicHash(nonstd::span<const std::string> list,
+                              const std::size_t prevHash = FNV1A_BASE_OFFSET);
+
+// This concatenates hashes of multiple sources
+std::size_t deterministicHash(nonstd::span<const common::Source> list);
diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index 7a19bcb941..58bc0a9ced 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -26,15 +26,17 @@
 #include <platform.hpp>
 #endif
 
+using boost::stacktrace::stacktrace;
 using std::move;
 using std::string;
 using std::stringstream;
 
+using common::getEnvVar;
+using common::getName;
 using common::is_stacktrace_enabled;
 
 AfError::AfError(const char *const func, const char *const file, const int line,
-                 const char *const message, af_err err,
-                 boost::stacktrace::stacktrace st)
+                 const char *const message, af_err err, stacktrace st)
     : logic_error(message)
     , functionName(func)
     , fileName(file)
@@ -43,8 +45,7 @@ AfError::AfError(const char *const func, const char *const file, const int line,
     , error(err) {}
 
 AfError::AfError(string func, string file, const int line,
-                 const string &message, af_err err,
-                 boost::stacktrace::stacktrace st)
+                 const string &message, af_err err, stacktrace st)
     : logic_error(message)
     , functionName(move(func))
     , fileName(move(file))
@@ -64,7 +65,7 @@ AfError::~AfError() noexcept = default;
 
 TypeError::TypeError(const char *const func, const char *const file,
                      const int line, const int index, const af_dtype type,
-                     boost::stacktrace::stacktrace st)
+                     stacktrace st)
     : AfError(func, file, line, "Invalid data type", AF_ERR_TYPE, move(st))
     , errTypeName(getName(type))
     , argIndex(index) {}
@@ -75,8 +76,7 @@ int TypeError::getArgIndex() const noexcept { return argIndex; }
 
 ArgumentError::ArgumentError(const char *const func, const char *const file,
                              const int line, const int index,
-                             const char *const expectString,
-                             boost::stacktrace::stacktrace st)
+                             const char *const expectString, stacktrace st)
     : AfError(func, file, line, "Invalid argument", AF_ERR_ARG, move(st))
     , expected(expectString)
     , argIndex(index) {}
@@ -89,7 +89,7 @@ int ArgumentError::getArgIndex() const noexcept { return argIndex; }
 
 SupportError::SupportError(const char *const func, const char *const file,
                            const int line, const char *const back,
-                           boost::stacktrace::stacktrace st)
+                           stacktrace st)
     : AfError(func, file, line, "Unsupported Error", AF_ERR_NOT_SUPPORTED,
               move(st))
     , backend(back) {}
@@ -99,7 +99,7 @@ const string &SupportError::getBackendName() const noexcept { return backend; }
 DimensionError::DimensionError(const char *const func, const char *const file,
                                const int line, const int index,
                                const char *const expectString,
-                               const boost::stacktrace::stacktrace &st)
+                               const stacktrace &st)
     : AfError(func, file, line, "Invalid size", AF_ERR_SIZE, st)
     , expected(expectString)
     , argIndex(index) {}
@@ -111,7 +111,7 @@ const string &DimensionError::getExpectedCondition() const noexcept {
 int DimensionError::getArgIndex() const noexcept { return argIndex; }
 
 af_err set_global_error_string(const string &msg, af_err err) {
-    std::string perr = getEnvVar("AF_PRINT_ERRORS");
+    string perr = getEnvVar("AF_PRINT_ERRORS");
     if (!perr.empty()) {
         if (perr != "0") { fprintf(stderr, "%s\n", msg.c_str()); }
     }
diff --git a/src/backend/common/graphics_common.cpp b/src/backend/common/graphics_common.cpp
index d1a572a153..75fe4c002c 100644
--- a/src/backend/common/graphics_common.cpp
+++ b/src/backend/common/graphics_common.cpp
@@ -15,6 +15,7 @@
 #include <mutex>
 #include <utility>
 
+using common::getEnvVar;
 using std::make_pair;
 using std::string;
 
diff --git a/src/backend/common/half.cpp b/src/backend/common/half.cpp
index 96c5ef4ff9..3e41699c72 100644
--- a/src/backend/common/half.cpp
+++ b/src/backend/common/half.cpp
@@ -1,9 +1,15 @@
 
 #include <common/half.hpp>
+#include <common/util.hpp>
 
 namespace common {
 std::ostream &operator<<(std::ostream &os, const half &val) {
     os << float(val);
     return os;
 }
+
+template<>
+std::string toString(const half val) {
+    return common::toString(static_cast<float>(val));
+}
 }  // namespace common
diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index 7904598eb8..bd5f143c28 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -47,6 +47,7 @@ using uint16_t = unsigned short;
 #include <type_traits>
 
 #include <limits>
+
 #endif
 
 namespace common {
diff --git a/src/backend/common/jit/Node.cpp b/src/backend/common/jit/Node.cpp
index c637926d79..71d88424f5 100644
--- a/src/backend/common/jit/Node.cpp
+++ b/src/backend/common/jit/Node.cpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include <common/defines.hpp>
+#include <common/deterministicHash.hpp>
 #include <common/jit/Node.hpp>
 #include <common/util.hpp>
 
diff --git a/src/backend/common/jit/NodeIO.hpp b/src/backend/common/jit/NodeIO.hpp
index 050c8e3a7c..bd4346f465 100644
--- a/src/backend/common/jit/NodeIO.hpp
+++ b/src/backend/common/jit/NodeIO.hpp
@@ -9,10 +9,9 @@
 
 #pragma once
 #include <common/jit/Node.hpp>
-#include <common/util.hpp>
 #include <spdlog/fmt/bundled/format.h>
 
-#include <common/TemplateArg.hpp>
+#include <common/util.hpp>
 
 template<>
 struct fmt::formatter<af::dtype> : fmt::formatter<char> {
@@ -69,9 +68,9 @@ struct fmt::formatter<common::Node> {
             if (isBuffer(node)) {
                 format_to(ctx.out(), "buffer ");
             } else if (isScalar(node)) {
-                format_to(ctx.out(), "scalar ", getOpEnumStr(node.getOp()));
+                format_to(ctx.out(), "scalar ", common::toString(node.getOp()));
             } else {
-                format_to(ctx.out(), "{} ", getOpEnumStr(node.getOp()));
+                format_to(ctx.out(), "{} ", common::toString(node.getOp()));
             }
         }
         if (type) format_to(ctx.out(), "{} ", node.getType());
diff --git a/src/backend/common/kernel_cache.cpp b/src/backend/common/kernel_cache.cpp
index ff2b53c787..1fb81ad293 100644
--- a/src/backend/common/kernel_cache.cpp
+++ b/src/backend/common/kernel_cache.cpp
@@ -10,13 +10,12 @@
 #if !defined(AF_CPU) && !defined(AF_ONEAPI)
 
 #include <common/compile_module.hpp>
+#include <common/deterministicHash.hpp>
 #include <common/kernel_cache.hpp>
-#include <common/util.hpp>
 #include <device_manager.hpp>
 #include <platform.hpp>
 
 #include <nonstd/span.hpp>
-#include <algorithm>
 #include <shared_mutex>
 #include <string>
 #include <unordered_map>
diff --git a/src/backend/common/kernel_cache.hpp b/src/backend/common/kernel_cache.hpp
index b021919a21..eb1b90f47b 100644
--- a/src/backend/common/kernel_cache.hpp
+++ b/src/backend/common/kernel_cache.hpp
@@ -14,6 +14,7 @@
 #include <Kernel.hpp>
 #include <Module.hpp>
 #include <backend.hpp>
+#include <common/Source.hpp>
 #include <common/TemplateTypename.hpp>
 #include <common/util.hpp>
 
diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp
index bac4cb573d..f6d39a864e 100644
--- a/src/backend/common/util.cpp
+++ b/src/backend/common/util.cpp
@@ -15,39 +15,53 @@
 #include <unistd.h>
 #endif
 
-#ifndef NOSPDLOG
 #include <common/Logger.hpp>
-#endif
-
+#include <common/TemplateArg.hpp>
 #include <common/defines.hpp>
 #include <common/util.hpp>
+#include <optypes.hpp>
 #include <af/defines.h>
 
 #include <nonstd/span.hpp>
 #include <sys/stat.h>
+
 #include <algorithm>
+#include <array>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <fstream>
 #include <numeric>
+#include <sstream>
 #include <string>
 #include <thread>
 #include <vector>
 
+#ifdef __has_include
+#if __has_include(<charconv>)
+#include <charconv>
+#endif
+#if __has_include(<version>)
+#include <version>
+#endif
+#endif
+
 using nonstd::span;
 using std::accumulate;
+using std::array;
 using std::hash;
 using std::ofstream;
 using std::once_flag;
 using std::rename;
 using std::size_t;
 using std::string;
+using std::stringstream;
 using std::thread;
 using std::to_string;
 using std::uint8_t;
 using std::vector;
 
+namespace common {
 // http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring/217605#217605
 // trim from start
 string& ltrim(string& s) {
@@ -237,31 +251,273 @@ string makeTempFilename() {
         hash<string>{}(to_string(threadID) + "_" + to_string(fileCount)));
 }
 
-size_t deterministicHash(const void* data, size_t byteSize, size_t prevHash) {
-    // Fowler-Noll-Vo "1a" 32 bit hash
-    // https://en.wikipedia.org/wiki/Fowler-Noll-Vo_hash_function
-    const auto* byteData = static_cast<const uint8_t*>(data);
-    return accumulate(
-        byteData, byteData + byteSize, prevHash,
-        [&](size_t hash, uint8_t data) { return (hash ^ data) * FNV1A_PRIME; });
+template<typename T>
+string toString(T value) {
+#ifdef __cpp_lib_to_chars
+    array<char, 128> out;
+    if (auto [ptr, ec] = std::to_chars(out.data(), out.data() + 128, value);
+        ec == std::errc()) {
+        return string(out.data(), ptr);
+    } else {
+        return string("#error invalid conversion");
+    }
+#else
+    stringstream ss;
+    ss.imbue(std::locale::classic());
+    ss << value;
+    return ss.str();
+#endif
+}
+
+template string toString<int>(int);
+template string toString<unsigned short>(unsigned short);
+template string toString<short>(short);
+template string toString<unsigned char>(unsigned char);
+template string toString<char>(char);
+template string toString<long>(long);
+template string toString<long long>(long long);
+template string toString<unsigned>(unsigned);
+template string toString<unsigned long>(unsigned long);
+template string toString<unsigned long long>(unsigned long long);
+template string toString<float>(float);
+template string toString<double>(double);
+template string toString<long double>(long double);
+
+template<>
+string toString(TemplateArg arg) {
+    return arg._tparam;
+}
+
+template<>
+string toString(bool val) {
+    return string(val ? "true" : "false");
+}
+
+template<>
+string toString(const char* str) {
+    return string(str);
+}
+
+template<>
+string toString(const string str) {
+    return str;
+}
+
+template<>
+string toString(af_op_t val) {
+    const char* retVal = NULL;
+#define CASE_STMT(v) \
+    case v: retVal = #v; break
+    switch (val) {
+        CASE_STMT(af_add_t);
+        CASE_STMT(af_sub_t);
+        CASE_STMT(af_mul_t);
+        CASE_STMT(af_div_t);
+
+        CASE_STMT(af_and_t);
+        CASE_STMT(af_or_t);
+        CASE_STMT(af_eq_t);
+        CASE_STMT(af_neq_t);
+        CASE_STMT(af_lt_t);
+        CASE_STMT(af_le_t);
+        CASE_STMT(af_gt_t);
+        CASE_STMT(af_ge_t);
+
+        CASE_STMT(af_bitnot_t);
+        CASE_STMT(af_bitor_t);
+        CASE_STMT(af_bitand_t);
+        CASE_STMT(af_bitxor_t);
+        CASE_STMT(af_bitshiftl_t);
+        CASE_STMT(af_bitshiftr_t);
+
+        CASE_STMT(af_min_t);
+        CASE_STMT(af_max_t);
+        CASE_STMT(af_cplx2_t);
+        CASE_STMT(af_atan2_t);
+        CASE_STMT(af_pow_t);
+        CASE_STMT(af_hypot_t);
+
+        CASE_STMT(af_sin_t);
+        CASE_STMT(af_cos_t);
+        CASE_STMT(af_tan_t);
+        CASE_STMT(af_asin_t);
+        CASE_STMT(af_acos_t);
+        CASE_STMT(af_atan_t);
+
+        CASE_STMT(af_sinh_t);
+        CASE_STMT(af_cosh_t);
+        CASE_STMT(af_tanh_t);
+        CASE_STMT(af_asinh_t);
+        CASE_STMT(af_acosh_t);
+        CASE_STMT(af_atanh_t);
+
+        CASE_STMT(af_exp_t);
+        CASE_STMT(af_expm1_t);
+        CASE_STMT(af_erf_t);
+        CASE_STMT(af_erfc_t);
+
+        CASE_STMT(af_log_t);
+        CASE_STMT(af_log10_t);
+        CASE_STMT(af_log1p_t);
+        CASE_STMT(af_log2_t);
+
+        CASE_STMT(af_sqrt_t);
+        CASE_STMT(af_cbrt_t);
+
+        CASE_STMT(af_abs_t);
+        CASE_STMT(af_cast_t);
+        CASE_STMT(af_cplx_t);
+        CASE_STMT(af_real_t);
+        CASE_STMT(af_imag_t);
+        CASE_STMT(af_conj_t);
+
+        CASE_STMT(af_floor_t);
+        CASE_STMT(af_ceil_t);
+        CASE_STMT(af_round_t);
+        CASE_STMT(af_trunc_t);
+        CASE_STMT(af_signbit_t);
+
+        CASE_STMT(af_rem_t);
+        CASE_STMT(af_mod_t);
+
+        CASE_STMT(af_tgamma_t);
+        CASE_STMT(af_lgamma_t);
+
+        CASE_STMT(af_notzero_t);
+
+        CASE_STMT(af_iszero_t);
+        CASE_STMT(af_isinf_t);
+        CASE_STMT(af_isnan_t);
+
+        CASE_STMT(af_sigmoid_t);
+
+        CASE_STMT(af_noop_t);
+
+        CASE_STMT(af_select_t);
+        CASE_STMT(af_not_select_t);
+        CASE_STMT(af_rsqrt_t);
+        CASE_STMT(af_moddims_t);
+
+        CASE_STMT(af_none_t);
+    }
+#undef CASE_STMT
+    return retVal;
+}
+
+template<>
+string toString(af_interp_type p) {
+    const char* retVal = NULL;
+#define CASE_STMT(v) \
+    case v: retVal = #v; break
+    switch (p) {
+        CASE_STMT(AF_INTERP_NEAREST);
+        CASE_STMT(AF_INTERP_LINEAR);
+        CASE_STMT(AF_INTERP_BILINEAR);
+        CASE_STMT(AF_INTERP_CUBIC);
+        CASE_STMT(AF_INTERP_LOWER);
+        CASE_STMT(AF_INTERP_LINEAR_COSINE);
+        CASE_STMT(AF_INTERP_BILINEAR_COSINE);
+        CASE_STMT(AF_INTERP_BICUBIC);
+        CASE_STMT(AF_INTERP_CUBIC_SPLINE);
+        CASE_STMT(AF_INTERP_BICUBIC_SPLINE);
+    }
+#undef CASE_STMT
+    return retVal;
 }
 
-size_t deterministicHash(const string& data, const size_t prevHash) {
-    return deterministicHash(data.data(), data.size(), prevHash);
+template<>
+string toString(af_border_type p) {
+    const char* retVal = NULL;
+#define CASE_STMT(v) \
+    case v: retVal = #v; break
+    switch (p) {
+        CASE_STMT(AF_PAD_ZERO);
+        CASE_STMT(AF_PAD_SYM);
+        CASE_STMT(AF_PAD_CLAMP_TO_EDGE);
+        CASE_STMT(AF_PAD_PERIODIC);
+    }
+#undef CASE_STMT
+    return retVal;
 }
 
-size_t deterministicHash(span<const string> list, const size_t prevHash) {
-    size_t hash = prevHash;
-    for (auto s : list) { hash = deterministicHash(s.data(), s.size(), hash); }
-    return hash;
+template<>
+string toString(af_moment_type p) {
+    const char* retVal = NULL;
+#define CASE_STMT(v) \
+    case v: retVal = #v; break
+    switch (p) {
+        CASE_STMT(AF_MOMENT_M00);
+        CASE_STMT(AF_MOMENT_M01);
+        CASE_STMT(AF_MOMENT_M10);
+        CASE_STMT(AF_MOMENT_M11);
+        CASE_STMT(AF_MOMENT_FIRST_ORDER);
+    }
+#undef CASE_STMT
+    return retVal;
 }
 
-size_t deterministicHash(span<const common::Source> list) {
-    // Combine the different source codes, via their hashes
-    size_t hash = FNV1A_BASE_OFFSET;
-    for (auto s : list) {
-        size_t h = s.hash ? s.hash : deterministicHash(s.ptr, s.length);
-        hash     = deterministicHash(&h, sizeof(size_t), hash);
+template<>
+string toString(af_match_type p) {
+    const char* retVal = NULL;
+#define CASE_STMT(v) \
+    case v: retVal = #v; break
+    switch (p) {
+        CASE_STMT(AF_SAD);
+        CASE_STMT(AF_ZSAD);
+        CASE_STMT(AF_LSAD);
+        CASE_STMT(AF_SSD);
+        CASE_STMT(AF_ZSSD);
+        CASE_STMT(AF_LSSD);
+        CASE_STMT(AF_NCC);
+        CASE_STMT(AF_ZNCC);
+        CASE_STMT(AF_SHD);
     }
-    return hash;
+#undef CASE_STMT
+    return retVal;
 }
+
+template<>
+string toString(af_flux_function p) {
+    const char* retVal = NULL;
+#define CASE_STMT(v) \
+    case v: retVal = #v; break
+    switch (p) {
+        CASE_STMT(AF_FLUX_QUADRATIC);
+        CASE_STMT(AF_FLUX_EXPONENTIAL);
+        CASE_STMT(AF_FLUX_DEFAULT);
+    }
+#undef CASE_STMT
+    return retVal;
+}
+
+template<>
+string toString(AF_BATCH_KIND val) {
+    const char* retVal = NULL;
+#define CASE_STMT(v) \
+    case v: retVal = #v; break
+    switch (val) {
+        CASE_STMT(AF_BATCH_NONE);
+        CASE_STMT(AF_BATCH_LHS);
+        CASE_STMT(AF_BATCH_RHS);
+        CASE_STMT(AF_BATCH_SAME);
+        CASE_STMT(AF_BATCH_DIFF);
+        CASE_STMT(AF_BATCH_UNSUPPORTED);
+    }
+#undef CASE_STMT
+    return retVal;
+}
+
+template<>
+string toString(af_homography_type val) {
+    const char* retVal = NULL;
+#define CASE_STMT(v) \
+    case v: retVal = #v; break
+    switch (val) {
+        CASE_STMT(AF_HOMOGRAPHY_RANSAC);
+        CASE_STMT(AF_HOMOGRAPHY_LMEDS);
+    }
+#undef CASE_STMT
+    return retVal;
+}
+
+}  // namespace common
diff --git a/src/backend/common/util.hpp b/src/backend/common/util.hpp
index fb6c195af6..896223e140 100644
--- a/src/backend/common/util.hpp
+++ b/src/backend/common/util.hpp
@@ -10,21 +10,12 @@
 /// This file contains platform independent utility functions
 #pragma once
 
+#include <optypes.hpp>
 #include <af/defines.h>
 
-#include <nonstd/span.hpp>
-#include <iosfwd>
 #include <string>
-#include <vector>
 
 namespace common {
-struct Source {
-    const char* ptr;           // Pointer to the kernel source
-    const std::size_t length;  // Length of the kernel source
-    const std::size_t hash;    // hash value for the source *ptr;
-};
-}  // namespace common
-
 /// The environment variable that determines where the runtime kernels
 /// will be stored on the file system
 constexpr const char* JIT_KERNEL_CACHE_DIRECTORY_ENV_NAME =
@@ -62,25 +53,9 @@ std::string makeTempFilename();
 
 const char* getName(af_dtype type);
 
-/// Return the FNV-1a hash of the provided bata.
-///
-/// \param[in] data Binary data to hash
-/// \param[in] byteSize Size of the data in bytes
-/// \param[in] optional prevHash Hash of previous parts when string is split
-///
-/// \returns An unsigned integer representing the hash of the data
-constexpr std::size_t FNV1A_BASE_OFFSET = 0x811C9DC5;
-constexpr std::size_t FNV1A_PRIME       = 0x01000193;
-std::size_t deterministicHash(const void* data, std::size_t byteSize,
-                              const std::size_t prevHash = FNV1A_BASE_OFFSET);
-
-// This is just a wrapper around the above function.
-std::size_t deterministicHash(const std::string& data,
-                              const std::size_t prevHash = FNV1A_BASE_OFFSET);
-
-// This concatenates strings in the vector and computes hash
-std::size_t deterministicHash(nonstd::span<const std::string> list,
-                              const std::size_t prevHash = FNV1A_BASE_OFFSET);
-
-// This concatenates hashes of multiple sources
-std::size_t deterministicHash(nonstd::span<const common::Source> list);
+std::string getOpEnumStr(af_op_t val);
+
+template<typename T>
+std::string toString(T value);
+
+}  // namespace common
diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index 3f83956b91..5bb28a41ec 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -21,6 +21,8 @@
 #include <sstream>
 #include <string>
 
+using common::getEnvVar;
+using common::ltrim;
 using common::memory::MemoryManagerBase;
 using std::endl;
 using std::ostringstream;
diff --git a/src/backend/cpu/queue.hpp b/src/backend/cpu/queue.hpp
index 2a0db9d638..97142f4f1a 100644
--- a/src/backend/cpu/queue.hpp
+++ b/src/backend/cpu/queue.hpp
@@ -56,7 +56,7 @@ class queue {
     queue()
         : count(0)
         , sync_calls(__SYNCHRONOUS_ARCH == 1 ||
-                     getEnvVar("AF_SYNCHRONOUS_CALLS") == "1") {}
+                     common::getEnvVar("AF_SYNCHRONOUS_CALLS") == "1") {}
 
     template<typename F, typename... Args>
     void enqueue(const F func, Args &&...args) {
diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp
index ee10077477..3f5bd17d84 100644
--- a/src/backend/cuda/compile_module.cpp
+++ b/src/backend/cuda/compile_module.cpp
@@ -12,6 +12,7 @@
 
 #include <Module.hpp>
 #include <common/Logger.hpp>
+#include <common/deterministicHash.hpp>
 #include <common/internal_enums.hpp>
 #include <common/util.hpp>
 #include <device_manager.hpp>
diff --git a/src/backend/cuda/cudnnModule.cpp b/src/backend/cuda/cudnnModule.cpp
index b76b0c65fe..4a2f3e792c 100644
--- a/src/backend/cuda/cudnnModule.cpp
+++ b/src/backend/cuda/cudnnModule.cpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <tuple>
 
+using common::int_version_to_string;
 using common::Version;
 using std::make_tuple;
 using std::string;
diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 221534f6dc..f556d08cce 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -46,6 +46,8 @@
 #include <thread>
 #include <utility>
 
+using common::getEnvVar;
+using common::int_version_to_string;
 using std::begin;
 using std::end;
 using std::find;
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 37ff605cb4..02cf3c367d 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 #include <Kernel.hpp>
+#include <common/deterministicHash.hpp>
 #include <common/half.hpp>
 #include <common/jit/ModdimNode.hpp>
 #include <common/jit/Node.hpp>
@@ -34,6 +35,7 @@
 #include <vector>
 
 using common::findModule;
+using common::getEnvVar;
 using common::getFuncName;
 using common::half;
 using common::ModdimNode;
@@ -42,6 +44,7 @@ using common::Node_ids;
 using common::Node_map_t;
 using common::Node_ptr;
 using common::NodeIterator;
+using common::saveKernel;
 
 using std::array;
 using std::equal;
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index fa412101f0..13d10564bf 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -60,6 +60,8 @@ using std::to_string;
 using std::unique_ptr;
 using std::vector;
 
+using common::getEnvVar;
+using common::int_version_to_string;
 using common::unique_handle;
 using common::memory::MemoryManagerBase;
 using cuda::Allocator;
diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp
index 4a85ce292e..83d66eb740 100644
--- a/src/backend/opencl/compile_module.cpp
+++ b/src/backend/opencl/compile_module.cpp
@@ -13,6 +13,7 @@
 #include <cl2hpp.hpp>
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
+#include <common/deterministicHash.hpp>
 #include <common/util.hpp>
 #include <debug_opencl.hpp>
 #include <err_opencl.hpp>
@@ -31,6 +32,7 @@
 
 using cl::Error;
 using cl::Program;
+using common::getEnvVar;
 using common::loggerFactory;
 using fmt::format;
 using nonstd::span;
diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index 6452ee590e..0a543f4297 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -44,6 +44,7 @@ using cl::CommandQueue;
 using cl::Context;
 using cl::Device;
 using cl::Platform;
+using common::getEnvVar;
 using std::begin;
 using std::end;
 using std::find;
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index d475f32b71..9a49c8c5f7 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 #include <common/compile_module.hpp>
+#include <common/deterministicHash.hpp>
 #include <common/jit/ModdimNode.hpp>
 #include <common/jit/Node.hpp>
 #include <common/jit/NodeIterator.hpp>
@@ -40,6 +41,7 @@ using common::Node_ids;
 using common::Node_map_t;
 using common::Node_ptr;
 using common::NodeIterator;
+using common::saveKernel;
 
 using cl::Kernel;
 using cl::NDRange;
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 0f0f19764b..6bcc2e55ae 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -66,6 +66,8 @@ using std::to_string;
 using std::unique_ptr;
 using std::vector;
 
+using common::getEnvVar;
+using common::ltrim;
 using common::memory::MemoryManagerBase;
 using opencl::Allocator;
 using opencl::AllocatorPinned;
diff --git a/src/backend/opencl/types.cpp b/src/backend/opencl/types.cpp
index a7d255a987..aba15fe693 100644
--- a/src/backend/opencl/types.cpp
+++ b/src/backend/opencl/types.cpp
@@ -10,6 +10,7 @@
 #include <types.hpp>
 
 #include <common/half.hpp>
+#include <common/util.hpp>
 #include <type_util.hpp>
 
 #include <cmath>
@@ -17,35 +18,39 @@
 #include <string>
 
 using common::half;
+using common::toString;
+
+using std::isinf;
+using std::stringstream;
 
 namespace opencl {
 
 template<typename T>
 inline std::string ToNumStr<T>::operator()(T val) {
     ToNum<T> toNum;
-    return std::to_string(toNum(val));
+    return toString(toNum(val));
 }
 
 template<>
 std::string ToNumStr<float>::operator()(float val) {
     static const char *PINF = "+INFINITY";
     static const char *NINF = "-INFINITY";
-    if (std::isinf(val)) { return val < 0.f ? NINF : PINF; }
-    return std::to_string(val);
+    if (isinf(val)) { return val < 0.f ? NINF : PINF; }
+    return toString(val);
 }
 
 template<>
 std::string ToNumStr<double>::operator()(double val) {
     static const char *PINF = "+INFINITY";
     static const char *NINF = "-INFINITY";
-    if (std::isinf(val)) { return val < 0. ? NINF : PINF; }
-    return std::to_string(val);
+    if (isinf(val)) { return val < 0. ? NINF : PINF; }
+    return toString(val);
 }
 
 template<>
 std::string ToNumStr<cfloat>::operator()(cfloat val) {
     ToNumStr<float> realStr;
-    std::stringstream s;
+    stringstream s;
     s << "{" << realStr(val.s[0]) << "," << realStr(val.s[1]) << "}";
     return s.str();
 }
@@ -53,7 +58,7 @@ std::string ToNumStr<cfloat>::operator()(cfloat val) {
 template<>
 std::string ToNumStr<cdouble>::operator()(cdouble val) {
     ToNumStr<double> realStr;
-    std::stringstream s;
+    stringstream s;
     s << "{" << realStr(val.s[0]) << "," << realStr(val.s[1]) << "}";
     return s.str();
 }
@@ -64,8 +69,8 @@ std::string ToNumStr<half>::operator()(half val) {
     using namespace common;
     static const char *PINF = "+INFINITY";
     static const char *NINF = "-INFINITY";
-    if (common::isinf(val)) { return val < 0.f ? NINF : PINF; }
-    return common::to_string(val);
+    if (isinf(val)) { return val < 0.f ? NINF : PINF; }
+    return toString(val);
 }
 
 template<>
@@ -73,8 +78,8 @@ template<>
 std::string ToNumStr<half>::operator()<float>(float val) {
     static const char *PINF = "+INFINITY";
     static const char *NINF = "-INFINITY";
-    if (common::isinf(half(val))) { return val < 0.f ? NINF : PINF; }
-    return std::to_string(val);
+    if (isinf(half(val))) { return val < 0.f ? NINF : PINF; }
+    return toString(val);
 }
 
 #define INSTANTIATE(TYPE) template struct ToNumStr<TYPE>

From a4bb0a5f19bc882c8fb952f7c8c5289cbde0a8cb Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 19 Nov 2022 09:57:47 -0500
Subject: [PATCH 137/473] Add compilers to GitHub actions matrix. Update Ubuntu
 versions

---
 .github/workflows/unix_cpu_build.yml | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 114799bbca..6085718c1d 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -20,11 +20,16 @@ jobs:
             matrix:
                 blas_backend: [Atlas, MKL, OpenBLAS]
                 os: [ubuntu-18.04, ubuntu-20.04, macos-latest]
+                compiler: [gcc, clang, icx]
                 exclude:
                     - os: macos-latest
                       blas_backend: Atlas
                     - os: macos-latest
                       blas_backend: MKL
+                    - blas_backend: Atlas
+                      compiler: icx
+                    - blas_backend: OpenBLAS
+                      compiler: icx
         steps:
             - name: Checkout Repository
               uses: actions/checkout@master
@@ -43,6 +48,7 @@ jobs:
               if: matrix.os != 'macos-latest'
               env:
                   OS_NAME: ${{ matrix.os }}
+                  CC: ${{ matrix.compiler }}
               run: |
                   cmake_suffix=$(if [ $OS_NAME == 'macos-latest' ]; then echo "Darwin-x86_64"; else echo "Linux-x86_64"; fi)
                   cmake_url=$(echo "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-${cmake_suffix}.tar.gz")
@@ -54,6 +60,17 @@ jobs:
                   cmake_osx_dir=$(echo "${cmake_install_dir}/CMake.app/Contents/bin")
                   cmake_dir=$(if [ $OS_NAME == 'macos-latest' ]; then echo "${cmake_osx_dir}"; else echo "${cmake_lnx_dir}"; fi)
                   echo "CMAKE_PROGRAM=$(pwd)/${cmake_dir}/cmake" >> $GITHUB_ENV
+                  case "$CC" in
+                    'gcc')
+                        echo "CXX=g++" >> $GITHUB_ENV
+                        ;;
+                    'clang')
+                        echo "CXX=clang++" >> $GITHUB_ENV
+                        ;;
+                    'icx')
+                        echo "CXX=icpx" >> $GITHUB_ENV
+                        ;;
+                  esac
 
             - name: Install Dependencies for Macos
               if: matrix.os == 'macos-latest'
@@ -62,7 +79,7 @@ jobs:
                   echo "CMAKE_PROGRAM=cmake" >> $GITHUB_ENV
 
             - name: Install Common Dependencies for Ubuntu
-              if: matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-18.04'
+              if: matrix.os == 'ubuntu-18.04' || matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04'
               run: |
                   sudo add-apt-repository ppa:mhier/libboost-latest
                   sudo apt-get -qq update
@@ -78,12 +95,15 @@ jobs:
 
             - name: Install MKL for Ubuntu
               if: matrix.os != 'macos-latest' && matrix.blas_backend == 'MKL'
+              env:
+                  CC: ${{ matrix.compiler }}
               run: |
                   wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
                   sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
                   sudo sh -c 'echo deb https://apt.repos.intel.com/oneapi all main > /etc/apt/sources.list.d/oneAPI.list'
                   sudo apt-get -qq update
                   sudo apt-get install -y intel-oneapi-mkl-devel
+                  if [ "$CC" == 'icx' ]; then sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp; fi
                   echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> ${GITHUB_ENV}
 
             - name: Install OpenBLAS for Ubuntu
@@ -94,6 +114,8 @@ jobs:
               env:
                   USE_MKL: ${{ matrix.blas_backend == 'MKL' }}
                   BLAS_BACKEND: ${{ matrix.blas_backend }}
+                  CC: ${{ matrix.compiler }}
+                  OS_NAME: ${{ matrix.os }}
               run: |
                   ref=$(echo ${GITHUB_REF} | awk '/refs\/pull\/[0-9]+\/merge/{print $0}')
                   prnum=$(echo $ref | awk '{split($0, a, "/"); print a[3]}')
@@ -103,6 +125,7 @@ jobs:
                   backend=$(if [ "$USE_MKL" == 1 ]; then echo "Intel-MKL"; else echo "FFTW/LAPACK/BLAS"; fi)
                   buildname="$buildname-cpu-$BLAS_BACKEND"
                   cmake_rpath=$(if [ $OS_NAME == 'macos-latest' ]; then echo "-DCMAKE_INSTALL_RPATH=/opt/arrayfire/lib"; fi)
+                  if [ "$CC" == 'icx' ]; then source /opt/intel/oneapi/setvars.sh intel64; fi
                   mkdir build && cd build && unset VCPKG_ROOT
                   ${CMAKE_PROGRAM} -G Ninja \
                       -DCMAKE_MAKE_PROGRAM:FILEPATH=${GITHUB_WORKSPACE}/ninja \
@@ -115,6 +138,9 @@ jobs:
                   echo "CTEST_DASHBOARD=${dashboard}" >> $GITHUB_ENV
 
             - name: Build and Test
+              env:
+                  CC: ${{ matrix.compiler }}
               run: |
                   cd ${GITHUB_WORKSPACE}/build
+                  if [ "$CC" == 'icx' ]; then source /opt/intel/oneapi/setvars.sh intel64; fi
                   ctest -D Experimental --track ${CTEST_DASHBOARD} -T Test -T Submit -R cpu -j2

From 45b6a3f585e92190802f4d360d75c0a12caf4782 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 21 Nov 2022 19:39:46 -0500
Subject: [PATCH 138/473] Add support for fast math compiler flags when
 building ArrayFire

---
 CMakeLists.txt                   |  1 +
 CMakeModules/InternalUtils.cmake | 22 +++++++++++++++++++++-
 test/CMakeLists.txt              |  6 ++++++
 test/approx1.cpp                 |  6 ++++++
 test/half.cpp                    |  2 ++
 test/imageio.cpp                 |  6 +++---
 test/ireduce.cpp                 |  4 ++++
 test/meanvar.cpp                 |  4 ++--
 test/median.cpp                  |  5 +++--
 test/reduce.cpp                  |  9 +++++++++
 test/replace.cpp                 |  1 +
 test/select.cpp                  |  1 +
 test/testHelpers.hpp             | 15 ++++++++++++---
 test/threading.cpp               |  2 +-
 14 files changed, 72 insertions(+), 12 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 099e9a72ae..2424d9162f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -92,6 +92,7 @@ option(AF_WITH_STATIC_MKL "Link against static Intel MKL libraries" OFF)
 option(AF_WITH_STATIC_CUDA_NUMERIC_LIBS "Link libafcuda with static numeric libraries(cublas, cufft, etc.)" OFF)
 option(AF_WITH_SPDLOG_HEADER_ONLY "Build ArrayFire with header only version of spdlog" OFF)
 option(AF_WITH_FMT_HEADER_ONLY "Build ArrayFire with header only version of fmt" OFF)
+option(AF_WITH_FAST_MATH "Use lower precision but high performance numeric optimizations" OFF)
 
 if(AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   option(AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS "Prune CUDA static libraries to reduce binary size.(WARNING: May break some libs on older CUDA toolkits for some compute arch)" OFF)
diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index dde0756aaa..f5bb077e57 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -25,6 +25,13 @@ if(WIN32)
   check_cxx_compiler_flag(/permissive- cxx_compliance)
 endif()
 
+check_cxx_compiler_flag(-ffast-math has_cxx_fast_math)
+check_cxx_compiler_flag("-fp-model fast" has_cxx_fp_model)
+check_cxx_compiler_flag(-fno-errno-math has_cxx_no_errno_math)
+check_cxx_compiler_flag(-fno-trapping-math  has_cxx_no_trapping_math)
+check_cxx_compiler_flag(-fno-signed-zeros  has_cxx_no_signed_zeros)
+check_cxx_compiler_flag(-mno-ieee-fp has_cxx_no_ieee_fp)
+
 function(arrayfire_set_default_cxx_flags target)
   target_compile_options(${target}
     PRIVATE
@@ -51,7 +58,19 @@ function(arrayfire_set_default_cxx_flags target)
               # ignored attribute warnings in the OpenCL
               # headers
               $<$<BOOL:${has_ignored_attributes_flag}>:-Wno-ignored-attributes>
-              $<$<BOOL:${has_all_warnings_flag}>:-Wall>>
+              $<$<BOOL:${has_all_warnings_flag}>:-Wall>
+
+              $<$<BOOL:${AF_WITH_FAST_MATH}>:
+                  $<$<BOOL:${has_cxx_fast_math}>:-ffast-math>
+                  $<$<BOOL:${has_cxx_no_errno_math}>:-fno-errno-math>
+                  $<$<BOOL:${has_cxx_no_trapping_math}>:-fno-trapping-math>
+                  $<$<BOOL:${has_cxx_no_signed_zeros}>:-fno-signed-zeros>
+                  $<$<BOOL:${has_cxx_no_ieee_fp}>:-mno-ieee-fp>
+                  >
+
+              $<$<NOT:$<BOOL:${AF_WITH_FAST_MATH}>>:
+                    $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>>
+                      >
     )
 
   target_compile_definitions(${target}
@@ -65,6 +84,7 @@ function(arrayfire_set_default_cxx_flags target)
 
       $<$<BOOL:${AF_WITH_LOGGING}>:           AF_WITH_LOGGING>
       $<$<BOOL:${AF_CACHE_KERNELS_TO_DISK}>:  AF_CACHE_KERNELS_TO_DISK>
+      $<$<BOOL:${AF_WITH_FAST_MATH}>:         AF_WITH_FAST_MATH>
   )
 endfunction()
 
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 7fcc708d32..8492ca574c 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -124,6 +124,10 @@ target_include_directories(arrayfire_test
 # The tautological-constant-compare warning is always thrown for std::nan
 # and std::info calls. Its unnecessarily verbose.
 target_compile_options(arrayfire_test
+  PUBLIC
+    # Intel compilers use fast math by default and ignore special floating point
+    # values like NaN and Infs.
+    $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>
   PRIVATE
     $<$<BOOL:${has_tautological_constant_compare_flag}>:-Wno-tautological-constant-compare>
     $<$<CXX_COMPILER_ID:MSVC>: /bigobj
@@ -137,6 +141,8 @@ if(WIN32)
 endif()
 
 target_compile_definitions(arrayfire_test
+  PUBLIC
+    $<$<BOOL:${AF_WITH_FAST_MATH}>:AF_WITH_FAST_MATH>
   PRIVATE
     TEST_RESULT_IMAGE_DIR="${CMAKE_BINARY_DIR}/test/"
     USE_MTX)
diff --git a/test/approx1.cpp b/test/approx1.cpp
index ed7bf83066..143f66bd71 100644
--- a/test/approx1.cpp
+++ b/test/approx1.cpp
@@ -777,6 +777,9 @@ TEST(Approx1, CPPUniformInvalidStepSize) {
 // specified by the user, ArrayFire will assume a regular grid with a
 // starting index of 0 and a step value of 1.
 TEST(Approx1, CPPInfCheck) {
+#ifdef __INTEL_LLVM_COMPILER
+    SKIP_IF_FAST_MATH_ENABLED();
+#endif
     array sampled(seq(0.0, 5.0, 0.5));
     sampled(0) = af::Inf;
     seq xo(0.0, 2.0, 0.25);
@@ -799,6 +802,9 @@ TEST(Approx1, CPPInfCheck) {
 }
 
 TEST(Approx1, CPPUniformInfCheck) {
+#ifdef __INTEL_LLVM_COMPILER
+    SKIP_IF_FAST_MATH_ENABLED();
+#endif
     array sampled(seq(10.0, 50.0, 10.0));
     sampled(0) = af::Inf;
     seq xo(0.0, 8.0, 2.0);
diff --git a/test/half.cpp b/test/half.cpp
index 18fcdb4077..33ae4eae4a 100644
--- a/test/half.cpp
+++ b/test/half.cpp
@@ -87,6 +87,7 @@ TEST(Half, arith) {
 
 TEST(Half, isInf) {
     SUPPORTED_TYPE_CHECK(af_half);
+    SKIP_IF_FAST_MATH_ENABLED();
     half_float::half hinf = std::numeric_limits<half_float::half>::infinity();
 
     vector<half_float::half> input(2, half_float::half(0));
@@ -105,6 +106,7 @@ TEST(Half, isInf) {
 
 TEST(Half, isNan) {
     SUPPORTED_TYPE_CHECK(af_half);
+    SKIP_IF_FAST_MATH_ENABLED();
     half_float::half hnan = std::numeric_limits<half_float::half>::quiet_NaN();
 
     vector<half_float::half> input(2, half_float::half(0));
diff --git a/test/imageio.cpp b/test/imageio.cpp
index 6d3de9f45b..4869e50e15 100644
--- a/test/imageio.cpp
+++ b/test/imageio.cpp
@@ -289,7 +289,7 @@ TEST(ImageIO, SaveImage16CPP) {
     dim4 dims(16, 24, 3);
 
     array input     = randu(dims, u16);
-    array input_255 = (input / 257).as(u16);
+    array input_255 = floor(input.as(f32) / 257);
 
     std::string testname  = getTestName() + "_" + getBackendName();
     std::string imagename = "saveImage16CPP_" + testname + ".png";
@@ -297,9 +297,9 @@ TEST(ImageIO, SaveImage16CPP) {
     saveImage(imagename.c_str(), input);
 
     array img = loadImage(imagename.c_str(), true);
-    ASSERT_EQ(img.type(), f32);  // loadImage should always return float
 
-    ASSERT_FALSE(anyTrue<bool>(abs(img - input_255)));
+    ASSERT_EQ(img.type(), f32);  // loadImage should always return float
+    ASSERT_IMAGES_NEAR(input_255, img, 0.001);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/test/ireduce.cpp b/test/ireduce.cpp
index 1e55b9ac23..2ebd951d46 100644
--- a/test/ireduce.cpp
+++ b/test/ireduce.cpp
@@ -192,6 +192,7 @@ TEST(IndexedReduce, MaxReduceDimensionHasSingleValue) {
 }
 
 TEST(IndexedReduce, MinNaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     float test_data[] = {1.f, NAN, 5.f, 0.1f, NAN, -0.5f, NAN, 0.f};
     int rows          = 4;
     int cols          = 2;
@@ -218,6 +219,7 @@ TEST(IndexedReduce, MinNaN) {
 }
 
 TEST(IndexedReduce, MaxNaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     float test_data[] = {1.f, NAN, 5.f, 0.1f, NAN, -0.5f, NAN, 0.f};
     int rows          = 4;
     int cols          = 2;
@@ -244,6 +246,7 @@ TEST(IndexedReduce, MaxNaN) {
 }
 
 TEST(IndexedReduce, MinCplxNaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     float real_wnan_data[] = {0.005f, NAN, -6.3f, NAN,      -0.5f,
                               NAN,    NAN, 0.2f,  -1205.4f, 8.9f};
 
@@ -279,6 +282,7 @@ TEST(IndexedReduce, MinCplxNaN) {
 }
 
 TEST(IndexedReduce, MaxCplxNaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     float real_wnan_data[] = {0.005f, NAN, -6.3f, NAN,      -0.5f,
                               NAN,    NAN, 0.2f,  -1205.4f, 8.9f};
 
diff --git a/test/meanvar.cpp b/test/meanvar.cpp
index 81f3fb8099..bd79c4015a 100644
--- a/test/meanvar.cpp
+++ b/test/meanvar.cpp
@@ -131,7 +131,7 @@ class MeanVarTyped : public ::testing::TestWithParam<meanvar_test<T>> {
             ASSERT_VEC_ARRAY_NEAR(test.variance_, outDim, var, 0.5f);
         } else if (is_same_type<float, outType<T>>::value ||
                    is_same_type<cfloat, outType<T>>::value) {
-            ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 0.001f);
+            ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 0.0016f);
             ASSERT_VEC_ARRAY_NEAR(test.variance_, outDim, var, 0.2f);
         } else {
             ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 0.00001f);
@@ -171,7 +171,7 @@ class MeanVarTyped : public ::testing::TestWithParam<meanvar_test<T>> {
             ASSERT_VEC_ARRAY_NEAR(test.variance_, outDim, var, 0.5f);
         } else if (is_same_type<float, outType<T>>::value ||
                    is_same_type<cfloat, outType<T>>::value) {
-            ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 0.001f);
+            ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 0.0016f);
             ASSERT_VEC_ARRAY_NEAR(test.variance_, outDim, var, 0.2f);
         } else {
             ASSERT_VEC_ARRAY_NEAR(test.mean_, outDim, mean, 0.00001f);
diff --git a/test/median.cpp b/test/median.cpp
index 332dbe8d70..c55251e66c 100644
--- a/test/median.cpp
+++ b/test/median.cpp
@@ -93,20 +93,21 @@ void median_test(int nx, int ny = 1, int nz = 1, int nw = 1) {
 
     if (sa.dims(dim) % 2 == 1) {
         mSeq[dim] = mSeq[dim] - 1.0;
+        sa        = sa.as((af_dtype)dtype_traits<To>::af_type);
         verify    = sa(mSeq[0], mSeq[1], mSeq[2], mSeq[3]);
     } else {
         dim_t sdim[4] = {0};
         sdim[dim]     = 1;
         sa            = sa.as((af_dtype)dtype_traits<To>::af_type);
         array sas     = shift(sa, sdim[0], sdim[1], sdim[2], sdim[3]);
-        verify        = ((sa + sas) / 2)(mSeq[0], mSeq[1], mSeq[2], mSeq[3]);
+        verify = ((sa + sas) / To(2))(mSeq[0], mSeq[1], mSeq[2], mSeq[3]);
     }
 
     // Test Part
     array out = median(a, dim);
 
     ASSERT_EQ(out.dims() == verify.dims(), true);
-    ASSERT_NEAR(0, sum<double>(abs(out - verify)), 1e-5);
+    ASSERT_ARRAYS_EQ(verify, out);
 }
 
 #define MEDIAN_FLAT(To, Ti)                                                    \
diff --git a/test/reduce.cpp b/test/reduce.cpp
index 31845b8d0c..5afdf70648 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -779,6 +779,7 @@ TEST(ReduceByKey, countReduceByKey) {
 }
 
 TEST(ReduceByKey, ReduceByKeyNans) {
+    SKIP_IF_FAST_MATH_ENABLED();
     const static int testSz      = 8;
     const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
     const float testVals[testSz] = {0, 7, NAN, 6, 2, 5, 3, 4};
@@ -1072,6 +1073,7 @@ TYPED_TEST(Reduce, Test_Any_Global) {
 }
 
 TEST(MinMax, MinMaxNaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     const int num      = 10000;
     array A            = randu(num);
     A(where(A < 0.25)) = NaN;
@@ -1095,6 +1097,7 @@ TEST(MinMax, MinMaxNaN) {
 }
 
 TEST(MinMax, MinCplxNaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     float real_wnan_data[] = {0.005f, NAN, -6.3f, NAN,      -0.5f,
                               NAN,    NAN, 0.2f,  -1205.4f, 8.9f};
 
@@ -1122,6 +1125,7 @@ TEST(MinMax, MinCplxNaN) {
 }
 
 TEST(MinMax, MaxCplxNaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     // 4th element is unusually large to cover the case where
     //  one part holds the largest value among the array,
     //  and the other part is NaN.
@@ -1158,6 +1162,7 @@ TEST(MinMax, MaxCplxNaN) {
 }
 
 TEST(Count, NaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     const int num = 10000;
     array A       = round(5 * randu(num));
     array B       = A;
@@ -1168,6 +1173,7 @@ TEST(Count, NaN) {
 }
 
 TEST(Sum, NaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     const int num      = 10000;
     array A            = randu(num);
     A(where(A < 0.25)) = NaN;
@@ -1187,6 +1193,7 @@ TEST(Sum, NaN) {
 }
 
 TEST(Product, NaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     const int num = 5;
     array A       = randu(num);
     A(2)          = NaN;
@@ -1206,6 +1213,7 @@ TEST(Product, NaN) {
 }
 
 TEST(AnyAll, NaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     const int num = 10000;
     array A       = (randu(num) > 0.5).as(f32);
     array B       = A;
@@ -2263,6 +2271,7 @@ TYPED_TEST(Reduce, Test_Any_Global_Array) {
 
 
 TEST(Reduce, Test_Sum_Global_Array_nanval) {
+    SKIP_IF_FAST_MATH_ENABLED();
     const int num = 100000;
     array a = af::randn(num, 2, 34, 4);
     a(1, 0, 0, 0) = NAN;
diff --git a/test/replace.cpp b/test/replace.cpp
index 1d0a758489..14e679436b 100644
--- a/test/replace.cpp
+++ b/test/replace.cpp
@@ -113,6 +113,7 @@ TYPED_TEST(Replace, Simple) { replaceTest<TypeParam>(dim4(1024, 1024)); }
 TYPED_TEST(Replace, Scalar) { replaceScalarTest<TypeParam>(dim4(5, 5)); }
 
 TEST(Replace, NaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     dim4 dims(1000, 1250);
     dtype ty = f32;
 
diff --git a/test/select.cpp b/test/select.cpp
index a147bb3039..0b6724d8fa 100644
--- a/test/select.cpp
+++ b/test/select.cpp
@@ -130,6 +130,7 @@ TYPED_TEST(Select, LeftScalar) {
 }
 
 TEST(Select, NaN) {
+    SKIP_IF_FAST_MATH_ENABLED();
     dim4 dims(1000, 1250);
     dtype ty = f32;
 
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index faf7162a3b..69240883ac 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -223,9 +223,18 @@ bool noDoubleTests(af::dtype ty);
 
 bool noHalfTests(af::dtype ty);
 
-#define SUPPORTED_TYPE_CHECK(type)                                        \
-    if (noDoubleTests((af_dtype)af::dtype_traits<type>::af_type)) return; \
-    if (noHalfTests((af_dtype)af::dtype_traits<type>::af_type)) return;
+#define SUPPORTED_TYPE_CHECK(type)                                \
+    if (noDoubleTests((af_dtype)af::dtype_traits<type>::af_type)) \
+        GTEST_SKIP() << "Device doesn't support Doubles";         \
+    if (noHalfTests((af_dtype)af::dtype_traits<type>::af_type))   \
+        GTEST_SKIP() << "Device doesn't support Half";
+
+#ifdef AF_WITH_FAST_MATH
+#define SKIP_IF_FAST_MATH_ENABLED() \
+    GTEST_SKIP() << "ArrayFire compiled with AF_WITH_FAST_MATH"
+#else
+#define SKIP_IF_FAST_MATH_ENABLED()
+#endif
 
 bool noImageIOTests();
 
diff --git a/test/threading.cpp b/test/threading.cpp
index f26047ce95..96dd894e4f 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -53,7 +53,7 @@ void calc(ArithOp opcode, array op1, array op2, float outValue,
     vector<float> out(res.elements());
     res.host((void*)out.data());
 
-    for (unsigned i = 0; i < out.size(); ++i) ASSERT_EQ(out[i], outValue);
+    for (unsigned i = 0; i < out.size(); ++i) ASSERT_FLOAT_EQ(out[i], outValue);
     af::sync();
 }
 

From dd6ac75471e27e776b8820b5846b40ba06f817d3 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 21 Nov 2022 19:40:13 -0500
Subject: [PATCH 139/473] Convert vector to array in addInterpEnumOptions. Fix
 clang warnings

---
 CMakeModules/InternalUtils.cmake             |  5 +++++
 examples/benchmarks/pi.cpp                   |  4 ++--
 src/api/c/blas.cpp                           |  8 ++++----
 src/backend/common/TemplateArg.hpp           |  2 +-
 src/backend/cuda/jit.cpp                     |  2 ++
 src/backend/cuda/join.cpp                    |  2 +-
 src/backend/cuda/kernel/memcopy.hpp          |  8 ++++----
 src/backend/oneapi/device_manager.cpp        |  2 ++
 src/backend/oneapi/kernel/mean.hpp           | 20 ++++++++++----------
 src/backend/oneapi/kernel/reduce_all.hpp     |  6 +++---
 src/backend/oneapi/kernel/reduce_dim.hpp     |  2 +-
 src/backend/oneapi/kernel/reduce_first.hpp   |  2 +-
 src/backend/oneapi/kernel/where.hpp          |  6 +++---
 src/backend/oneapi/platform.cpp              |  4 +++-
 src/backend/opencl/jit.cpp                   |  2 ++
 src/backend/opencl/jit/kernel_generators.hpp |  5 +++--
 src/backend/opencl/join.cpp                  |  2 +-
 src/backend/opencl/kernel/homography.hpp     |  2 +-
 src/backend/opencl/kernel/interp.hpp         |  4 ++--
 src/backend/opencl/kernel/memcopy.hpp        |  7 ++++---
 test/CMakeLists.txt                          | 18 +++---------------
 test/arrayfire_test.cpp                      |  4 ++--
 22 files changed, 60 insertions(+), 57 deletions(-)

diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index f5bb077e57..c698e3d290 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -31,6 +31,8 @@ check_cxx_compiler_flag(-fno-errno-math has_cxx_no_errno_math)
 check_cxx_compiler_flag(-fno-trapping-math  has_cxx_no_trapping_math)
 check_cxx_compiler_flag(-fno-signed-zeros  has_cxx_no_signed_zeros)
 check_cxx_compiler_flag(-mno-ieee-fp has_cxx_no_ieee_fp)
+check_cxx_compiler_flag(-Wno-unqualified-std-cast-call has_cxx_unqualified_std_cast_call)
+check_cxx_compiler_flag(-Werror=reorder-ctor has_cxx_error_reorder_ctor)
 
 function(arrayfire_set_default_cxx_flags target)
   target_compile_options(${target}
@@ -46,6 +48,7 @@ function(arrayfire_set_default_cxx_flags target)
                                           /wd4668
                                           /wd4710
                                           /wd4505
+                                          /we5038
                                           /bigobj
                                           /EHsc
                                           # MSVC incorrectly sets the cplusplus to 199711L even if the compiler supports
@@ -59,6 +62,8 @@ function(arrayfire_set_default_cxx_flags target)
               # headers
               $<$<BOOL:${has_ignored_attributes_flag}>:-Wno-ignored-attributes>
               $<$<BOOL:${has_all_warnings_flag}>:-Wall>
+              $<$<BOOL:${has_cxx_unqualified_std_cast_call}>:-Wno-unqualified-std-cast-call>
+              $<$<BOOL:${has_cxx_error_reorder_ctor}>:-Werror=reorder-ctor>
 
               $<$<BOOL:${AF_WITH_FAST_MATH}>:
                   $<$<BOOL:${has_cxx_fast_math}>:-ffast-math>
diff --git a/examples/benchmarks/pi.cpp b/examples/benchmarks/pi.cpp
index 8913f36bc1..d4a550b78a 100644
--- a/examples/benchmarks/pi.cpp
+++ b/examples/benchmarks/pi.cpp
@@ -35,8 +35,8 @@ static double pi_device() {
 static double pi_host() {
     int count = 0;
     for (int i = 0; i < samples; ++i) {
-        float x = float(rand()) / RAND_MAX;
-        float y = float(rand()) / RAND_MAX;
+        float x = float(rand()) / float(RAND_MAX);
+        float y = float(rand()) / float(RAND_MAX);
         if (sqrt(x * x + y * y) < 1) count++;
     }
     return 4.0 * count / samples;
diff --git a/src/api/c/blas.cpp b/src/api/c/blas.cpp
index d34d55fd4a..0afd4f79b2 100644
--- a/src/api/c/blas.cpp
+++ b/src/api/c/blas.cpp
@@ -254,8 +254,8 @@ af_err af_matmul(af_array *out, const af_array lhs, const af_array rhs,
                 break;
             }
             case c32: {
-                cfloat alpha = {1.f, 0.f};
-                cfloat beta  = {0.f, 0.f};
+                cfloat alpha{1.f, 0.f};
+                cfloat beta{0.f, 0.f};
 
                 AF_CHECK(af_gemm(&gemm_out, optLhs, optRhs, &alpha, lhs, rhs,
                                  &beta));
@@ -269,8 +269,8 @@ af_err af_matmul(af_array *out, const af_array lhs, const af_array rhs,
                 break;
             }
             case c64: {
-                cdouble alpha = {1.0, 0.0};
-                cdouble beta  = {0.0, 0.0};
+                cdouble alpha{1.0, 0.0};
+                cdouble beta{0.0, 0.0};
                 AF_CHECK(af_gemm(&gemm_out, optLhs, optRhs, &alpha, lhs, rhs,
                                  &beta));
                 break;
diff --git a/src/backend/common/TemplateArg.hpp b/src/backend/common/TemplateArg.hpp
index 3a92bf643e..a26df012ca 100644
--- a/src/backend/common/TemplateArg.hpp
+++ b/src/backend/common/TemplateArg.hpp
@@ -16,7 +16,7 @@
 #include <utility>
 
 template<typename T>
-class TemplateTypename;
+struct TemplateTypename;
 
 struct TemplateArg {
     std::string _tparam;
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 02cf3c367d..4dab53a877 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -334,6 +334,7 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
     assert(outputs.size() == output_nodes.size());
     dim_t* outDims{outputs[0].dims};
     dim_t* outStrides{outputs[0].strides};
+#ifndef NDEBUG
     for_each(
         begin(outputs)++, end(outputs),
         [outDims, outStrides](Param<T>& output) {
@@ -341,6 +342,7 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                    equal(output.strides, output.strides + AF_MAX_DIMS,
                          outStrides));
         });
+#endif
 
     dim_t ndims{outDims[3] > 1   ? 4
                 : outDims[2] > 1 ? 3
diff --git a/src/backend/cuda/join.cpp b/src/backend/cuda/join.cpp
index a605867863..7f65773d0a 100644
--- a/src/backend/cuda/join.cpp
+++ b/src/backend/cuda/join.cpp
@@ -53,7 +53,7 @@ Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     //              will be called twice
     if (fdims.dims[jdim] == sdims.dims[jdim]) {
         const size_t L2CacheSize{getL2CacheSize(getActiveDeviceId())};
-        if (!(first.isReady() | second.isReady()) ||
+        if (!(first.isReady() || second.isReady()) ||
             (fdims.elements() * sizeof(T) * 2 * 2 < L2CacheSize)) {
             // Both arrays have same size & everything fits into the cache,
             // so treat in 1 JIT kernel, iso individual copies which is
diff --git a/src/backend/cuda/kernel/memcopy.hpp b/src/backend/cuda/kernel/memcopy.hpp
index 7a971bddb0..1592d62ec9 100644
--- a/src/backend/cuda/kernel/memcopy.hpp
+++ b/src/backend/cuda/kernel/memcopy.hpp
@@ -194,10 +194,10 @@ void copy(Param<outType> dst, CParam<inType> src, dim_t ondims,
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
     auto copy{common::getKernel(
-        th.loop0              ? "cuda::scaledCopyLoop0"
-        : th.loop2 | th.loop3 ? "cuda::scaledCopyLoop123"
-        : th.loop1            ? "cuda::scaledCopyLoop1"
-                              : "cuda::scaledCopy",
+        th.loop0                 ? "cuda::scaledCopyLoop0"
+        : (th.loop2 || th.loop3) ? "cuda::scaledCopyLoop123"
+        : th.loop1               ? "cuda::scaledCopyLoop1"
+                                 : "cuda::scaledCopy",
         std::array{copy_cuh_src},
         TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>(),
                      TemplateArg(same_dims), TemplateArg(factor != 1.0)))};
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index ed97248dcb..4588369637 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -30,10 +30,12 @@
 #include <string>
 #include <vector>
 
+using common::getEnvVar;
 using std::begin;
 using std::end;
 using std::find;
 using std::make_unique;
+using std::move;
 using std::string;
 using std::stringstream;
 using std::unique_ptr;
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 8a2e07d93c..17d2eb2164 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -74,21 +74,21 @@ class meanDimKernelSMEM {
                       local_accessor<compute_t<Tw>, 1> s_idx,
                       sycl::stream debug, bool input_weight, bool output_weight)
         : out_(out)
-        , oInfo_(oInfo)
         , owt_(owt)
-        , owInfo_(owInfo)
         , in_(in)
-        , iInfo_(iInfo)
         , iwt_(iwt)
+        , oInfo_(oInfo)
+        , owInfo_(owInfo)
+        , iInfo_(iInfo)
         , iwInfo_(iwInfo)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , offset_dim_(offset_dim)
         , s_val_(s_val)
         , s_idx_(s_idx)
-        , debug_(debug)
         , input_weight_(input_weight)
-        , output_weight_(output_weight) {}
+        , output_weight_(output_weight)
+        , debug_(debug) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -335,12 +335,12 @@ class meanFirstKernelSMEM {
                         sycl::stream debug, bool input_weight,
                         bool output_weight)
         : out_(out)
-        , oInfo_(oInfo)
         , owt_(owt)
-        , owInfo_(owInfo)
         , in_(in)
-        , iInfo_(iInfo)
         , iwt_(iwt)
+        , oInfo_(oInfo)
+        , owInfo_(owInfo)
+        , iInfo_(iInfo)
         , iwInfo_(iwInfo)
         , DIMX_(DIMX)
         , groups_x_(groups_x)
@@ -348,9 +348,9 @@ class meanFirstKernelSMEM {
         , repeat_(repeat)
         , s_val_(s_val)
         , s_idx_(s_idx)
-        , debug_(debug)
         , input_weight_(input_weight)
-        , output_weight_(output_weight) {}
+        , output_weight_(output_weight)
+        , debug_(debug) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index 8ad65d7948..1a318e8bc5 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -56,16 +56,16 @@ class reduceAllKernelSMEM {
                         local_accessor<compute_t<To>, 1> s_ptr,
                         local_accessor<bool, 1> amLast, sycl::stream debug)
         : out_(out)
-        , oInfo_(oInfo)
         , retCount_(retCount)
         , tmp_(tmp)
-        , tmpInfo_(tmpInfo)
         , in_(in)
+        , oInfo_(oInfo)
+        , tmpInfo_(tmpInfo)
         , iInfo_(iInfo)
         , DIMX_(DIMX)
+        , repeat_(repeat)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
-        , repeat_(repeat)
         , change_nan_(change_nan)
         , nanval_(nanval)
         , s_ptr_(s_ptr)
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index b5e4252651..6efb6851b1 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -49,8 +49,8 @@ class reduceDimKernelSMEM {
                         sycl::stream debug)
         : out_(out)
         , oInfo_(oInfo)
-        , in_(in)
         , iInfo_(iInfo)
+        , in_(in)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , offset_dim_(offset_dim)
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index 6bfe177148..a4094f8cb9 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -49,8 +49,8 @@ class reduceFirstKernelSMEM {
                           sycl::stream debug)
         : out_(out)
         , oInfo_(oInfo)
-        , in_(in)
         , iInfo_(iInfo)
+        , in_(in)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , repeat_(repeat)
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index cb8887fb84..4158641dce 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -38,12 +38,12 @@ class whereKernel {
                 read_accessor<T> in_acc, KParam iInfo, uint groups_x,
                 uint groups_y, uint lim, sycl::stream debug)
         : out_acc_(out_acc)
-        , oInfo_(oInfo)
         , otmp_acc_(otmp_acc)
-        , otInfo_(otInfo)
         , rtmp_acc_(rtmp_acc)
-        , rtInfo_(rtInfo)
         , in_acc_(in_acc)
+        , oInfo_(oInfo)
+        , otInfo_(otInfo)
+        , rtInfo_(rtInfo)
         , iInfo_(iInfo)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index d32d9e8d46..c16a4afff9 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -59,6 +59,8 @@ using std::to_string;
 using std::unique_ptr;
 using std::vector;
 
+using common::getEnvVar;
+using common::ltrim;
 using common::memory::MemoryManagerBase;
 using oneapi::Allocator;
 using oneapi::AllocatorPinned;
@@ -316,7 +318,7 @@ sycl::info::device_type getDeviceType() {
 }
 
 bool isHostUnifiedMemory(const sycl::device& device) {
-    return device.get_info<sycl::info::device::host_unified_memory>();
+    return device.has(sycl::aspect::usm_host_allocations);
 }
 
 bool OneAPICPUOffload(bool forceOffloadOSX) {
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 9a49c8c5f7..dddf1ecd0d 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -295,6 +295,7 @@ void evalNodes(vector<Param>& outputs, const vector<Node*>& output_nodes) {
     KParam& out_info{outputs[0].info};
     dim_t* outDims{out_info.dims};
     dim_t* outStrides{out_info.strides};
+#ifndef NDEBUG
     for_each(begin(outputs)++, end(outputs),
              [outDims, outStrides](Param& output) {
                  assert(equal(output.info.dims, output.info.dims + AF_MAX_DIMS,
@@ -302,6 +303,7 @@ void evalNodes(vector<Param>& outputs, const vector<Node*>& output_nodes) {
                         equal(output.info.strides,
                               output.info.strides + AF_MAX_DIMS, outStrides));
              });
+#endif
 
     dim_t ndims{outDims[3] > 1   ? 4
                 : outDims[2] > 1 ? 3
diff --git a/src/backend/opencl/jit/kernel_generators.hpp b/src/backend/opencl/jit/kernel_generators.hpp
index fe87ebc21b..5c111fdedb 100644
--- a/src/backend/opencl/jit/kernel_generators.hpp
+++ b/src/backend/opencl/jit/kernel_generators.hpp
@@ -16,8 +16,9 @@ namespace opencl {
 namespace {
 
 /// Creates a string that will be used to declare the parameter of kernel
-void generateParamDeclaration(std::stringstream& kerStream, int id,
-                              bool is_linear, const std::string& m_type_str) {
+inline void generateParamDeclaration(std::stringstream& kerStream, int id,
+                                     bool is_linear,
+                                     const std::string& m_type_str) {
     if (is_linear) {
         kerStream << "__global " << m_type_str << " *in" << id
                   << ", dim_t iInfo" << id << "_offset, \n";
diff --git a/src/backend/opencl/join.cpp b/src/backend/opencl/join.cpp
index 2d166b693e..7eda4fc307 100644
--- a/src/backend/opencl/join.cpp
+++ b/src/backend/opencl/join.cpp
@@ -51,7 +51,7 @@ Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     //              will be called twice
     if (fdims.dims[jdim] == sdims.dims[jdim]) {
         const size_t L2CacheSize{getL2CacheSize(opencl::getDevice())};
-        if (!(first.isReady() | second.isReady()) ||
+        if (!(first.isReady() || second.isReady()) ||
             (fdims.elements() * sizeof(T) * 2 * 2 < L2CacheSize)) {
             // Both arrays have same size & everything fits into the cache,
             // so thread in 1 JIT kernel, iso individual copies which is
diff --git a/src/backend/opencl/kernel/homography.hpp b/src/backend/opencl/kernel/homography.hpp
index 34f1b2c7e9..2c192ef6b7 100644
--- a/src/backend/opencl/kernel/homography.hpp
+++ b/src/backend/opencl/kernel/homography.hpp
@@ -193,7 +193,7 @@ int computeH(Param bestH, Param H, Param err, Param x_src, Param y_src,
                                      sizeof(unsigned), &inliersH);
 
         bufferFree(totalInliers.data);
-    } else if (htype == AF_HOMOGRAPHY_RANSAC) {
+    } else /* if (htype == AF_HOMOGRAPHY_RANSAC) */ {
         unsigned blockIdx;
         inliersH = kernel::ireduceAll<unsigned, af_max_t>(&blockIdx, inliers);
 
diff --git a/src/backend/opencl/kernel/interp.hpp b/src/backend/opencl/kernel/interp.hpp
index 370e500322..0c3a744c42 100644
--- a/src/backend/opencl/kernel/interp.hpp
+++ b/src/backend/opencl/kernel/interp.hpp
@@ -12,14 +12,14 @@
 #include <common/TemplateArg.hpp>
 #include <af/defines.h>
 
+#include <array>
 #include <string>
-#include <vector>
 
 namespace opencl {
 namespace kernel {
 
 static void addInterpEnumOptions(std::vector<std::string>& options) {
-    std::vector<std::string> enOpts = {
+    static std::array<std::string, 10> enOpts = {
         DefineKeyValue(AF_INTERP_NEAREST, static_cast<int>(AF_INTERP_NEAREST)),
         DefineKeyValue(AF_INTERP_LINEAR, static_cast<int>(AF_INTERP_LINEAR)),
         DefineKeyValue(AF_INTERP_BILINEAR,
diff --git a/src/backend/opencl/kernel/memcopy.hpp b/src/backend/opencl/kernel/memcopy.hpp
index e4091fea53..c63d1e42b3 100644
--- a/src/backend/opencl/kernel/memcopy.hpp
+++ b/src/backend/opencl/kernel/memcopy.hpp
@@ -47,9 +47,10 @@ typedef struct {
 //  - maximum obtained vectorization.
 //  - All the parameters are updated accordingly
 //
-static unsigned vectorizeShape(const unsigned maxVectorWidth, int dims[4],
-                               int istrides[4], int& indims, dim_t& ioffset,
-                               int ostrides[4], dim_t& ooffset) {
+static inline unsigned vectorizeShape(const unsigned maxVectorWidth,
+                                      int dims[4], int istrides[4], int& indims,
+                                      dim_t& ioffset, int ostrides[4],
+                                      dim_t& ooffset) {
     unsigned vectorWidth{1};
     if ((maxVectorWidth != 1) & (istrides[0] == 1) & (ostrides[0] == 1)) {
         // - Only adjacent items can be vectorized into a base vector type
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 8492ca574c..5177293c9f 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -107,8 +107,6 @@ if(AF_BUILD_UNIFIED)
   list(APPEND enabled_backends "unified")
 endif(AF_BUILD_UNIFIED)
 
-check_cxx_compiler_flag("-Wtautological-constant-compare" has_tautological_constant_compare_flag)
-
 add_library(arrayfire_test STATIC
   testHelpers.hpp
   arrayfire_test.cpp)
@@ -127,9 +125,10 @@ target_compile_options(arrayfire_test
   PUBLIC
     # Intel compilers use fast math by default and ignore special floating point
     # values like NaN and Infs.
-    $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>
+    $<$<COMPILE_LANGUAGE:CXX>:
+      $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>
+      $<$<BOOL:${has_cxx_unqualified_std_cast_call}>:-Wno-unqualified-std-cast-call>>
   PRIVATE
-    $<$<BOOL:${has_tautological_constant_compare_flag}>:-Wno-tautological-constant-compare>
     $<$<CXX_COMPILER_ID:MSVC>: /bigobj
                                /EHsc>
   )
@@ -196,11 +195,8 @@ function(make_test)
         arrayfire_test
       )
 
-    # The tautological-constant-compare warning is always thrown for std::nan
-    # and std::info calls. Its unnecessarily verbose.
     target_compile_options(${target}
       PRIVATE
-        $<$<BOOL:${has_tautological_constant_compare_flag}>:-Wno-tautological-constant-compare>
         $<$<CXX_COMPILER_ID:MSVC>: /bigobj
                                    /EHsc>
       )
@@ -364,14 +360,6 @@ if(CUDA_FOUND)
     set(cuda_test_backends "cuda" "unified")
     if(${backend} IN_LIST cuda_test_backends)
       set(target test_cuda_${backend})
-      if(${CMAKE_VERSION} VERSION_LESS 3.5.2)
-        cuda_include_directories(
-          ${ArrayFire_SOURCE_DIR}/include
-          ${ArrayFire_BINARY_DIR}/include
-          ${ArrayFire_SOURCE_DIR}/extern/half/include
-          ${CMAKE_CURRENT_SOURCE_DIR}
-        )
-      endif()
       add_executable(${target} cuda.cu)
       target_include_directories(${target} PRIVATE
         ${ArrayFire_SOURCE_DIR}/extern/half/include
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index fda3d887d6..b9e73b0458 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -1119,7 +1119,6 @@ bool compareArraysRMSD(dim_t data_size, T *gold, T *data, double tolerance) {
 INSTANTIATE(float);
 INSTANTIATE(double);
 INSTANTIATE(char);
-INSTANTIATE(unsigned char);
 #undef INSTANTIATE
 
 TestOutputArrayInfo::TestOutputArrayInfo()
@@ -1368,7 +1367,8 @@ af::array cpu_randu(const af::dim4 dims) {
 
     std::vector<BT> out(elements);
     for (size_t i = 0; i < elements; i++) {
-        out[i] = isTypeFloat ? (BT)(rand()) / RAND_MAX : rand() % 100;
+        out[i] = isTypeFloat ? (BT)(rand()) / static_cast<double>(RAND_MAX)
+                             : rand() % 100;
     }
 
     return af::array(dims, (T *)&out[0]);

From 65e67404e570ff64ce3969c06c3c459d9f9aff95 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 21 Nov 2022 20:28:05 -0500
Subject: [PATCH 140/473] Refactor GitHub workflows

---
 .github/workflows/clang-format-lint.yml | 38 -------------------
 .github/workflows/docs_build.yml        | 44 ----------------------
 .github/workflows/unix_cpu_build.yml    | 49 +++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 82 deletions(-)
 delete mode 100644 .github/workflows/clang-format-lint.yml
 delete mode 100644 .github/workflows/docs_build.yml

diff --git a/.github/workflows/clang-format-lint.yml b/.github/workflows/clang-format-lint.yml
deleted file mode 100644
index 25e79545ac..0000000000
--- a/.github/workflows/clang-format-lint.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-    branches:
-    - master
-
-name: ci
-
-jobs:
-  clang-format:
-      name: Clang Format Lint
-      runs-on: ubuntu-latest
-      steps:
-          - name: Checkout Respository
-            uses: actions/checkout@master
-
-          - name: Check Sources
-            uses: DoozyX/clang-format-lint-action@v0.14
-            with:
-              source: './src'
-              extensions: 'h,cpp,hpp'
-              clangFormatVersion: 14
-
-          - name: Check Tests
-            uses: DoozyX/clang-format-lint-action@v0.14
-            with:
-              source: './test'
-              extensions: 'h,cpp,hpp'
-              clangFormatVersion: 14
-
-          - name: Check Examples
-            uses: DoozyX/clang-format-lint-action@v0.14
-            with:
-              source: './examples'
-              extensions: 'h,cpp,hpp'
-              clangFormatVersion: 14
diff --git a/.github/workflows/docs_build.yml b/.github/workflows/docs_build.yml
deleted file mode 100644
index 38091d113a..0000000000
--- a/.github/workflows/docs_build.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-on:
-  push:
-    branches:
-    - master
-  pull_request:
-    branches:
-    - master
-
-name: ci
-
-jobs:
-    build_documentation:
-        name: Documentation
-        runs-on: ubuntu-18.04
-        env:
-          DOXYGEN_VER: 1.8.18
-        steps:
-            - name: Checkout Repository
-              uses: actions/checkout@master
-
-            - name: Install Doxygen
-              run: |
-                  wget --quiet https://sourceforge.net/projects/doxygen/files/rel-${DOXYGEN_VER}/doxygen-${DOXYGEN_VER}.linux.bin.tar.gz
-                  mkdir doxygen
-                  tar -xf doxygen-${DOXYGEN_VER}.linux.bin.tar.gz -C doxygen --strip 1
-
-            - name: Install Boost
-              run: |
-                  sudo add-apt-repository ppa:mhier/libboost-latest
-                  sudo apt-get -qq update
-                  sudo apt-get install -y libboost1.74-dev
-
-            - name: Configure
-              run: |
-                  mkdir build && cd build && unset VCPKG_ROOT
-                  cmake -DAF_BUILD_CPU:BOOL=OFF -DAF_BUILD_CUDA:BOOL=OFF \
-                        -DAF_BUILD_OPENCL:BOOL=OFF -DAF_BUILD_UNIFIED:BOOL=OFF \
-                        -DAF_BUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTING:BOOL=OFF \
-                        -DDOXYGEN_EXECUTABLE:FILEPATH=${GITHUB_WORKSPACE}/doxygen/bin/doxygen ..
-
-            - name: Build
-              run: |
-                  cd ${GITHUB_WORKSPACE}/build
-                  cmake --build . --target docs
diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 6085718c1d..3c0e566d6f 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -9,9 +9,58 @@ on:
 name: ci
 
 jobs:
+    clang-format:
+        name: Clang Format Lint
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout Respository
+              uses: actions/checkout@master
+
+            - name: Check Sources
+              uses: DoozyX/clang-format-lint-action@v0.14
+              with:
+                source: './src ./test ./examples'
+                extensions: 'h,cpp,hpp'
+                clangFormatVersion: 14
+
+    documentation:
+        name: Documentation
+        runs-on: ubuntu-18.04
+        env:
+          DOXYGEN_VER: 1.8.18
+        steps:
+            - name: Checkout Repository
+              uses: actions/checkout@master
+
+            - name: Install Doxygen
+              run: |
+                  wget --quiet https://sourceforge.net/projects/doxygen/files/rel-${DOXYGEN_VER}/doxygen-${DOXYGEN_VER}.linux.bin.tar.gz
+                  mkdir doxygen
+                  tar -xf doxygen-${DOXYGEN_VER}.linux.bin.tar.gz -C doxygen --strip 1
+
+            - name: Install Boost
+              run: |
+                  sudo add-apt-repository ppa:mhier/libboost-latest
+                  sudo apt-get -qq update
+                  sudo apt-get install -y libboost1.74-dev
+
+            - name: Configure
+              run: |
+                  mkdir build && cd build && unset VCPKG_ROOT
+                  cmake -DAF_BUILD_CPU:BOOL=OFF -DAF_BUILD_CUDA:BOOL=OFF \
+                        -DAF_BUILD_OPENCL:BOOL=OFF -DAF_BUILD_UNIFIED:BOOL=OFF \
+                        -DAF_BUILD_EXAMPLES:BOOL=OFF -DBUILD_TESTING:BOOL=OFF \
+                        -DDOXYGEN_EXECUTABLE:FILEPATH=${GITHUB_WORKSPACE}/doxygen/bin/doxygen ..
+
+            - name: Build
+              run: |
+                  cd ${GITHUB_WORKSPACE}/build
+                  cmake --build . --target docs
+
     build_cpu:
         name: CPU
         runs-on: ${{ matrix.os }}
+        needs: [clang-format, documentation]
         env:
           NINJA_VER: 1.10.2
           CMAKE_VER: 3.10.2

From ad47660dc574f884e2f4e84473b177fb4f489d58 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 22 Nov 2022 14:29:59 -0500
Subject: [PATCH 141/473] Add CMake targets and exports for afoneapi

The oneAPI target was not creating CMake configuration files. This caused a
problem with the print_info target when no other backends were built because
CMake didn't include the ArrayFire include directories and libraries.
---
 CMakeLists.txt                    |  2 +-
 src/backend/oneapi/CMakeLists.txt | 13 +++++++++++++
 test/CMakeLists.txt               |  2 ++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2424d9162f..b779d929f5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -470,7 +470,7 @@ install(DIRECTORY "${ArrayFire_SOURCE_DIR}/LICENSES/"
     DESTINATION LICENSES
     COMPONENT licenses)
 
-foreach(backend CPU CUDA OpenCL Unified)
+foreach(backend CPU CUDA OpenCL oneAPI Unified)
   string(TOUPPER ${backend} upper_backend)
   string(TOLOWER ${backend} lower_backend)
   if(AF_BUILD_${upper_backend})
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 0561573a44..4fb6f3c0a9 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -265,6 +265,19 @@ target_link_libraries(afoneapi
     -fsycl
   )
 
+af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR})
+
+install(TARGETS afoneapi
+  EXPORT ArrayFireoneAPITargets
+  COMPONENT oneapi
+  PUBLIC_HEADER DESTINATION af
+  RUNTIME DESTINATION ${AF_INSTALL_BIN_DIR}
+  LIBRARY DESTINATION ${AF_INSTALL_LIB_DIR}
+  ARCHIVE DESTINATION ${AF_INSTALL_LIB_DIR}
+  FRAMEWORK DESTINATION framework
+  INCLUDES DESTINATION ${AF_INSTALL_INC_DIR}
+)
+
 source_group(include REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/include/*)
 source_group(api\\cpp REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/cpp/*)
 source_group(api\\c   REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/c/*)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 5177293c9f..50fcadaf5b 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -478,6 +478,8 @@ elseif(AF_BUILD_CUDA)
   target_link_libraries(print_info ArrayFire::afcuda)
 elseif(AF_BUILD_CPU)
   target_link_libraries(print_info ArrayFire::afcpu)
+elseif(AF_BUILD_ONEAPI)
+  target_link_libraries(print_info ArrayFire::afoneapi)
 endif()
 
 make_test(SRC jit_test_api.cpp)

From 5cf3169d86a39da47446d67be54924816406ca99 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 23 Nov 2022 18:04:05 -0500
Subject: [PATCH 142/473] Remove reinterpret casts for conversions to void*

---
 src/api/c/device.cpp             |  5 +++--
 src/api/c/error.cpp              |  5 ++++-
 src/api/c/print.cpp              |  7 ++++---
 src/api/c/sparse.cpp             |  2 +-
 src/api/unified/error.cpp        |  9 ++++++---
 src/backend/common/ArrayInfo.cpp |  5 +++--
 src/backend/cuda/Kernel.hpp      |  2 +-
 src/backend/oneapi/Array.cpp     | 12 +++++++-----
 src/backend/opencl/api.cpp       | 14 +++++++++++++-
 9 files changed, 42 insertions(+), 19 deletions(-)

diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp
index 57c61be4c3..b619a867f2 100644
--- a/src/api/c/device.cpp
+++ b/src/api/c/device.cpp
@@ -167,8 +167,9 @@ af_err af_info_string(char** str, const bool verbose) {
     UNUSED(verbose);  // TODO(umar): Add something useful
     try {
         std::string infoStr = getDeviceInfo();
-        af_alloc_host(reinterpret_cast<void**>(str),
-                      sizeof(char) * (infoStr.size() + 1));
+        void* halloc_ptr    = nullptr;
+        af_alloc_host(&halloc_ptr, sizeof(char) * (infoStr.size() + 1));
+        memcpy(str, &halloc_ptr, sizeof(void*));
 
         // Need to do a deep copy
         // str.c_str wont cut it
diff --git a/src/api/c/error.cpp b/src/api/c/error.cpp
index 8ede0ee9c0..4dd1ff190f 100644
--- a/src/api/c/error.cpp
+++ b/src/api/c/error.cpp
@@ -13,6 +13,7 @@
 #include <af/util.h>
 
 #include <algorithm>
+#include <cstring>
 #include <string>
 
 void af_get_last_error(char **str, dim_t *len) {
@@ -26,7 +27,9 @@ void af_get_last_error(char **str, dim_t *len) {
         return;
     }
 
-    af_alloc_host(reinterpret_cast<void **>(str), sizeof(char) * (slen + 1));
+    void *halloc_ptr = nullptr;
+    af_alloc_host(&halloc_ptr, sizeof(char) * (slen + 1));
+    memcpy(str, &halloc_ptr, sizeof(void *));
     global_error_string.copy(*str, slen);
 
     (*str)[slen]        = '\0';
diff --git a/src/api/c/print.cpp b/src/api/c/print.cpp
index ef749e970f..85f30dc028 100644
--- a/src/api/c/print.cpp
+++ b/src/api/c/print.cpp
@@ -278,9 +278,10 @@ af_err af_array_to_string(char **output, const char *exp, const af_array arr,
                 default: TYPE_ERROR(1, type);
             }
         }
-        std::string str = ss.str();
-        af_alloc_host(reinterpret_cast<void **>(output),
-                      sizeof(char) * (str.size() + 1));
+        std::string str  = ss.str();
+        void *halloc_ptr = nullptr;
+        af_alloc_host(&halloc_ptr, sizeof(char) * (str.size() + 1));
+        memcpy(output, &halloc_ptr, sizeof(void *));
         str.copy(*output, str.size());
         (*output)[str.size()] = '\0';  // don't forget the terminating 0
     }
diff --git a/src/api/c/sparse.cpp b/src/api/c/sparse.cpp
index d1a737f488..714a0c1d15 100644
--- a/src/api/c/sparse.cpp
+++ b/src/api/c/sparse.cpp
@@ -31,7 +31,7 @@ using detail::sparseConvertDenseToStorage;
 const SparseArrayBase &getSparseArrayBase(const af_array in,
                                           bool device_check) {
     const SparseArrayBase *base =
-        static_cast<SparseArrayBase *>(reinterpret_cast<void *>(in));
+        static_cast<SparseArrayBase *>(static_cast<void *>(in));
 
     if (!base->isSparse()) {
         AF_ERROR(
diff --git a/src/api/unified/error.cpp b/src/api/unified/error.cpp
index de6fad63e9..9fd89c0166 100644
--- a/src/api/unified/error.cpp
+++ b/src/api/unified/error.cpp
@@ -28,8 +28,9 @@ void af_get_last_error(char **str, dim_t *len) {
             return;
         }
 
-        af_alloc_host(reinterpret_cast<void **>(str),
-                      sizeof(char) * (slen + 1));
+        void *in = nullptr;
+        af_alloc_host(&in, sizeof(char) * (slen + 1));
+        memcpy(str, &in, sizeof(void *));
         global_error_string.copy(*str, slen);
 
         (*str)[slen]        = '\0';
@@ -39,7 +40,9 @@ void af_get_last_error(char **str, dim_t *len) {
     } else {
         // If false, the error is coming from active backend.
         typedef void (*af_func)(char **, dim_t *);
-        auto func = reinterpret_cast<af_func>(LOAD_SYMBOL());
+        void *vfn    = LOAD_SYMBOL();
+        af_func func = nullptr;
+        memcpy(&func, vfn, sizeof(void *));
         func(str, len);
     }
 }
diff --git a/src/backend/common/ArrayInfo.cpp b/src/backend/common/ArrayInfo.cpp
index c2c6a842f2..f079bac8ef 100644
--- a/src/backend/common/ArrayInfo.cpp
+++ b/src/backend/common/ArrayInfo.cpp
@@ -11,6 +11,7 @@
 #include <common/err_common.hpp>
 #include <common/traits.hpp>
 #include <algorithm>
+#include <cstring>
 #include <functional>
 #include <numeric>
 
@@ -173,8 +174,8 @@ dim4 toStride(const vector<af_seq> &seqs, const af::dim4 &parentDims) {
 
 const ArrayInfo &getInfo(const af_array arr, bool sparse_check,
                          bool device_check) {
-    const ArrayInfo *info =
-        static_cast<ArrayInfo *>(reinterpret_cast<void *>(arr));
+    const ArrayInfo *info = nullptr;
+    memcpy(&info, &arr, sizeof(af_array));
 
     // Check Sparse -> If false, then both standard Array<T> and SparseArray<T>
     // are accepted Otherwise only regular Array<T> is accepted
diff --git a/src/backend/cuda/Kernel.hpp b/src/backend/cuda/Kernel.hpp
index 1e2459bc73..a728940d97 100644
--- a/src/backend/cuda/Kernel.hpp
+++ b/src/backend/cuda/Kernel.hpp
@@ -29,7 +29,7 @@ struct Enqueuer {
     template<typename... Args>
     void operator()(std::string name, void* ker, const EnqueueArgs& qArgs,
                     Args... args) {
-        void* params[] = {reinterpret_cast<void*>(&args)...};
+        void* params[] = {static_cast<void*>(&args)...};
         for (auto& event : qArgs.mEvents) {
             CU_CHECK(cuStreamWaitEvent(qArgs.mStream, event, 0));
         }
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index bd5676fd01..24330ee3ae 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -179,14 +179,16 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
                 const T *const in_data, bool is_device)
     : info(getActiveDeviceId(), dims, offset_, strides,
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(is_device ? (new buffer<T>(*reinterpret_cast<buffer<T> *>(
-                           const_cast<T *>(in_data))))
-                     : (memAlloc<T>(info.elements()).release()),
-           bufferFree<T>)
+    , data()
     , data_dims(dims)
     , node()
     , owner(true) {
-    if (!is_device) {
+    if (is_device) {
+        buffer<T> *ptr;
+        std::memcpy(&ptr, in_data, sizeof(buffer<T> *));
+        data = make_shared<buffer<T>>(*ptr);
+    } else {
+        data = memAlloc<T>(info.elements());
         getQueue()
             .submit(
                 [&](sycl::handler &h) { h.copy(in_data, data->get_access(h)); })
diff --git a/src/backend/opencl/api.cpp b/src/backend/opencl/api.cpp
index 04b73eff4f..df3f6783a1 100644
--- a/src/backend/opencl/api.cpp
+++ b/src/backend/opencl/api.cpp
@@ -1,11 +1,23 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
 #include <af/array.h>
 #include <af/opencl.h>
+#include <cstring>
 
 namespace af {
 template<>
 AFAPI cl_mem *array::device() const {
     auto *mem_ptr = new cl_mem;
-    af_err err = af_get_device_ptr(reinterpret_cast<void **>(mem_ptr), get());
+    void *dptr    = nullptr;
+    af_err err    = af_get_device_ptr(&dptr, get());
+    memcpy(mem_ptr, &dptr, sizeof(void *));
     if (err != AF_SUCCESS) {
         throw af::exception("Failed to get cl_mem from array object");
     }

From 9890fb0ef77ba1665aa90309e314e6cc9c71a92c Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 23 Nov 2022 18:30:23 -0500
Subject: [PATCH 143/473] Rename version.hpp to build_version.hpp

---
 CMakeLists.txt                                        | 5 -----
 CMakeModules/Version.cmake                            | 4 ++--
 CMakeModules/{version.hpp.in => build_version.hpp.in} | 2 +-
 src/api/c/version.cpp                                 | 2 +-
 src/api/unified/CMakeLists.txt                        | 1 -
 src/backend/common/CMakeLists.txt                     | 3 +--
 src/backend/common/jit/Node.cpp                       | 2 +-
 src/backend/cpu/CMakeLists.txt                        | 2 +-
 src/backend/cpu/platform.cpp                          | 2 +-
 src/backend/cuda/CMakeLists.txt                       | 2 +-
 src/backend/cuda/device_manager.cpp                   | 2 +-
 src/backend/cuda/platform.cpp                         | 2 +-
 src/backend/oneapi/CMakeLists.txt                     | 2 +-
 src/backend/oneapi/device_manager.cpp                 | 2 +-
 src/backend/oneapi/platform.cpp                       | 2 +-
 src/backend/opencl/CMakeLists.txt                     | 2 +-
 src/backend/opencl/device_manager.cpp                 | 2 +-
 src/backend/opencl/platform.cpp                       | 2 +-
 18 files changed, 17 insertions(+), 24 deletions(-)
 rename CMakeModules/{version.hpp.in => build_version.hpp.in} (92%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b779d929f5..f67cecee36 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -302,11 +302,6 @@ af_dep_check_and_populate(${assets_prefix}
 )
 set(ASSETS_DIR ${${assets_prefix}_SOURCE_DIR})
 
-configure_file(
-    ${ArrayFire_SOURCE_DIR}/CMakeModules/version.hpp.in
-    ${ArrayFire_BINARY_DIR}/version.hpp
-)
-
 # when crosscompiling use the bin2cpp file from the native bin directory
 if(CMAKE_CROSSCOMPILING)
   set(NATIVE_BIN_DIR "NATIVE_BIN_DIR-NOTFOUND"
diff --git a/CMakeModules/Version.cmake b/CMakeModules/Version.cmake
index 54c0ac8174..2269bd73f2 100644
--- a/CMakeModules/Version.cmake
+++ b/CMakeModules/Version.cmake
@@ -49,6 +49,6 @@ configure_file(
 )
 
 configure_file(
-    ${ArrayFire_SOURCE_DIR}/CMakeModules/version.hpp.in
-    ${ArrayFire_BINARY_DIR}/src/backend/version.hpp
+    ${ArrayFire_SOURCE_DIR}/CMakeModules/build_version.hpp.in
+    ${ArrayFire_BINARY_DIR}/src/backend/build_version.hpp
 )
diff --git a/CMakeModules/version.hpp.in b/CMakeModules/build_version.hpp.in
similarity index 92%
rename from CMakeModules/version.hpp.in
rename to CMakeModules/build_version.hpp.in
index f4c9ec6150..d3b881f8d9 100644
--- a/CMakeModules/version.hpp.in
+++ b/CMakeModules/build_version.hpp.in
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2022, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
diff --git a/src/api/c/version.cpp b/src/api/c/version.cpp
index ce471bd9d1..47b6952427 100644
--- a/src/api/c/version.cpp
+++ b/src/api/c/version.cpp
@@ -7,7 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <version.hpp>
+#include <build_version.hpp>
 #include <af/util.h>
 
 af_err af_get_version(int *major, int *minor, int *patch) {
diff --git a/src/api/unified/CMakeLists.txt b/src/api/unified/CMakeLists.txt
index 67b6b80dd2..a17c6618f1 100644
--- a/src/api/unified/CMakeLists.txt
+++ b/src/api/unified/CMakeLists.txt
@@ -78,7 +78,6 @@ target_include_directories(af
   PRIVATE
     ${ArrayFire_SOURCE_DIR}/src/api/c
     ${ArrayFire_SOURCE_DIR}/src/api/unified
-    ${ArrayFire_BINARY_DIR}
     $<TARGET_PROPERTY:afcommon_interface,INTERFACE_INCLUDE_DIRECTORIES>
     $<$<BOOL:${OpenCL_FOUND}>: $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>>
     $<$<BOOL:${CUDA_FOUND}>:  ${CUDA_INCLUDE_DIRS}>
diff --git a/src/backend/common/CMakeLists.txt b/src/backend/common/CMakeLists.txt
index 1487d99c44..7b26e11194 100644
--- a/src/backend/common/CMakeLists.txt
+++ b/src/backend/common/CMakeLists.txt
@@ -78,7 +78,6 @@ target_sources(afcommon_interface
     ${CMAKE_CURRENT_SOURCE_DIR}/unique_handle.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/util.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/util.hpp
-    ${ArrayFire_BINARY_DIR}/version.hpp
   )
 
 if(WIN32)
@@ -115,7 +114,7 @@ endif()
 target_include_directories(afcommon_interface
   INTERFACE
     ${ArrayFire_SOURCE_DIR}/src/backend
-    ${ArrayFire_BINARY_DIR})
+    ${ArrayFire_BINARY_DIR}/src/backend)
 
 target_include_directories(afcommon_interface
   SYSTEM INTERFACE
diff --git a/src/backend/common/jit/Node.cpp b/src/backend/common/jit/Node.cpp
index 71d88424f5..ed24b9c1f8 100644
--- a/src/backend/common/jit/Node.cpp
+++ b/src/backend/common/jit/Node.cpp
@@ -7,12 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <build_version.hpp>
 #include <common/defines.hpp>
 #include <common/deterministicHash.hpp>
 #include <common/jit/Node.hpp>
 #include <common/util.hpp>
 
-#include <version.hpp>
 #include <sstream>
 #include <string>
 #include <vector>
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index 04d0d3390b..83005c8e62 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -356,5 +356,5 @@ source_group(api\\cpp REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/cpp/*)
 source_group(api\\c   REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/c/*)
 source_group(backend  REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/backend/common/*|${CMAKE_CURRENT_SOURCE_DIR}/*)
 source_group(backend\\kernel  REGULAR_EXPRESSION ${CMAKE_CURRENT_SOURCE_DIR}/kernel/*)
-source_group("generated files" FILES ${ArrayFire_BINARY_DIR}/version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h)
+source_group("generated files" FILES ${ArrayFire_BINARY_DIR}/src/backend/build_version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h)
 source_group("" FILES CMakeLists.txt)
diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index 5bb28a41ec..8676054136 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -7,12 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <build_version.hpp>
 #include <common/MemoryManagerBase.hpp>
 #include <common/defines.hpp>
 #include <common/host_memory.hpp>
 #include <device_manager.hpp>
 #include <platform.hpp>
-#include <version.hpp>
 #include <af/version.h>
 
 #include <cctype>
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index e1f47b2947..4d3ac5051e 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -861,7 +861,7 @@ source_group(api\\cpp REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/cpp/*)
 source_group(api\\c   REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/c/*)
 source_group(backend  REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/backend/common/*|${CMAKE_CURRENT_SOURCE_DIR}/*)
 source_group(backend\\kernel  REGULAR_EXPRESSION ${CMAKE_CURRENT_SOURCE_DIR}/kernel/*|${CMAKE_CURRENT_SOURCE_DIR}/kernel/thrust_sort_by_key/*|${CMAKE_CURRENT_SOURCE_DIR}/kernel/scan_by_key/*)
-source_group("generated files"  FILES ${ArrayFire_BINARY_DIR}/version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h
+source_group("generated files"  FILES ${ArrayFire_BINARY_DIR}/src/backend/build_version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h
                                 REGULAR_EXPRESSION ${CMAKE_CURRENT_BINARY_DIR}/${kernel_headers_dir}/*)
 source_group("" FILES CMakeLists.txt)
 
diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index f556d08cce..4b946a7fee 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -12,6 +12,7 @@
 #endif
 
 #include <GraphicsResourceManager.hpp>
+#include <build_version.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
 #include <common/MemoryManagerBase.hpp>
@@ -26,7 +27,6 @@
 #include <memory.hpp>
 #include <platform.hpp>
 #include <spdlog/spdlog.h>
-#include <version.hpp>
 #include <af/cuda.h>
 #include <af/version.h>
 // cuda_gl_interop.h does not include OpenGL headers for ARM
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 13d10564bf..7e82f76843 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -17,6 +17,7 @@
 #endif
 
 #include <GraphicsResourceManager.hpp>
+#include <build_version.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
@@ -36,7 +37,6 @@
 #include <memory.hpp>
 #include <spdlog/spdlog.h>
 #include <utility.hpp>
-#include <version.hpp>
 #include <af/cuda.h>
 #include <af/device.h>
 #include <af/version.h>
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 4fb6f3c0a9..f67764e21d 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -283,5 +283,5 @@ source_group(api\\cpp REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/cpp/*)
 source_group(api\\c   REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/c/*)
 source_group(backend  REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/backend/common/*|${CMAKE_CURRENT_SOURCE_DIR}/*)
 source_group(backend\\kernel  REGULAR_EXPRESSION ${CMAKE_CURRENT_SOURCE_DIR}/kernel/*)
-source_group("generated files" FILES ${ArrayFire_BINARY_DIR}/version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h)
+source_group("generated files" FILES ${ArrayFire_BINARY_DIR}/src/backend/build_version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h)
 source_group("" FILES CMakeLists.txt)
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index 4588369637..48201b7ebc 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -19,8 +19,8 @@
 #include <err_oneapi.hpp>
 #include <platform.hpp>  //TODO: blas.hpp? y tho, also Array.hpp
 //#include <errorcodes.hpp>
+#include <build_version.hpp>
 #include <memory.hpp>
-#include <version.hpp>
 #include <af/oneapi.h>
 #include <af/version.h>
 
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index c16a4afff9..4e22f742ae 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -9,6 +9,7 @@
 
 #include <GraphicsResourceManager.hpp>
 #include <blas.hpp>
+#include <build_version.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
 #include <common/graphics_common.hpp>
@@ -18,7 +19,6 @@
 #include <err_oneapi.hpp>
 #include <errorcodes.hpp>
 #include <memory.hpp>
-#include <version.hpp>
 #include <af/version.h>
 
 #ifdef OS_MAC
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index a827c55193..8df8ff6aaa 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -544,4 +544,4 @@ source_group(api\\cpp REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/cpp/*)
 source_group(api\\c   REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/api/c/*)
 source_group(backend  REGULAR_EXPRESSION ${ArrayFire_SOURCE_DIR}/src/backend/common/*|${CMAKE_CURRENT_SOURCE_DIR}/*)
 source_group(backend\\kernel  REGULAR_EXPRESSION ${CMAKE_CURRENT_SOURCE_DIR}/kernel/*|${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/*|${CMAKE_CURRENT_SOURCE_DIR}/kernel/scan_by_key/*)
-source_group("generated files" FILES ${ArrayFire_BINARY_DIR}/version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h)
+source_group("generated files" FILES ${ArrayFire_BINARY_DIR}/src/backend/build_version.hpp ${ArrayFire_BINARY_DIR}/include/af/version.h)
diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index 0a543f4297..a9cfbc02e2 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -13,6 +13,7 @@
 
 #include <GraphicsResourceManager.hpp>
 #include <blas.hpp>
+#include <build_version.hpp>
 #include <clfft.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
@@ -22,7 +23,6 @@
 #include <device_manager.hpp>
 #include <err_opencl.hpp>
 #include <errorcodes.hpp>
-#include <version.hpp>
 #include <af/opencl.h>
 #include <af/version.h>
 #include <memory>
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 6bcc2e55ae..04859ad40a 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -13,6 +13,7 @@
 
 #include <GraphicsResourceManager.hpp>
 #include <blas.hpp>
+#include <build_version.hpp>
 #include <clfft.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
@@ -22,7 +23,6 @@
 #include <err_opencl.hpp>
 #include <errorcodes.hpp>
 #include <platform.hpp>
-#include <version.hpp>
 #include <af/version.h>
 
 #ifdef OS_MAC

From d59f70d1547294db6ec27a1a460a2d01e5b4ada3 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 23 Nov 2022 18:50:17 -0500
Subject: [PATCH 144/473] Fix target_include_directory to specify system
 headers

---
 CMakeLists.txt                                |  7 ++++-
 CMakeModules/build_clFFT.cmake                |  5 ++++
 src/api/c/CMakeLists.txt                      |  2 +-
 src/api/cpp/CMakeLists.txt                    |  8 +++--
 src/api/unified/CMakeLists.txt                |  5 +++-
 src/backend/common/CMakeLists.txt             | 14 ++++-----
 src/backend/cpu/CMakeLists.txt                |  8 +++--
 .../cpu/kernel/sort_by_key/CMakeLists.txt     |  8 +++--
 src/backend/cuda/CMakeLists.txt               |  9 ++++--
 src/backend/oneapi/CMakeLists.txt             |  6 +++-
 src/backend/opencl/CMakeLists.txt             |  3 +-
 .../opencl/kernel/scan_by_key/CMakeLists.txt  |  2 +-
 .../opencl/kernel/sort_by_key/CMakeLists.txt  | 30 +++++++++----------
 test/CMakeLists.txt                           | 18 +++++++----
 14 files changed, 79 insertions(+), 46 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f67cecee36..440f28ae18 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -261,7 +261,7 @@ else()
   )
   add_subdirectory(${${spdlog_prefix}_SOURCE_DIR} ${${spdlog_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
 
-  target_include_directories(af_spdlog INTERFACE "${${spdlog_prefix}_SOURCE_DIR}/include")
+  target_include_directories(af_spdlog SYSTEM INTERFACE "${${spdlog_prefix}_SOURCE_DIR}/include")
   if(TARGET fmt::fmt)
     set_target_properties(af_spdlog
       PROPERTIES
@@ -294,6 +294,11 @@ if(NOT TARGET nonstd::span-lite)
     REF "ccf2351"
     )
   add_subdirectory(${span-lite_SOURCE_DIR} EXCLUDE_FROM_ALL)
+  get_property(span_include_dir
+    TARGET span-lite
+    PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
+  set_target_properties(span-lite
+    PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${span_include_dir}")
 endif()
 
 af_dep_check_and_populate(${assets_prefix}
diff --git a/CMakeModules/build_clFFT.cmake b/CMakeModules/build_clFFT.cmake
index d4f3081e63..b3e56137bf 100644
--- a/CMakeModules/build_clFFT.cmake
+++ b/CMakeModules/build_clFFT.cmake
@@ -13,6 +13,11 @@ af_dep_check_and_populate(${clfft_prefix}
 set(current_build_type ${BUILD_SHARED_LIBS})
 set(BUILD_SHARED_LIBS OFF)
 add_subdirectory(${${clfft_prefix}_SOURCE_DIR}/src ${${clfft_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
+get_property(clfft_include_dir
+  TARGET clFFT
+  PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
+set_target_properties(clFFT
+  PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${clfft_include_dir}")
 
 # OpenCL targets need this flag to avoid ignored attribute warnings in the
 # OpenCL headers
diff --git a/src/api/c/CMakeLists.txt b/src/api/c/CMakeLists.txt
index 0830402a1f..8dcf7c3d5b 100644
--- a/src/api/c/CMakeLists.txt
+++ b/src/api/c/CMakeLists.txt
@@ -175,7 +175,7 @@ if(FreeImage_FOUND AND AF_WITH_IMAGEIO)
     target_compile_definitions(c_api_interface INTERFACE FREEIMAGE_STATIC)
     target_link_libraries(c_api_interface INTERFACE FreeImage::FreeImage_STATIC)
   else ()
-    target_include_directories(c_api_interface INTERFACE $<TARGET_PROPERTY:FreeImage::FreeImage,INTERFACE_INCLUDE_DIRECTORIES>)
+    target_include_directories(c_api_interface SYSTEM INTERFACE $<TARGET_PROPERTY:FreeImage::FreeImage,INTERFACE_INCLUDE_DIRECTORIES>)
     if (WIN32 AND AF_INSTALL_STANDALONE)
       install(FILES $<TARGET_FILE:FreeImage::FreeImage>
         DESTINATION ${AF_INSTALL_BIN_DIR}
diff --git a/src/api/cpp/CMakeLists.txt b/src/api/cpp/CMakeLists.txt
index 1df8c7ff77..e33a8b320d 100644
--- a/src/api/cpp/CMakeLists.txt
+++ b/src/api/cpp/CMakeLists.txt
@@ -89,8 +89,10 @@ target_sources(cpp_api_interface
     ${CMAKE_CURRENT_SOURCE_DIR}/ycbcr_rgb.cpp
 )
 
+target_include_directories(cpp_api_interface
+  SYSTEM INTERFACE
+    ${ArrayFire_SOURCE_DIR}/extern/half/include)
+
 target_include_directories(cpp_api_interface
   INTERFACE
-    ${CMAKE_SOURCE_DIR}/src/api/c
-    ${ArrayFire_SOURCE_DIR}/extern/half/include
-)
+    ${CMAKE_SOURCE_DIR}/src/api/c)
diff --git a/src/api/unified/CMakeLists.txt b/src/api/unified/CMakeLists.txt
index a17c6618f1..ca6805c7a4 100644
--- a/src/api/unified/CMakeLists.txt
+++ b/src/api/unified/CMakeLists.txt
@@ -77,7 +77,10 @@ target_include_directories(af
     $<INSTALL_INTERFACE:${AF_INSTALL_INC_DIR}>
   PRIVATE
     ${ArrayFire_SOURCE_DIR}/src/api/c
-    ${ArrayFire_SOURCE_DIR}/src/api/unified
+    ${ArrayFire_SOURCE_DIR}/src/api/unified)
+
+target_include_directories(af
+  SYSTEM PRIVATE
     $<TARGET_PROPERTY:afcommon_interface,INTERFACE_INCLUDE_DIRECTORIES>
     $<$<BOOL:${OpenCL_FOUND}>: $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>>
     $<$<BOOL:${CUDA_FOUND}>:  ${CUDA_INCLUDE_DIRS}>
diff --git a/src/backend/common/CMakeLists.txt b/src/backend/common/CMakeLists.txt
index 7b26e11194..795e5df44c 100644
--- a/src/backend/common/CMakeLists.txt
+++ b/src/backend/common/CMakeLists.txt
@@ -111,25 +111,25 @@ if(AF_BUILD_FORGE AND NOT Forge_FOUND)
   add_dependencies(afcommon_interface forge)
 endif()
 
+target_include_directories(afcommon_interface
+  SYSTEM INTERFACE
+    $<$<PLATFORM_ID:Darwin>:${OPENGL_INCLUDE_DIR}>)
+
 target_include_directories(afcommon_interface
   INTERFACE
     ${ArrayFire_SOURCE_DIR}/src/backend
     ${ArrayFire_BINARY_DIR}/src/backend)
 
-target_include_directories(afcommon_interface
-  SYSTEM INTERFACE
-    $<$<PLATFORM_ID:Darwin>:${OPENGL_INCLUDE_DIR}>
-  )
 if(TARGET Forge::forge)
   target_include_directories(afcommon_interface
     SYSTEM INTERFACE
-    $<TARGET_PROPERTY:Forge::forge,INCLUDE_DIRECTORIES>
+      $<TARGET_PROPERTY:Forge::forge,INCLUDE_DIRECTORIES>
   )
 else()
   target_include_directories(afcommon_interface
     SYSTEM INTERFACE
-    ${${forge_prefix}_SOURCE_DIR}/include
-    ${${forge_prefix}_BINARY_DIR}/include
+      ${${forge_prefix}_SOURCE_DIR}/include
+      ${${forge_prefix}_BINARY_DIR}/include
   )
 endif()
 
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index 83005c8e62..fc84101de4 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -292,9 +292,11 @@ target_include_directories(afcpu
     $<INSTALL_INTERFACE:${AF_INSTALL_INC_DIR}>
   PRIVATE
     ${CMAKE_CURRENT_SOURCE_DIR}
-    ${${threads_prefix}_SOURCE_DIR}/include
-    ${CBLAS_INCLUDE_DIR}
-  )
+    ${${threads_prefix}_SOURCE_DIR}/include)
+
+target_include_directories(afcpu
+  SYSTEM PRIVATE
+    ${CBLAS_INCLUDE_DIR})
 
 target_compile_definitions(afcpu
   PRIVATE
diff --git a/src/backend/cpu/kernel/sort_by_key/CMakeLists.txt b/src/backend/cpu/kernel/sort_by_key/CMakeLists.txt
index 3c894b37f5..752501fabc 100644
--- a/src/backend/cpu/kernel/sort_by_key/CMakeLists.txt
+++ b/src/backend/cpu/kernel/sort_by_key/CMakeLists.txt
@@ -29,20 +29,22 @@ foreach(SBK_TYPE ${SBK_TYPES})
       FOLDER "Generated Targets")
 
   arrayfire_set_default_cxx_flags(cpu_sort_by_key_${SBK_TYPE})
-  # TODO(umar): This should just use the include directories from the
-  # afcpu_static target
+
   target_include_directories(cpu_sort_by_key_${SBK_TYPE}
     PUBLIC
       .
       ../../api/c
       ${ArrayFire_SOURCE_DIR}/include
       ${ArrayFire_BINARY_DIR}/include
-      $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
     PRIVATE
       ../common
       ..
       threads)
 
+  target_include_directories(cpu_sort_by_key_${SBK_TYPE}
+    SYSTEM PRIVATE
+      $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>)
+
   set_target_properties(cpu_sort_by_key_${SBK_TYPE} PROPERTIES POSITION_INDEPENDENT_CODE ON)
   target_sources(cpu_sort_by_key
     INTERFACE $<TARGET_OBJECTS:cpu_sort_by_key_${SBK_TYPE}>)
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 4d3ac5051e..8490c541a0 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -668,13 +668,16 @@ target_include_directories (afcuda
     $<BUILD_INTERFACE:${ArrayFire_BINARY_DIR}/include>
     $<INSTALL_INTERFACE:${AF_INSTALL_INC_DIR}>
   PRIVATE
-    $<$<BOOL:${AF_WITH_CUDNN}>:${cuDNN_INCLUDE_DIRS}>
-    ${CUDA_INCLUDE_DIRS}
     ${ArrayFire_SOURCE_DIR}/src/api/c
     ${CMAKE_CURRENT_SOURCE_DIR}
     ${CMAKE_CURRENT_SOURCE_DIR}/kernel
     ${CMAKE_CURRENT_SOURCE_DIR}/jit
-    ${CMAKE_CURRENT_BINARY_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR})
+
+target_include_directories (afcuda
+  SYSTEM PRIVATE
+    $<$<BOOL:${AF_WITH_CUDNN}>:${cuDNN_INCLUDE_DIRS}>
+    ${CUDA_INCLUDE_DIRS}
 )
 
 target_link_libraries(afcuda
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index f67764e21d..3036a20b2f 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -239,6 +239,10 @@ add_library(ArrayFire::afoneapi ALIAS afoneapi)
 
 arrayfire_set_default_cxx_flags(afoneapi)
 
+target_include_directories(afoneapi
+  SYSTEM PRIVATE
+    ${SYCL_INCLUDE_DIR})
+
 target_include_directories(afoneapi
   PUBLIC
     $<BUILD_INTERFACE:${ArrayFire_SOURCE_DIR}/include>
@@ -246,7 +250,7 @@ target_include_directories(afoneapi
     $<INSTALL_INTERFACE:${AF_INSTALL_INC_DIR}>
   PRIVATE
     ${CMAKE_CURRENT_SOURCE_DIR}
-    ${SYCL_INCLUDE_DIR}
+
   )
 
 target_compile_options(afoneapi
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 8df8ff6aaa..069609b95e 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -480,8 +480,9 @@ if(LAPACK_FOUND OR BUILD_WITH_MKL)
     endif()
 
     target_include_directories(afopencl
-      PRIVATE
+      SYSTEM PRIVATE
         ${CBLAS_INCLUDE_DIR})
+
     target_link_libraries(afopencl
       PRIVATE
         ${CBLAS_LIBRARIES}
diff --git a/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt b/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
index e5d0de3a97..316e946a31 100644
--- a/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
+++ b/src/backend/opencl/kernel/scan_by_key/CMakeLists.txt
@@ -40,7 +40,7 @@ foreach(SBK_BINARY_OP ${SBK_BINARY_OPS})
         $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>
         $<TARGET_PROPERTY:OpenCL::cl2hpp,INTERFACE_INCLUDE_DIRECTORIES>
         $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
-        $<TARGET_PROPERTY:nonstd::span-lite,INTERFACE_INCLUDE_DIRECTORIES>
+        $<TARGET_PROPERTY:nonstd::span-lite,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES>
         ${ArrayFire_BINARY_DIR}/include
       )
     if(TARGET Forge::forge)
diff --git a/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt b/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
index 2853d75cd9..e2ad168138 100644
--- a/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
+++ b/src/backend/opencl/kernel/sort_by_key/CMakeLists.txt
@@ -22,46 +22,46 @@ foreach(SBK_TYPE ${SBK_TYPES})
     add_dependencies(opencl_sort_by_key_${SBK_TYPE}
                         ${cl_kernel_targets} OpenCL::cl2hpp Boost::boost)
 
+    target_include_directories(opencl_sort_by_key_${SBK_TYPE}
+      SYSTEM PRIVATE
+        ${span-lite_SOURCE_DIR}/include
+        $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>
+        $<TARGET_PROPERTY:OpenCL::cl2hpp,INTERFACE_INCLUDE_DIRECTORIES>
+        $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
+        $<TARGET_PROPERTY:af_spdlog,INTERFACE_INCLUDE_DIRECTORIES>)
+
     target_include_directories(opencl_sort_by_key_${SBK_TYPE}
       PRIVATE
         .
         ..
-        magma
         ../../api/c
         ../common
         ../../../include
-        ${span-lite_SOURCE_DIR}/include
+        magma
+        ${ArrayFire_BINARY_DIR}/include
         ${CMAKE_CURRENT_BINARY_DIR})
 
-    target_include_directories(opencl_sort_by_key_${SBK_TYPE}
-      SYSTEM PRIVATE
-        $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>
-        $<TARGET_PROPERTY:OpenCL::cl2hpp,INTERFACE_INCLUDE_DIRECTORIES>
-        $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
-        $<TARGET_PROPERTY:af_spdlog,INTERFACE_INCLUDE_DIRECTORIES>
-        ${ArrayFire_BINARY_DIR}/include
-      )
     if(TARGET Forge::forge)
       target_include_directories(opencl_sort_by_key_${SBK_TYPE}
         SYSTEM INTERFACE
-        $<TARGET_PROPERTY:Forge::forge,INCLUDE_DIRECTORIES>
+          $<TARGET_PROPERTY:Forge::forge,INCLUDE_DIRECTORIES>
       )
     else()
       target_include_directories(opencl_sort_by_key_${SBK_TYPE}
         SYSTEM INTERFACE
-        ${${forge_prefix}_SOURCE_DIR}/include
-        ${${forge_prefix}_BINARY_DIR}/include
+          ${${forge_prefix}_SOURCE_DIR}/include
+          ${${forge_prefix}_BINARY_DIR}/include
       )
     endif()
     if(TARGET glad::glad)
       target_include_directories(opencl_sort_by_key_${SBK_TYPE}
         SYSTEM INTERFACE
-        $<TARGET_PROPERTY:glad::glad,INTERFACE_INCLUDE_DIRECTORIES>
+          $<TARGET_PROPERTY:glad::glad,INTERFACE_INCLUDE_DIRECTORIES>
       )
     else()
       target_include_directories(opencl_sort_by_key_${SBK_TYPE}
         SYSTEM INTERFACE
-        $<TARGET_PROPERTY:af_glad,INTERFACE_INCLUDE_DIRECTORIES>
+          $<TARGET_PROPERTY:af_glad,INTERFACE_INCLUDE_DIRECTORIES>
       )
     endif()
 
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 50fcadaf5b..1ff1d94041 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -115,7 +115,10 @@ target_include_directories(arrayfire_test
   PRIVATE
     ${CMAKE_CURRENT_LIST_DIR}
     ${ArrayFire_SOURCE_DIR}/include
-    ${ArrayFire_BINARY_DIR}/include
+    ${ArrayFire_BINARY_DIR}/include)
+
+target_include_directories(arrayfire_test
+  SYSTEM PRIVATE
     ${ArrayFire_SOURCE_DIR}/extern/half/include
   )
 
@@ -185,9 +188,10 @@ function(make_test)
     add_executable(${target} ${mt_args_SRC})
     target_include_directories(${target}
       PRIVATE
-        ${ArrayFire_SOURCE_DIR}/extern/half/include
         ${CMAKE_SOURCE_DIR}
         ${CMAKE_CURRENT_SOURCE_DIR}
+      SYSTEM PRIVATE
+        ${ArrayFire_SOURCE_DIR}/extern/half/include
       )
     target_link_libraries(${target}
       PRIVATE
@@ -361,10 +365,12 @@ if(CUDA_FOUND)
     if(${backend} IN_LIST cuda_test_backends)
       set(target test_cuda_${backend})
       add_executable(${target} cuda.cu)
-      target_include_directories(${target} PRIVATE
-        ${ArrayFire_SOURCE_DIR}/extern/half/include
-        ${CMAKE_SOURCE_DIR}
-        ${CMAKE_CURRENT_SOURCE_DIR})
+      target_include_directories(${target}
+        PRIVATE
+          ${CMAKE_SOURCE_DIR}
+          ${CMAKE_CURRENT_SOURCE_DIR}
+        SYSTEM PRIVATE
+          ${ArrayFire_SOURCE_DIR}/extern/half/include)
       if(${backend} STREQUAL "unified")
         target_link_libraries(${target}
           ArrayFire::af)

From 021fab268972edb47dab371abc36003fe1b64d67 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 23 Nov 2022 18:53:20 -0500
Subject: [PATCH 145/473] Fix cl2hpp deprecated header warning

---
 src/backend/opencl/cl2hpp.hpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/backend/opencl/cl2hpp.hpp b/src/backend/opencl/cl2hpp.hpp
index ef6f80037b..729710d420 100644
--- a/src/backend/opencl/cl2hpp.hpp
+++ b/src/backend/opencl/cl2hpp.hpp
@@ -19,6 +19,14 @@ AF_DEPRECATED_WARNINGS_OFF
 #if __GNUC__ >= 8
 #pragma GCC diagnostic ignored "-Wcatch-value="
 #endif
+#ifdef __has_include
+#if __has_include(<CL/opencl.hpp>)
+#include <CL/opencl.hpp>
+#else
 #include <CL/cl2.hpp>
+#endif
+#else
+#include <CL/cl2.hpp>
+#endif
 AF_DEPRECATED_WARNINGS_ON
 #pragma GCC diagnostic pop

From 5d13f3835cfea7c57626c7934a0d000c2112a9fa Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 25 Nov 2022 17:28:18 -0500
Subject: [PATCH 146/473] Pass fast math flags to nvcc, NVRTC and OpenCL

---
 src/backend/cuda/CMakeLists.txt       |  1 +
 src/backend/cuda/compile_module.cpp   |  4 +++
 src/backend/cuda/kernel/jit.cuh       |  5 +++
 src/backend/cuda/math.hpp             | 35 ++++++++++-----------
 src/backend/opencl/compile_module.cpp |  4 +++
 src/backend/opencl/math.hpp           | 45 ++++++++++-----------------
 test/reduce.cpp                       |  1 +
 7 files changed, 48 insertions(+), 47 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 8490c541a0..ece17d962f 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -636,6 +636,7 @@ endif()
 
 target_compile_options(afcuda
   PRIVATE
+    $<$<BOOL:${AF_WITH_FAST_MATH}>:$<$<COMPILE_LANGUAGE:CUDA>:-use_fast_math>>
     $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>
     $<$<COMPILE_LANGUAGE:CUDA>:-Xcudafe --diag_suppress=unrecognized_gcc_pragma>
     $<$<COMPILE_LANGUAGE:CUDA>: $<$<CXX_COMPILER_ID:MSVC>:  -Xcompiler=/wd4251
diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp
index 3f5bd17d84..de22e8c493 100644
--- a/src/backend/cuda/compile_module.cpp
+++ b/src/backend/cuda/compile_module.cpp
@@ -261,6 +261,10 @@ Module compileModule(const string &moduleKey, span<const string> sources,
         arch.data(),
         "--std=c++14",
         "--device-as-default-execution-space",
+#ifdef AF_WITH_FAST_MATH
+        "--use_fast_math",
+        "-DAF_WITH_FAST_MATH",
+#endif
 #if !(defined(NDEBUG) || defined(__aarch64__) || defined(__LP64__))
         "--device-debug",
         "--generate-line-info"
diff --git a/src/backend/cuda/kernel/jit.cuh b/src/backend/cuda/kernel/jit.cuh
index 4681c151ed..cf69146114 100644
--- a/src/backend/cuda/kernel/jit.cuh
+++ b/src/backend/cuda/kernel/jit.cuh
@@ -59,8 +59,13 @@ typedef cuDoubleComplex cdouble;
 #define __rem(lhs, rhs) ((lhs) % (rhs))
 #define __mod(lhs, rhs) ((lhs) % (rhs))
 
+#ifdef AF_WITH_FAST_MATH
+#define __pow(lhs, rhs) \
+    static_cast<double>(pow(static_cast<double>(lhs), static_cast<double>(rhs)));
+#else
 #define __pow(lhs, rhs) \
     __float2int_rn(pow(__int2float_rn((int)lhs), __int2float_rn((int)rhs)))
+#endif
 #define __powll(lhs, rhs) \
     __double2ll_rn(pow(__ll2double_rn(lhs), __ll2double_rn(rhs)))
 #define __powul(lhs, rhs) \
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index 5987017fa7..23aa1a449b 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -32,6 +32,12 @@
 
 namespace cuda {
 
+#ifdef AF_WITH_FAST_MATH
+constexpr bool fast_math = true;
+#else
+constexpr bool fast_math = false;
+#endif
+
 template<typename T>
 static inline __DH__ T abs(T val) {
     return ::abs(val);
@@ -138,29 +144,22 @@ __DH__ static To scalar(Ti real, Ti imag) {
 }
 
 #ifndef __CUDA_ARCH__
+
 template<typename T>
 inline T maxval() {
-    return std::numeric_limits<T>::max();
+    if constexpr (std::is_floating_point_v<T> && !fast_math) {
+        return std::numeric_limits<T>::infinity();
+    } else {
+        return std::numeric_limits<T>::max();
+    }
 }
 template<typename T>
 inline T minval() {
-    return std::numeric_limits<T>::min();
-}
-template<>
-inline float maxval() {
-    return std::numeric_limits<float>::infinity();
-}
-template<>
-inline double maxval() {
-    return std::numeric_limits<double>::infinity();
-}
-template<>
-inline float minval() {
-    return -std::numeric_limits<float>::infinity();
-}
-template<>
-inline double minval() {
-    return -std::numeric_limits<double>::infinity();
+    if constexpr (std::is_floating_point_v<T> && !fast_math) {
+        return -std::numeric_limits<T>::infinity();
+    } else {
+        return std::numeric_limits<T>::lowest();
+    }
 }
 #else
 template<typename T>
diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp
index 83d66eb740..f931bb554a 100644
--- a/src/backend/opencl/compile_module.cpp
+++ b/src/backend/opencl/compile_module.cpp
@@ -126,6 +126,10 @@ Program buildProgram(span<const string> kernelSources,
         ostringstream options;
         for (auto &opt : compileOpts) { options << opt; }
 
+#ifdef AF_WITH_FAST_MATH
+        options << " -cl-fast-relaxed-math -DAF_WITH_FAST_MATH";
+#endif
+
         retVal.build({device}, (cl_std + defaults + options.str()).c_str());
     } catch (Error &err) {
         if (err.err() == CL_BUILD_PROGRAM_FAILURE) {
diff --git a/src/backend/opencl/math.hpp b/src/backend/opencl/math.hpp
index e1e9c28f12..e7cf8d1928 100644
--- a/src/backend/opencl/math.hpp
+++ b/src/backend/opencl/math.hpp
@@ -106,40 +106,27 @@ static To scalar(Ti real, Ti imag) {
     return cval;
 }
 
+#ifdef AF_WITH_FAST_MATH
+constexpr bool fast_math = true;
+#else
+constexpr bool fast_math = false;
+#endif
+
 template<typename T>
 inline T maxval() {
-    return std::numeric_limits<T>::max();
+    if constexpr (std::is_floating_point_v<T> && !fast_math) {
+        return std::numeric_limits<T>::infinity();
+    } else {
+        return std::numeric_limits<T>::max();
+    }
 }
 template<typename T>
 inline T minval() {
-    return std::numeric_limits<T>::min();
-}
-template<>
-inline float maxval() {
-    return std::numeric_limits<float>::infinity();
-}
-template<>
-inline double maxval() {
-    return std::numeric_limits<double>::infinity();
-}
-
-template<>
-inline common::half maxval() {
-    return std::numeric_limits<common::half>::infinity();
-}
-
-template<>
-inline float minval() {
-    return -std::numeric_limits<float>::infinity();
-}
-
-template<>
-inline double minval() {
-    return -std::numeric_limits<double>::infinity();
-}
-template<>
-inline common::half minval() {
-    return -std::numeric_limits<common::half>::infinity();
+    if constexpr (std::is_floating_point_v<T> && !fast_math) {
+        return -std::numeric_limits<T>::infinity();
+    } else {
+        return std::numeric_limits<T>::lowest();
+    }
 }
 
 static inline double real(cdouble in) { return in.s[0]; }
diff --git a/test/reduce.cpp b/test/reduce.cpp
index 5afdf70648..ef5b33bb1c 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -2296,6 +2296,7 @@ TEST(Reduce, Test_Sum_Global_Array_nanval) {
 }
 
 TEST(Reduce, nanval_issue_3255) {
+    SKIP_IF_FAST_MATH_ENABLED();
     char *info_str;
     af_array  ikeys, ivals, okeys, ovals;
     dim_t dims[1] = {8};

From 218173939f60fa4eb2d46e3738309b2f39744f78 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 25 Nov 2022 17:30:36 -0500
Subject: [PATCH 147/473] Set cublasMathMode and Atomic mode when
 AF_WITH_FAST_MATH is set

---
 src/backend/cuda/platform.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 7e82f76843..d3b7c2efd9 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -94,6 +94,12 @@ unique_handle<cublasHandle_t> *cublasManager(const int deviceId) {
         // call outside of call_once scope.
         CUBLAS_CHECK(
             cublasSetStream(handles[deviceId], cuda::getStream(deviceId)));
+#ifdef AF_WITH_FAST_MATH
+        CUBLAS_CHECK(
+            cublasSetMathMode(handles[deviceId], CUBLAS_TF32_TENSOR_OP_MATH));
+        CUBLAS_CHECK(
+            cublasSetAtomicsMode(handles[deviceId], CUBLAS_ATOMICS_ALLOWED));
+#endif
     });
 
     return &handles[deviceId];

From 921799bfe324956602a0df9b3f034e67108ff416 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Sat, 12 Nov 2022 12:18:07 -0500
Subject: [PATCH 148/473] wrap ported to oneapi. most tests fail due to missing
 jit

---
 src/backend/oneapi/CMakeLists.txt          |   2 +
 src/backend/oneapi/kernel/wrap.hpp         | 162 +++++++++++++++++++
 src/backend/oneapi/kernel/wrap_dilated.hpp | 177 +++++++++++++++++++++
 src/backend/oneapi/wrap.cpp                |  10 +-
 4 files changed, 345 insertions(+), 6 deletions(-)
 create mode 100755 src/backend/oneapi/kernel/wrap.hpp
 create mode 100755 src/backend/oneapi/kernel/wrap_dilated.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 3036a20b2f..826f144a83 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -233,6 +233,8 @@ target_sources(afoneapi
     kernel/transpose_inplace.hpp
     kernel/triangle.hpp
     kernel/where.hpp
+    kernel/wrap.hpp
+    kernel/wrap_dilated.hpp
 )
 
 add_library(ArrayFire::afoneapi ALIAS afoneapi)
diff --git a/src/backend/oneapi/kernel/wrap.hpp b/src/backend/oneapi/kernel/wrap.hpp
new file mode 100755
index 0000000000..0cac661ba6
--- /dev/null
+++ b/src/backend/oneapi/kernel/wrap.hpp
@@ -0,0 +1,162 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <kernel/default_config.hpp>
+#include <math.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
+                                      sycl::access::target::local>;
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T>
+class wrapCreateKernel {
+   public:
+    wrapCreateKernel(write_accessor<T> optrAcc, KParam out,
+                     read_accessor<T> iptrAcc, KParam in, const int wx,
+                     const int wy, const int sx, const int sy, const int px,
+                     const int py, const int nx, const int ny, int groups_x,
+                     int groups_y, const bool is_column)
+        : optrAcc_(optrAcc)
+        , out_(out)
+        , iptrAcc_(iptrAcc)
+        , in_(in)
+        , wx_(wx)
+        , wy_(wy)
+        , sx_(sx)
+        , sy_(sy)
+        , px_(px)
+        , py_(py)
+        , nx_(nx)
+        , ny_(ny)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , is_column_(is_column) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        int idx2 = g.get_group_id(0) / groups_x_;
+        int idx3 = g.get_group_id(1) / groups_y_;
+
+        int groupId_x = g.get_group_id(0) - idx2 * groups_x_;
+        int groupId_y = g.get_group_id(1) - idx3 * groups_y_;
+
+        int oidx0 = it.get_local_id(0) + g.get_local_range(0) * groupId_x;
+        int oidx1 = it.get_local_id(1) + g.get_local_range(1) * groupId_y;
+
+        T *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] +
+                  idx3 * out_.strides[3] + out_.offset;
+        T *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] +
+                  idx3 * in_.strides[3] + in_.offset;
+
+        if (oidx0 >= out_.dims[0] || oidx1 >= out_.dims[1]) return;
+
+        int pidx0 = oidx0 + px_;
+        int pidx1 = oidx1 + py_;
+
+        // The last time a value appears in_ the unwrapped index is padded_index
+        // / stride Each previous index has the value appear "stride" locations
+        // earlier We work our way back from the last index
+
+        const int x_end = fmin(pidx0 / sx_, nx_ - 1);
+        const int y_end = fmin(pidx1 / sy_, ny_ - 1);
+
+        const int x_off = pidx0 - sx_ * x_end;
+        const int y_off = pidx1 - sy_ * y_end;
+
+        T val   = (T)0;
+        int idx = 1;
+
+        for (int y = y_end, yo = y_off; y >= 0 && yo < wy_; yo += sy_, y--) {
+            int win_end_y = yo * wx_;
+            int dim_end_y = y * nx_;
+
+            for (int x = x_end, xo = x_off; x >= 0 && xo < wx_;
+                 xo += sx_, x--) {
+                int win_end = win_end_y + xo;
+                int dim_end = dim_end_y + x;
+
+                if (is_column_) {
+                    idx = dim_end * in_.strides[1] + win_end;
+                } else {
+                    idx = dim_end + win_end * in_.strides[1];
+                }
+
+                // No need to include anything special for complex
+                // Add for complex numbers is just vector add of reals
+                // Might need to change if we generalize add to more binary ops
+                val = val + iptr[idx];
+            }
+        }
+
+        optr[oidx1 * out_.strides[1] + oidx0] = val;
+    }
+
+   private:
+    write_accessor<T> optrAcc_;
+    KParam out_;
+    read_accessor<T> iptrAcc_;
+    KParam in_;
+    const int wx_;
+    const int wy_;
+    const int sx_;
+    const int sy_;
+    const int px_;
+    const int py_;
+    const int nx_;
+    const int ny_;
+    int groups_x_;
+    int groups_y_;
+    const bool is_column_;
+};
+
+template<typename T>
+void wrap(Param<T> out, const Param<T> in, const dim_t wx, const dim_t wy,
+          const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
+          const bool is_column) {
+    dim_t nx = (out.info.dims[0] + 2 * px - wx) / sx + 1;
+    dim_t ny = (out.info.dims[1] + 2 * py - wy) / sy + 1;
+
+    auto local = sycl::range{THREADS_X, THREADS_Y};
+
+    dim_t groups_x = divup(out.info.dims[0], local[0]);
+    dim_t groups_y = divup(out.info.dims[1], local[1]);
+
+    auto global = sycl::range{groups_x * local[0] * out.info.dims[2],
+                              groups_y * local[1]};
+
+    auto Q = getQueue();
+    Q.submit([&](sycl::handler &h) {
+        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor inAcc{*in.data, h, sycl::read_only};
+        h.parallel_for(sycl::nd_range{global, local},
+                       wrapCreateKernel<T>(outAcc, out.info, inAcc, in.info, wx,
+                                           wy, sx, sy, px, py, nx, ny, groups_x,
+                                           groups_y, is_column));
+    });
+    ONEAPI_DEBUG_FINISH(Q);
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/wrap_dilated.hpp b/src/backend/oneapi/kernel/wrap_dilated.hpp
new file mode 100755
index 0000000000..12760a57c6
--- /dev/null
+++ b/src/backend/oneapi/kernel/wrap_dilated.hpp
@@ -0,0 +1,177 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <kernel/default_config.hpp>
+#include <math.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
+                                      sycl::access::target::local>;
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T>
+class wrapDilatedCreateKernel {
+   public:
+    wrapDilatedCreateKernel(write_accessor<T> optrAcc, KParam out,
+                            read_accessor<T> iptrAcc, KParam in, const int wx,
+                            const int wy, const int sx, const int sy,
+                            const int px, const int py, const int dx,
+                            const int dy, const int nx, const int ny,
+                            int groups_x, int groups_y, const bool is_column)
+        : optrAcc_(optrAcc)
+        , out_(out)
+        , iptrAcc_(iptrAcc)
+        , in_(in)
+        , wx_(wx)
+        , wy_(wy)
+        , sx_(sx)
+        , sy_(sy)
+        , px_(px)
+        , py_(py)
+        , dx_(dx)
+        , dy_(dy)
+        , nx_(nx)
+        , ny_(ny)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , is_column_(is_column) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        int idx2 = g.get_group_id(0) / groups_x_;
+        int idx3 = g.get_group_id(1) / groups_y_;
+
+        int groupId_x = g.get_group_id(0) - idx2 * groups_x_;
+        int groupId_y = g.get_group_id(1) - idx3 * groups_y_;
+
+        int oidx0 = it.get_local_id(0) + g.get_local_range(0) * groupId_x;
+        int oidx1 = it.get_local_id(1) + g.get_local_range(1) * groupId_y;
+
+        T *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] +
+                  idx3 * out_.strides[3];
+        T *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] +
+                  idx3 * in_.strides[3] + in_.offset;
+
+        if (oidx0 >= out_.dims[0] || oidx1 >= out_.dims[1]) return;
+
+        int eff_wx = wx_ + (wx_ - 1) * (dx_ - 1);
+        int eff_wy = wy_ + (wy_ - 1) * (dy_ - 1);
+
+        int pidx0 = oidx0 + px_;
+        int pidx1 = oidx1 + py_;
+
+        // The last time a value appears in_ the unwrapped index is padded_index
+        // / stride Each previous index has the value appear "stride" locations
+        // earlier We work our way back from the last index
+
+        const int y_start = (pidx1 < eff_wy) ? 0 : (pidx1 - eff_wy) / sy_ + 1;
+        const int y_end   = fmin(pidx1 / sy_ + 1, ny_);
+
+        const int x_start = (pidx0 < eff_wx) ? 0 : (pidx0 - eff_wx) / sx_ + 1;
+        const int x_end   = fmin(pidx0 / sx_ + 1, nx_);
+
+        T val   = (T)0;
+        int idx = 1;
+
+        for (int y = y_start; y < y_end; y++) {
+            int fy      = (pidx1 - y * sy_);
+            bool yvalid = (fy % dy_ == 0) && (y < ny_);
+            fy /= dy_;
+
+            int win_end_y = fy * wx_;
+            int dim_end_y = y * nx_;
+
+            for (int x = x_start; x < x_end; x++) {
+                int fx      = (pidx0 - x * sx_);
+                bool xvalid = (fx % dx_ == 0) && (x < nx_);
+                fx /= dx_;
+
+                int win_end = win_end_y + fx;
+                int dim_end = dim_end_y + x;
+
+                if (is_column_) {
+                    idx = dim_end * in_.strides[1] + win_end;
+                } else {
+                    idx = dim_end + win_end * in_.strides[1];
+                }
+
+                T ival;
+                ival = (yvalid && xvalid) ? iptr[idx] : (T)0;
+                val  = val + ival;
+            }
+        }
+
+        optr[oidx1 * out_.strides[1] + oidx0] = val;
+    }
+
+   private:
+    write_accessor<T> optrAcc_;
+    KParam out_;
+    read_accessor<T> iptrAcc_;
+    KParam in_;
+    const int wx_;
+    const int wy_;
+    const int sx_;
+    const int sy_;
+    const int px_;
+    const int py_;
+    const int dx_;
+    const int dy_;
+    const int nx_;
+    const int ny_;
+    int groups_x_;
+    int groups_y_;
+    const bool is_column_;
+};
+
+template<typename T>
+void wrap_dilated(Param<T> out, const Param<T> in, const dim_t wx,
+                  const dim_t wy, const dim_t sx, const dim_t sy,
+                  const dim_t px, const dim_t py, const dim_t dx,
+                  const dim_t dy, const bool is_column) {
+    dim_t nx = 1 + (out.info.dims[0] + 2 * px - (((wx - 1) * dx) + 1)) / sx;
+    dim_t ny = 1 + (out.info.dims[1] + 2 * py - (((wy - 1) * dy) + 1)) / sy;
+
+    auto local = sycl::range{THREADS_X, THREADS_Y};
+
+    dim_t groups_x = divup(out.info.dims[0], local[0]);
+    dim_t groups_y = divup(out.info.dims[1], local[1]);
+
+    auto global = sycl::range{local[0] * groups_x * out.info.dims[2],
+                              local[1] * groups_y * out.info.dims[3]};
+
+    auto Q = getQueue();
+    Q.submit([&](sycl::handler &h) {
+        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor inAcc{*in.data, h, sycl::read_only};
+        h.parallel_for(sycl::nd_range{global, local},
+                       wrapDilatedCreateKernel<T>(
+                           outAcc, out.info, inAcc, in.info, wx, wy, sx, sy, px,
+                           py, dx, dy, nx, ny, groups_x, groups_y, is_column));
+    });
+    ONEAPI_DEBUG_FINISH(Q);
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/wrap.cpp b/src/backend/oneapi/wrap.cpp
index e3a9b2fc1f..b00b61efef 100644
--- a/src/backend/oneapi/wrap.cpp
+++ b/src/backend/oneapi/wrap.cpp
@@ -11,7 +11,8 @@
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/wrap.hpp>
+#include <kernel/wrap.hpp>
+#include <kernel/wrap_dilated.hpp>
 #include <math.hpp>
 #include <wrap.hpp>
 #include <stdexcept>
@@ -24,8 +25,7 @@ template<typename T>
 void wrap(Array<T> &out, const Array<T> &in, const dim_t wx, const dim_t wy,
           const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
           const bool is_column) {
-    ONEAPI_NOT_SUPPORTED("wrap Not supported");
-    // kernel::wrap<T>(out, in, wx, wy, sx, sy, px, py, is_column);
+    kernel::wrap<T>(out, in, wx, wy, sx, sy, px, py, is_column);
 }
 
 #define INSTANTIATE(T)                                                        \
@@ -57,9 +57,7 @@ Array<T> wrap_dilated(const Array<T> &in, const dim_t ox, const dim_t oy,
     af::dim4 odims(ox, oy, idims[2], idims[3]);
     Array<T> out = createValueArray<T>(odims, scalar<T>(0));
 
-    // kernel::wrap_dilated<T>(out, in, wx, wy, sx, sy, px, py, dx, dy,
-    // is_column);
-    ONEAPI_NOT_SUPPORTED("wrap_dilated Not supported");
+    kernel::wrap_dilated<T>(out, in, wx, wy, sx, sy, px, py, dx, dy, is_column);
     return out;
 }
 

From 052778ff48c83500d400ec3d63004524e864f405 Mon Sep 17 00:00:00 2001
From: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Date: Fri, 11 Nov 2022 14:29:50 -0500
Subject: [PATCH 149/473] unwrap ported to oneapi. all tests pass

Co-authored-by: Umar Arshad <umar@arrayfire.com>
---
 src/backend/oneapi/CMakeLists.txt    |   1 +
 src/backend/oneapi/kernel/unwrap.hpp | 170 +++++++++++++++++++++++++++
 src/backend/oneapi/unwrap.cpp        |   7 +-
 3 files changed, 174 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 src/backend/oneapi/CMakeLists.txt
 create mode 100755 src/backend/oneapi/kernel/unwrap.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
old mode 100644
new mode 100755
index 826f144a83..dcff3b35e9
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -232,6 +232,7 @@ target_sources(afoneapi
     kernel/transpose.hpp
     kernel/transpose_inplace.hpp
     kernel/triangle.hpp
+    kernel/unwrap.hpp
     kernel/where.hpp
     kernel/wrap.hpp
     kernel/wrap_dilated.hpp
diff --git a/src/backend/oneapi/kernel/unwrap.hpp b/src/backend/oneapi/kernel/unwrap.hpp
new file mode 100755
index 0000000000..475e55b66c
--- /dev/null
+++ b/src/backend/oneapi/kernel/unwrap.hpp
@@ -0,0 +1,170 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <kernel/default_config.hpp>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+class unwrapCreateKernel {
+   public:
+    unwrapCreateKernel(sycl::accessor<T, 1, sycl::access::mode::write> d_out,
+                       const KParam out,
+                       sycl::accessor<T, 1, sycl::access::mode::read> d_in,
+                       const KParam in, const int wx, const int wy,
+                       const int sx, const int sy, const int px, const int py,
+                       const int dx, const int dy, const int nx, const int reps,
+                       const bool IS_COLUMN)
+        : d_out_(d_out)
+        , out_(out)
+        , d_in_(d_in)
+        , in_(in)
+        , wx_(wx)
+        , wy_(wy)
+        , sx_(sx)
+        , sy_(sy)
+        , px_(px)
+        , py_(py)
+        , dx_(dx)
+        , dy_(dy)
+        , nx_(nx)
+        , reps_(reps)
+        , IS_COLUMN_(IS_COLUMN) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        // Compute channel and volume
+        const int w = g.get_group_id(1) / in_.dims[2];
+        const int z = g.get_group_id(1) - w * in_.dims[2];
+
+        if (w >= in_.dims[3] || z >= in_.dims[2]) return;
+
+        // Compute offset for channel and volume
+        const int cOut = w * out_.strides[3] + z * out_.strides[2];
+        const int cIn  = w * in_.strides[3] + z * in_.strides[2];
+
+        // Compute the output column index
+        const int id = IS_COLUMN_ ? (g.get_group_id(0) * g.get_local_range(1) +
+                                     it.get_local_id(1))
+                                  : it.get_global_id(0);
+
+        if (id >= (IS_COLUMN_ ? out_.dims[1] : out_.dims[0])) return;
+
+        // Compute the starting index of window in_ x and y of input
+        const int startx = (id % nx_) * sx_;
+        const int starty = (id / nx_) * sy_;
+
+        const int spx = startx - px_;
+        const int spy = starty - py_;
+
+        // Offset the global pointers to the respective starting indices
+        T *optr = d_out_.get_pointer() + cOut +
+                  id * (IS_COLUMN_ ? out_.strides[1] : 1);
+        const T *iptr = d_in_.get_pointer() + cIn + in_.offset;
+
+        bool cond = (spx >= 0 && spx + (wx_ * dx_) < in_.dims[0] && spy >= 0 &&
+                     spy + (wy_ * dy_) < in_.dims[1]);
+
+        // Compute output index local to column
+        int outIdx = IS_COLUMN_ ? it.get_local_id(0) : it.get_local_id(1);
+        const int oStride =
+            IS_COLUMN_ ? it.get_local_range(0) : it.get_local_range(1);
+
+        for (int i = 0; i < reps_; i++) {
+            if (outIdx >= (IS_COLUMN_ ? out_.dims[0] : out_.dims[1])) return;
+
+            // Compute input index local to window
+            const int y = outIdx / wx_;
+            const int x = outIdx % wx_;
+
+            const int xpad = spx + x * dx_;
+            const int ypad = spy + y * dy_;
+
+            // Copy
+            T val = (T)0;
+            if (cond || (xpad >= 0 && xpad < in_.dims[0] && ypad >= 0 &&
+                         ypad < in_.dims[1])) {
+                const int inIdx = ypad * in_.strides[1] + xpad * in_.strides[0];
+                val             = iptr[inIdx];
+            }
+
+            if (IS_COLUMN_) {
+                optr[outIdx] = val;
+            } else {
+                optr[outIdx * out_.strides[1]] = val;
+            }
+
+            outIdx += oStride;
+        }
+    }
+
+   private:
+    sycl::accessor<T, 1, sycl::access::mode::write> d_out_;
+    const KParam out_;
+    sycl::accessor<T, 1, sycl::access::mode::read> d_in_;
+    const KParam in_;
+    const int wx_;
+    const int wy_;
+    const int sx_;
+    const int sy_;
+    const int px_;
+    const int py_;
+    const int dx_;
+    const int dy_;
+    const int nx_;
+    const int reps_;
+    const bool IS_COLUMN_;
+};
+
+template<typename T>
+void unwrap(Param<T> out, const Param<T> in, const dim_t wx, const dim_t wy,
+            const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
+            const dim_t dx, const dim_t dy, const dim_t nx,
+            const bool IS_COLUMN) {
+    dim_t TX = 1, TY = 1;
+    dim_t BX       = 1;
+    const dim_t BY = out.info.dims[2] * out.info.dims[3];
+    int reps       = 1;
+
+    if (IS_COLUMN) {
+        TX   = std::min(THREADS_PER_BLOCK, nextpow2(out.info.dims[0]));
+        TY   = THREADS_PER_BLOCK / TX;
+        BX   = divup(out.info.dims[1], TY);
+        reps = divup((wx * wy), TX);
+    } else {
+        TX   = THREADS_X;
+        TY   = THREADS_X;
+        BX   = divup(out.info.dims[0], TX);
+        reps = divup((wx * wy), TY);
+    }
+
+    auto local  = sycl::range(TX, TY);
+    auto global = sycl::range(local[0] * BX, local[1] * BY);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_out{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor d_in{*in.data, h, sycl::read_only};
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            unwrapCreateKernel<T>(d_out, out.info, d_in, in.info, wx, wy, sx,
+                                  sy, px, py, dx, dy, nx, reps, IS_COLUMN));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/unwrap.cpp b/src/backend/oneapi/unwrap.cpp
index cbb2910ef7..bfb21aef17 100644
--- a/src/backend/oneapi/unwrap.cpp
+++ b/src/backend/oneapi/unwrap.cpp
@@ -10,7 +10,7 @@
 #include <Array.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/unwrap.hpp>
+#include <kernel/unwrap.hpp>
 #include <unwrap.hpp>
 #include <stdexcept>
 
@@ -32,9 +32,8 @@ Array<T> unwrap(const Array<T> &in, const dim_t wx, const dim_t wy,
     if (!is_column) { std::swap(odims[0], odims[1]); }
 
     Array<T> outArray = createEmptyArray<T>(odims);
-    ONEAPI_NOT_SUPPORTED("unwrap Not supported");
-    // kernel::unwrap<T>(outArray, in, wx, wy, sx, sy, px, py, dx, dy, nx,
-    //                   is_column);
+    kernel::unwrap<T>(outArray, in, wx, wy, sx, sy, px, py, dx, dy, nx,
+                      is_column);
 
     return outArray;
 }

From 138f12e9f181b8a7bd013323137931aec0f3bd59 Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Tue, 29 Nov 2022 12:52:55 -0500
Subject: [PATCH 150/473] convolve{1,2,3} oneapi port redo (#3327)

* convolve{1,2,3}. fails separable, unwrap, double, jit. as expected


Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
---
 src/backend/oneapi/convolve.cpp         |  58 ++++++-
 src/backend/oneapi/kernel/convolve.hpp  | 145 +++++++++++++++++
 src/backend/oneapi/kernel/convolve1.hpp | 174 +++++++++++++++++++++
 src/backend/oneapi/kernel/convolve2.hpp | 193 +++++++++++++++++++++++
 src/backend/oneapi/kernel/convolve3.hpp | 199 ++++++++++++++++++++++++
 5 files changed, 765 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 src/backend/oneapi/convolve.cpp
 create mode 100755 src/backend/oneapi/kernel/convolve.hpp
 create mode 100755 src/backend/oneapi/kernel/convolve1.hpp
 create mode 100755 src/backend/oneapi/kernel/convolve2.hpp
 create mode 100755 src/backend/oneapi/kernel/convolve3.hpp

diff --git a/src/backend/oneapi/convolve.cpp b/src/backend/oneapi/convolve.cpp
old mode 100644
new mode 100755
index 94e6d48d09..a7a2fc9aee
--- a/src/backend/oneapi/convolve.cpp
+++ b/src/backend/oneapi/convolve.cpp
@@ -14,6 +14,7 @@
 #include <common/moddims.hpp>
 #include <convolve.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/convolve.hpp>
 #include <reorder.hpp>
 #include <transpose.hpp>
 #include <unwrap.hpp>
@@ -33,8 +34,56 @@ namespace oneapi {
 template<typename T, typename accT>
 Array<T> convolve(Array<T> const &signal, Array<accT> const &filter,
                   AF_BATCH_KIND kind, const int rank, const bool expand) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> out = createEmptyArray<T>(dim4(1));
+    const dim4 &sDims = signal.dims();
+    const dim4 &fDims = filter.dims();
+
+    dim4 oDims(1);
+    if (expand) {
+        for (int d = 0; d < AF_MAX_DIMS; ++d) {
+            if (kind == AF_BATCH_NONE || kind == AF_BATCH_RHS) {
+                oDims[d] = sDims[d] + fDims[d] - 1;
+            } else {
+                oDims[d] = (d < rank ? sDims[d] + fDims[d] - 1 : sDims[d]);
+            }
+        }
+    } else {
+        oDims = sDims;
+        if (kind == AF_BATCH_RHS) {
+            for (int i = rank; i < AF_MAX_DIMS; ++i) { oDims[i] = fDims[i]; }
+        }
+    }
+
+    Array<T> out    = createEmptyArray<T>(oDims);
+    bool callKernel = true;
+
+    dim_t MCFL2 = kernel::MAX_CONV2_FILTER_LEN;
+    dim_t MCFL3 = kernel::MAX_CONV3_FILTER_LEN;
+    switch (rank) {
+        case 1:
+            if (fDims[0] > kernel::MAX_CONV1_FILTER_LEN) { callKernel = false; }
+            break;
+        case 2:
+            if ((fDims[0] * fDims[1]) > (MCFL2 * MCFL2)) { callKernel = false; }
+            break;
+        case 3:
+            if ((fDims[0] * fDims[1] * fDims[2]) > (MCFL3 * MCFL3 * MCFL3)) {
+                callKernel = false;
+            }
+            break;
+        default: AF_ERROR("rank only supports values 1-3.", AF_ERR_UNKNOWN);
+    }
+
+    if (!callKernel) {
+        char errMessage[256];
+        snprintf(errMessage, sizeof(errMessage),
+                 "\nOneAPI N Dimensional Convolution doesn't support "
+                 "%llux%llux%llu kernel\n",
+                 fDims[0], fDims[1], fDims[2]);
+        ONEAPI_NOT_SUPPORTED(errMessage);
+    }
+
+    kernel::convolve_nd<T, accT>(out, signal, filter, kind, rank, expand);
+
     return out;
 }
 
@@ -60,8 +109,9 @@ template<typename T>
 Array<T> convolve2_unwrap(const Array<T> &signal, const Array<T> &filter,
                           const dim4 &stride, const dim4 &padding,
                           const dim4 &dilation) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> out = createEmptyArray<T>(dim4(1));
+    Array<T> out =
+        convolve2_unwrap<T>(signal, filter, stride, padding, dilation);
+
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/convolve.hpp b/src/backend/oneapi/kernel/convolve.hpp
new file mode 100755
index 0000000000..39abe603ad
--- /dev/null
+++ b/src/backend/oneapi/kernel/convolve.hpp
@@ -0,0 +1,145 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <af/defines.h>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+// below shared MAX_*_LEN's are calculated based on
+// a maximum shared memory configuration of 48KB per block
+// considering complex types as well
+constexpr int MAX_CONV1_FILTER_LEN = 129;
+constexpr int MAX_CONV2_FILTER_LEN = 17;
+constexpr int MAX_CONV3_FILTER_LEN = 5;
+
+constexpr int MAX_SCONV_FILTER_LEN = 31;
+
+constexpr int THREADS   = 256;
+constexpr int THREADS_X = 16;
+constexpr int THREADS_Y = 16;
+constexpr int CUBE_X    = 8;
+constexpr int CUBE_Y    = 8;
+constexpr int CUBE_Z    = 4;
+
+template<typename aT>
+struct conv_kparam_t {
+    sycl::range<3> global{0, 0, 0};
+    sycl::range<3> local{0, 0, 0};
+    size_t loc_size;
+    int nBBS0;
+    int nBBS1;
+    bool outHasNoOffset;
+    bool inHasNoOffset;
+    bool launchMoreBlocks;
+    int o[3];
+    int s[3];
+    sycl::buffer<aT> *impulse;
+};
+
+template<typename T>
+T binOp(T lhs, T rhs) {
+    return lhs * rhs;
+}
+
+template<typename aT>
+void prepareKernelArgs(conv_kparam_t<aT> &param, dim_t *oDims,
+                       const dim_t *fDims, const int rank) {
+    using sycl::range;
+
+    int batchDims[4] = {1, 1, 1, 1};
+    for (int i = rank; i < 4; ++i) {
+        batchDims[i] = (param.launchMoreBlocks ? 1 : oDims[i]);
+    }
+
+    if (rank == 1) {
+        param.local    = range<3>{THREADS, 1, 1};
+        param.nBBS0    = divup(oDims[0], THREADS);
+        param.nBBS1    = batchDims[2];
+        param.global   = range<3>(param.nBBS0 * THREADS * batchDims[1],
+                                param.nBBS1 * batchDims[3], 1);
+        param.loc_size = (THREADS + 2 * (fDims[0] - 1));
+    } else if (rank == 2) {
+        param.local  = range<3>{THREADS_X, THREADS_Y, 1};
+        param.nBBS0  = divup(oDims[0], THREADS_X);
+        param.nBBS1  = divup(oDims[1], THREADS_Y);
+        param.global = range<3>(param.nBBS0 * THREADS_X * batchDims[2],
+                                param.nBBS1 * THREADS_Y * batchDims[3], 1);
+    } else if (rank == 3) {
+        param.local    = range<3>{CUBE_X, CUBE_Y, CUBE_Z};
+        param.nBBS0    = divup(oDims[0], CUBE_X);
+        param.nBBS1    = divup(oDims[1], CUBE_Y);
+        int blk_z      = divup(oDims[2], CUBE_Z);
+        param.global   = range<3>(param.nBBS0 * CUBE_X * batchDims[3],
+                                param.nBBS1 * CUBE_Y, blk_z * CUBE_Z);
+        param.loc_size = (CUBE_X + 2 * (fDims[0] - 1)) *
+                         (CUBE_Y + 2 * (fDims[1] - 1)) *
+                         (CUBE_Z + 2 * (fDims[2] - 1));
+    }
+}
+
+template<typename T>
+void memcpyBuffer(sycl::buffer<T, 1> &dest, sycl::buffer<T, 1> &src,
+                  const size_t n, const size_t srcOffset) {
+    getQueue().submit([&](auto &h) {
+        sycl::accessor srcAcc{src, h, sycl::range{n}, sycl::id{srcOffset},
+                              sycl::read_only};
+        sycl::accessor destAcc{
+            dest,         h, sycl::range{n}, sycl::id{0}, sycl::write_only,
+            sycl::no_init};
+        h.copy(srcAcc, destAcc);
+    });
+}
+
+template<typename T>
+using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
+                                      sycl::access::target::local>;
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+#include "convolve1.hpp"
+#include "convolve2.hpp"
+#include "convolve3.hpp"
+
+template<typename T, typename aT>
+void convolve_nd(Param<T> out, const Param<T> signal, const Param<aT> filter,
+                 AF_BATCH_KIND kind, const int rank, const bool expand) {
+    conv_kparam_t<aT> param;
+
+    for (int i = 0; i < 3; ++i) {
+        param.o[i] = 0;
+        param.s[i] = 0;
+    }
+    param.launchMoreBlocks = kind == AF_BATCH_SAME || kind == AF_BATCH_RHS;
+    param.outHasNoOffset   = kind == AF_BATCH_LHS || kind == AF_BATCH_NONE;
+    param.inHasNoOffset    = kind != AF_BATCH_SAME;
+
+    prepareKernelArgs<aT>(param, out.info.dims, filter.info.dims, rank);
+
+    switch (rank) {
+        case 1: conv1<T, aT>(param, out, signal, filter, expand); break;
+        case 2: conv2<T, aT>(param, out, signal, filter, expand); break;
+        case 3: conv3<T, aT>(param, out, signal, filter, expand); break;
+    }
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/convolve1.hpp b/src/backend/oneapi/kernel/convolve1.hpp
new file mode 100755
index 0000000000..1383bb4591
--- /dev/null
+++ b/src/backend/oneapi/kernel/convolve1.hpp
@@ -0,0 +1,174 @@
+template<typename T, typename aT>
+class conv1HelperCreateKernel {
+   public:
+    conv1HelperCreateKernel(write_accessor<T> out, KParam oInfo,
+                            read_accessor<T> signal, KParam sInfo,
+                            local_accessor<aT> localMem,
+                            read_accessor<aT> impulse, KParam fInfo, int nBBS0,
+                            int nBBS1, int ostep1, int ostep2, int ostep3,
+                            int sstep1, int sstep2, int sstep3,
+                            const bool expand)
+        : out_(out)
+        , oInfo_(oInfo)
+        , signal_(signal)
+        , sInfo_(sInfo)
+        , localMem_(localMem)
+        , impulse_(impulse)
+        , fInfo_(fInfo)
+        , nBBS0_(nBBS0)
+        , nBBS1_(nBBS1)
+        , ostep1_(ostep1)
+        , ostep2_(ostep2)
+        , ostep3_(ostep3)
+        , sstep1_(sstep1)
+        , sstep2_(sstep2)
+        , sstep3_(sstep3)
+        , expand_(expand) {}
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g = it.get_group();
+
+        int fLen          = fInfo_.dims[0];
+        int padding       = fLen - 1;
+        int shrdLen       = g.get_local_range(0) + 2 * padding;
+        const unsigned b1 = g.get_group_id(0) / nBBS0_;
+        const unsigned b0 = g.get_group_id(0) - nBBS0_ * b1;
+        const unsigned b3 = g.get_group_id(1) / nBBS1_;
+        const unsigned b2 = g.get_group_id(1) - nBBS1_ * b3;
+
+        T *dst =
+            out_.get_pointer() +
+            (b1 * oInfo_.strides[1] + /* activated with batched input signal_ */
+             ostep1_ *
+                 oInfo_.strides[1] +  /* activated with batched input filter */
+             b2 * oInfo_.strides[2] + /* activated with batched input signal_ */
+             ostep2_ *
+                 oInfo_.strides[2] +  /* activated with batched input filter */
+             b3 * oInfo_.strides[3] + /* activated with batched input signal_ */
+             ostep3_ *
+                 oInfo_.strides[3]); /* activated with batched input filter */
+
+        T const *src =
+            signal_.get_pointer() + sInfo_.offset +
+            (b1 * sInfo_.strides[1] + /* activated with batched input signal_ */
+             sstep1_ *
+                 sInfo_.strides[1] +  /* activated with batched input filter */
+             b2 * sInfo_.strides[2] + /* activated with batched input signal_ */
+             sstep2_ *
+                 sInfo_.strides[2] +  /* activated with batched input filter */
+             b3 * sInfo_.strides[3] + /* activated with batched input signal_ */
+             sstep3_ *
+                 sInfo_.strides[3]); /* activated with batched input filter */
+
+        int gx = g.get_local_range(0) * b0;
+
+        for (int i = it.get_local_id(0); i < shrdLen;
+             i += g.get_local_range(0)) {
+            int idx      = gx - padding + i;
+            localMem_[i] = (idx >= 0 && idx < sInfo_.dims[0])
+                               ? src[idx * sInfo_.strides[0]]
+                               : (T)(0);
+        }
+        it.barrier();
+        gx += it.get_local_id(0);
+
+        if (gx >= 0 && gx < oInfo_.dims[0]) {
+            int lx   = it.get_local_id(0) + padding + (expand_ ? 0 : fLen >> 1);
+            aT accum = (aT)(0);
+            for (int f = 0; f < fLen; ++f) {
+                // binOp will do MUL_OP for convolution operation
+                accum = accum + binOp((aT)localMem_[lx - f], (aT)impulse_[f]);
+            }
+            dst[gx] = (T)accum;
+        }
+    }
+
+   private:
+    write_accessor<T> out_;
+    KParam oInfo_;
+    read_accessor<T> signal_;
+    KParam sInfo_;
+    local_accessor<aT> localMem_;
+    read_accessor<aT> impulse_;
+    KParam fInfo_;
+    int nBBS0_;
+    int nBBS1_;
+    int ostep1_;
+    int ostep2_;
+    int ostep3_;
+    int sstep1_;
+    int sstep2_;
+    int sstep3_;
+    const bool expand_;
+};
+
+template<typename T, typename aT>
+void conv1Helper(const conv_kparam_t<aT> &param, Param<T> &out,
+                 const Param<T> &signal, const Param<aT> &filter,
+                 const int rank, const bool expand) {
+    auto Q = getQueue();
+    Q.submit([&](auto &h) {
+        sycl::accessor<aT, 1, sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            localMem(param.loc_size, h);
+        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor signalAcc{*signal.data, h, sycl::read_only};
+        sycl::accessor impulseAcc{*param.impulse, h, sycl::read_only};
+        h.parallel_for(
+            sycl::nd_range{param.global, param.local},
+            conv1HelperCreateKernel<T, aT>(
+                outAcc, out.info, signalAcc, signal.info, localMem, impulseAcc,
+                filter.info, param.nBBS0, param.nBBS1, param.o[0], param.o[1],
+                param.o[2], param.s[0], param.s[1], param.s[2], expand));
+    });
+    ONEAPI_DEBUG_FINISH(Q);
+}
+
+template<typename T, typename aT>
+void conv1(conv_kparam_t<aT> &p, Param<T> &out, const Param<T> &sig,
+           const Param<aT> &filt, const bool expand) {
+    const size_t se_size = filt.info.dims[0];
+    sycl::buffer<aT> impulse{sycl::range(filt.info.dims[0])};
+    int f0Off = filt.info.offset;
+    for (int b3 = 0; b3 < filt.info.dims[3]; ++b3) {
+        int f3Off = b3 * filt.info.strides[3];
+
+        for (int b2 = 0; b2 < filt.info.dims[2]; ++b2) {
+            int f2Off = b2 * filt.info.strides[2];
+
+            for (int b1 = 0; b1 < filt.info.dims[1]; ++b1) {
+                int f1Off = b1 * filt.info.strides[1];
+
+                const size_t srcOffset = f0Off + f1Off + f2Off + f3Off;
+                memcpyBuffer(impulse, *filt.data, se_size, srcOffset);
+                p.impulse = &impulse;
+
+                p.o[0] = (p.outHasNoOffset ? 0 : b1);
+                p.o[1] = (p.outHasNoOffset ? 0 : b2);
+                p.o[2] = (p.outHasNoOffset ? 0 : b3);
+                p.s[0] = (p.inHasNoOffset ? 0 : b1);
+                p.s[1] = (p.inHasNoOffset ? 0 : b2);
+                p.s[2] = (p.inHasNoOffset ? 0 : b3);
+
+                conv1Helper<T, aT>(p, out, sig, filt, 1, expand);
+            }
+        }
+    }
+}
+
+#define INSTANTIATE_CONV1(T, aT)                                    \
+    template void conv1<T, aT>(conv_kparam_t<aT> &, Param<T> &,     \
+                               const Param<T> &, const Param<aT> &, \
+                               const bool);
+
+INSTANTIATE_CONV1(cdouble, cdouble)
+INSTANTIATE_CONV1(cfloat, cfloat)
+INSTANTIATE_CONV1(double, double)
+INSTANTIATE_CONV1(float, float)
+INSTANTIATE_CONV1(uint, float)
+INSTANTIATE_CONV1(int, float)
+INSTANTIATE_CONV1(uchar, float)
+INSTANTIATE_CONV1(char, float)
+INSTANTIATE_CONV1(ushort, float)
+INSTANTIATE_CONV1(short, float)
+INSTANTIATE_CONV1(uintl, float)
+INSTANTIATE_CONV1(intl, float)
diff --git a/src/backend/oneapi/kernel/convolve2.hpp b/src/backend/oneapi/kernel/convolve2.hpp
new file mode 100755
index 0000000000..5232b225ff
--- /dev/null
+++ b/src/backend/oneapi/kernel/convolve2.hpp
@@ -0,0 +1,193 @@
+template<typename T, typename aT>
+class conv2HelperCreateKernel {
+   public:
+    conv2HelperCreateKernel(write_accessor<T> out, KParam oInfo,
+                            read_accessor<T> signal, KParam sInfo,
+                            read_accessor<aT> impulse, KParam fInfo, int nBBS0,
+                            int nBBS1, int ostep2, int ostep3, int sstep2,
+                            int sstep3, local_accessor<aT> localMem,
+                            const int f0, const int f1, const bool expand)
+        : out_(out)
+        , oInfo_(oInfo)
+        , signal_(signal)
+        , sInfo_(sInfo)
+        , impulse_(impulse)
+        , fInfo_(fInfo)
+        , nBBS0_(nBBS0)
+        , nBBS1_(nBBS1)
+        , ostep2_(ostep2)
+        , ostep3_(ostep3)
+        , sstep2_(sstep2)
+        , sstep3_(sstep3)
+        , localMem_(localMem)
+        , f0_(f0)
+        , f1_(f1)
+        , expand_(expand) {}
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g = it.get_group();
+
+        int radius0  = f0_ - 1;
+        int radius1  = f1_ - 1;
+        int padding0 = 2 * radius0;
+        int padding1 = 2 * radius1;
+        int shrdLen0 = g.get_local_range(0) + padding0;
+        int shrdLen1 = g.get_local_range(1) + padding1;
+
+        unsigned b0 = g.get_group_id(0) / nBBS0_;
+        unsigned b1 = g.get_group_id(1) / nBBS1_;
+
+        T *dst =
+            out_.get_pointer() +
+            (b0 * oInfo_.strides[2] + /* activated with batched input signal_ */
+             ostep2_ *
+                 oInfo_.strides[2] +  /* activated with batched input filter */
+             b1 * oInfo_.strides[3] + /* activated with batched input signal_ */
+             ostep3_ *
+                 oInfo_.strides[3]); /* activated with batched input filter */
+
+        const T *src =
+            signal_.get_pointer() + sInfo_.offset +
+            (b0 * sInfo_.strides[2] + /* activated with batched input signal_ */
+             sstep2_ *
+                 sInfo_.strides[2] +  /* activated with batched input filter */
+             b1 * sInfo_.strides[3] + /* activated with batched input signal_ */
+             sstep3_ *
+                 sInfo_.strides[3]); /* activated with batched input filter */
+
+        int lx = it.get_local_id(0);
+        int ly = it.get_local_id(1);
+        int gx = g.get_local_range(0) * (g.get_group_id(0) - b0 * nBBS0_) + lx;
+        int gy = g.get_local_range(1) * (g.get_group_id(1) - b1 * nBBS1_) + ly;
+
+        // below loops are traditional loops, they only run multiple
+        // times filter length is more than launch size
+        int s0 = sInfo_.strides[0];
+        int s1 = sInfo_.strides[1];
+        int d0 = sInfo_.dims[0];
+        int d1 = sInfo_.dims[1];
+        for (int b = ly, gy2 = gy; b < shrdLen1;
+             b += g.get_local_range(1), gy2 += g.get_local_range(1)) {
+            int j     = gy2 - radius1;
+            bool is_j = j >= 0 && j < d1;
+            // move row_set g.get_local_range(1) along coloumns
+            for (int a = lx, gx2 = gx; a < shrdLen0;
+                 a += g.get_local_range(0), gx2 += g.get_local_range(0)) {
+                int i     = gx2 - radius0;
+                bool is_i = i >= 0 && i < d0;
+                localMem_[b * shrdLen0 + a] =
+                    (is_i && is_j ? src[i * s0 + j * s1] : (T)(0));
+            }
+        }
+        it.barrier();
+
+        if (gx < oInfo_.dims[0] && gy < oInfo_.dims[1]) {
+            int ci = lx + radius0 + (expand_ ? 0 : f0_ >> 1);
+            int cj = ly + radius1 + (expand_ ? 0 : f1_ >> 1);
+
+            aT accum = (aT)(0);
+            for (int fj = 0; fj < f1_; ++fj) {
+                for (int fi = 0; fi < f0_; ++fi) {
+                    aT f_val = impulse_[fj * f0_ + fi];
+                    T s_val  = localMem_[(cj - fj) * shrdLen0 + (ci - fi)];
+
+                    // binOp will do MUL_OP for convolution operation
+                    accum = accum + binOp((aT)s_val, (aT)f_val);
+                }
+            }
+            dst[gy * oInfo_.strides[1] + gx] = (T)accum;
+        }
+    }
+
+   private:
+    write_accessor<T> out_;
+    KParam oInfo_;
+    read_accessor<T> signal_;
+    KParam sInfo_;
+    read_accessor<aT> impulse_;
+    KParam fInfo_;
+    int nBBS0_;
+    int nBBS1_;
+    int ostep2_;
+    int ostep3_;
+    int sstep2_;
+    int sstep3_;
+    local_accessor<aT> localMem_;
+    const int f0_;
+    const int f1_;
+    const bool expand_;
+};
+
+template<typename T, typename aT>
+void conv2Helper(const conv_kparam_t<aT> &param, Param<T> out,
+                 const Param<T> signal, const Param<aT> filter,
+                 const bool expand) {
+    constexpr bool IsComplex =
+        std::is_same<T, cfloat>::value || std::is_same<T, cdouble>::value;
+
+    const int f0 = filter.info.dims[0];
+    const int f1 = filter.info.dims[1];
+    const size_t LOC_SIZE =
+        (THREADS_X + 2 * (f0 - 1)) * (THREADS_Y + 2 * (f1 - 1));
+
+    auto Q = getQueue();
+    Q.submit([&](auto &h) {
+        sycl::accessor<aT, 1, sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            localMem(LOC_SIZE, h);
+        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor signalAcc{*signal.data, h, sycl::read_only};
+        sycl::accessor impulseAcc{*param.impulse, h, sycl::read_only};
+        h.parallel_for(
+            sycl::nd_range{param.global, param.local},
+            conv2HelperCreateKernel<T, aT>(
+                outAcc, out.info, signalAcc, signal.info, impulseAcc,
+                filter.info, param.nBBS0, param.nBBS1, param.o[1], param.o[2],
+                param.s[1], param.s[2], localMem, f0, f1, expand));
+    });
+    ONEAPI_DEBUG_FINISH(Q);
+}
+
+template<typename T, typename aT>
+void conv2(conv_kparam_t<aT> &p, Param<T> &out, const Param<T> &sig,
+           const Param<aT> &filt, const bool expand) {
+    size_t se_size = filt.info.dims[0] * filt.info.dims[1];
+    sycl::buffer<aT> impulse{sycl::range(se_size)};
+    int f0Off = filt.info.offset;
+
+    for (int b3 = 0; b3 < filt.info.dims[3]; ++b3) {
+        int f3Off = b3 * filt.info.strides[3];
+
+        for (int b2 = 0; b2 < filt.info.dims[2]; ++b2) {
+            int f2Off = b2 * filt.info.strides[2];
+
+            const size_t srcOffset = f2Off + f3Off + f0Off;
+            memcpyBuffer(impulse, *filt.data, se_size, srcOffset);
+            p.impulse = &impulse;
+
+            p.o[1] = (p.outHasNoOffset ? 0 : b2);
+            p.o[2] = (p.outHasNoOffset ? 0 : b3);
+            p.s[1] = (p.inHasNoOffset ? 0 : b2);
+            p.s[2] = (p.inHasNoOffset ? 0 : b3);
+
+            conv2Helper<T, aT>(p, out, sig, filt, expand);
+        }
+    }
+}
+
+#define INSTANTIATE_CONV2(T, aT)                                    \
+    template void conv2<T, aT>(conv_kparam_t<aT> &, Param<T> &,     \
+                               const Param<T> &, const Param<aT> &, \
+                               const bool);
+
+INSTANTIATE_CONV2(char, float)
+INSTANTIATE_CONV2(cfloat, cfloat)
+INSTANTIATE_CONV2(cdouble, cdouble)
+INSTANTIATE_CONV2(float, float)
+INSTANTIATE_CONV2(double, double)
+INSTANTIATE_CONV2(short, float)
+INSTANTIATE_CONV2(int, float)
+INSTANTIATE_CONV2(intl, float)
+INSTANTIATE_CONV2(ushort, float)
+INSTANTIATE_CONV2(uint, float)
+INSTANTIATE_CONV2(uintl, float)
+INSTANTIATE_CONV2(uchar, float)
diff --git a/src/backend/oneapi/kernel/convolve3.hpp b/src/backend/oneapi/kernel/convolve3.hpp
new file mode 100755
index 0000000000..d9a93affef
--- /dev/null
+++ b/src/backend/oneapi/kernel/convolve3.hpp
@@ -0,0 +1,199 @@
+int index(int i, int j, int k, int jstride, int kstride) {
+    return i + j * jstride + k * kstride;
+}
+
+template<typename T, typename aT>
+class conv3HelperCreateKernel {
+   public:
+    conv3HelperCreateKernel(write_accessor<T> out, KParam oInfo,
+                            read_accessor<T> signal, KParam sInfo,
+                            local_accessor<aT> localMem,
+                            read_accessor<aT> impulse, KParam fInfo, int nBBS0,
+                            int nBBS1, int ostep1, int ostep2, int ostep3,
+                            int sstep1, int sstep2, int sstep3,
+                            const bool EXPAND)
+        : out_(out)
+        , oInfo_(oInfo)
+        , signal_(signal)
+        , sInfo_(sInfo)
+        , localMem_(localMem)
+        , impulse_(impulse)
+        , fInfo_(fInfo)
+        , nBBS0_(nBBS0)
+        , nBBS1_(nBBS1)
+        , ostep1_(ostep1)
+        , ostep2_(ostep2)
+        , ostep3_(ostep3)
+        , sstep1_(sstep1)
+        , sstep2_(sstep2)
+        , sstep3_(sstep3)
+        , EXPAND_(EXPAND) {}
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g = it.get_group();
+        int fLen0     = fInfo_.dims[0];
+        int fLen1     = fInfo_.dims[1];
+        int fLen2     = fInfo_.dims[2];
+        int radius0   = fLen0 - 1;
+        int radius1   = fLen1 - 1;
+        int radius2   = fLen2 - 1;
+        int shrdLen0  = g.get_local_range(0) + 2 * radius0;
+        int shrdLen1  = g.get_local_range(1) + 2 * radius1;
+        int shrdLen2  = g.get_local_range(2) + 2 * radius2;
+        int skStride  = shrdLen0 * shrdLen1;
+        int fStride   = fLen0 * fLen1;
+        unsigned b2   = g.get_group_id(0) / nBBS0_;
+
+        T *dst =
+            out_.get_pointer() +
+            (b2 * oInfo_.strides[3] + /* activated with batched input signal_ */
+             ostep3_ *
+                 oInfo_.strides[3]); /* activated with batched input filter */
+
+        const T *src =
+            signal_.get_pointer() + sInfo_.offset +
+            (b2 * sInfo_.strides[3] + /* activated with batched input signal_ */
+             sstep3_ *
+                 sInfo_.strides[3]); /* activated with batched input filter */
+
+        int lx  = it.get_local_id(0);
+        int ly  = it.get_local_id(1);
+        int lz  = it.get_local_id(2);
+        int gx  = g.get_local_range(0) * (g.get_group_id(0) - b2 * nBBS0_) + lx;
+        int gy  = g.get_local_range(1) * g.get_group_id(1) + ly;
+        int gz  = g.get_local_range(2) * g.get_group_id(2) + lz;
+        int lx2 = lx + g.get_local_range(0);
+        int ly2 = ly + g.get_local_range(1);
+        int lz2 = lz + g.get_local_range(2);
+        int gx2 = gx + g.get_local_range(0);
+        int gy2 = gy + g.get_local_range(1);
+        int gz2 = gz + g.get_local_range(2);
+
+        int s0 = sInfo_.strides[0];
+        int s1 = sInfo_.strides[1];
+        int s2 = sInfo_.strides[2];
+        int d0 = sInfo_.dims[0];
+        int d1 = sInfo_.dims[1];
+        int d2 = sInfo_.dims[2];
+
+        for (int c = lz, gz2 = gz; c < shrdLen2;
+             c += g.get_local_range(2), gz2 += g.get_local_range(2)) {
+            int k     = gz2 - radius2;
+            bool is_k = k >= 0 && k < d2;
+            for (int b = ly, gy2 = gy; b < shrdLen1;
+                 b += g.get_local_range(1), gy2 += g.get_local_range(1)) {
+                int j     = gy2 - radius1;
+                bool is_j = j >= 0 && j < d1;
+                for (int a = lx, gx2 = gx; a < shrdLen0;
+                     a += g.get_local_range(0), gx2 += g.get_local_range(0)) {
+                    int i     = gx2 - radius0;
+                    bool is_i = i >= 0 && i < d0;
+                    localMem_[c * skStride + b * shrdLen0 + a] =
+                        (is_i && is_j && is_k ? src[i * s0 + j * s1 + k * s2]
+                                              : (T)(0));
+                }
+            }
+        }
+        it.barrier();
+
+        if (gx < oInfo_.dims[0] && gy < oInfo_.dims[1] && gz < oInfo_.dims[2]) {
+            int ci = lx + radius0 + (EXPAND_ ? 0 : fLen0 >> 1);
+            int cj = ly + radius1 + (EXPAND_ ? 0 : fLen1 >> 1);
+            int ck = lz + radius2 + (EXPAND_ ? 0 : fLen2 >> 1);
+
+            aT accum = (aT)(0);
+            for (int fk = 0; fk < fLen2; ++fk) {
+                for (int fj = 0; fj < fLen1; ++fj) {
+                    for (int fi = 0; fi < fLen0; ++fi) {
+                        aT f_val = impulse_[index(fi, fj, fk, fLen0, fStride)];
+                        T s_val  = localMem_[index(ci - fi, cj - fj, ck - fk,
+                                                   shrdLen0, skStride)];
+
+                        // binOp will do MUL_OP for convolution operation
+                        accum = accum + binOp((aT)s_val, (aT)f_val);
+                    }
+                }
+            }
+            dst[index(gx, gy, gz, oInfo_.strides[1], oInfo_.strides[2])] =
+                (T)accum;
+        }
+    }
+
+   private:
+    write_accessor<T> out_;
+    KParam oInfo_;
+    read_accessor<T> signal_;
+    KParam sInfo_;
+    local_accessor<aT> localMem_;
+    read_accessor<aT> impulse_;
+    KParam fInfo_;
+    int nBBS0_;
+    int nBBS1_;
+    int ostep1_;
+    int ostep2_;
+    int ostep3_;
+    int sstep1_;
+    int sstep2_;
+    int sstep3_;
+    const bool EXPAND_;
+};
+
+template<typename T, typename aT>
+void conv3Helper(const conv_kparam_t<aT> &param, Param<T> &out,
+                 const Param<T> &signal, const Param<aT> &impulse,
+                 const int rank, const bool EXPAND) {
+    auto Q = getQueue();
+    Q.submit([&](auto &h) {
+        sycl::accessor<aT, 1, sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            localMem(param.loc_size, h);
+        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor signalAcc{*signal.data, h, sycl::read_only};
+        sycl::accessor impulseAcc{*param.impulse, h, sycl::read_only};
+        h.parallel_for(
+            sycl::nd_range{param.global, param.local},
+            conv3HelperCreateKernel<T, aT>(
+                outAcc, out.info, signalAcc, signal.info, localMem, impulseAcc,
+                impulse.info, param.nBBS0, param.nBBS1, param.o[0], param.o[1],
+                param.o[2], param.s[0], param.s[1], param.s[2], EXPAND));
+    });
+    ONEAPI_DEBUG_FINISH(Q);
+}
+
+template<typename T, typename aT>
+void conv3(conv_kparam_t<aT> &p, Param<T> &out, const Param<T> &sig,
+           const Param<aT> &filt, const bool expand) {
+    size_t se_size = filt.info.dims[0] * filt.info.dims[1] * filt.info.dims[2];
+    sycl::buffer<aT> impulse{sycl::range(se_size)};
+    int f0Off = filt.info.offset;
+
+    for (int b3 = 0; b3 < filt.info.dims[3]; ++b3) {
+        int f3Off = b3 * filt.info.strides[3];
+
+        const size_t srcOffset = f3Off + f0Off;
+        memcpyBuffer(impulse, *filt.data, se_size, srcOffset);
+        p.impulse = &impulse;
+
+        p.o[2] = (p.outHasNoOffset ? 0 : b3);
+        p.s[2] = (p.inHasNoOffset ? 0 : b3);
+
+        conv3Helper<T, aT>(p, out, sig, filt, 3, expand);
+    }
+}
+
+#define INSTANTIATE_CONV3(T, aT)                                    \
+    template void conv3<T, aT>(conv_kparam_t<aT> &, Param<T> &,     \
+                               const Param<T> &, const Param<aT> &, \
+                               const bool);
+
+INSTANTIATE_CONV3(cdouble, cdouble)
+INSTANTIATE_CONV3(cfloat, cfloat)
+INSTANTIATE_CONV3(double, double)
+INSTANTIATE_CONV3(float, float)
+INSTANTIATE_CONV3(uint, float)
+INSTANTIATE_CONV3(int, float)
+INSTANTIATE_CONV3(uchar, float)
+INSTANTIATE_CONV3(char, float)
+INSTANTIATE_CONV3(ushort, float)
+INSTANTIATE_CONV3(short, float)
+INSTANTIATE_CONV3(uintl, float)
+INSTANTIATE_CONV3(intl, float)

From a230ef46c27588cffe204bb5c465dba30bc08cdd Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Wed, 21 Dec 2022 18:02:03 -0500
Subject: [PATCH 151/473] reorder oneapi port (#3332)

* reorder ported to oneapi

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Co-authored-by: Umar Arshad <umar@arrayfire.com>
Co-authored-by: syurkevi <stefan@arrayfire.com>
---
 src/backend/oneapi/CMakeLists.txt     |   1 +
 src/backend/oneapi/kernel/reorder.hpp | 130 ++++++++++++++++++++++++++
 src/backend/oneapi/reorder.cpp        |   6 +-
 3 files changed, 133 insertions(+), 4 deletions(-)
 create mode 100755 src/backend/oneapi/kernel/reorder.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index dcff3b35e9..9abca35940 100755
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -227,6 +227,7 @@ target_sources(afoneapi
     kernel/reduce_all.hpp
     kernel/reduce_first.hpp
     kernel/reduce_dim.hpp
+    kernel/reorder.hpp
     kernel/scan_first.hpp
     kernel/scan_dim.hpp
     kernel/transpose.hpp
diff --git a/src/backend/oneapi/kernel/reorder.hpp b/src/backend/oneapi/kernel/reorder.hpp
new file mode 100755
index 0000000000..2eb7484db2
--- /dev/null
+++ b/src/backend/oneapi/kernel/reorder.hpp
@@ -0,0 +1,130 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+// #include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
+                                      sycl::access::target::local>;
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T>
+class reorderCreateKernel {
+   public:
+    reorderCreateKernel(write_accessor<T> out, read_accessor<T> in,
+                        const KParam op, const KParam ip, const int d0,
+                        const int d1, const int d2, const int d3,
+                        const int blocksPerMatX, const int blocksPerMatY)
+        : out_(out)
+        , in_(in)
+        , op_(op)
+        , ip_(ip)
+        , d0_(d0)
+        , d1_(d1)
+        , d2_(d2)
+        , d3_(d3)
+        , blocksPerMatX_(blocksPerMatX)
+        , blocksPerMatY_(blocksPerMatY) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        auto g = it.get_group();
+
+        const int oz = g.get_group_id(0) / blocksPerMatX_;
+        const int ow = g.get_group_id(1) / blocksPerMatY_;
+
+        const int blockIdx_x = g.get_group_id(0) - oz * blocksPerMatX_;
+        const int blockIdx_y = g.get_group_id(1) - ow * blocksPerMatY_;
+
+        const int xx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+        const int yy = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+        bool valid = (xx < op_.dims[0] && yy < op_.dims[1] &&
+                      oz < op_.dims[2] && ow < op_.dims[3]);
+
+        const int incy = blocksPerMatY_ * g.get_local_range(1);
+        const int incx = blocksPerMatX_ * g.get_local_range(0);
+
+        const int o_off    = ow * op_.strides[3] + oz * op_.strides[2];
+        const int rdims[4] = {d0_, d1_, d2_, d3_};
+        int ods[4]         = {xx, yy, oz, ow};
+        int ids[4]         = {0};
+
+        ids[rdims[3]] = ow;
+        ids[rdims[2]] = oz;
+
+        for (int oy = yy; oy < op_.dims[1]; oy += incy) {
+            ids[rdims[1]] = oy;
+            for (int ox = xx; ox < op_.dims[0]; ox += incx) {
+                ids[rdims[0]] = ox;
+
+                const int oIdx = o_off + oy * op_.strides[1] + ox;
+
+                const int iIdx = ids[3] * ip_.strides[3] +
+                                 ids[2] * ip_.strides[2] +
+                                 ids[1] * ip_.strides[1] + ids[0];
+
+                if (valid) { out_[oIdx] = in_[ip_.offset + iIdx]; }
+            }
+        }
+    }
+
+   private:
+    write_accessor<T> out_;
+    read_accessor<T> in_;
+    const KParam op_;
+    const KParam ip_;
+    const int d0_;
+    const int d1_;
+    const int d2_;
+    const int d3_;
+    const int blocksPerMatX_;
+    const int blocksPerMatY_;
+};
+
+template<typename T>
+void reorder(Param<T> out, const Param<T> in, const dim_t* rdims) {
+    constexpr int TX    = 32;
+    constexpr int TY    = 8;
+    constexpr int TILEX = 512;
+    constexpr int TILEY = 32;
+
+    auto local = sycl::range{TX, TY};
+
+    int blocksPerMatX = divup(out.info.dims[0], TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], TILEY);
+    auto global       = sycl::range{local[0] * blocksPerMatX * out.info.dims[2],
+                              local[1] * blocksPerMatY * out.info.dims[3]};
+
+    getQueue().submit([&](sycl::handler& h) {
+        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor inAcc{*in.data, h, sycl::read_only};
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       reorderCreateKernel<T>(
+                           outAcc, inAcc, out.info, in.info, rdims[0], rdims[1],
+                           rdims[2], rdims[3], blocksPerMatX, blocksPerMatY));
+    });
+}
+}  // namespace kernel
+}  // namespace oneapi
diff --git a/src/backend/oneapi/reorder.cpp b/src/backend/oneapi/reorder.cpp
index fe5bf98854..fc5c7f26a7 100644
--- a/src/backend/oneapi/reorder.cpp
+++ b/src/backend/oneapi/reorder.cpp
@@ -10,7 +10,7 @@
 #include <Array.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/reorder.hpp>
+#include <kernel/reorder.hpp>
 #include <reorder.hpp>
 #include <stdexcept>
 
@@ -19,15 +19,13 @@ using common::half;
 namespace oneapi {
 template<typename T>
 Array<T> reorder(const Array<T> &in, const af::dim4 &rdims) {
-    ONEAPI_NOT_SUPPORTED("reorder Not supported");
-
     const af::dim4 &iDims = in.dims();
     af::dim4 oDims(0);
     for (int i = 0; i < 4; i++) { oDims[i] = iDims[rdims[i]]; }
 
     Array<T> out = createEmptyArray<T>(oDims);
 
-    // kernel::reorder<T>(out, in, rdims.get());
+    kernel::reorder<T>(out, in, rdims.get());
 
     return out;
 }

From 60231723cb7ce7c57f5af10040b8f21eb0411c22 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 15 Dec 2022 19:15:59 -0500
Subject: [PATCH 152/473] Put all internal symbols in the arrayfire namespace

There were some conflicts in the new cuda and oneapi version. This needed to be
done because the namespaces we used can conflict with other libraries.
---
 CMakeModules/FileToString.cmake               |   7 +-
 src/api/c/CMakeLists.txt                      |   1 +
 src/api/c/anisotropic_diffusion.cpp           |   2 +-
 src/api/c/array.cpp                           |  84 +------
 src/api/c/assign.cpp                          |  19 +-
 src/api/c/binary.cpp                          |  22 +-
 src/api/c/blas.cpp                            |  25 +-
 src/api/c/canny.cpp                           |  10 +-
 src/api/c/cast.cpp                            |   4 +-
 src/api/c/cholesky.cpp                        |   1 +
 src/api/c/clamp.cpp                           |   2 +-
 src/api/c/complex.cpp                         |   2 +-
 src/api/c/confidence_connected.cpp            |  12 +-
 src/api/c/convolve.cpp                        |  10 +-
 src/api/c/corrcoef.cpp                        |   2 +-
 src/api/c/covariance.cpp                      |   2 +-
 src/api/c/data.cpp                            |  15 +-
 src/api/c/deconvolution.cpp                   |   2 +-
 src/api/c/device.cpp                          |   9 +-
 src/api/c/diff.cpp                            |   2 +
 src/api/c/error.cpp                           |   2 +-
 src/api/c/exampleFunction.cpp                 |   2 +-
 src/api/c/fftconvolve.cpp                     |   2 +-
 src/api/c/flip.cpp                            |   5 +-
 src/api/c/gradient.cpp                        |   1 +
 src/api/c/handle.cpp                          | 116 +++++++++
 src/api/c/handle.hpp                          |  24 +-
 src/api/c/hist.cpp                            |  11 +-
 src/api/c/histeq.cpp                          |   4 +-
 src/api/c/histogram.cpp                       |   4 +-
 src/api/c/image.cpp                           |  10 +-
 src/api/c/imageio.cpp                         |  17 ++
 src/api/c/imageio2.cpp                        |  13 ++
 src/api/c/imageio_helper.h                    |   3 +
 src/api/c/imgproc_common.hpp                  |   2 +
 src/api/c/index.cpp                           |  10 +-
 src/api/c/indexing_common.hpp                 |   2 +
 src/api/c/internal.cpp                        |   2 +-
 src/api/c/join.cpp                            |   2 +-
 src/api/c/mean.cpp                            |   6 +-
 src/api/c/memory.cpp                          |   2 +-
 src/api/c/memoryapi.hpp                       |   2 +-
 src/api/c/moddims.cpp                         |   6 +-
 src/api/c/morph.cpp                           |   4 +-
 src/api/c/pinverse.cpp                        |   4 +-
 src/api/c/plot.cpp                            |  16 +-
 src/api/c/print.cpp                           |   6 +-
 src/api/c/random.cpp                          |  20 +-
 src/api/c/reduce.cpp                          |   2 +-
 src/api/c/reorder.cpp                         |   2 +-
 src/api/c/replace.cpp                         |   3 +-
 src/api/c/rgb_gray.cpp                        |   4 +-
 src/api/c/sat.cpp                             |   3 +-
 src/api/c/select.cpp                          |   2 +-
 src/api/c/sparse.cpp                          | 220 ++++++++++--------
 src/api/c/sparse_handle.hpp                   |   6 +
 src/api/c/stdev.cpp                           |   2 +-
 src/api/c/surface.cpp                         |  17 +-
 src/api/c/tile.cpp                            |   5 +-
 src/api/c/topk.cpp                            |   2 +-
 src/api/c/transpose.cpp                       |   2 +-
 src/api/c/unary.cpp                           |   2 +-
 src/api/c/var.cpp                             |   4 +-
 src/api/c/vector_field.cpp                    |  16 +-
 src/api/c/window.cpp                          |   4 +-
 src/api/cpp/array.cpp                         |  30 +--
 src/api/unified/device.cpp                    |  10 +-
 src/api/unified/symbol_manager.cpp            |  16 +-
 src/api/unified/symbol_manager.hpp            |  28 ++-
 src/backend/common/AllocatorInterface.hpp     |   4 +-
 src/backend/common/ArrayInfo.cpp              |  28 ++-
 src/backend/common/Binary.hpp                 |   2 +
 src/backend/common/DefaultMemoryManager.cpp   |   2 +
 src/backend/common/DefaultMemoryManager.hpp   |   4 +-
 src/backend/common/DependencyModule.cpp       |   4 +-
 src/backend/common/DependencyModule.hpp       |   2 +
 src/backend/common/EventBase.hpp              |   2 +
 src/backend/common/FFTPlanCache.hpp           |   2 +
 src/backend/common/HandleBase.hpp             |   2 +
 src/backend/common/InteropManager.hpp         |  20 +-
 src/backend/common/KernelInterface.hpp        |   2 +
 src/backend/common/Logger.cpp                 |   2 +
 src/backend/common/Logger.hpp                 |   2 +
 src/backend/common/MemoryManagerBase.hpp      |   4 +-
 src/backend/common/MersenneTwister.hpp        |   2 +
 src/backend/common/ModuleInterface.hpp        |   2 +
 src/backend/common/Source.hpp                 |   2 +
 src/backend/common/SparseArray.cpp            |   2 +
 src/backend/common/SparseArray.hpp            |   2 +
 src/backend/common/TemplateArg.hpp            |   7 +-
 src/backend/common/TemplateTypename.hpp       |   4 +-
 src/backend/common/Transform.hpp              |   2 +
 src/backend/common/cast.cpp                   |   8 +-
 src/backend/common/cast.hpp                   |  10 +-
 src/backend/common/compile_module.hpp         |   2 +
 src/backend/common/complex.hpp                |   2 +
 src/backend/common/defines.hpp                |   2 +
 src/backend/common/deterministicHash.cpp      |   2 +-
 src/backend/common/deterministicHash.hpp      |   3 +-
 src/backend/common/err_common.cpp             |   8 +-
 src/backend/common/err_common.hpp             |   4 +-
 src/backend/common/forge_loader.hpp           |  10 +-
 src/backend/common/graphics_common.cpp        |  13 +-
 src/backend/common/graphics_common.hpp        |  15 +-
 src/backend/common/half.cpp                   |   2 +
 src/backend/common/half.hpp                   | 109 +++++----
 src/backend/common/host_memory.cpp            |   2 +
 src/backend/common/host_memory.hpp            |   4 +-
 src/backend/common/indexing_helpers.hpp       |   2 +
 src/backend/common/jit/BinaryNode.cpp         |   2 +
 src/backend/common/jit/BinaryNode.hpp         |   2 +
 src/backend/common/jit/BufferNodeBase.hpp     |   2 +
 src/backend/common/jit/ModdimNode.hpp         |   2 +
 src/backend/common/jit/NaryNode.hpp           |   2 +
 src/backend/common/jit/Node.cpp               |   9 +-
 src/backend/common/jit/Node.hpp               |  18 +-
 src/backend/common/jit/NodeIO.hpp             |  14 +-
 src/backend/common/jit/NodeIterator.hpp       |   4 +-
 src/backend/common/jit/ScalarNode.hpp         |   2 +
 src/backend/common/jit/ShiftNodeBase.hpp      |   2 +
 src/backend/common/jit/UnaryNode.hpp          |   2 +
 src/backend/common/kernel_cache.cpp           |   2 +
 src/backend/common/kernel_cache.hpp           |   5 +-
 src/backend/common/kernel_type.hpp            |   2 +
 src/backend/common/moddims.cpp                |  16 +-
 src/backend/common/moddims.hpp                |   2 +
 src/backend/common/module_loading.hpp         |   2 +
 src/backend/common/module_loading_unix.cpp    |   2 +
 src/backend/common/module_loading_windows.cpp |   2 +
 src/backend/common/sparse_helpers.hpp         |   2 +
 src/backend/common/tile.hpp                   |   2 +
 src/backend/common/traits.hpp                 |   6 +-
 src/backend/common/unique_handle.hpp          |   6 +-
 src/backend/common/util.cpp                   |   2 +
 src/backend/common/util.hpp                   |   2 +
 src/backend/cpu/Array.cpp                     |  14 +-
 src/backend/cpu/Array.hpp                     |   2 +
 src/backend/cpu/Event.cpp                     |   2 +
 src/backend/cpu/Event.hpp                     |   2 +
 src/backend/cpu/Param.hpp                     |   2 +
 src/backend/cpu/ParamIterator.hpp             |   2 +
 src/backend/cpu/anisotropic_diffusion.cpp     |   2 +
 src/backend/cpu/anisotropic_diffusion.hpp     |   2 +
 src/backend/cpu/approx.cpp                    |   2 +
 src/backend/cpu/approx.hpp                    |   2 +
 src/backend/cpu/arith.hpp                     |   2 +
 src/backend/cpu/assign.cpp                    |   4 +-
 src/backend/cpu/assign.hpp                    |   2 +
 src/backend/cpu/backend.hpp                   |   2 +-
 src/backend/cpu/bilateral.cpp                 |   2 +
 src/backend/cpu/bilateral.hpp                 |   4 +-
 src/backend/cpu/binary.hpp                    |   2 +
 src/backend/cpu/blas.cpp                      |   8 +-
 src/backend/cpu/blas.hpp                      |   2 +
 src/backend/cpu/canny.cpp                     |   2 +
 src/backend/cpu/canny.hpp                     |   2 +
 src/backend/cpu/cast.hpp                      |  18 +-
 src/backend/cpu/cholesky.cpp                  |   4 +
 src/backend/cpu/cholesky.hpp                  |   2 +
 src/backend/cpu/complex.hpp                   |   2 +
 src/backend/cpu/convolve.cpp                  |   8 +-
 src/backend/cpu/convolve.hpp                  |   2 +
 src/backend/cpu/copy.cpp                      |   7 +-
 src/backend/cpu/copy.hpp                      |   2 +
 src/backend/cpu/device_manager.cpp            |   6 +-
 src/backend/cpu/device_manager.hpp            |   8 +-
 src/backend/cpu/diagonal.cpp                  |   9 +-
 src/backend/cpu/diagonal.hpp                  |   2 +
 src/backend/cpu/diff.cpp                      |   2 +
 src/backend/cpu/diff.hpp                      |   2 +
 src/backend/cpu/exampleFunction.cpp           |   2 +
 src/backend/cpu/exampleFunction.hpp           |   4 +-
 src/backend/cpu/fast.cpp                      |   2 +
 src/backend/cpu/fast.hpp                      |   2 +
 src/backend/cpu/fft.cpp                       |   2 +
 src/backend/cpu/fft.hpp                       |   2 +
 src/backend/cpu/fftconvolve.cpp               |   2 +
 src/backend/cpu/fftconvolve.hpp               |   4 +-
 src/backend/cpu/flood_fill.cpp                |   2 +
 src/backend/cpu/flood_fill.hpp                |   2 +
 src/backend/cpu/gradient.cpp                  |   2 +
 src/backend/cpu/gradient.hpp                  |   4 +-
 src/backend/cpu/harris.cpp                    |   2 +
 src/backend/cpu/harris.hpp                    |   4 +-
 src/backend/cpu/hist_graphics.cpp             |   8 +-
 src/backend/cpu/hist_graphics.hpp             |   4 +-
 src/backend/cpu/histogram.cpp                 |   4 +-
 src/backend/cpu/histogram.hpp                 |   4 +-
 src/backend/cpu/homography.cpp                |   2 +
 src/backend/cpu/homography.hpp                |   4 +-
 src/backend/cpu/hsv_rgb.cpp                   |   2 +
 src/backend/cpu/hsv_rgb.hpp                   |   2 +
 src/backend/cpu/identity.cpp                  |   5 +-
 src/backend/cpu/identity.hpp                  |   4 +-
 src/backend/cpu/iir.cpp                       |   2 +
 src/backend/cpu/iir.hpp                       |   4 +-
 src/backend/cpu/image.cpp                     |   8 +-
 src/backend/cpu/image.hpp                     |   4 +-
 src/backend/cpu/index.cpp                     |   5 +-
 src/backend/cpu/index.hpp                     |   4 +-
 src/backend/cpu/inverse.cpp                   |   4 +
 src/backend/cpu/inverse.hpp                   |   4 +-
 src/backend/cpu/iota.cpp                      |   5 +-
 src/backend/cpu/iota.hpp                      |   4 +-
 src/backend/cpu/ireduce.cpp                   |   4 +-
 src/backend/cpu/ireduce.hpp                   |   2 +
 src/backend/cpu/jit/BinaryNode.hpp            |   3 +-
 src/backend/cpu/jit/BufferNode.hpp            |   2 +
 src/backend/cpu/jit/Node.hpp                  |   4 +-
 src/backend/cpu/jit/ScalarNode.hpp            |   3 +-
 src/backend/cpu/jit/UnaryNode.hpp             |   5 +-
 src/backend/cpu/join.cpp                      |   4 +-
 src/backend/cpu/join.hpp                      |   2 +
 src/backend/cpu/kernel/Array.hpp              |  16 +-
 .../cpu/kernel/anisotropic_diffusion.hpp      |   2 +
 src/backend/cpu/kernel/approx.hpp             |   2 +
 src/backend/cpu/kernel/assign.hpp             |   2 +
 src/backend/cpu/kernel/bilateral.hpp          |   2 +
 src/backend/cpu/kernel/canny.hpp              |   2 +
 src/backend/cpu/kernel/convolve.hpp           |   2 +
 src/backend/cpu/kernel/copy.hpp               |   2 +
 src/backend/cpu/kernel/diagonal.hpp           |   2 +
 src/backend/cpu/kernel/diff.hpp               |   2 +
 src/backend/cpu/kernel/dot.hpp                |   2 +
 src/backend/cpu/kernel/exampleFunction.hpp    |   2 +
 src/backend/cpu/kernel/fast.hpp               |   2 +
 src/backend/cpu/kernel/fftconvolve.hpp        |   2 +
 src/backend/cpu/kernel/flood_fill.hpp         |   2 +
 src/backend/cpu/kernel/gradient.hpp           |   2 +
 src/backend/cpu/kernel/harris.hpp             |   2 +
 src/backend/cpu/kernel/histogram.hpp          |   2 +
 src/backend/cpu/kernel/hsv_rgb.hpp            |   2 +
 src/backend/cpu/kernel/identity.hpp           |   2 +
 src/backend/cpu/kernel/iir.hpp                |   2 +
 src/backend/cpu/kernel/index.hpp              |   2 +
 src/backend/cpu/kernel/interp.hpp             |   2 +
 src/backend/cpu/kernel/iota.hpp               |   2 +
 src/backend/cpu/kernel/ireduce.hpp            |   2 +
 src/backend/cpu/kernel/join.hpp               |   2 +
 src/backend/cpu/kernel/lookup.hpp             |   2 +
 src/backend/cpu/kernel/lu.hpp                 |   2 +
 src/backend/cpu/kernel/match_template.hpp     |   2 +
 src/backend/cpu/kernel/mean.hpp               |   2 +
 src/backend/cpu/kernel/meanshift.hpp          |   2 +
 src/backend/cpu/kernel/medfilt.hpp            |   2 +
 src/backend/cpu/kernel/moments.hpp            |   2 +
 src/backend/cpu/kernel/morph.hpp              |   2 +
 src/backend/cpu/kernel/nearest_neighbour.hpp  |   2 +
 src/backend/cpu/kernel/orb.hpp                |   2 +
 src/backend/cpu/kernel/pad_array_borders.hpp  |   2 +
 src/backend/cpu/kernel/random_engine.hpp      |  19 +-
 .../cpu/kernel/random_engine_mersenne.hpp     |   2 +
 .../cpu/kernel/random_engine_philox.hpp       |   2 +
 .../cpu/kernel/random_engine_threefry.hpp     |   2 +
 src/backend/cpu/kernel/range.hpp              |   2 +
 src/backend/cpu/kernel/reduce.hpp             |   2 +
 src/backend/cpu/kernel/regions.hpp            |   2 +
 src/backend/cpu/kernel/reorder.hpp            |   2 +
 src/backend/cpu/kernel/resize.hpp             |   2 +
 src/backend/cpu/kernel/rotate.hpp             |   2 +
 src/backend/cpu/kernel/scan.hpp               |   2 +
 src/backend/cpu/kernel/scan_by_key.hpp        |   2 +
 src/backend/cpu/kernel/select.hpp             |   2 +
 src/backend/cpu/kernel/shift.hpp              |   2 +
 src/backend/cpu/kernel/sift.hpp               |   2 +
 src/backend/cpu/kernel/sobel.hpp              |   2 +
 src/backend/cpu/kernel/sort.hpp               |   2 +
 src/backend/cpu/kernel/sort_by_key.hpp        |   2 +
 .../kernel/sort_by_key/sort_by_key_impl.cpp   |   2 +
 src/backend/cpu/kernel/sort_by_key_impl.hpp   |   3 +
 src/backend/cpu/kernel/sort_helper.hpp        |   2 +
 src/backend/cpu/kernel/sparse.hpp             |   2 +
 src/backend/cpu/kernel/sparse_arith.hpp       |   2 +
 src/backend/cpu/kernel/susan.hpp              |   2 +
 src/backend/cpu/kernel/tile.hpp               |   2 +
 src/backend/cpu/kernel/transform.hpp          |   2 +
 src/backend/cpu/kernel/transpose.hpp          |   2 +
 src/backend/cpu/kernel/triangle.hpp           |   2 +
 src/backend/cpu/kernel/unwrap.hpp             |   2 +
 src/backend/cpu/kernel/wrap.hpp               |   2 +
 src/backend/cpu/logic.hpp                     |   2 +
 src/backend/cpu/lookup.cpp                    |   4 +-
 src/backend/cpu/lookup.hpp                    |   4 +-
 src/backend/cpu/lu.cpp                        |   6 +
 src/backend/cpu/lu.hpp                        |   2 +
 src/backend/cpu/match_template.cpp            |   2 +
 src/backend/cpu/match_template.hpp            |   4 +-
 src/backend/cpu/math.cpp                      |   2 +
 src/backend/cpu/math.hpp                      |  10 +-
 src/backend/cpu/mean.cpp                      |   4 +-
 src/backend/cpu/mean.hpp                      |   2 +
 src/backend/cpu/meanshift.cpp                 |   2 +
 src/backend/cpu/meanshift.hpp                 |   4 +-
 src/backend/cpu/medfilt.cpp                   |   2 +
 src/backend/cpu/medfilt.hpp                   |   2 +
 src/backend/cpu/memory.cpp                    |   6 +-
 src/backend/cpu/memory.hpp                    |   4 +-
 src/backend/cpu/moments.cpp                   |   2 +
 src/backend/cpu/moments.hpp                   |   4 +-
 src/backend/cpu/morph.cpp                     |   2 +
 src/backend/cpu/morph.hpp                     |   2 +
 src/backend/cpu/nearest_neighbour.cpp         |   2 +
 src/backend/cpu/nearest_neighbour.hpp         |   4 +-
 src/backend/cpu/orb.cpp                       |   2 +
 src/backend/cpu/orb.hpp                       |   4 +-
 src/backend/cpu/platform.cpp                  |  13 +-
 src/backend/cpu/platform.hpp                  |  15 +-
 src/backend/cpu/plot.cpp                      |   7 +-
 src/backend/cpu/plot.hpp                      |   4 +-
 src/backend/cpu/print.hpp                     |   4 +-
 src/backend/cpu/qr.cpp                        |   6 +
 src/backend/cpu/qr.hpp                        |   2 +
 src/backend/cpu/queue.hpp                     |   2 +
 src/backend/cpu/random_engine.cpp             |   4 +-
 src/backend/cpu/random_engine.hpp             |   2 +
 src/backend/cpu/range.cpp                     |   4 +-
 src/backend/cpu/range.hpp                     |   4 +-
 src/backend/cpu/reduce.cpp                    |  11 +-
 src/backend/cpu/reduce.hpp                    |   2 +
 src/backend/cpu/regions.cpp                   |   2 +
 src/backend/cpu/regions.hpp                   |   4 +-
 src/backend/cpu/reorder.cpp                   |   4 +-
 src/backend/cpu/reorder.hpp                   |   4 +-
 src/backend/cpu/reshape.cpp                   |   4 +-
 src/backend/cpu/resize.cpp                    |   2 +
 src/backend/cpu/resize.hpp                    |   4 +-
 src/backend/cpu/rotate.cpp                    |   2 +
 src/backend/cpu/rotate.hpp                    |   4 +-
 src/backend/cpu/scan.cpp                      |   2 +
 src/backend/cpu/scan.hpp                      |   4 +-
 src/backend/cpu/scan_by_key.cpp               |   2 +
 src/backend/cpu/scan_by_key.hpp               |   4 +-
 src/backend/cpu/select.cpp                    |   4 +-
 src/backend/cpu/select.hpp                    |   2 +
 src/backend/cpu/set.cpp                       |   2 +
 src/backend/cpu/set.hpp                       |   2 +
 src/backend/cpu/shift.cpp                     |   2 +
 src/backend/cpu/shift.hpp                     |   4 +-
 src/backend/cpu/sift.cpp                      |   2 +
 src/backend/cpu/sift.hpp                      |   4 +-
 src/backend/cpu/sobel.cpp                     |   2 +
 src/backend/cpu/sobel.hpp                     |   4 +-
 src/backend/cpu/solve.cpp                     |   6 +
 src/backend/cpu/solve.hpp                     |   2 +
 src/backend/cpu/sort.cpp                      |   2 +
 src/backend/cpu/sort.hpp                      |   4 +-
 src/backend/cpu/sort_by_key.cpp               |   2 +
 src/backend/cpu/sort_by_key.hpp               |   4 +-
 src/backend/cpu/sort_index.cpp                |   2 +
 src/backend/cpu/sort_index.hpp                |   4 +-
 src/backend/cpu/sparse.cpp                    |  10 +-
 src/backend/cpu/sparse.hpp                    |   2 +
 src/backend/cpu/sparse_arith.cpp              |   8 +-
 src/backend/cpu/sparse_arith.hpp              |   2 +
 src/backend/cpu/sparse_blas.cpp               |   2 +
 src/backend/cpu/sparse_blas.hpp               |   4 +-
 src/backend/cpu/surface.cpp                   |   7 +-
 src/backend/cpu/surface.hpp                   |   4 +-
 src/backend/cpu/susan.cpp                     |   2 +
 src/backend/cpu/susan.hpp                     |   4 +-
 src/backend/cpu/svd.cpp                       |   6 +
 src/backend/cpu/svd.hpp                       |   2 +
 src/backend/cpu/tile.cpp                      |   4 +-
 src/backend/cpu/tile.hpp                      |   4 +-
 src/backend/cpu/topk.cpp                      |   4 +-
 src/backend/cpu/topk.hpp                      |   4 +-
 src/backend/cpu/transform.cpp                 |   2 +
 src/backend/cpu/transform.hpp                 |   4 +-
 src/backend/cpu/transpose.cpp                 |   4 +-
 src/backend/cpu/transpose.hpp                 |   2 +
 src/backend/cpu/triangle.cpp                  |   4 +-
 src/backend/cpu/triangle.hpp                  |   2 +
 src/backend/cpu/types.hpp                     |   7 +-
 src/backend/cpu/unary.hpp                     |   2 +
 src/backend/cpu/unwrap.cpp                    |   4 +-
 src/backend/cpu/unwrap.hpp                    |   4 +-
 src/backend/cpu/utility.hpp                   |   2 +
 src/backend/cpu/vector_field.cpp              |   7 +-
 src/backend/cpu/vector_field.hpp              |   4 +-
 src/backend/cpu/where.cpp                     |   2 +
 src/backend/cpu/where.hpp                     |   4 +-
 src/backend/cpu/wrap.cpp                      |   4 +-
 src/backend/cpu/wrap.hpp                      |   2 +
 src/backend/cuda/Array.cpp                    |  24 +-
 src/backend/cuda/Array.hpp                    |   3 +
 src/backend/cuda/CMakeLists.txt               |   4 +-
 src/backend/cuda/EnqueueArgs.hpp              |   2 +
 src/backend/cuda/Event.cpp                    |   2 +
 src/backend/cuda/Event.hpp                    |   2 +
 src/backend/cuda/GraphicsResourceManager.cpp  |   2 +
 src/backend/cuda/GraphicsResourceManager.hpp  |   2 +
 src/backend/cuda/Kernel.cpp                   |  15 +-
 src/backend/cuda/Kernel.hpp                   |   2 +
 src/backend/cuda/LookupTable1D.hpp            |   2 +
 src/backend/cuda/Module.hpp                   |   2 +
 src/backend/cuda/Param.hpp                    |   2 +
 src/backend/cuda/ThrustAllocator.cuh          |   3 +
 src/backend/cuda/ThrustArrayFirePolicy.hpp    |  16 +-
 src/backend/cuda/all.cu                       |   6 +-
 src/backend/cuda/anisotropic_diffusion.cpp    |   2 +
 src/backend/cuda/anisotropic_diffusion.hpp    |   4 +-
 src/backend/cuda/any.cu                       |   6 +-
 src/backend/cuda/approx.cpp                   |   2 +
 src/backend/cuda/approx.hpp                   |   2 +
 src/backend/cuda/arith.hpp                    |   2 +
 src/backend/cuda/assign.cpp                   |   4 +-
 src/backend/cuda/assign.hpp                   |   4 +-
 src/backend/cuda/assign_kernel_param.hpp      |   2 +
 src/backend/cuda/backend.hpp                  |   6 +-
 src/backend/cuda/bilateral.cpp                |   2 +
 src/backend/cuda/bilateral.hpp                |   4 +-
 src/backend/cuda/binary.hpp                   |   2 +
 src/backend/cuda/blas.cu                      |   6 +-
 src/backend/cuda/blas.hpp                     |   2 +
 src/backend/cuda/canny.cpp                    |   2 +
 src/backend/cuda/canny.hpp                    |   2 +
 src/backend/cuda/cast.hpp                     |   2 +
 src/backend/cuda/cholesky.cpp                 |   2 +
 src/backend/cuda/cholesky.hpp                 |   2 +
 src/backend/cuda/compile_module.cpp           |  22 +-
 src/backend/cuda/complex.hpp                  |   2 +
 src/backend/cuda/convolve.cpp                 |   4 +-
 src/backend/cuda/convolve.hpp                 |   2 +
 src/backend/cuda/convolveNN.cpp               |  10 +-
 src/backend/cuda/copy.cpp                     |  15 +-
 src/backend/cuda/copy.hpp                     |   2 +
 src/backend/cuda/count.cu                     |   6 +-
 src/backend/cuda/cublas.cpp                   |   2 +
 src/backend/cuda/cublas.hpp                   |   4 +-
 src/backend/cuda/cudaDataType.hpp             |   2 +
 src/backend/cuda/cudnn.cpp                    |   2 +
 src/backend/cuda/cudnn.hpp                    |  10 +-
 src/backend/cuda/cudnnModule.cpp              |   6 +-
 src/backend/cuda/cudnnModule.hpp              |   2 +
 src/backend/cuda/cufft.cu                     |   6 +-
 src/backend/cuda/cufft.hpp                    |  26 ++-
 src/backend/cuda/cusolverDn.cpp               |   2 +
 src/backend/cuda/cusolverDn.hpp               |   4 +-
 src/backend/cuda/cusparse.cpp                 |   2 +
 src/backend/cuda/cusparse.hpp                 |  14 +-
 src/backend/cuda/cusparseModule.cpp           |   2 +
 src/backend/cuda/cusparseModule.hpp           |   4 +-
 .../cuda/cusparse_descriptor_helpers.hpp      |   2 +
 src/backend/cuda/debug_cuda.hpp               |  44 ++--
 src/backend/cuda/device_manager.cpp           |  12 +-
 src/backend/cuda/device_manager.hpp           |   8 +-
 src/backend/cuda/diagonal.cpp                 |   4 +-
 src/backend/cuda/diagonal.hpp                 |   2 +
 src/backend/cuda/diff.cpp                     |   2 +
 src/backend/cuda/diff.hpp                     |   2 +
 src/backend/cuda/dims_param.hpp               |   2 +
 src/backend/cuda/exampleFunction.cpp          |   2 +
 src/backend/cuda/exampleFunction.hpp          |   4 +-
 src/backend/cuda/fast.cu                      |   2 +
 src/backend/cuda/fast.hpp                     |   4 +-
 src/backend/cuda/fast_pyramid.cpp             |   2 +
 src/backend/cuda/fast_pyramid.hpp             |   4 +-
 src/backend/cuda/fft.cu                       |   8 +-
 src/backend/cuda/fft.hpp                      |   2 +
 src/backend/cuda/fftconvolve.cpp              |   2 +
 src/backend/cuda/fftconvolve.hpp              |   4 +-
 src/backend/cuda/flood_fill.cpp               |   2 +
 src/backend/cuda/flood_fill.hpp               |   2 +
 src/backend/cuda/gradient.cpp                 |   2 +
 src/backend/cuda/gradient.hpp                 |   4 +-
 src/backend/cuda/harris.cu                    |   2 +
 src/backend/cuda/harris.hpp                   |   4 +-
 src/backend/cuda/hist_graphics.cpp            |  10 +-
 src/backend/cuda/hist_graphics.hpp            |   4 +-
 src/backend/cuda/histogram.cpp                |   4 +-
 src/backend/cuda/histogram.hpp                |   4 +-
 src/backend/cuda/homography.cu                |   2 +
 src/backend/cuda/homography.hpp               |   4 +-
 src/backend/cuda/hsv_rgb.cpp                  |   2 +
 src/backend/cuda/hsv_rgb.hpp                  |   2 +
 src/backend/cuda/identity.cpp                 |   4 +-
 src/backend/cuda/identity.hpp                 |   4 +-
 src/backend/cuda/iir.cpp                      |   2 +
 src/backend/cuda/iir.hpp                      |   4 +-
 src/backend/cuda/image.cpp                    |   9 +-
 src/backend/cuda/image.hpp                    |   4 +-
 src/backend/cuda/index.cpp                    |   4 +-
 src/backend/cuda/index.hpp                    |   4 +-
 src/backend/cuda/inverse.cpp                  |   2 +
 src/backend/cuda/inverse.hpp                  |   4 +-
 src/backend/cuda/iota.cpp                     |   4 +-
 src/backend/cuda/iota.hpp                     |   4 +-
 src/backend/cuda/ireduce.cpp                  |   4 +-
 src/backend/cuda/ireduce.hpp                  |   2 +
 src/backend/cuda/jit.cpp                      |  26 ++-
 src/backend/cuda/jit/BufferNode.hpp           |   6 +-
 src/backend/cuda/jit/kernel_generators.hpp    |   2 +
 src/backend/cuda/join.cpp                     |   8 +-
 src/backend/cuda/join.hpp                     |   2 +
 .../cuda/kernel/anisotropic_diffusion.cuh     |  61 +++--
 .../cuda/kernel/anisotropic_diffusion.hpp     |  10 +-
 src/backend/cuda/kernel/approx.hpp            |  20 +-
 src/backend/cuda/kernel/approx1.cuh           |   2 +
 src/backend/cuda/kernel/approx2.cuh           |   2 +
 src/backend/cuda/kernel/assign.cuh            |   7 +-
 src/backend/cuda/kernel/assign.hpp            |  11 +-
 src/backend/cuda/kernel/atomics.hpp           |   2 +
 src/backend/cuda/kernel/bilateral.cuh         |  23 +-
 src/backend/cuda/kernel/bilateral.hpp         |   7 +-
 src/backend/cuda/kernel/canny.cuh             |  93 ++++----
 src/backend/cuda/kernel/canny.hpp             |  10 +-
 src/backend/cuda/kernel/config.hpp            |   2 +
 src/backend/cuda/kernel/convolve.hpp          |  14 +-
 src/backend/cuda/kernel/convolve1.cuh         |  16 +-
 src/backend/cuda/kernel/convolve2.cuh         |  19 +-
 src/backend/cuda/kernel/convolve3.cuh         |  17 +-
 .../cuda/kernel/convolve_separable.cpp        |   2 +
 .../cuda/kernel/convolve_separable.cuh        |   8 +-
 src/backend/cuda/kernel/copy.cuh              |  11 +-
 src/backend/cuda/kernel/diagonal.cuh          |   2 +
 src/backend/cuda/kernel/diagonal.hpp          |  20 +-
 src/backend/cuda/kernel/diff.cuh              |   2 +
 src/backend/cuda/kernel/diff.hpp              |  11 +-
 src/backend/cuda/kernel/exampleFunction.cuh   |   4 +-
 src/backend/cuda/kernel/exampleFunction.hpp   |   6 +-
 src/backend/cuda/kernel/fast.hpp              |   9 +-
 src/backend/cuda/kernel/fftconvolve.cuh       |   2 +
 src/backend/cuda/kernel/fftconvolve.hpp       |  10 +-
 src/backend/cuda/kernel/flood_fill.cuh        |  64 ++---
 src/backend/cuda/kernel/flood_fill.hpp        |  14 +-
 src/backend/cuda/kernel/gradient.cuh          |  11 +-
 src/backend/cuda/kernel/gradient.hpp          |  17 +-
 src/backend/cuda/kernel/harris.hpp            |  18 +-
 src/backend/cuda/kernel/histogram.cuh         |  16 +-
 src/backend/cuda/kernel/histogram.hpp         |   4 +-
 src/backend/cuda/kernel/homography.hpp        |  18 +-
 src/backend/cuda/kernel/hsv_rgb.cuh           |   7 +-
 src/backend/cuda/kernel/hsv_rgb.hpp           |  11 +-
 src/backend/cuda/kernel/identity.cuh          |   2 +
 src/backend/cuda/kernel/identity.hpp          |  15 +-
 src/backend/cuda/kernel/iir.cuh               |   2 +
 src/backend/cuda/kernel/iir.hpp               |   4 +-
 src/backend/cuda/kernel/index.cuh             |   7 +-
 src/backend/cuda/kernel/index.hpp             |  14 +-
 src/backend/cuda/kernel/interp.hpp            |   2 +
 src/backend/cuda/kernel/iota.cuh              |   2 +
 src/backend/cuda/kernel/iota.hpp              |  14 +-
 src/backend/cuda/kernel/ireduce.cuh           |   4 +-
 src/backend/cuda/kernel/ireduce.hpp           |  36 ++-
 src/backend/cuda/kernel/jit.cuh               |  28 +--
 src/backend/cuda/kernel/lookup.cuh            |   2 +
 src/backend/cuda/kernel/lookup.hpp            |   8 +-
 src/backend/cuda/kernel/lu_split.cuh          |   2 +
 src/backend/cuda/kernel/lu_split.hpp          |   4 +-
 src/backend/cuda/kernel/match_template.cuh    |   9 +-
 src/backend/cuda/kernel/match_template.hpp    |   4 +-
 src/backend/cuda/kernel/mean.hpp              |  63 +++--
 src/backend/cuda/kernel/meanshift.cuh         |   9 +-
 src/backend/cuda/kernel/meanshift.hpp         |   4 +-
 src/backend/cuda/kernel/medfilt.cuh           |  26 +--
 src/backend/cuda/kernel/medfilt.hpp           |  12 +-
 src/backend/cuda/kernel/memcopy.cuh           |   2 +
 src/backend/cuda/kernel/memcopy.hpp           |  23 +-
 src/backend/cuda/kernel/moments.cuh           |   8 +-
 src/backend/cuda/kernel/moments.hpp           |   8 +-
 src/backend/cuda/kernel/morph.cuh             |   6 +-
 src/backend/cuda/kernel/morph.hpp             |   6 +-
 src/backend/cuda/kernel/nearest_neighbour.hpp |   2 +
 src/backend/cuda/kernel/orb.hpp               |  21 +-
 src/backend/cuda/kernel/orb_patch.hpp         |   2 +
 src/backend/cuda/kernel/pad_array_borders.cuh |  20 +-
 src/backend/cuda/kernel/pad_array_borders.hpp |   4 +-
 src/backend/cuda/kernel/random_engine.hpp     |   2 +
 .../cuda/kernel/random_engine_mersenne.hpp    |   2 +
 .../cuda/kernel/random_engine_philox.hpp      |   2 +
 .../cuda/kernel/random_engine_threefry.hpp    |   2 +
 src/backend/cuda/kernel/range.cuh             |   2 +
 src/backend/cuda/kernel/range.hpp             |  14 +-
 src/backend/cuda/kernel/reduce.hpp            |  23 +-
 src/backend/cuda/kernel/reduce_by_key.hpp     |   2 +
 src/backend/cuda/kernel/regions.hpp           |  40 ++--
 src/backend/cuda/kernel/reorder.cuh           |   2 +
 src/backend/cuda/kernel/reorder.hpp           |  15 +-
 src/backend/cuda/kernel/resize.cuh            |  36 ++-
 src/backend/cuda/kernel/resize.hpp            |   4 +-
 src/backend/cuda/kernel/rotate.cuh            |   4 +-
 src/backend/cuda/kernel/rotate.hpp            |   4 +-
 .../kernel/scan_by_key/scan_by_key_impl.cpp   |   2 +
 src/backend/cuda/kernel/scan_dim.cuh          |   2 +
 src/backend/cuda/kernel/scan_dim.hpp          |  26 +--
 src/backend/cuda/kernel/scan_dim_by_key.cuh   |   2 +
 src/backend/cuda/kernel/scan_dim_by_key.hpp   |   2 +
 .../cuda/kernel/scan_dim_by_key_impl.hpp      |  11 +-
 src/backend/cuda/kernel/scan_first.cuh        |   2 +
 src/backend/cuda/kernel/scan_first.hpp        |  20 +-
 src/backend/cuda/kernel/scan_first_by_key.cuh |  14 +-
 src/backend/cuda/kernel/scan_first_by_key.hpp |   2 +
 .../cuda/kernel/scan_first_by_key_impl.hpp    |  11 +-
 src/backend/cuda/kernel/select.cuh            |   2 +
 src/backend/cuda/kernel/select.hpp            |  13 +-
 src/backend/cuda/kernel/shared.hpp            |   4 +
 src/backend/cuda/kernel/shfl_intrinsics.hpp   |  39 ++--
 src/backend/cuda/kernel/sift.hpp              |  48 ++--
 src/backend/cuda/kernel/sobel.cuh             |  19 +-
 src/backend/cuda/kernel/sobel.hpp             |   4 +-
 src/backend/cuda/kernel/sort.hpp              |   2 +
 src/backend/cuda/kernel/sort_by_key.hpp       |   2 +
 src/backend/cuda/kernel/sparse.cuh            |   2 +
 src/backend/cuda/kernel/sparse.hpp            |   4 +-
 src/backend/cuda/kernel/sparse_arith.cuh      |   2 +
 src/backend/cuda/kernel/sparse_arith.hpp      |  34 +--
 src/backend/cuda/kernel/susan.cuh             |   2 +
 src/backend/cuda/kernel/susan.hpp             |  14 +-
 .../cuda/kernel/thrust_sort_by_key.hpp        |   2 +
 .../thrust_sort_by_key_impl.cu                |   2 +
 .../cuda/kernel/thrust_sort_by_key_impl.hpp   |   2 +
 src/backend/cuda/kernel/tile.cuh              |   2 +
 src/backend/cuda/kernel/tile.hpp              |  14 +-
 src/backend/cuda/kernel/topk.hpp              |   2 +
 src/backend/cuda/kernel/transform.cuh         |  20 +-
 src/backend/cuda/kernel/transform.hpp         |   4 +-
 src/backend/cuda/kernel/transpose.cuh         |   7 +-
 src/backend/cuda/kernel/transpose.hpp         |  11 +-
 src/backend/cuda/kernel/transpose_inplace.cuh |   4 +-
 src/backend/cuda/kernel/transpose_inplace.hpp |   4 +-
 src/backend/cuda/kernel/triangle.cuh          |   2 +
 src/backend/cuda/kernel/triangle.hpp          |  11 +-
 src/backend/cuda/kernel/unwrap.cuh            |   2 +
 src/backend/cuda/kernel/unwrap.hpp            |  11 +-
 src/backend/cuda/kernel/where.cuh             |   9 +-
 src/backend/cuda/kernel/where.hpp             |  16 +-
 src/backend/cuda/kernel/wrap.cuh              |   2 +
 src/backend/cuda/kernel/wrap.hpp              |  20 +-
 src/backend/cuda/logic.hpp                    |   2 +
 src/backend/cuda/lookup.cpp                   |   4 +-
 src/backend/cuda/lookup.hpp                   |   4 +-
 src/backend/cuda/lu.cpp                       |   2 +
 src/backend/cuda/lu.hpp                       |   2 +
 src/backend/cuda/match_template.cpp           |   2 +
 src/backend/cuda/match_template.hpp           |   4 +-
 src/backend/cuda/math.hpp                     |  17 +-
 src/backend/cuda/max.cu                       |   6 +-
 src/backend/cuda/mean.cu                      |   6 +-
 src/backend/cuda/mean.hpp                     |   2 +
 src/backend/cuda/meanshift.cpp                |   2 +
 src/backend/cuda/meanshift.hpp                |   4 +-
 src/backend/cuda/medfilt.cpp                  |   2 +
 src/backend/cuda/medfilt.hpp                  |   2 +
 src/backend/cuda/memory.cpp                   |  16 +-
 src/backend/cuda/memory.hpp                   |   6 +-
 src/backend/cuda/min.cu                       |   6 +-
 src/backend/cuda/minmax_op.hpp                |   2 +
 src/backend/cuda/moments.cpp                  |   2 +
 src/backend/cuda/moments.hpp                  |   4 +-
 src/backend/cuda/morph.cpp                    |   2 +
 src/backend/cuda/morph.hpp                    |   2 +
 src/backend/cuda/nearest_neighbour.cu         |   2 +
 src/backend/cuda/nearest_neighbour.hpp        |   4 +-
 src/backend/cuda/orb.cu                       |   2 +
 src/backend/cuda/orb.hpp                      |   4 +-
 src/backend/cuda/pad_array_borders.cpp        |   2 +
 src/backend/cuda/platform.cpp                 |  54 ++---
 src/backend/cuda/platform.hpp                 |  15 +-
 src/backend/cuda/plot.cpp                     |   9 +-
 src/backend/cuda/plot.hpp                     |   4 +-
 src/backend/cuda/print.hpp                    |   2 +
 src/backend/cuda/product.cu                   |   6 +-
 src/backend/cuda/qr.cpp                       |   2 +
 src/backend/cuda/qr.hpp                       |   2 +
 src/backend/cuda/random_engine.cu             |   4 +-
 src/backend/cuda/random_engine.hpp            |   2 +
 src/backend/cuda/range.cpp                    |   4 +-
 src/backend/cuda/range.hpp                    |   4 +-
 src/backend/cuda/reduce.hpp                   |   2 +
 src/backend/cuda/reduce_impl.hpp              |   2 +
 src/backend/cuda/regions.cu                   |   2 +
 src/backend/cuda/regions.hpp                  |   4 +-
 src/backend/cuda/reorder.cpp                  |   4 +-
 src/backend/cuda/reorder.hpp                  |   4 +-
 src/backend/cuda/reshape.cpp                  |   4 +-
 src/backend/cuda/resize.cpp                   |   2 +
 src/backend/cuda/resize.hpp                   |   4 +-
 src/backend/cuda/rotate.cpp                   |   2 +
 src/backend/cuda/rotate.hpp                   |   4 +-
 src/backend/cuda/scalar.hpp                   |   2 +
 src/backend/cuda/scan.cpp                     |   2 +
 src/backend/cuda/scan.hpp                     |   4 +-
 src/backend/cuda/scan_by_key.cpp              |   2 +
 src/backend/cuda/scan_by_key.hpp              |   4 +-
 src/backend/cuda/select.cpp                   |   8 +-
 src/backend/cuda/select.hpp                   |   2 +
 src/backend/cuda/set.cu                       |   4 +-
 src/backend/cuda/set.hpp                      |   2 +
 src/backend/cuda/shift.cpp                    |   8 +-
 src/backend/cuda/shift.hpp                    |   4 +-
 src/backend/cuda/sift.cu                      |   2 +
 src/backend/cuda/sift.hpp                     |   4 +-
 src/backend/cuda/sobel.cpp                    |   2 +
 src/backend/cuda/sobel.hpp                    |   4 +-
 src/backend/cuda/solve.cu                     |   8 +-
 src/backend/cuda/solve.hpp                    |   2 +
 src/backend/cuda/sort.cu                      |   2 +
 src/backend/cuda/sort.hpp                     |   4 +-
 src/backend/cuda/sort_by_key.cu               |   2 +
 src/backend/cuda/sort_by_key.hpp              |   4 +-
 src/backend/cuda/sort_index.cu                |   2 +
 src/backend/cuda/sort_index.hpp               |   4 +-
 src/backend/cuda/sparse.cu                    |   8 +-
 src/backend/cuda/sparse.hpp                   |   2 +
 src/backend/cuda/sparse_arith.cu              |   8 +-
 src/backend/cuda/sparse_arith.hpp             |   2 +
 src/backend/cuda/sparse_blas.cu               |   2 +
 src/backend/cuda/sparse_blas.hpp              |   4 +-
 src/backend/cuda/sum.cu                       |   6 +-
 src/backend/cuda/surface.cpp                  |   9 +-
 src/backend/cuda/surface.hpp                  |   4 +-
 src/backend/cuda/susan.cpp                    |   2 +
 src/backend/cuda/susan.hpp                    |   4 +-
 src/backend/cuda/svd.cpp                      |   2 +
 src/backend/cuda/svd.hpp                      |   2 +
 src/backend/cuda/threadsMgt.hpp               |  10 +-
 src/backend/cuda/thrust_utils.hpp             |  27 ++-
 src/backend/cuda/tile.cpp                     |   4 +-
 src/backend/cuda/tile.hpp                     |   4 +-
 src/backend/cuda/topk.cu                      |   4 +-
 src/backend/cuda/topk.hpp                     |   4 +-
 src/backend/cuda/transform.cpp                |   2 +
 src/backend/cuda/transform.hpp                |   4 +-
 src/backend/cuda/transpose.cpp                |   4 +-
 src/backend/cuda/transpose.hpp                |   2 +
 src/backend/cuda/transpose_inplace.cpp        |   4 +-
 src/backend/cuda/triangle.cpp                 |   4 +-
 src/backend/cuda/triangle.hpp                 |   2 +
 src/backend/cuda/types.hpp                    |  14 +-
 src/backend/cuda/unary.hpp                    |   8 +-
 src/backend/cuda/unwrap.cpp                   |   4 +-
 src/backend/cuda/unwrap.hpp                   |   4 +-
 src/backend/cuda/utility.cpp                  |   2 +
 src/backend/cuda/utility.hpp                  |   2 +
 src/backend/cuda/vector_field.cpp             |   9 +-
 src/backend/cuda/vector_field.hpp             |   4 +-
 src/backend/cuda/where.cpp                    |   2 +
 src/backend/cuda/where.hpp                    |   4 +-
 src/backend/cuda/wrap.cpp                     |   4 +-
 src/backend/cuda/wrap.hpp                     |   2 +
 src/backend/opencl/Array.cpp                  |  12 +-
 src/backend/opencl/Array.hpp                  |   2 +
 src/backend/opencl/CMakeLists.txt             |   2 +-
 src/backend/opencl/Event.cpp                  |   2 +
 src/backend/opencl/Event.hpp                  |   2 +
 .../opencl/GraphicsResourceManager.cpp        |   2 +
 .../opencl/GraphicsResourceManager.hpp        |   2 +
 src/backend/opencl/Kernel.cpp                 |   2 +
 src/backend/opencl/Kernel.hpp                 |   2 +
 src/backend/opencl/Module.hpp                 |   2 +
 src/backend/opencl/Param.cpp                  |   2 +
 src/backend/opencl/Param.hpp                  |   2 +
 src/backend/opencl/all.cpp                    |   4 +-
 src/backend/opencl/anisotropic_diffusion.cpp  |   2 +
 src/backend/opencl/anisotropic_diffusion.hpp  |   4 +-
 src/backend/opencl/any.cpp                    |   4 +-
 src/backend/opencl/approx.cpp                 |   2 +
 src/backend/opencl/approx.hpp                 |   2 +
 src/backend/opencl/arith.hpp                  |   2 +
 src/backend/opencl/assign.cpp                 |   4 +-
 src/backend/opencl/assign.hpp                 |   4 +-
 src/backend/opencl/backend.hpp                |   2 +-
 src/backend/opencl/bilateral.cpp              |   2 +
 src/backend/opencl/bilateral.hpp              |   4 +-
 src/backend/opencl/binary.hpp                 |   2 +
 src/backend/opencl/blas.cpp                   |   4 +-
 src/backend/opencl/blas.hpp                   |   2 +
 src/backend/opencl/canny.cpp                  |   2 +
 src/backend/opencl/canny.hpp                  |   2 +
 src/backend/opencl/cast.hpp                   |   2 +
 src/backend/opencl/cholesky.cpp               |   4 +
 src/backend/opencl/cholesky.hpp               |   2 +
 src/backend/opencl/clfft.cpp                  |   2 +
 src/backend/opencl/clfft.hpp                  |   2 +
 src/backend/opencl/compile_module.cpp         |  44 ++--
 src/backend/opencl/complex.hpp                |   2 +
 src/backend/opencl/convolve.cpp               |   8 +-
 src/backend/opencl/convolve.hpp               |   2 +
 src/backend/opencl/convolve_separable.cpp     |   2 +
 src/backend/opencl/copy.cpp                   |   6 +-
 src/backend/opencl/copy.hpp                   |   2 +
 src/backend/opencl/count.cpp                  |   4 +-
 src/backend/opencl/cpu/cpu_blas.cpp           |   4 +-
 src/backend/opencl/cpu/cpu_blas.hpp           |   4 +-
 src/backend/opencl/cpu/cpu_cholesky.cpp       |   2 +
 src/backend/opencl/cpu/cpu_cholesky.hpp       |   2 +
 src/backend/opencl/cpu/cpu_helper.hpp         |   4 +-
 src/backend/opencl/cpu/cpu_inverse.cpp        |   2 +
 src/backend/opencl/cpu/cpu_inverse.hpp        |   4 +-
 src/backend/opencl/cpu/cpu_lu.cpp             |   2 +
 src/backend/opencl/cpu/cpu_lu.hpp             |   2 +
 src/backend/opencl/cpu/cpu_qr.cpp             |   2 +
 src/backend/opencl/cpu/cpu_qr.hpp             |   2 +
 src/backend/opencl/cpu/cpu_solve.cpp          |   2 +
 src/backend/opencl/cpu/cpu_solve.hpp          |   2 +
 src/backend/opencl/cpu/cpu_sparse_blas.cpp    |   4 +-
 src/backend/opencl/cpu/cpu_sparse_blas.hpp    |   8 +-
 src/backend/opencl/cpu/cpu_svd.cpp            |   2 +
 src/backend/opencl/cpu/cpu_svd.hpp            |   2 +
 src/backend/opencl/cpu/cpu_triangle.hpp       |   2 +
 src/backend/opencl/device_manager.cpp         |   6 +-
 src/backend/opencl/device_manager.hpp         |  33 +--
 src/backend/opencl/diagonal.cpp               |   4 +-
 src/backend/opencl/diagonal.hpp               |   2 +
 src/backend/opencl/diff.cpp                   |   2 +
 src/backend/opencl/diff.hpp                   |   2 +
 src/backend/opencl/exampleFunction.cpp        |   2 +
 src/backend/opencl/exampleFunction.hpp        |   4 +-
 src/backend/opencl/fast.cpp                   |   2 +
 src/backend/opencl/fast.hpp                   |   4 +-
 src/backend/opencl/fft.cpp                    |   2 +
 src/backend/opencl/fft.hpp                    |   2 +
 src/backend/opencl/fftconvolve.cpp            |   2 +
 src/backend/opencl/fftconvolve.hpp            |   4 +-
 src/backend/opencl/flood_fill.cpp             |   2 +
 src/backend/opencl/flood_fill.hpp             |   2 +
 src/backend/opencl/gradient.cpp               |   2 +
 src/backend/opencl/gradient.hpp               |   4 +-
 src/backend/opencl/harris.cpp                 |   2 +
 src/backend/opencl/harris.hpp                 |   4 +-
 src/backend/opencl/hist_graphics.cpp          |   7 +-
 src/backend/opencl/hist_graphics.hpp          |   4 +-
 src/backend/opencl/histogram.cpp              |   4 +-
 src/backend/opencl/histogram.hpp              |   4 +-
 src/backend/opencl/homography.cpp             |   2 +
 src/backend/opencl/homography.hpp             |   4 +-
 src/backend/opencl/hsv_rgb.cpp                |   2 +
 src/backend/opencl/hsv_rgb.hpp                |   2 +
 src/backend/opencl/identity.cpp               |   4 +-
 src/backend/opencl/identity.hpp               |   4 +-
 src/backend/opencl/iir.cpp                    |   2 +
 src/backend/opencl/iir.hpp                    |   4 +-
 src/backend/opencl/image.cpp                  |   7 +-
 src/backend/opencl/image.hpp                  |   4 +-
 src/backend/opencl/index.cpp                  |   4 +-
 src/backend/opencl/index.hpp                  |   4 +-
 src/backend/opencl/inverse.cpp                |   4 +
 src/backend/opencl/inverse.hpp                |   4 +-
 src/backend/opencl/iota.cpp                   |   4 +-
 src/backend/opencl/iota.hpp                   |   4 +-
 src/backend/opencl/ireduce.cpp                |   4 +-
 src/backend/opencl/ireduce.hpp                |   2 +
 src/backend/opencl/jit.cpp                    |  20 +-
 src/backend/opencl/jit/BufferNode.hpp         |   4 +-
 src/backend/opencl/jit/kernel_generators.hpp  |   2 +
 src/backend/opencl/join.cpp                   |   8 +-
 src/backend/opencl/join.hpp                   |   2 +
 .../opencl/kernel/anisotropic_diffusion.hpp   |   2 +
 src/backend/opencl/kernel/approx.hpp          |   2 +
 src/backend/opencl/kernel/assign.hpp          |   2 +
 src/backend/opencl/kernel/bilateral.hpp       |   2 +
 src/backend/opencl/kernel/canny.hpp           |   2 +
 src/backend/opencl/kernel/config.cpp          |   2 +
 src/backend/opencl/kernel/config.hpp          |   2 +
 src/backend/opencl/kernel/convolve.hpp        |   2 +
 src/backend/opencl/kernel/convolve/conv1.cpp  |   2 +
 .../opencl/kernel/convolve/conv2_b8.cpp       |   2 +
 .../opencl/kernel/convolve/conv2_c32.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_c64.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_f32.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_f64.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_impl.hpp     |   2 +
 .../opencl/kernel/convolve/conv2_s16.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_s32.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_s64.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_u16.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_u32.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_u64.cpp      |   2 +
 .../opencl/kernel/convolve/conv2_u8.cpp       |   2 +
 src/backend/opencl/kernel/convolve/conv3.cpp  |   2 +
 .../opencl/kernel/convolve/conv_common.hpp    |   2 +
 .../opencl/kernel/convolve_separable.cpp      |   2 +
 .../opencl/kernel/convolve_separable.hpp      |   2 +
 src/backend/opencl/kernel/cscmm.hpp           |   4 +-
 src/backend/opencl/kernel/cscmv.hpp           |   4 +-
 src/backend/opencl/kernel/csrmm.hpp           |   4 +-
 src/backend/opencl/kernel/csrmv.hpp           |   4 +-
 src/backend/opencl/kernel/diagonal.hpp        |   6 +-
 src/backend/opencl/kernel/diff.hpp            |   2 +
 src/backend/opencl/kernel/exampleFunction.hpp |   2 +
 src/backend/opencl/kernel/fast.hpp            |   2 +
 src/backend/opencl/kernel/fftconvolve.hpp     |   2 +
 src/backend/opencl/kernel/flood_fill.hpp      |   2 +
 src/backend/opencl/kernel/gradient.hpp        |   6 +-
 src/backend/opencl/kernel/harris.hpp          |   2 +
 src/backend/opencl/kernel/histogram.hpp       |   2 +
 src/backend/opencl/kernel/homography.hpp      |   2 +
 src/backend/opencl/kernel/hsv_rgb.hpp         |   2 +
 src/backend/opencl/kernel/identity.hpp        |   6 +-
 src/backend/opencl/kernel/iir.hpp             |   4 +-
 src/backend/opencl/kernel/index.hpp           |   2 +
 src/backend/opencl/kernel/interp.hpp          |   2 +
 src/backend/opencl/kernel/iota.hpp            |   2 +
 src/backend/opencl/kernel/ireduce.hpp         |   6 +-
 src/backend/opencl/kernel/laset.hpp           |   4 +-
 src/backend/opencl/kernel/laset_band.hpp      |   4 +-
 src/backend/opencl/kernel/laswp.hpp           |   2 +
 src/backend/opencl/kernel/lookup.hpp          |   2 +
 src/backend/opencl/kernel/lu_split.hpp        |   6 +-
 src/backend/opencl/kernel/match_template.hpp  |   2 +
 src/backend/opencl/kernel/mean.hpp            |   2 +
 src/backend/opencl/kernel/meanshift.hpp       |   2 +
 src/backend/opencl/kernel/medfilt.hpp         |   2 +
 src/backend/opencl/kernel/memcopy.hpp         |   2 +
 src/backend/opencl/kernel/moments.hpp         |   2 +
 src/backend/opencl/kernel/morph.hpp           |   2 +
 .../opencl/kernel/nearest_neighbour.hpp       |   2 +
 src/backend/opencl/kernel/orb.hpp             |   2 +
 .../opencl/kernel/pad_array_borders.hpp       |   2 +
 src/backend/opencl/kernel/random_engine.hpp   |   2 +
 src/backend/opencl/kernel/range.hpp           |   2 +
 src/backend/opencl/kernel/reduce.hpp          |   8 +-
 src/backend/opencl/kernel/reduce_by_key.hpp   |  14 +-
 src/backend/opencl/kernel/regions.hpp         |   2 +
 src/backend/opencl/kernel/reorder.hpp         |   2 +
 src/backend/opencl/kernel/resize.hpp          |   2 +
 src/backend/opencl/kernel/rotate.hpp          |   2 +
 .../kernel/scan_by_key/scan_by_key_impl.cpp   |   2 +
 src/backend/opencl/kernel/scan_dim.hpp        |   4 +-
 src/backend/opencl/kernel/scan_dim_by_key.hpp |   2 +
 .../opencl/kernel/scan_dim_by_key_impl.hpp    |   4 +-
 src/backend/opencl/kernel/scan_first.hpp      |   4 +-
 .../opencl/kernel/scan_first_by_key.hpp       |   2 +
 .../opencl/kernel/scan_first_by_key_impl.hpp  |   4 +-
 src/backend/opencl/kernel/select.hpp          |   4 +-
 src/backend/opencl/kernel/sift.hpp            |   2 +
 src/backend/opencl/kernel/sobel.hpp           |   2 +
 src/backend/opencl/kernel/sort.hpp            |   2 +
 src/backend/opencl/kernel/sort_by_key.hpp     |   2 +
 .../kernel/sort_by_key/sort_by_key_impl.cpp   |   2 +
 .../opencl/kernel/sort_by_key_impl.hpp        |   4 +-
 src/backend/opencl/kernel/sort_helper.hpp     |   2 +
 src/backend/opencl/kernel/sparse.hpp          |   2 +
 src/backend/opencl/kernel/sparse_arith.hpp    |   4 +-
 src/backend/opencl/kernel/susan.hpp           |   2 +
 src/backend/opencl/kernel/swapdblk.hpp        |   2 +
 src/backend/opencl/kernel/tile.hpp            |   2 +
 src/backend/opencl/kernel/transform.hpp       |   2 +
 src/backend/opencl/kernel/transpose.hpp       |   4 +-
 .../opencl/kernel/transpose_inplace.hpp       |   4 +-
 src/backend/opencl/kernel/triangle.hpp        |   4 +-
 src/backend/opencl/kernel/unwrap.hpp          |   2 +
 src/backend/opencl/kernel/where.hpp           |   4 +-
 src/backend/opencl/kernel/wrap.hpp            |   2 +
 src/backend/opencl/logic.hpp                  |   2 +
 src/backend/opencl/lookup.cpp                 |   4 +-
 src/backend/opencl/lookup.hpp                 |   4 +-
 src/backend/opencl/lu.cpp                     |   4 +
 src/backend/opencl/lu.hpp                     |   2 +
 src/backend/opencl/magma/geqrf2.cpp           |   2 +-
 src/backend/opencl/magma/getrs.cpp            |   2 +-
 src/backend/opencl/magma/labrd.cpp            |   2 +-
 src/backend/opencl/magma/laset.cpp            |  10 +-
 src/backend/opencl/magma/laswp.cpp            |   3 +-
 src/backend/opencl/magma/magma_blas.h         |   4 +-
 src/backend/opencl/magma/magma_blas_clblast.h |   6 +-
 src/backend/opencl/magma/magma_data.h         |   4 +-
 src/backend/opencl/magma/swapdblk.cpp         |   4 +-
 src/backend/opencl/magma/transpose.cpp        |   4 +-
 .../opencl/magma/transpose_inplace.cpp        |   4 +-
 src/backend/opencl/match_template.cpp         |   2 +
 src/backend/opencl/match_template.hpp         |   4 +-
 src/backend/opencl/math.cpp                   |   2 +
 src/backend/opencl/math.hpp                   |  16 +-
 src/backend/opencl/max.cpp                    |   4 +-
 src/backend/opencl/mean.cpp                   |   4 +-
 src/backend/opencl/mean.hpp                   |   2 +
 src/backend/opencl/meanshift.cpp              |   2 +
 src/backend/opencl/meanshift.hpp              |   4 +-
 src/backend/opencl/medfilt.cpp                |   2 +
 src/backend/opencl/medfilt.hpp                |   2 +
 src/backend/opencl/memory.cpp                 |   4 +-
 src/backend/opencl/memory.hpp                 |   6 +-
 src/backend/opencl/min.cpp                    |   4 +-
 src/backend/opencl/moments.cpp                |   2 +
 src/backend/opencl/moments.hpp                |   4 +-
 src/backend/opencl/morph.cpp                  |   2 +
 src/backend/opencl/morph.hpp                  |   2 +
 src/backend/opencl/nearest_neighbour.cpp      |   2 +
 src/backend/opencl/nearest_neighbour.hpp      |   4 +-
 src/backend/opencl/orb.cpp                    |   2 +
 src/backend/opencl/orb.hpp                    |   4 +-
 src/backend/opencl/platform.cpp               |  16 +-
 src/backend/opencl/platform.hpp               |  15 +-
 src/backend/opencl/plot.cpp                   |   6 +-
 src/backend/opencl/plot.hpp                   |   4 +-
 src/backend/opencl/print.hpp                  |   2 +
 src/backend/opencl/product.cpp                |   4 +-
 src/backend/opencl/qr.cpp                     |   4 +
 src/backend/opencl/qr.hpp                     |   2 +
 src/backend/opencl/random_engine.cpp          |   4 +-
 src/backend/opencl/random_engine.hpp          |   2 +
 src/backend/opencl/range.cpp                  |   4 +-
 src/backend/opencl/range.hpp                  |   4 +-
 src/backend/opencl/reduce.hpp                 |   2 +
 src/backend/opencl/reduce_impl.hpp            |   2 +
 src/backend/opencl/regions.cpp                |   2 +
 src/backend/opencl/regions.hpp                |   4 +-
 src/backend/opencl/reorder.cpp                |   4 +-
 src/backend/opencl/reorder.hpp                |   4 +-
 src/backend/opencl/reshape.cpp                |   4 +-
 src/backend/opencl/resize.cpp                 |   2 +
 src/backend/opencl/resize.hpp                 |   4 +-
 src/backend/opencl/rotate.cpp                 |   2 +
 src/backend/opencl/rotate.hpp                 |   4 +-
 src/backend/opencl/scalar.hpp                 |   2 +
 src/backend/opencl/scan.cpp                   |   2 +
 src/backend/opencl/scan.hpp                   |   4 +-
 src/backend/opencl/scan_by_key.cpp            |   2 +
 src/backend/opencl/scan_by_key.hpp            |   4 +-
 src/backend/opencl/select.cpp                 |   6 +-
 src/backend/opencl/select.hpp                 |   2 +
 src/backend/opencl/set.cpp                    |   2 +
 src/backend/opencl/set.hpp                    |   2 +
 src/backend/opencl/shift.cpp                  |   8 +-
 src/backend/opencl/shift.hpp                  |   4 +-
 src/backend/opencl/sift.cpp                   |   2 +
 src/backend/opencl/sift.hpp                   |   4 +-
 src/backend/opencl/sobel.cpp                  |   2 +
 src/backend/opencl/sobel.hpp                  |   4 +-
 src/backend/opencl/solve.cpp                  |   4 +
 src/backend/opencl/solve.hpp                  |   2 +
 src/backend/opencl/sort.cpp                   |   2 +
 src/backend/opencl/sort.hpp                   |   4 +-
 src/backend/opencl/sort_by_key.cpp            |   2 +
 src/backend/opencl/sort_by_key.hpp            |   4 +-
 src/backend/opencl/sort_index.cpp             |   4 +-
 src/backend/opencl/sort_index.hpp             |   4 +-
 src/backend/opencl/sparse.cpp                 |   2 +
 src/backend/opencl/sparse.hpp                 |   2 +
 src/backend/opencl/sparse_arith.cpp           |   2 +
 src/backend/opencl/sparse_arith.hpp           |   2 +
 src/backend/opencl/sparse_blas.cpp            |   2 +
 src/backend/opencl/sparse_blas.hpp            |   4 +-
 src/backend/opencl/sum.cpp                    |   4 +-
 src/backend/opencl/surface.cpp                |   6 +-
 src/backend/opencl/surface.hpp                |   4 +-
 src/backend/opencl/susan.cpp                  |   2 +
 src/backend/opencl/susan.hpp                  |   4 +-
 src/backend/opencl/svd.cpp                    |   4 +
 src/backend/opencl/svd.hpp                    |   2 +
 src/backend/opencl/threadsMgt.hpp             |   4 +-
 src/backend/opencl/tile.cpp                   |   4 +-
 src/backend/opencl/tile.hpp                   |   4 +-
 src/backend/opencl/topk.cpp                   |   4 +-
 src/backend/opencl/topk.hpp                   |   8 +-
 src/backend/opencl/traits.hpp                 |  19 +-
 src/backend/opencl/transform.cpp              |   2 +
 src/backend/opencl/transform.hpp              |   4 +-
 src/backend/opencl/transpose.cpp              |   4 +-
 src/backend/opencl/transpose.hpp              |   2 +
 src/backend/opencl/transpose_inplace.cpp      |   4 +-
 src/backend/opencl/triangle.cpp               |   4 +-
 src/backend/opencl/triangle.hpp               |   2 +
 src/backend/opencl/types.cpp                  |   6 +-
 src/backend/opencl/types.hpp                  |   6 +-
 src/backend/opencl/unary.hpp                  |   8 +-
 src/backend/opencl/unwrap.cpp                 |   4 +-
 src/backend/opencl/unwrap.hpp                 |   4 +-
 src/backend/opencl/vector_field.cpp           |   6 +-
 src/backend/opencl/vector_field.hpp           |   4 +-
 src/backend/opencl/where.cpp                  |   2 +
 src/backend/opencl/where.hpp                  |   4 +-
 src/backend/opencl/wrap.cpp                   |   4 +-
 src/backend/opencl/wrap.hpp                   |   2 +
 1065 files changed, 4081 insertions(+), 1883 deletions(-)
 create mode 100644 src/api/c/handle.cpp

diff --git a/CMakeModules/FileToString.cmake b/CMakeModules/FileToString.cmake
index 6092c9176c..5491c8b126 100644
--- a/CMakeModules/FileToString.cmake
+++ b/CMakeModules/FileToString.cmake
@@ -45,6 +45,7 @@ function(FILE_TO_STRING)
         endif(RTCS_NULLTERM)
 
         string(REPLACE "." "_" var_name ${var_name})
+        string(REPLACE "\ " "_" namespace_name ${RTCS_NAMESPACE})
 
         set(_output_path "${CMAKE_CURRENT_BINARY_DIR}/${RTCS_OUTPUT_DIR}")
         if(RTCS_WITH_EXTENSION)
@@ -66,9 +67,9 @@ function(FILE_TO_STRING)
 
         list(APPEND _output_files ${_output_file})
     endforeach()
-    add_custom_target(${RTCS_NAMESPACE}_${RTCS_OUTPUT_DIR}_bin_target DEPENDS ${_output_files})
-    set_target_properties(${RTCS_NAMESPACE}_${RTCS_OUTPUT_DIR}_bin_target PROPERTIES FOLDER "Generated Targets")
+    add_custom_target(${namespace_name}_${RTCS_OUTPUT_DIR}_bin_target DEPENDS ${_output_files})
+    set_target_properties(${namespace_name}_${RTCS_OUTPUT_DIR}_bin_target PROPERTIES FOLDER "Generated Targets")
 
     set("${RTCS_VARNAME}" ${_output_files} PARENT_SCOPE)
-    set("${RTCS_TARGETS}" ${RTCS_NAMESPACE}_${RTCS_OUTPUT_DIR}_bin_target PARENT_SCOPE)
+    set("${RTCS_TARGETS}" ${namespace_name}_${RTCS_OUTPUT_DIR}_bin_target PARENT_SCOPE)
 endfunction(FILE_TO_STRING)
diff --git a/src/api/c/CMakeLists.txt b/src/api/c/CMakeLists.txt
index 8dcf7c3d5b..870d687382 100644
--- a/src/api/c/CMakeLists.txt
+++ b/src/api/c/CMakeLists.txt
@@ -88,6 +88,7 @@ target_sources(c_api_interface
     ${CMAKE_CURRENT_SOURCE_DIR}/gaussian_kernel.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/gradient.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/hamming.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/handle.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/handle.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/harris.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/hist.cpp
diff --git a/src/api/c/anisotropic_diffusion.cpp b/src/api/c/anisotropic_diffusion.cpp
index fd2f83c5c1..3c77f8644c 100644
--- a/src/api/c/anisotropic_diffusion.cpp
+++ b/src/api/c/anisotropic_diffusion.cpp
@@ -24,7 +24,7 @@
 #include <type_traits>
 
 using af::dim4;
-using common::cast;
+using arrayfire::common::cast;
 using detail::arithOp;
 using detail::Array;
 using detail::createEmptyArray;
diff --git a/src/api/c/array.cpp b/src/api/c/array.cpp
index 8cb79bfae8..e9a0f68603 100644
--- a/src/api/c/array.cpp
+++ b/src/api/c/array.cpp
@@ -17,8 +17,14 @@
 #include <af/sparse.h>
 
 using af::dim4;
-using common::half;
-using common::SparseArrayBase;
+using arrayfire::copyData;
+using arrayfire::copySparseArray;
+using arrayfire::getSparseArrayBase;
+using arrayfire::releaseHandle;
+using arrayfire::releaseSparseHandle;
+using arrayfire::retainSparseHandle;
+using arrayfire::common::half;
+using arrayfire::common::SparseArrayBase;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
@@ -27,48 +33,6 @@ using detail::uint;
 using detail::uintl;
 using detail::ushort;
 
-af_array createHandle(const dim4 &d, af_dtype dtype) {
-    // clang-format off
-    switch (dtype) {
-        case f32: return createHandle<float  >(d);
-        case c32: return createHandle<cfloat >(d);
-        case f64: return createHandle<double >(d);
-        case c64: return createHandle<cdouble>(d);
-        case b8:  return createHandle<char   >(d);
-        case s32: return createHandle<int    >(d);
-        case u32: return createHandle<uint   >(d);
-        case u8:  return createHandle<uchar  >(d);
-        case s64: return createHandle<intl   >(d);
-        case u64: return createHandle<uintl  >(d);
-        case s16: return createHandle<short  >(d);
-        case u16: return createHandle<ushort >(d);
-        case f16: return createHandle<half   >(d);
-        default: TYPE_ERROR(3, dtype);
-    }
-    // clang-format on
-}
-
-af_array createHandleFromValue(const dim4 &d, double val, af_dtype dtype) {
-    // clang-format off
-    switch (dtype) {
-        case f32: return createHandleFromValue<float  >(d, val);
-        case c32: return createHandleFromValue<cfloat >(d, val);
-        case f64: return createHandleFromValue<double >(d, val);
-        case c64: return createHandleFromValue<cdouble>(d, val);
-        case b8:  return createHandleFromValue<char   >(d, val);
-        case s32: return createHandleFromValue<int    >(d, val);
-        case u32: return createHandleFromValue<uint   >(d, val);
-        case u8:  return createHandleFromValue<uchar  >(d, val);
-        case s64: return createHandleFromValue<intl   >(d, val);
-        case u64: return createHandleFromValue<uintl  >(d, val);
-        case s16: return createHandleFromValue<short  >(d, val);
-        case u16: return createHandleFromValue<ushort >(d, val);
-        case f16: return createHandleFromValue<half   >(d, val);
-        default: TYPE_ERROR(3, dtype);
-    }
-    // clang-format on
-}
-
 af_err af_get_data_ptr(void *data, const af_array arr) {
     try {
         af_dtype type = getInfo(arr).getType();
@@ -291,38 +255,6 @@ af_err af_release_array(af_array arr) {
     return AF_SUCCESS;
 }
 
-af_array retain(const af_array in) {
-    const ArrayInfo &info = getInfo(in, false, false);
-    af_dtype ty           = info.getType();
-
-    if (info.isSparse()) {
-        switch (ty) {
-            case f32: return retainSparseHandle<float>(in);
-            case f64: return retainSparseHandle<double>(in);
-            case c32: return retainSparseHandle<detail::cfloat>(in);
-            case c64: return retainSparseHandle<detail::cdouble>(in);
-            default: TYPE_ERROR(1, ty);
-        }
-    } else {
-        switch (ty) {
-            case f32: return retainHandle<float>(in);
-            case f64: return retainHandle<double>(in);
-            case s32: return retainHandle<int>(in);
-            case u32: return retainHandle<uint>(in);
-            case u8: return retainHandle<uchar>(in);
-            case c32: return retainHandle<detail::cfloat>(in);
-            case c64: return retainHandle<detail::cdouble>(in);
-            case b8: return retainHandle<char>(in);
-            case s64: return retainHandle<intl>(in);
-            case u64: return retainHandle<uintl>(in);
-            case s16: return retainHandle<short>(in);
-            case u16: return retainHandle<ushort>(in);
-            case f16: return retainHandle<half>(in);
-            default: TYPE_ERROR(1, ty);
-        }
-    }
-}
-
 af_err af_retain_array(af_array *out, const af_array in) {
     try {
         *out = retain(in);
diff --git a/src/api/c/assign.cpp b/src/api/c/assign.cpp
index 20aa69e629..e53b43a6c5 100644
--- a/src/api/c/assign.cpp
+++ b/src/api/c/assign.cpp
@@ -30,12 +30,13 @@ using std::swap;
 using std::vector;
 
 using af::dim4;
-using common::convert2Canonical;
-using common::createSpanIndex;
-using common::half;
-using common::if_complex;
-using common::if_real;
-using common::modDims;
+using arrayfire::common::convert2Canonical;
+using arrayfire::common::createSpanIndex;
+using arrayfire::common::half;
+using arrayfire::common::if_complex;
+using arrayfire::common::if_real;
+using arrayfire::common::modDims;
+using arrayfire::common::tile;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
@@ -77,9 +78,9 @@ static void assign(Array<Tout>& out, const vector<af_seq> seqs,
 
         // If both out and in are vectors of equal elements,
         // reshape in to out dims
-        Array<Tin> in_ =
-            in.elements() == 1 ? common::tile(in, oDims) : modDims(in, oDims);
-        auto dst = createSubArray<Tout>(out, seqs, false);
+        Array<Tin> in_ = in.elements() == 1 ? arrayfire::common::tile(in, oDims)
+                                            : modDims(in, oDims);
+        auto dst       = createSubArray<Tout>(out, seqs, false);
 
         copyArray<Tin, Tout>(dst, in_);
     } else {
diff --git a/src/api/c/binary.cpp b/src/api/c/binary.cpp
index fc24fd64eb..b9f9393421 100644
--- a/src/api/c/binary.cpp
+++ b/src/api/c/binary.cpp
@@ -30,9 +30,13 @@
 
 using af::dim4;
 using af::dtype;
-using common::half;
-using common::modDims;
-using common::tile;
+using arrayfire::castSparse;
+using arrayfire::getSparseArray;
+using arrayfire::getSparseArrayBase;
+using arrayfire::common::half;
+using arrayfire::common::modDims;
+using arrayfire::common::SparseArrayBase;
+using arrayfire::common::tile;
 using detail::arithOp;
 using detail::arithOpD;
 using detail::Array;
@@ -84,8 +88,10 @@ static inline af_array arithOpBroadcast(const af_array lhs,
         }
     }
 
-    Array<T> lhst = common::tile<T>(modDims(getArray<T>(lhs), lshape), ltile);
-    Array<T> rhst = common::tile<T>(modDims(getArray<T>(rhs), rshape), rtile);
+    Array<T> lhst =
+        arrayfire::common::tile<T>(modDims(getArray<T>(lhs), lshape), ltile);
+    Array<T> rhst =
+        arrayfire::common::tile<T>(modDims(getArray<T>(rhs), rshape), rtile);
 
     return getHandle(arithOp<T, op>(lhst, rhst, odims));
 }
@@ -199,8 +205,8 @@ template<af_op_t op>
 static af_err af_arith_sparse(af_array *out, const af_array lhs,
                               const af_array rhs) {
     try {
-        const common::SparseArrayBase linfo = getSparseArrayBase(lhs);
-        const common::SparseArrayBase rinfo = getSparseArrayBase(rhs);
+        const SparseArrayBase linfo = getSparseArrayBase(lhs);
+        const SparseArrayBase rinfo = getSparseArrayBase(rhs);
 
         ARG_ASSERT(1, (linfo.getStorage() == rinfo.getStorage()));
         ARG_ASSERT(1, (linfo.dims() == rinfo.dims()));
@@ -227,7 +233,7 @@ static af_err af_arith_sparse_dense(af_array *out, const af_array lhs,
                                     const af_array rhs,
                                     const bool reverse = false) {
     try {
-        const common::SparseArrayBase linfo = getSparseArrayBase(lhs);
+        const SparseArrayBase linfo = getSparseArrayBase(lhs);
         if (linfo.ndims() > 2) {
             AF_ERROR(
                 "Sparse-Dense arithmetic operations cannot be used in batch "
diff --git a/src/api/c/blas.cpp b/src/api/c/blas.cpp
index 0afd4f79b2..0946d42083 100644
--- a/src/api/c/blas.cpp
+++ b/src/api/c/blas.cpp
@@ -25,13 +25,16 @@
 #include <af/defines.h>
 #include <af/dim4.hpp>
 
-using common::half;
-using common::SparseArrayBase;
+using arrayfire::getSparseArray;
+using arrayfire::getSparseArrayBase;
+using arrayfire::common::half;
+using arrayfire::common::SparseArrayBase;
 using detail::cdouble;
 using detail::cfloat;
 using detail::gemm;
 using detail::matmul;
 
+namespace {
 template<typename T>
 static inline af_array sparseMatmul(const af_array lhs, const af_array rhs,
                                     af_mat_prop optLhs, af_mat_prop optRhs) {
@@ -54,6 +57,16 @@ static inline af_array dot(const af_array lhs, const af_array rhs,
         dot<T>(getArray<T>(lhs), getArray<T>(rhs), optLhs, optRhs));
 }
 
+template<typename T>
+static inline T dotAll(af_array out) {
+    T res{};
+    AF_CHECK(af_eval(out));
+    AF_CHECK(af_get_data_ptr((void *)&res, out));
+    return res;
+}
+
+}  // namespace
+
 af_err af_sparse_matmul(af_array *out, const af_array lhs, const af_array rhs,
                         const af_mat_prop optLhs, const af_mat_prop optRhs) {
     try {
@@ -327,14 +340,6 @@ af_err af_dot(af_array *out, const af_array lhs, const af_array rhs,
     return AF_SUCCESS;
 }
 
-template<typename T>
-static inline T dotAll(af_array out) {
-    T res{};
-    AF_CHECK(af_eval(out));
-    AF_CHECK(af_get_data_ptr((void *)&res, out));
-    return res;
-}
-
 af_err af_dot_all(double *rval, double *ival, const af_array lhs,
                   const af_array rhs, const af_mat_prop optLhs,
                   const af_mat_prop optRhs) {
diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp
index 625ce748fa..ae1fa8add9 100644
--- a/src/api/c/canny.cpp
+++ b/src/api/c/canny.cpp
@@ -36,8 +36,8 @@
 #include <vector>
 
 using af::dim4;
-using common::cast;
-using common::tile;
+using arrayfire::common::cast;
+using arrayfire::common::tile;
 using detail::arithOp;
 using detail::Array;
 using detail::convolve2;
@@ -62,6 +62,7 @@ using std::make_pair;
 using std::pair;
 using std::vector;
 
+namespace {
 Array<float> gradientMagnitude(const Array<float>& gx, const Array<float>& gy,
                                const bool& isf) {
     using detail::abs;
@@ -138,7 +139,8 @@ Array<float> otsuThreshold(const Array<float>& in, const unsigned NUM_BINS,
 
     ireduce<af_max_t, float>(thresh, locs, sigmas, 0);
 
-    return cast<float, uint>(common::tile(locs, dim4(inDims[0], inDims[1])));
+    return cast<float, uint>(
+        arrayfire::common::tile(locs, dim4(inDims[0], inDims[1])));
 }
 
 Array<float> normalize(const Array<float>& supEdges, const float minVal,
@@ -219,6 +221,8 @@ af_array cannyHelper(const Array<T>& in, const float t1,
     return getHandle(edgeTrackingByHysteresis(swpair.first, swpair.second));
 }
 
+}  // namespace
+
 af_err af_canny(af_array* out, const af_array in, const af_canny_threshold ct,
                 const float t1, const float t2, const unsigned sw,
                 const bool isf) {
diff --git a/src/api/c/cast.cpp b/src/api/c/cast.cpp
index c4f66cdf34..20e47a1a2d 100644
--- a/src/api/c/cast.cpp
+++ b/src/api/c/cast.cpp
@@ -22,7 +22,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::castSparse;
+using arrayfire::getHandle;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
diff --git a/src/api/c/cholesky.cpp b/src/api/c/cholesky.cpp
index 4dd8fdc20f..1a662c649f 100644
--- a/src/api/c/cholesky.cpp
+++ b/src/api/c/cholesky.cpp
@@ -17,6 +17,7 @@
 #include <af/defines.h>
 #include <af/lapack.h>
 
+using arrayfire::getArray;
 using detail::cdouble;
 using detail::cfloat;
 
diff --git a/src/api/c/clamp.cpp b/src/api/c/clamp.cpp
index f0da3323eb..fb821d3bf3 100644
--- a/src/api/c/clamp.cpp
+++ b/src/api/c/clamp.cpp
@@ -22,7 +22,7 @@
 #include <af/defines.h>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
diff --git a/src/api/c/complex.cpp b/src/api/c/complex.cpp
index 1732aaf4bc..c7a4c4e2bc 100644
--- a/src/api/c/complex.cpp
+++ b/src/api/c/complex.cpp
@@ -22,7 +22,7 @@
 #include <complex.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::conj;
diff --git a/src/api/c/confidence_connected.cpp b/src/api/c/confidence_connected.cpp
index b42decc227..ceb8ca7b75 100644
--- a/src/api/c/confidence_connected.cpp
+++ b/src/api/c/confidence_connected.cpp
@@ -24,8 +24,10 @@
 #include <type_traits>
 
 using af::dim4;
-using common::cast;
-using common::createSpanIndex;
+using arrayfire::common::cast;
+using arrayfire::common::convRange;
+using arrayfire::common::createSpanIndex;
+using arrayfire::common::integralImage;
 using detail::arithOp;
 using detail::Array;
 using detail::createValueArray;
@@ -122,10 +124,10 @@ af_array ccHelper(const Array<T>& img, const Array<uint>& seedx,
     Array<uint> x_     = arithOp<uint, af_add_t>(seedx, radii, seedDims);
     Array<uint> _y     = arithOp<uint, af_sub_t>(seedy, radiip, seedDims);
     Array<uint> y_     = arithOp<uint, af_add_t>(seedy, radii, seedDims);
-    Array<CT> in       = common::convRange<CT, T>(img, CT(1), CT(2));
+    Array<CT> in       = convRange<CT, T>(img, CT(1), CT(2));
     Array<CT> in_2     = arithOp<CT, af_mul_t>(in, in, inDims);
-    Array<CT> I1       = common::integralImage<CT>(in);
-    Array<CT> I2       = common::integralImage<CT>(in_2);
+    Array<CT> I1       = integralImage<CT>(in);
+    Array<CT> I2       = integralImage<CT>(in_2);
     Array<CT> S1       = sum(I1, _x, x_, _y, y_);
     Array<CT> S2       = sum(I2, _x, x_, _y, y_);
     CT totSum          = getScalar<CT>(reduce_all<af_add_t, CT, CT>(S1));
diff --git a/src/api/c/convolve.cpp b/src/api/c/convolve.cpp
index 9a496633b0..abbcd2f71b 100644
--- a/src/api/c/convolve.cpp
+++ b/src/api/c/convolve.cpp
@@ -25,8 +25,8 @@
 #include <cstdio>
 
 using af::dim4;
-using common::cast;
-using common::half;
+using arrayfire::common::cast;
+using arrayfire::common::half;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
@@ -54,8 +54,10 @@ inline af_array convolve2(const af_array &s, const af_array &c_f,
     const Array<accT> signal    = castArray<accT>(s);
 
     if (colFilter.isScalar() && rowFilter.isScalar()) {
-        Array<accT> colArray = common::tile(colFilter, signal.dims());
-        Array<accT> rowArray = common::tile(rowFilter, signal.dims());
+        Array<accT> colArray =
+            arrayfire::common::tile(colFilter, signal.dims());
+        Array<accT> rowArray =
+            arrayfire::common::tile(rowFilter, signal.dims());
 
         Array<accT> filter =
             arithOp<accT, af_mul_t>(colArray, rowArray, signal.dims());
diff --git a/src/api/c/corrcoef.cpp b/src/api/c/corrcoef.cpp
index 0efc503cd4..fd767fb0ba 100644
--- a/src/api/c/corrcoef.cpp
+++ b/src/api/c/corrcoef.cpp
@@ -24,7 +24,7 @@
 #include <cmath>
 
 using af::dim4;
-using common::cast;
+using arrayfire::common::cast;
 using detail::arithOp;
 using detail::Array;
 using detail::getScalar;
diff --git a/src/api/c/covariance.cpp b/src/api/c/covariance.cpp
index 80108c4b0b..f364558b11 100644
--- a/src/api/c/covariance.cpp
+++ b/src/api/c/covariance.cpp
@@ -23,7 +23,7 @@
 #include "stats.h"
 
 using af::dim4;
-using common::cast;
+using arrayfire::common::cast;
 using detail::arithOp;
 using detail::Array;
 using detail::createValueArray;
diff --git a/src/api/c/data.cpp b/src/api/c/data.cpp
index f231c7b300..60ede3d4f6 100644
--- a/src/api/c/data.cpp
+++ b/src/api/c/data.cpp
@@ -26,7 +26,7 @@
 #include <af/util.h>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::createValueArray;
@@ -40,19 +40,6 @@ using detail::uint;
 using detail::uintl;
 using detail::ushort;
 
-dim4 verifyDims(const unsigned ndims, const dim_t *const dims) {
-    DIM_ASSERT(1, ndims >= 1);
-
-    dim4 d(1, 1, 1, 1);
-
-    for (unsigned i = 0; i < ndims; i++) {
-        d[i] = dims[i];
-        DIM_ASSERT(2, dims[i] >= 1);
-    }
-
-    return d;
-}
-
 // Strong Exception Guarantee
 af_err af_constant(af_array *result, const double value, const unsigned ndims,
                    const dim_t *const dims, const af_dtype type) {
diff --git a/src/api/c/deconvolution.cpp b/src/api/c/deconvolution.cpp
index 21180b2d8b..d5327d1efe 100644
--- a/src/api/c/deconvolution.cpp
+++ b/src/api/c/deconvolution.cpp
@@ -33,7 +33,7 @@
 #include <vector>
 
 using af::dim4;
-using common::cast;
+using arrayfire::common::cast;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp
index b619a867f2..1b6ef9fb93 100644
--- a/src/api/c/device.cpp
+++ b/src/api/c/device.cpp
@@ -28,10 +28,11 @@
 #include <string>
 
 using af::dim4;
-using common::getCacheDirectory;
-using common::getEnvVar;
-using common::half;
-using common::JIT_KERNEL_CACHE_DIRECTORY_ENV_NAME;
+using arrayfire::getSparseArray;
+using arrayfire::common::getCacheDirectory;
+using arrayfire::common::getEnvVar;
+using arrayfire::common::half;
+using arrayfire::common::JIT_KERNEL_CACHE_DIRECTORY_ENV_NAME;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/diff.cpp b/src/api/c/diff.cpp
index 3fb1cee150..c579f0b53e 100644
--- a/src/api/c/diff.cpp
+++ b/src/api/c/diff.cpp
@@ -16,6 +16,8 @@
 #include <af/defines.h>
 
 using af::dim4;
+using arrayfire::getArray;
+using arrayfire::getHandle;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
diff --git a/src/api/c/error.cpp b/src/api/c/error.cpp
index 4dd1ff190f..91a84b3ff3 100644
--- a/src/api/c/error.cpp
+++ b/src/api/c/error.cpp
@@ -39,7 +39,7 @@ void af_get_last_error(char **str, dim_t *len) {
 }
 
 af_err af_set_enable_stacktrace(int is_enabled) {
-    common::is_stacktrace_enabled() = is_enabled;
+    arrayfire::common::is_stacktrace_enabled() = is_enabled;
 
     return AF_SUCCESS;
 }
diff --git a/src/api/c/exampleFunction.cpp b/src/api/c/exampleFunction.cpp
index a304a6d963..4a7a52f6bd 100644
--- a/src/api/c/exampleFunction.cpp
+++ b/src/api/c/exampleFunction.cpp
@@ -41,7 +41,7 @@ af_array example(const af_array& a, const af_array& b,
     // getArray<T> function is defined in handle.hpp
     // and it returns backend specific Array, namely one of the following
     //      * cpu::Array<T>
-    //      * cuda::Array<T>
+    //      * arrayfire::cuda::Array<T>
     //      * opencl::Array<T>
     // getHandle<T> function is defined in handle.hpp takes one of the
     // above backend specific detail::Array<T> and returns the
diff --git a/src/api/c/fftconvolve.cpp b/src/api/c/fftconvolve.cpp
index 58cbc9e2c4..bbcb2d2a1d 100644
--- a/src/api/c/fftconvolve.cpp
+++ b/src/api/c/fftconvolve.cpp
@@ -26,7 +26,7 @@
 #include <vector>
 
 using af::dim4;
-using common::cast;
+using arrayfire::common::cast;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
diff --git a/src/api/c/flip.cpp b/src/api/c/flip.cpp
index 4b0bf15ef2..080af47aac 100644
--- a/src/api/c/flip.cpp
+++ b/src/api/c/flip.cpp
@@ -18,8 +18,9 @@
 #include <cassert>
 
 using af::dim4;
-using common::flip;
-using common::half;
+using arrayfire::getArray;
+using arrayfire::common::flip;
+using arrayfire::common::half;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/gradient.cpp b/src/api/c/gradient.cpp
index 419039ad11..e99f4e6e64 100644
--- a/src/api/c/gradient.cpp
+++ b/src/api/c/gradient.cpp
@@ -16,6 +16,7 @@
 #include <af/image.h>
 
 using af::dim4;
+using arrayfire::getArray;
 using detail::cdouble;
 using detail::cfloat;
 
diff --git a/src/api/c/handle.cpp b/src/api/c/handle.cpp
new file mode 100644
index 0000000000..392e120fca
--- /dev/null
+++ b/src/api/c/handle.cpp
@@ -0,0 +1,116 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <handle.hpp>
+
+#include <backend.hpp>
+#include <sparse_handle.hpp>
+
+#include <af/dim4.hpp>
+
+using af::dim4;
+using arrayfire::common::half;
+using detail::cdouble;
+using detail::cfloat;
+using detail::intl;
+using detail::uchar;
+using detail::uint;
+using detail::uintl;
+using detail::ushort;
+
+namespace arrayfire {
+
+af_array retain(const af_array in) {
+    const ArrayInfo &info = getInfo(in, false, false);
+    af_dtype ty           = info.getType();
+
+    if (info.isSparse()) {
+        switch (ty) {
+            case f32: return retainSparseHandle<float>(in);
+            case f64: return retainSparseHandle<double>(in);
+            case c32: return retainSparseHandle<detail::cfloat>(in);
+            case c64: return retainSparseHandle<detail::cdouble>(in);
+            default: TYPE_ERROR(1, ty);
+        }
+    } else {
+        switch (ty) {
+            case f32: return retainHandle<float>(in);
+            case f64: return retainHandle<double>(in);
+            case s32: return retainHandle<int>(in);
+            case u32: return retainHandle<uint>(in);
+            case u8: return retainHandle<uchar>(in);
+            case c32: return retainHandle<detail::cfloat>(in);
+            case c64: return retainHandle<detail::cdouble>(in);
+            case b8: return retainHandle<char>(in);
+            case s64: return retainHandle<intl>(in);
+            case u64: return retainHandle<uintl>(in);
+            case s16: return retainHandle<short>(in);
+            case u16: return retainHandle<ushort>(in);
+            case f16: return retainHandle<half>(in);
+            default: TYPE_ERROR(1, ty);
+        }
+    }
+}
+
+af_array createHandle(const dim4 &d, af_dtype dtype) {
+    // clang-format off
+    switch (dtype) {
+        case f32: return createHandle<float  >(d);
+        case c32: return createHandle<cfloat >(d);
+        case f64: return createHandle<double >(d);
+        case c64: return createHandle<cdouble>(d);
+        case b8:  return createHandle<char   >(d);
+        case s32: return createHandle<int    >(d);
+        case u32: return createHandle<uint   >(d);
+        case u8:  return createHandle<uchar  >(d);
+        case s64: return createHandle<intl   >(d);
+        case u64: return createHandle<uintl  >(d);
+        case s16: return createHandle<short  >(d);
+        case u16: return createHandle<ushort >(d);
+        case f16: return createHandle<half   >(d);
+        default: TYPE_ERROR(3, dtype);
+    }
+    // clang-format on
+}
+
+af_array createHandleFromValue(const dim4 &d, double val, af_dtype dtype) {
+    // clang-format off
+    switch (dtype) {
+        case f32: return createHandleFromValue<float  >(d, val);
+        case c32: return createHandleFromValue<cfloat >(d, val);
+        case f64: return createHandleFromValue<double >(d, val);
+        case c64: return createHandleFromValue<cdouble>(d, val);
+        case b8:  return createHandleFromValue<char   >(d, val);
+        case s32: return createHandleFromValue<int    >(d, val);
+        case u32: return createHandleFromValue<uint   >(d, val);
+        case u8:  return createHandleFromValue<uchar  >(d, val);
+        case s64: return createHandleFromValue<intl   >(d, val);
+        case u64: return createHandleFromValue<uintl  >(d, val);
+        case s16: return createHandleFromValue<short  >(d, val);
+        case u16: return createHandleFromValue<ushort >(d, val);
+        case f16: return createHandleFromValue<half   >(d, val);
+        default: TYPE_ERROR(3, dtype);
+    }
+    // clang-format on
+}
+
+dim4 verifyDims(const unsigned ndims, const dim_t *const dims) {
+    DIM_ASSERT(1, ndims >= 1);
+
+    dim4 d(1, 1, 1, 1);
+
+    for (unsigned i = 0; i < ndims; i++) {
+        d[i] = dims[i];
+        DIM_ASSERT(2, dims[i] >= 1);
+    }
+
+    return d;
+}
+
+}  // namespace arrayfire
diff --git a/src/api/c/handle.hpp b/src/api/c/handle.hpp
index 2499c9781a..4b73293cb3 100644
--- a/src/api/c/handle.hpp
+++ b/src/api/c/handle.hpp
@@ -20,8 +20,7 @@
 #include <af/defines.h>
 #include <af/dim4.hpp>
 
-const ArrayInfo &getInfo(const af_array arr, bool sparse_check = true,
-                         bool device_check = true);
+namespace arrayfire {
 
 af_array retain(const af_array in);
 
@@ -31,10 +30,14 @@ af_array createHandle(const af::dim4 &d, af_dtype dtype);
 
 af_array createHandleFromValue(const af::dim4 &d, double val, af_dtype dtype);
 
+namespace common {
+const ArrayInfo &getInfo(const af_array arr, bool sparse_check = true,
+                         bool device_check = true);
+
 template<typename To>
 detail::Array<To> castArray(const af_array &in);
 
-namespace {
+}  // namespace common
 
 template<typename T>
 const detail::Array<T> &getArray(const af_array &arr) {
@@ -119,4 +122,17 @@ detail::Array<T> &getCopyOnWriteArray(const af_array &arr) {
     return *A;
 }
 
-}  // namespace
+}  // namespace arrayfire
+
+using arrayfire::copyArray;
+using arrayfire::copyData;
+using arrayfire::createHandle;
+using arrayfire::createHandleFromData;
+using arrayfire::createHandleFromValue;
+using arrayfire::getArray;
+using arrayfire::getHandle;
+using arrayfire::releaseHandle;
+using arrayfire::retain;
+using arrayfire::verifyDims;
+using arrayfire::common::castArray;
+using arrayfire::common::getInfo;
diff --git a/src/api/c/hist.cpp b/src/api/c/hist.cpp
index 4b74e33cdf..350d97416d 100644
--- a/src/api/c/hist.cpp
+++ b/src/api/c/hist.cpp
@@ -18,6 +18,12 @@
 #include <reduce.hpp>
 #include <af/graphics.h>
 
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+using arrayfire::common::getGLType;
+using arrayfire::common::makeContextCurrent;
+using arrayfire::common::step_round;
 using detail::Array;
 using detail::copy_histogram;
 using detail::forgeManager;
@@ -25,13 +31,12 @@ using detail::getScalar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
-using graphics::ForgeManager;
 
 template<typename T>
 fg_chart setup_histogram(fg_window const window, const af_array in,
                          const double minval, const double maxval,
                          const af_cell* const props) {
-    ForgeModule& _ = graphics::forgePlugin();
+    ForgeModule& _ = forgePlugin();
 
     const Array<T> histogramInput = getArray<T>(in);
     dim_t nBins                   = histogramInput.elements();
@@ -133,7 +138,7 @@ af_err af_draw_hist(const af_window window, const af_array X,
         }
         auto gridDims = forgeManager().getWindowGrid(window);
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         if (props->col > -1 && props->row > -1) {
             FG_CHECK(_.fg_draw_chart_to_cell(
                 window, gridDims.first, gridDims.second,
diff --git a/src/api/c/histeq.cpp b/src/api/c/histeq.cpp
index 8fef8a2684..da2a7579d8 100644
--- a/src/api/c/histeq.cpp
+++ b/src/api/c/histeq.cpp
@@ -23,8 +23,8 @@
 #include <af/index.h>
 
 using af::dim4;
-using common::cast;
-using common::modDims;
+using arrayfire::common::cast;
+using arrayfire::common::modDims;
 using detail::arithOp;
 using detail::Array;
 using detail::createValueArray;
diff --git a/src/api/c/histogram.cpp b/src/api/c/histogram.cpp
index f04f4a23df..aa2744bb6c 100644
--- a/src/api/c/histogram.cpp
+++ b/src/api/c/histogram.cpp
@@ -79,8 +79,8 @@ af_err af_histogram(af_array *out, const af_array in, const unsigned nbins,
                                           info.isLinear());
                 break;
             case f16:
-                output = histogram<common::half>(in, nbins, minval, maxval,
-                                                 info.isLinear());
+                output = histogram<arrayfire::common::half>(
+                    in, nbins, minval, maxval, info.isLinear());
                 break;
             default: TYPE_ERROR(1, type);
         }
diff --git a/src/api/c/image.cpp b/src/api/c/image.cpp
index 4b93727d01..533612f45d 100644
--- a/src/api/c/image.cpp
+++ b/src/api/c/image.cpp
@@ -27,7 +27,12 @@
 #include <limits>
 
 using af::dim4;
-using common::cast;
+using arrayfire::common::cast;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+using arrayfire::common::getGLType;
+using arrayfire::common::makeContextCurrent;
 using detail::arithOp;
 using detail::Array;
 using detail::copy_image;
@@ -36,7 +41,6 @@ using detail::forgeManager;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
-using graphics::ForgeManager;
 
 template<typename T>
 Array<T> normalizePerType(const Array<T>& in) {
@@ -101,7 +105,7 @@ af_err af_draw_image(const af_window window, const af_array in,
             default: TYPE_ERROR(1, type);
         }
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         auto gridDims  = forgeManager().getWindowGrid(window);
         FG_CHECK(_.fg_set_window_colormap(window, (fg_color_map)props->cmap));
         if (props->col > -1 && props->row > -1) {
diff --git a/src/api/c/imageio.cpp b/src/api/c/imageio.cpp
index ba0a024d9e..41e713e631 100644
--- a/src/api/c/imageio.cpp
+++ b/src/api/c/imageio.cpp
@@ -35,6 +35,16 @@
 #include <string>
 
 using af::dim4;
+using arrayfire::AFFI_GRAY;
+using arrayfire::AFFI_RGB;
+using arrayfire::AFFI_RGBA;
+using arrayfire::bitmap_ptr;
+using arrayfire::channel_split;
+using arrayfire::FI_CHANNELS;
+using arrayfire::FreeImage_Module;
+using arrayfire::FreeImageErrorHandler;
+using arrayfire::getFreeImagePlugin;
+using arrayfire::make_bitmap_ptr;
 using detail::pinnedAlloc;
 using detail::pinnedFree;
 using detail::uchar;
@@ -43,6 +53,8 @@ using detail::ushort;
 using std::string;
 using std::swap;
 
+namespace arrayfire {
+
 template<typename T, FI_CHANNELS fi_color, FI_CHANNELS fo_color>
 static af_err readImage(af_array* rImage, const uchar* pSrcLine,
                         const int nSrcPitch, const uint fi_w, const uint fi_h) {
@@ -213,11 +225,14 @@ static af_err readImage(af_array* rImage, const uchar* pSrcLine,
     return err;
 }
 
+}  // namespace arrayfire
+
 ////////////////////////////////////////////////////////////////////////////////
 // File IO
 ////////////////////////////////////////////////////////////////////////////////
 // Load image from disk.
 af_err af_load_image(af_array* out, const char* filename, const bool isColor) {
+    using arrayfire::readImage;
     try {
         ARG_ASSERT(1, filename != NULL);
 
@@ -707,6 +722,7 @@ af_err af_save_image(const char* filename, const af_array in_) {
 ////////////////////////////////////////////////////////////////////////////////
 /// Load image from memory.
 af_err af_load_image_memory(af_array* out, const void* ptr) {
+    using arrayfire::readImage;
     try {
         ARG_ASSERT(1, ptr != NULL);
 
@@ -1075,4 +1091,5 @@ af_err af_delete_image_memory(void *ptr) {
     AF_RETURN_ERROR("ArrayFire compiled without Image IO (FreeImage) support",
                     AF_ERR_NOT_CONFIGURED);
 }
+}  // namespace arrayfire
 #endif  // WITH_FREEIMAGE
diff --git a/src/api/c/imageio2.cpp b/src/api/c/imageio2.cpp
index f1edab6d7e..7130202397 100644
--- a/src/api/c/imageio2.cpp
+++ b/src/api/c/imageio2.cpp
@@ -32,12 +32,23 @@
 #include <string>
 
 using af::dim4;
+using arrayfire::AFFI_GRAY;
+using arrayfire::AFFI_RGB;
+using arrayfire::AFFI_RGBA;
+using arrayfire::bitmap_ptr;
+using arrayfire::channel_split;
+using arrayfire::FI_CHANNELS;
+using arrayfire::FreeImage_Module;
+using arrayfire::FreeImageErrorHandler;
+using arrayfire::getFreeImagePlugin;
+using arrayfire::make_bitmap_ptr;
 using detail::pinnedAlloc;
 using detail::pinnedFree;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
 
+namespace {
 template<typename T, FI_CHANNELS fi_color>
 static af_err readImage_t(af_array* rImage, const uchar* pSrcLine,
                           const int nSrcPitch, const uint fi_w,
@@ -116,6 +127,8 @@ FREE_IMAGE_TYPE getFIT(FI_CHANNELS channels, af_dtype type) {
     return FIT_BITMAP;
 }
 
+}  // namespace
+
 ////////////////////////////////////////////////////////////////////////////////
 // File IO
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/src/api/c/imageio_helper.h b/src/api/c/imageio_helper.h
index 787a391e59..e9ef818bf3 100644
--- a/src/api/c/imageio_helper.h
+++ b/src/api/c/imageio_helper.h
@@ -21,6 +21,8 @@
 #include <functional>
 #include <memory>
 
+namespace arrayfire {
+
 class FreeImage_Module {
     common::DependencyModule module;
 
@@ -102,3 +104,4 @@ static af_err channel_split(const af_array rgb, const af::dim4 &dims,
 }
 
 #endif
+}
diff --git a/src/api/c/imgproc_common.hpp b/src/api/c/imgproc_common.hpp
index 214fbe6c7a..f4abcb0907 100644
--- a/src/api/c/imgproc_common.hpp
+++ b/src/api/c/imgproc_common.hpp
@@ -19,6 +19,7 @@
 
 #include <cmath>
 
+namespace arrayfire {
 namespace common {
 
 template<typename To, typename Ti = To>
@@ -78,3 +79,4 @@ detail::Array<To> convRange(const detail::Array<Ti>& in,
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/api/c/index.cpp b/src/api/c/index.cpp
index 0f36e0b463..1c7484f2bf 100644
--- a/src/api/c/index.cpp
+++ b/src/api/c/index.cpp
@@ -32,10 +32,10 @@ using std::swap;
 using std::vector;
 
 using af::dim4;
-using common::convert2Canonical;
-using common::createSpanIndex;
-using common::flat;
-using common::half;
+using arrayfire::common::convert2Canonical;
+using arrayfire::common::createSpanIndex;
+using arrayfire::common::flat;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::index;
@@ -45,6 +45,7 @@ using detail::uint;
 using detail::uintl;
 using detail::ushort;
 
+namespace arrayfire {
 namespace common {
 af_index_t createSpanIndex() {
     static af_index_t s = [] {
@@ -64,6 +65,7 @@ af_seq convert2Canonical(const af_seq s, const dim_t len) {
     return af_seq{begin, end, s.step};
 }
 }  // namespace common
+}  // namespace arrayfire
 
 template<typename T>
 static af_array indexBySeqs(const af_array& src,
diff --git a/src/api/c/indexing_common.hpp b/src/api/c/indexing_common.hpp
index ae5ea3958a..85a5d9562a 100644
--- a/src/api/c/indexing_common.hpp
+++ b/src/api/c/indexing_common.hpp
@@ -11,6 +11,7 @@
 
 #include <af/index.h>
 
+namespace arrayfire {
 namespace common {
 /// Creates a af_index_t object that represents a af_span value
 af_index_t createSpanIndex();
@@ -39,3 +40,4 @@ af_index_t createSpanIndex();
 /// s{-1, 2, -1};    will return the sequence af_seq(9,2,-1)
 af_seq convert2Canonical(const af_seq s, const dim_t len);
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/api/c/internal.cpp b/src/api/c/internal.cpp
index 219942cc1e..38c0c96dfe 100644
--- a/src/api/c/internal.cpp
+++ b/src/api/c/internal.cpp
@@ -20,7 +20,7 @@
 #include <cstring>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::createStridedArray;
diff --git a/src/api/c/join.cpp b/src/api/c/join.cpp
index a31a728874..4c47fbe495 100644
--- a/src/api/c/join.cpp
+++ b/src/api/c/join.cpp
@@ -20,7 +20,7 @@
 #include <vector>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/mean.cpp b/src/api/c/mean.cpp
index 2dfb7bdbf2..af9021983e 100644
--- a/src/api/c/mean.cpp
+++ b/src/api/c/mean.cpp
@@ -23,7 +23,7 @@
 #include "stats.h"
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
@@ -160,7 +160,9 @@ af_err af_mean_all(double *realVal, double *imagVal, const af_array in) {
             case u16: *realVal = mean<ushort, float>(in); break;
             case u8: *realVal = mean<uchar, float>(in); break;
             case b8: *realVal = mean<char, float>(in); break;
-            case f16: *realVal = mean<common::half, float>(in); break;
+            case f16:
+                *realVal = mean<arrayfire::common::half, float>(in);
+                break;
             case c32: {
                 cfloat tmp = mean<cfloat, cfloat>(in);
                 *realVal   = real(tmp);
diff --git a/src/api/c/memory.cpp b/src/api/c/memory.cpp
index 2958d6c90c..a689f92a91 100644
--- a/src/api/c/memory.cpp
+++ b/src/api/c/memory.cpp
@@ -26,7 +26,7 @@
 #include <utility>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::createDeviceDataArray;
diff --git a/src/api/c/memoryapi.hpp b/src/api/c/memoryapi.hpp
index 945b0fb287..a52947dce0 100644
--- a/src/api/c/memoryapi.hpp
+++ b/src/api/c/memoryapi.hpp
@@ -22,7 +22,7 @@
  * on a af_memory_manager via calls to a MemoryManagerBase
  */
 class MemoryManagerFunctionWrapper final
-    : public common::memory::MemoryManagerBase {
+    : public arrayfire::common::MemoryManagerBase {
     af_memory_manager handle_;
 
    public:
diff --git a/src/api/c/moddims.cpp b/src/api/c/moddims.cpp
index 5f07c6bf8b..4f6f0f310d 100644
--- a/src/api/c/moddims.cpp
+++ b/src/api/c/moddims.cpp
@@ -18,7 +18,7 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
@@ -30,11 +30,11 @@ using detail::ushort;
 namespace {
 template<typename T>
 af_array modDims(const af_array in, const dim4& newDims) {
-    return getHandle(common::modDims(getArray<T>(in), newDims));
+    return getHandle(arrayfire::common::modDims(getArray<T>(in), newDims));
 }
 template<typename T>
 af_array flat(const af_array in) {
-    return getHandle(common::flat(getArray<T>(in)));
+    return getHandle(arrayfire::common::flat(getArray<T>(in)));
 }
 }  // namespace
 
diff --git a/src/api/c/morph.cpp b/src/api/c/morph.cpp
index 948effd652..efaf6cc53a 100644
--- a/src/api/c/morph.cpp
+++ b/src/api/c/morph.cpp
@@ -24,8 +24,8 @@
 #include <af/image.h>
 
 using af::dim4;
-using common::cast;
-using common::flip;
+using arrayfire::common::cast;
+using arrayfire::common::flip;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
diff --git a/src/api/c/pinverse.cpp b/src/api/c/pinverse.cpp
index 05d2d92fba..55c5cf8d7d 100644
--- a/src/api/c/pinverse.cpp
+++ b/src/api/c/pinverse.cpp
@@ -32,8 +32,8 @@
 
 using af::dim4;
 using af::dtype_traits;
-using common::cast;
-using common::modDims;
+using arrayfire::common::cast;
+using arrayfire::common::modDims;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
diff --git a/src/api/c/plot.cpp b/src/api/c/plot.cpp
index 677fda370a..b60448593f 100644
--- a/src/api/c/plot.cpp
+++ b/src/api/c/plot.cpp
@@ -23,6 +23,13 @@
 #include <transpose.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+using arrayfire::common::getFGMarker;
+using arrayfire::common::getGLType;
+using arrayfire::common::makeContextCurrent;
+using arrayfire::common::step_round;
 using detail::Array;
 using detail::copy_plot;
 using detail::forgeManager;
@@ -30,14 +37,13 @@ using detail::reduce;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
-using namespace graphics;
 
 // Requires in_ to be in either [order, n] or [n, order] format
 template<typename T, int order>
 fg_chart setup_plot(fg_window window, const af_array in_,
                     const af_cell* const props, fg_plot_type ptype,
                     fg_marker_type mtype) {
-    ForgeModule& _ = graphics::forgePlugin();
+    ForgeModule& _ = forgePlugin();
 
     Array<T> in = getArray<T>(in_);
 
@@ -168,7 +174,7 @@ af_err plotWrapper(const af_window window, const af_array in,
 
         auto gridDims = forgeManager().getWindowGrid(window);
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         if (props->col > -1 && props->row > -1) {
             FG_CHECK(_.fg_draw_chart_to_cell(
                 window, gridDims.first, gridDims.second,
@@ -240,7 +246,7 @@ af_err plotWrapper(const af_window window, const af_array X, const af_array Y,
         }
         auto gridDims = forgeManager().getWindowGrid(window);
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         if (props->col > -1 && props->row > -1) {
             FG_CHECK(_.fg_draw_chart_to_cell(
                 window, gridDims.first, gridDims.second,
@@ -307,7 +313,7 @@ af_err plotWrapper(const af_window window, const af_array X, const af_array Y,
         }
         auto gridDims = forgeManager().getWindowGrid(window);
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         if (props->col > -1 && props->row > -1) {
             FG_CHECK(_.fg_draw_chart_to_cell(
                 window, gridDims.first, gridDims.second,
diff --git a/src/api/c/print.cpp b/src/api/c/print.cpp
index 85f30dc028..48fea73b48 100644
--- a/src/api/c/print.cpp
+++ b/src/api/c/print.cpp
@@ -30,7 +30,9 @@
 
 #include <af/index.h>
 
-using common::half;
+using arrayfire::getSparseArray;
+using arrayfire::common::half;
+using arrayfire::common::SparseArray;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
@@ -115,7 +117,7 @@ static void print(const char *exp, af_array arr, const int precision,
 template<typename T>
 static void printSparse(const char *exp, af_array arr, const int precision,
                         std::ostream &os = std::cout, bool transpose = true) {
-    common::SparseArray<T> sparse = getSparseArray<T>(arr);
+    SparseArray<T> sparse = getSparseArray<T>(arr);
     std::string name("No Name Sparse Array");
 
     if (exp != NULL) { name = std::string(exp); }
diff --git a/src/api/c/random.cpp b/src/api/c/random.cpp
index 8d65c4b718..f1a85b2891 100644
--- a/src/api/c/random.cpp
+++ b/src/api/c/random.cpp
@@ -23,16 +23,16 @@
 #include <memory>
 
 using af::dim4;
-using common::half;
-using common::mask;
-using common::MaxBlocks;
-using common::MtStateLength;
-using common::pos;
-using common::recursion_tbl;
-using common::sh1;
-using common::sh2;
-using common::TableLength;
-using common::temper_tbl;
+using arrayfire::common::half;
+using arrayfire::common::mask;
+using arrayfire::common::MaxBlocks;
+using arrayfire::common::MtStateLength;
+using arrayfire::common::pos;
+using arrayfire::common::recursion_tbl;
+using arrayfire::common::sh1;
+using arrayfire::common::sh2;
+using arrayfire::common::TableLength;
+using arrayfire::common::temper_tbl;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/reduce.cpp b/src/api/c/reduce.cpp
index 1849255257..8e1e670506 100644
--- a/src/api/c/reduce.cpp
+++ b/src/api/c/reduce.cpp
@@ -21,7 +21,7 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/reorder.cpp b/src/api/c/reorder.cpp
index c367430809..b283c800bf 100644
--- a/src/api/c/reorder.cpp
+++ b/src/api/c/reorder.cpp
@@ -20,7 +20,7 @@
 #include <af/data.h>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/replace.cpp b/src/api/c/replace.cpp
index bd4814157a..b8fdd75e02 100644
--- a/src/api/c/replace.cpp
+++ b/src/api/c/replace.cpp
@@ -22,7 +22,8 @@
 #include <select.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::getCopyOnWriteArray;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
diff --git a/src/api/c/rgb_gray.cpp b/src/api/c/rgb_gray.cpp
index 3c189af5df..3bea06e855 100644
--- a/src/api/c/rgb_gray.cpp
+++ b/src/api/c/rgb_gray.cpp
@@ -23,7 +23,7 @@
 #include <math.hpp>
 
 using af::dim4;
-using common::cast;
+using arrayfire::common::cast;
 using detail::arithOp;
 using detail::Array;
 using detail::createEmptyArray;
@@ -75,7 +75,7 @@ static af_array gray2rgb(const af_array& in, const float r, const float g,
                          const float b) {
     if (r == 1.0 && g == 1.0 && b == 1.0) {
         dim4 tileDims(1, 1, 3, 1);
-        return getHandle(common::tile(getArray<T>(in), tileDims));
+        return getHandle(arrayfire::common::tile(getArray<T>(in), tileDims));
     }
 
     af_array mod_input = 0;
diff --git a/src/api/c/sat.cpp b/src/api/c/sat.cpp
index 8012cfaaba..3ff72abacc 100644
--- a/src/api/c/sat.cpp
+++ b/src/api/c/sat.cpp
@@ -14,6 +14,7 @@
 #include <af/image.h>
 
 using af::dim4;
+using arrayfire::common::integralImage;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
@@ -24,7 +25,7 @@ using detail::ushort;
 
 template<typename To, typename Ti>
 inline af_array sat(const af_array& in) {
-    return getHandle<To>(common::integralImage<To, Ti>(getArray<Ti>(in)));
+    return getHandle<To>(integralImage<To, Ti>(getArray<Ti>(in)));
 }
 
 af_err af_sat(af_array* out, const af_array in) {
diff --git a/src/api/c/select.cpp b/src/api/c/select.cpp
index 31d7facbcd..dec47166e7 100644
--- a/src/api/c/select.cpp
+++ b/src/api/c/select.cpp
@@ -20,7 +20,7 @@
 #include <af/defines.h>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/sparse.cpp b/src/api/c/sparse.cpp
index 714a0c1d15..917864dcaf 100644
--- a/src/api/c/sparse.cpp
+++ b/src/api/c/sparse.cpp
@@ -20,14 +20,21 @@
 #include <af/sparse.h>
 
 using af::dim4;
-using common::createEmptySparseArray;
-using common::SparseArray;
-using common::SparseArrayBase;
+using arrayfire::getSparseArray;
+using arrayfire::retainSparseHandle;
+using arrayfire::common::createArrayDataSparseArray;
+using arrayfire::common::createDeviceDataSparseArray;
+using arrayfire::common::createEmptySparseArray;
+using arrayfire::common::createHostDataSparseArray;
+using arrayfire::common::SparseArray;
+using arrayfire::common::SparseArrayBase;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::sparseConvertDenseToStorage;
 
+namespace arrayfire {
+
 const SparseArrayBase &getSparseArrayBase(const af_array in,
                                           bool device_check) {
     const SparseArrayBase *base =
@@ -54,12 +61,119 @@ template<typename T>
 af_array createSparseArrayFromData(const dim4 &dims, const af_array values,
                                    const af_array rowIdx, const af_array colIdx,
                                    const af::storage stype) {
-    SparseArray<T> sparse = common::createArrayDataSparseArray(
+    SparseArray<T> sparse = createArrayDataSparseArray(
         dims, getArray<T>(values), getArray<int>(rowIdx), getArray<int>(colIdx),
         stype);
     return getHandle(sparse);
 }
 
+template<typename T>
+af_array createSparseArrayFromPtr(const af::dim4 &dims, const dim_t nNZ,
+                                  const T *const values,
+                                  const int *const rowIdx,
+                                  const int *const colIdx,
+                                  const af::storage stype,
+                                  const af::source source) {
+    if (nNZ) {
+        switch (source) {
+            case afHost:
+                return getHandle(createHostDataSparseArray(
+                    dims, nNZ, values, rowIdx, colIdx, stype));
+                break;
+            case afDevice:
+                return getHandle(createDeviceDataSparseArray(
+                    dims, nNZ, const_cast<T *>(values),
+                    const_cast<int *>(rowIdx), const_cast<int *>(colIdx),
+                    stype));
+                break;
+        }
+    }
+
+    return getHandle(createEmptySparseArray<T>(dims, nNZ, stype));
+}
+
+template<typename T>
+af_array createSparseArrayFromDense(const af_array _in,
+                                    const af_storage stype) {
+    const Array<T> in = getArray<T>(_in);
+
+    switch (stype) {
+        case AF_STORAGE_CSR:
+            return getHandle(
+                sparseConvertDenseToStorage<T, AF_STORAGE_CSR>(in));
+        case AF_STORAGE_COO:
+            return getHandle(
+                sparseConvertDenseToStorage<T, AF_STORAGE_COO>(in));
+        case AF_STORAGE_CSC:
+            // return getHandle(sparseConvertDenseToStorage<T,
+            // AF_STORAGE_CSC>(in));
+        default:
+            AF_ERROR("Storage type is out of range/unsupported", AF_ERR_ARG);
+    }
+}
+
+template<typename T>
+af_array sparseConvertStorage(const af_array in_,
+                              const af_storage destStorage) {
+    const SparseArray<T> in = getSparseArray<T>(in_);
+
+    if (destStorage == AF_STORAGE_DENSE) {
+        // Returns a regular af_array, not sparse
+        switch (in.getStorage()) {
+            case AF_STORAGE_CSR:
+                return getHandle(
+                    detail::sparseConvertStorageToDense<T, AF_STORAGE_CSR>(in));
+            case AF_STORAGE_COO:
+                return getHandle(
+                    detail::sparseConvertStorageToDense<T, AF_STORAGE_COO>(in));
+            default:
+                AF_ERROR("Invalid storage type of input array", AF_ERR_ARG);
+        }
+    } else if (destStorage == AF_STORAGE_CSR) {
+        // Returns a sparse af_array
+        switch (in.getStorage()) {
+            case AF_STORAGE_CSR: return retainSparseHandle<T>(in_);
+            case AF_STORAGE_COO:
+                return getHandle(
+                    detail::sparseConvertStorageToStorage<T, AF_STORAGE_CSR,
+                                                          AF_STORAGE_COO>(in));
+            default:
+                AF_ERROR("Invalid storage type of input array", AF_ERR_ARG);
+        }
+    } else if (destStorage == AF_STORAGE_COO) {
+        // Returns a sparse af_array
+        switch (in.getStorage()) {
+            case AF_STORAGE_CSR:
+                return getHandle(
+                    detail::sparseConvertStorageToStorage<T, AF_STORAGE_COO,
+                                                          AF_STORAGE_CSR>(in));
+            case AF_STORAGE_COO: return retainSparseHandle<T>(in_);
+            default:
+                AF_ERROR("Invalid storage type of input array", AF_ERR_ARG);
+        }
+    }
+
+    // Shoud never come here
+    return NULL;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Get Functions
+////////////////////////////////////////////////////////////////////////////////
+template<typename T>
+af_array getSparseValues(const af_array in) {
+    return getHandle(getSparseArray<T>(in).getValues());
+}
+
+}  // namespace arrayfire
+
+using arrayfire::createSparseArrayFromData;
+using arrayfire::createSparseArrayFromDense;
+using arrayfire::createSparseArrayFromPtr;
+using arrayfire::getSparseArrayBase;
+using arrayfire::getSparseValues;
+using arrayfire::sparseConvertStorage;
+
 af_err af_create_sparse_array(af_array *out, const dim_t nRows,
                               const dim_t nCols, const af_array values,
                               const af_array rowIdx, const af_array colIdx,
@@ -132,31 +246,6 @@ af_err af_create_sparse_array(af_array *out, const dim_t nRows,
     return AF_SUCCESS;
 }
 
-template<typename T>
-af_array createSparseArrayFromPtr(const af::dim4 &dims, const dim_t nNZ,
-                                  const T *const values,
-                                  const int *const rowIdx,
-                                  const int *const colIdx,
-                                  const af::storage stype,
-                                  const af::source source) {
-    if (nNZ) {
-        switch (source) {
-            case afHost:
-                return getHandle(common::createHostDataSparseArray(
-                    dims, nNZ, values, rowIdx, colIdx, stype));
-                break;
-            case afDevice:
-                return getHandle(common::createDeviceDataSparseArray(
-                    dims, nNZ, const_cast<T *>(values),
-                    const_cast<int *>(rowIdx), const_cast<int *>(colIdx),
-                    stype));
-                break;
-        }
-    }
-
-    return getHandle(createEmptySparseArray<T>(dims, nNZ, stype));
-}
-
 af_err af_create_sparse_array_from_ptr(
     af_array *out, const dim_t nRows, const dim_t nCols, const dim_t nNZ,
     const void *const values, const int *const rowIdx, const int *const colIdx,
@@ -211,26 +300,6 @@ af_err af_create_sparse_array_from_ptr(
     return AF_SUCCESS;
 }
 
-template<typename T>
-af_array createSparseArrayFromDense(const af_array _in,
-                                    const af_storage stype) {
-    const Array<T> in = getArray<T>(_in);
-
-    switch (stype) {
-        case AF_STORAGE_CSR:
-            return getHandle(
-                sparseConvertDenseToStorage<T, AF_STORAGE_CSR>(in));
-        case AF_STORAGE_COO:
-            return getHandle(
-                sparseConvertDenseToStorage<T, AF_STORAGE_COO>(in));
-        case AF_STORAGE_CSC:
-            // return getHandle(sparseConvertDenseToStorage<T,
-            // AF_STORAGE_CSC>(in));
-        default:
-            AF_ERROR("Storage type is out of range/unsupported", AF_ERR_ARG);
-    }
-}
-
 af_err af_create_sparse_array_from_dense(af_array *out, const af_array in,
                                          const af_storage stype) {
     try {
@@ -274,51 +343,6 @@ af_err af_create_sparse_array_from_dense(af_array *out, const af_array in,
     return AF_SUCCESS;
 }
 
-template<typename T>
-af_array sparseConvertStorage(const af_array in_,
-                              const af_storage destStorage) {
-    const SparseArray<T> in = getSparseArray<T>(in_);
-
-    if (destStorage == AF_STORAGE_DENSE) {
-        // Returns a regular af_array, not sparse
-        switch (in.getStorage()) {
-            case AF_STORAGE_CSR:
-                return getHandle(
-                    detail::sparseConvertStorageToDense<T, AF_STORAGE_CSR>(in));
-            case AF_STORAGE_COO:
-                return getHandle(
-                    detail::sparseConvertStorageToDense<T, AF_STORAGE_COO>(in));
-            default:
-                AF_ERROR("Invalid storage type of input array", AF_ERR_ARG);
-        }
-    } else if (destStorage == AF_STORAGE_CSR) {
-        // Returns a sparse af_array
-        switch (in.getStorage()) {
-            case AF_STORAGE_CSR: return retainSparseHandle<T>(in_);
-            case AF_STORAGE_COO:
-                return getHandle(
-                    detail::sparseConvertStorageToStorage<T, AF_STORAGE_CSR,
-                                                          AF_STORAGE_COO>(in));
-            default:
-                AF_ERROR("Invalid storage type of input array", AF_ERR_ARG);
-        }
-    } else if (destStorage == AF_STORAGE_COO) {
-        // Returns a sparse af_array
-        switch (in.getStorage()) {
-            case AF_STORAGE_CSR:
-                return getHandle(
-                    detail::sparseConvertStorageToStorage<T, AF_STORAGE_COO,
-                                                          AF_STORAGE_CSR>(in));
-            case AF_STORAGE_COO: return retainSparseHandle<T>(in_);
-            default:
-                AF_ERROR("Invalid storage type of input array", AF_ERR_ARG);
-        }
-    }
-
-    // Shoud never come here
-    return NULL;
-}
-
 af_err af_sparse_convert_to(af_array *out, const af_array in,
                             const af_storage destStorage) {
     try {
@@ -398,14 +422,6 @@ af_err af_sparse_to_dense(af_array *out, const af_array in) {
     return AF_SUCCESS;
 }
 
-////////////////////////////////////////////////////////////////////////////////
-// Get Functions
-////////////////////////////////////////////////////////////////////////////////
-template<typename T>
-af_array getSparseValues(const af_array in) {
-    return getHandle(getSparseArray<T>(in).getValues());
-}
-
 af_err af_sparse_get_info(af_array *values, af_array *rows, af_array *cols,
                           af_storage *stype, const af_array in) {
     try {
diff --git a/src/api/c/sparse_handle.hpp b/src/api/c/sparse_handle.hpp
index 72b251473b..e99bbb36e5 100644
--- a/src/api/c/sparse_handle.hpp
+++ b/src/api/c/sparse_handle.hpp
@@ -20,6 +20,8 @@
 
 #include <common/SparseArray.hpp>
 
+namespace arrayfire {
+
 const common::SparseArrayBase &getSparseArrayBase(const af_array in,
                                                   bool device_check = true);
 
@@ -86,3 +88,7 @@ static af_array copySparseArray(const af_array in) {
     const common::SparseArray<T> &inArray = getSparseArray<T>(in);
     return getHandle<T>(common::copySparseArray<T>(inArray));
 }
+
+}  // namespace arrayfire
+
+using arrayfire::getHandle;
diff --git a/src/api/c/stdev.cpp b/src/api/c/stdev.cpp
index 3be779e544..7f64bf3355 100644
--- a/src/api/c/stdev.cpp
+++ b/src/api/c/stdev.cpp
@@ -26,7 +26,7 @@
 #include "stats.h"
 
 using af::dim4;
-using common::cast;
+using arrayfire::common::cast;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/surface.cpp b/src/api/c/surface.cpp
index 58cc9476aa..62ef46e0e2 100644
--- a/src/api/c/surface.cpp
+++ b/src/api/c/surface.cpp
@@ -24,7 +24,13 @@
 #include <surface.hpp>
 
 using af::dim4;
-using common::modDims;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+using arrayfire::common::getGLType;
+using arrayfire::common::makeContextCurrent;
+using arrayfire::common::modDims;
+using arrayfire::common::step_round;
 using detail::Array;
 using detail::copy_surface;
 using detail::createEmptyArray;
@@ -34,13 +40,12 @@ using detail::reduce_all;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
-using namespace graphics;
 
 template<typename T>
 fg_chart setup_surface(fg_window window, const af_array xVals,
                        const af_array yVals, const af_array zVals,
                        const af_cell* const props) {
-    ForgeModule& _ = graphics::forgePlugin();
+    ForgeModule& _ = forgePlugin();
     Array<T> xIn   = getArray<T>(xVals);
     Array<T> yIn   = getArray<T>(yVals);
     Array<T> zIn   = getArray<T>(zVals);
@@ -58,13 +63,13 @@ fg_chart setup_surface(fg_window window, const af_array xVals,
         xIn = modDims(xIn, xIn.elements());
         // Now tile along second dimension
         dim4 x_tdims(1, Y_dims[0], 1, 1);
-        xIn = common::tile(xIn, x_tdims);
+        xIn = arrayfire::common::tile(xIn, x_tdims);
 
         // Convert yIn to a row vector
         yIn = modDims(yIn, dim4(1, yIn.elements()));
         // Now tile along first dimension
         dim4 y_tdims(X_dims[0], 1, 1, 1);
-        yIn = common::tile(yIn, y_tdims);
+        yIn = arrayfire::common::tile(yIn, y_tdims);
     }
 
     // Flatten xIn, yIn and zIn into row vectors
@@ -191,7 +196,7 @@ af_err af_draw_surface(const af_window window, const af_array xVals,
         }
         auto gridDims = forgeManager().getWindowGrid(window);
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         if (props->col > -1 && props->row > -1) {
             FG_CHECK(_.fg_draw_chart_to_cell(
                 window, gridDims.first, gridDims.second,
diff --git a/src/api/c/tile.cpp b/src/api/c/tile.cpp
index 443419b540..ce512e9958 100644
--- a/src/api/c/tile.cpp
+++ b/src/api/c/tile.cpp
@@ -20,7 +20,8 @@
 #include <af/data.h>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
+using arrayfire::common::tile;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
@@ -33,7 +34,7 @@ using detail::ushort;
 
 template<typename T>
 static inline af_array tile(const af_array in, const af::dim4 &tileDims) {
-    return getHandle(common::tile<T>(getArray<T>(in), tileDims));
+    return getHandle(arrayfire::common::tile<T>(getArray<T>(in), tileDims));
 }
 
 af_err af_tile(af_array *out, const af_array in, const af::dim4 &tileDims) {
diff --git a/src/api/c/topk.cpp b/src/api/c/topk.cpp
index 9375d857c0..c8a303afea 100644
--- a/src/api/c/topk.cpp
+++ b/src/api/c/topk.cpp
@@ -17,7 +17,7 @@
 #include <handle.hpp>
 #include <topk.hpp>
 
-using common::half;
+using arrayfire::common::half;
 using detail::createEmptyArray;
 using detail::uint;
 
diff --git a/src/api/c/transpose.cpp b/src/api/c/transpose.cpp
index a92fe77e91..82ae18fef2 100644
--- a/src/api/c/transpose.cpp
+++ b/src/api/c/transpose.cpp
@@ -19,7 +19,7 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
diff --git a/src/api/c/unary.cpp b/src/api/c/unary.cpp
index 95e48d75bc..af18031eab 100644
--- a/src/api/c/unary.cpp
+++ b/src/api/c/unary.cpp
@@ -31,7 +31,7 @@
 #include <af/defines.h>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
diff --git a/src/api/c/var.cpp b/src/api/c/var.cpp
index efbbfc8a70..c82c1ca0cd 100644
--- a/src/api/c/var.cpp
+++ b/src/api/c/var.cpp
@@ -26,8 +26,8 @@
 #include <tuple>
 
 using af::dim4;
-using common::cast;
-using common::half;
+using arrayfire::common::cast;
+using arrayfire::common::half;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
diff --git a/src/api/c/vector_field.cpp b/src/api/c/vector_field.cpp
index fa48328462..a6bd0e07cc 100644
--- a/src/api/c/vector_field.cpp
+++ b/src/api/c/vector_field.cpp
@@ -23,6 +23,12 @@
 #include <vector>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+using arrayfire::common::getGLType;
+using arrayfire::common::makeContextCurrent;
+using arrayfire::common::step_round;
 using detail::Array;
 using detail::copy_vector_field;
 using detail::createEmptyArray;
@@ -34,14 +40,12 @@ using detail::uint;
 using detail::ushort;
 using std::vector;
 
-using namespace graphics;
-
 template<typename T>
 fg_chart setup_vector_field(fg_window window, const vector<af_array>& points,
                             const vector<af_array>& directions,
                             const af_cell* const props,
                             const bool transpose_ = true) {
-    ForgeModule& _ = graphics::forgePlugin();
+    ForgeModule& _ = forgePlugin();
     vector<Array<T>> pnts;
     vector<Array<T>> dirs;
 
@@ -184,7 +188,7 @@ af_err vectorFieldWrapper(const af_window window, const af_array points,
         }
         auto gridDims = forgeManager().getWindowGrid(window);
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         if (props->col > -1 && props->row > -1) {
             FG_CHECK(_.fg_draw_chart_to_cell(
                 window, gridDims.first, gridDims.second,
@@ -291,7 +295,7 @@ af_err vectorFieldWrapper(const af_window window, const af_array xPoints,
         }
         auto gridDims = forgeManager().getWindowGrid(window);
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         if (props->col > -1 && props->row > -1) {
             FG_CHECK(_.fg_draw_chart_to_cell(
                 window, gridDims.first, gridDims.second,
@@ -386,7 +390,7 @@ af_err vectorFieldWrapper(const af_window window, const af_array xPoints,
 
         auto gridDims = forgeManager().getWindowGrid(window);
 
-        ForgeModule& _ = graphics::forgePlugin();
+        ForgeModule& _ = forgePlugin();
         if (props->col > -1 && props->row > -1) {
             FG_CHECK(_.fg_draw_chart_to_cell(
                 window, gridDims.first, gridDims.second,
diff --git a/src/api/c/window.cpp b/src/api/c/window.cpp
index 5f9d6e1c43..fe9fea5ba0 100644
--- a/src/api/c/window.cpp
+++ b/src/api/c/window.cpp
@@ -15,8 +15,10 @@
 #include <common/graphics_common.hpp>
 #include <platform.hpp>
 
+using arrayfire::common::ForgeManager;
+using arrayfire::common::forgePlugin;
+using arrayfire::common::step_round;
 using detail::forgeManager;
-using namespace graphics;
 
 af_err af_create_window(af_window* out, const int width, const int height,
                         const char* const title) {
diff --git a/src/api/cpp/array.cpp b/src/api/cpp/array.cpp
index 832c2999e5..1d61c63c2d 100644
--- a/src/api/cpp/array.cpp
+++ b/src/api/cpp/array.cpp
@@ -36,6 +36,7 @@
 #ifdef AF_UNIFIED
 #include <symbol_manager.hpp>
 #include <af/backend.h>
+using arrayfire::common::getFunctionPointer;
 #endif
 
 #include <memory>
@@ -255,43 +256,46 @@ array::~array() {
         std::add_pointer<decltype(af_release_array)>::type;
 
     if (get()) {
-        af_backend backend = unified::getActiveBackend();
+        af_backend backend = arrayfire::unified::getActiveBackend();
         af_err err         = af_get_backend_id(&backend, get());
         if (!err) {
             switch (backend) {
                 case AF_BACKEND_CPU: {
-                    static auto *cpu_handle = unified::getActiveHandle();
+                    static auto *cpu_handle =
+                        arrayfire::unified::getActiveHandle();
                     static auto release_func =
                         reinterpret_cast<af_release_array_ptr>(
-                            common::getFunctionPointer(cpu_handle,
-                                                       "af_release_array"));
+                            getFunctionPointer(cpu_handle, "af_release_array"));
                     release_func(get());
                     break;
                 }
                 case AF_BACKEND_OPENCL: {
-                    static auto *opencl_handle = unified::getActiveHandle();
+                    static auto *opencl_handle =
+                        arrayfire::unified::getActiveHandle();
                     static auto release_func =
                         reinterpret_cast<af_release_array_ptr>(
-                            common::getFunctionPointer(opencl_handle,
-                                                       "af_release_array"));
+                            getFunctionPointer(opencl_handle,
+                                               "af_release_array"));
                     release_func(get());
                     break;
                 }
                 case AF_BACKEND_CUDA: {
-                    static auto *cuda_handle = unified::getActiveHandle();
+                    static auto *cuda_handle =
+                        arrayfire::unified::getActiveHandle();
                     static auto release_func =
                         reinterpret_cast<af_release_array_ptr>(
-                            common::getFunctionPointer(cuda_handle,
-                                                       "af_release_array"));
+                            getFunctionPointer(cuda_handle,
+                                               "af_release_array"));
                     release_func(get());
                     break;
                 }
                 case AF_BACKEND_ONEAPI: {
-                    static auto *oneapi_handle = unified::getActiveHandle();
+                    static auto *oneapi_handle =
+                        arrayfire::unified::getActiveHandle();
                     static auto release_func =
                         reinterpret_cast<af_release_array_ptr>(
-                            common::getFunctionPointer(oneapi_handle,
-                                                       "af_release_array"));
+                            getFunctionPointer(oneapi_handle,
+                                               "af_release_array"));
                     release_func(get());
                     break;
                 }
diff --git a/src/api/unified/device.cpp b/src/api/unified/device.cpp
index 826d44a83d..96b14d621e 100644
--- a/src/api/unified/device.cpp
+++ b/src/api/unified/device.cpp
@@ -14,16 +14,18 @@
 #include "symbol_manager.hpp"
 
 af_err af_set_backend(const af_backend bknd) {
-    return unified::setBackend(bknd);
+    return arrayfire::unified::setBackend(bknd);
 }
 
 af_err af_get_backend_count(unsigned *num_backends) {
-    *num_backends = unified::AFSymbolManager::getInstance().getBackendCount();
+    *num_backends =
+        arrayfire::unified::AFSymbolManager::getInstance().getBackendCount();
     return AF_SUCCESS;
 }
 
 af_err af_get_available_backends(int *result) {
-    *result = unified::AFSymbolManager::getInstance().getAvailableBackends();
+    *result = arrayfire::unified::AFSymbolManager::getInstance()
+                  .getAvailableBackends();
     return AF_SUCCESS;
 }
 
@@ -39,7 +41,7 @@ af_err af_get_device_id(int *device, const af_array in) {
 }
 
 af_err af_get_active_backend(af_backend *result) {
-    *result = unified::getActiveBackend();
+    *result = arrayfire::unified::getActiveBackend();
     return AF_SUCCESS;
 }
 
diff --git a/src/api/unified/symbol_manager.cpp b/src/api/unified/symbol_manager.cpp
index a2efc6ee59..d3aed5f498 100644
--- a/src/api/unified/symbol_manager.cpp
+++ b/src/api/unified/symbol_manager.cpp
@@ -26,16 +26,17 @@
 #include <dlfcn.h>
 #endif
 
-using common::getEnvVar;
-using common::getErrorMessage;
-using common::getFunctionPointer;
-using common::loadLibrary;
-using common::loggerFactory;
-
+using arrayfire::common::getEnvVar;
+using arrayfire::common::getErrorMessage;
+using arrayfire::common::getFunctionPointer;
+using arrayfire::common::loadLibrary;
+using arrayfire::common::loggerFactory;
+using arrayfire::common::unloadLibrary;
 using std::extent;
 using std::function;
 using std::string;
 
+namespace arrayfire {
 namespace unified {
 
 #if defined(OS_WIN)
@@ -222,7 +223,7 @@ AFSymbolManager::AFSymbolManager()
 
 AFSymbolManager::~AFSymbolManager() {
     for (auto& bkndHandle : bkndHandles) {
-        if (bkndHandle) { common::unloadLibrary(bkndHandle); }
+        if (bkndHandle) { unloadLibrary(bkndHandle); }
     }
 }
 
@@ -252,3 +253,4 @@ af_err setBackend(af::Backend bknd) {
 }
 
 }  // namespace unified
+}  // namespace arrayfire
diff --git a/src/api/unified/symbol_manager.hpp b/src/api/unified/symbol_manager.hpp
index 3106bfa2ae..df5d77705c 100644
--- a/src/api/unified/symbol_manager.hpp
+++ b/src/api/unified/symbol_manager.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <unordered_map>
 
+namespace arrayfire {
 namespace unified {
 
 const int NUM_BACKENDS = 4;
@@ -123,6 +124,7 @@ bool checkArrays(af_backend activeBackend, T a, Args... arg) {
 }
 
 }  // namespace unified
+}  // namespace arrayfire
 
 /// Checks if the active backend and the af_arrays are the same.
 ///
@@ -133,27 +135,28 @@ bool checkArrays(af_backend activeBackend, T a, Args... arg) {
 /// \param[in] Any number of af_arrays or pointer to af_arrays
 #define CHECK_ARRAYS(...)                                                     \
     do {                                                                      \
-        af_backend backendId = unified::getActiveBackend();                   \
-        if (!unified::checkArrays(backendId, __VA_ARGS__))                    \
+        af_backend backendId = arrayfire::unified::getActiveBackend();        \
+        if (!arrayfire::unified::checkArrays(backendId, __VA_ARGS__))         \
             AF_RETURN_ERROR("Input array does not belong to current backend", \
                             AF_ERR_ARR_BKND_MISMATCH);                        \
     } while (0)
 
 #define CALL(FUNCTION, ...)                                                      \
     using af_func                  = std::add_pointer<decltype(FUNCTION)>::type; \
-    thread_local af_backend index_ = unified::getActiveBackend();                \
-    if (unified::getActiveHandle()) {                                            \
-        thread_local af_func func = (af_func)common::getFunctionPointer(         \
-            unified::getActiveHandle(), __func__);                               \
+    thread_local af_backend index_ = arrayfire::unified::getActiveBackend();     \
+    if (arrayfire::unified::getActiveHandle()) {                                 \
+        thread_local af_func func =                                              \
+            (af_func)arrayfire::common::getFunctionPointer(                      \
+                arrayfire::unified::getActiveHandle(), __func__);                \
         if (!func) {                                                             \
             AF_RETURN_ERROR(                                                     \
                 "requested symbol name could not be found in loaded library.",   \
                 AF_ERR_LOAD_LIB);                                                \
         }                                                                        \
-        if (index_ != unified::getActiveBackend()) {                             \
-            index_ = unified::getActiveBackend();                                \
-            func   = (af_func)common::getFunctionPointer(                        \
-                  unified::getActiveHandle(), __func__);                         \
+        if (index_ != arrayfire::unified::getActiveBackend()) {                  \
+            index_ = arrayfire::unified::getActiveBackend();                     \
+            func   = (af_func)arrayfire::common::getFunctionPointer(             \
+                  arrayfire::unified::getActiveHandle(), __func__);              \
         }                                                                        \
         return func(__VA_ARGS__);                                                \
     } else {                                                                     \
@@ -163,5 +166,6 @@ bool checkArrays(af_backend activeBackend, T a, Args... arg) {
 
 #define CALL_NO_PARAMS(FUNCTION) CALL(FUNCTION)
 
-#define LOAD_SYMBOL() \
-    common::getFunctionPointer(unified::getActiveHandle(), __FUNCTION__)
+#define LOAD_SYMBOL()                      \
+    arrayfire::common::getFunctionPointer( \
+        arrayfire::unified::getActiveHandle(), __FUNCTION__)
diff --git a/src/backend/common/AllocatorInterface.hpp b/src/backend/common/AllocatorInterface.hpp
index 0a7d34393f..0df799efdb 100644
--- a/src/backend/common/AllocatorInterface.hpp
+++ b/src/backend/common/AllocatorInterface.hpp
@@ -15,8 +15,8 @@
 namespace spdlog {
 class logger;
 }
+namespace arrayfire {
 namespace common {
-namespace memory {
 
 /**
  * An interface that provides backend-specific memory management functions,
@@ -39,5 +39,5 @@ class AllocatorInterface {
     std::shared_ptr<spdlog::logger> logger;
 };
 
-}  // namespace memory
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/ArrayInfo.cpp b/src/backend/common/ArrayInfo.cpp
index f079bac8ef..b83380fe88 100644
--- a/src/backend/common/ArrayInfo.cpp
+++ b/src/backend/common/ArrayInfo.cpp
@@ -87,23 +87,27 @@ bool ArrayInfo::isVector() const {
     return singular_dims == AF_MAX_DIMS - 1 && non_singular_dims == 1;
 }
 
-bool ArrayInfo::isComplex() const { return common::isComplex(type); }
+bool ArrayInfo::isComplex() const { return arrayfire::common::isComplex(type); }
 
-bool ArrayInfo::isReal() const { return common::isReal(type); }
+bool ArrayInfo::isReal() const { return arrayfire::common::isReal(type); }
 
-bool ArrayInfo::isDouble() const { return common::isDouble(type); }
+bool ArrayInfo::isDouble() const { return arrayfire::common::isDouble(type); }
 
-bool ArrayInfo::isSingle() const { return common::isSingle(type); }
+bool ArrayInfo::isSingle() const { return arrayfire::common::isSingle(type); }
 
-bool ArrayInfo::isHalf() const { return common::isHalf(type); }
+bool ArrayInfo::isHalf() const { return arrayfire::common::isHalf(type); }
 
-bool ArrayInfo::isRealFloating() const { return common::isRealFloating(type); }
+bool ArrayInfo::isRealFloating() const {
+    return arrayfire::common::isRealFloating(type);
+}
 
-bool ArrayInfo::isFloating() const { return common::isFloating(type); }
+bool ArrayInfo::isFloating() const {
+    return arrayfire::common::isFloating(type);
+}
 
-bool ArrayInfo::isInteger() const { return common::isInteger(type); }
+bool ArrayInfo::isInteger() const { return arrayfire::common::isInteger(type); }
 
-bool ArrayInfo::isBool() const { return common::isBool(type); }
+bool ArrayInfo::isBool() const { return arrayfire::common::isBool(type); }
 
 bool ArrayInfo::isLinear() const {
     if (ndims() == 1) { return dim_strides[0] == 1; }
@@ -172,6 +176,9 @@ dim4 toStride(const vector<af_seq> &seqs, const af::dim4 &parentDims) {
     return out;
 }
 
+namespace arrayfire {
+namespace common {
+
 const ArrayInfo &getInfo(const af_array arr, bool sparse_check,
                          bool device_check) {
     const ArrayInfo *info = nullptr;
@@ -188,3 +195,6 @@ const ArrayInfo &getInfo(const af_array arr, bool sparse_check,
 
     return *info;
 }
+
+}  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/Binary.hpp b/src/backend/common/Binary.hpp
index ca500ac865..6ad8654f83 100644
--- a/src/backend/common/Binary.hpp
+++ b/src/backend/common/Binary.hpp
@@ -18,6 +18,7 @@
 
 #include "optypes.hpp"
 
+namespace arrayfire {
 namespace common {
 
 using namespace detail;  // NOLINT
@@ -124,3 +125,4 @@ SPECIALIZE_COMPLEX_MAX(cdouble, double)
 #undef SPECIALIZE_COMPLEX_MAX
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/DefaultMemoryManager.cpp b/src/backend/common/DefaultMemoryManager.cpp
index 3ac5ab7324..d4aae2138e 100644
--- a/src/backend/common/DefaultMemoryManager.cpp
+++ b/src/backend/common/DefaultMemoryManager.cpp
@@ -28,6 +28,7 @@ using std::stoi;
 using std::string;
 using std::vector;
 
+namespace arrayfire {
 namespace common {
 
 DefaultMemoryManager::memory_info &
@@ -374,3 +375,4 @@ void DefaultMemoryManager::setMemStepSize(size_t new_step_size) {
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/DefaultMemoryManager.hpp b/src/backend/common/DefaultMemoryManager.hpp
index 83af36d390..60fa10a8c9 100644
--- a/src/backend/common/DefaultMemoryManager.hpp
+++ b/src/backend/common/DefaultMemoryManager.hpp
@@ -16,6 +16,7 @@
 #include <unordered_map>
 #include <vector>
 
+namespace arrayfire {
 namespace common {
 
 constexpr unsigned MAX_BUFFERS = 1000;
@@ -23,7 +24,7 @@ constexpr size_t ONE_GB        = 1 << 30;
 
 using uptr_t = std::unique_ptr<void, std::function<void(void *)>>;
 
-class DefaultMemoryManager final : public common::memory::MemoryManagerBase {
+class DefaultMemoryManager final : public common::MemoryManagerBase {
     size_t mem_step_size;
     unsigned max_buffers;
 
@@ -134,3 +135,4 @@ class DefaultMemoryManager final : public common::memory::MemoryManagerBase {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/DependencyModule.cpp b/src/backend/common/DependencyModule.cpp
index bdb5b27e0a..6511c54e67 100644
--- a/src/backend/common/DependencyModule.cpp
+++ b/src/backend/common/DependencyModule.cpp
@@ -20,7 +20,7 @@
 #include <dlfcn.h>
 #endif
 
-using common::Version;
+using arrayfire::common::Version;
 using std::make_tuple;
 using std::string;
 using std::to_string;
@@ -87,6 +87,7 @@ vector<string> libNames(const std::string& name, const string& suffix,
 #error "Unsupported platform"
 #endif
 
+namespace arrayfire {
 namespace common {
 
 DependencyModule::DependencyModule(const char* plugin_file_name,
@@ -168,3 +169,4 @@ spdlog::logger* DependencyModule::getLogger() const noexcept {
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/DependencyModule.hpp b/src/backend/common/DependencyModule.hpp
index 923ba96a47..41cc64569e 100644
--- a/src/backend/common/DependencyModule.hpp
+++ b/src/backend/common/DependencyModule.hpp
@@ -22,6 +22,7 @@
 namespace spdlog {
 class logger;
 }
+namespace arrayfire {
 namespace common {
 
 using Version = std::tuple<int, int, int>;  // major, minor, patch
@@ -75,6 +76,7 @@ class DependencyModule {
 };
 
 }  // namespace common
+}  // namespace arrayfire
 
 /// Creates a function pointer
 #define MODULE_MEMBER(NAME) decltype(&::NAME) NAME
diff --git a/src/backend/common/EventBase.hpp b/src/backend/common/EventBase.hpp
index 82ad049061..6356e4e1af 100644
--- a/src/backend/common/EventBase.hpp
+++ b/src/backend/common/EventBase.hpp
@@ -9,6 +9,7 @@
 #pragma once
 #include <utility>
 
+namespace arrayfire {
 namespace common {
 
 template<typename NativeEventPolicy>
@@ -81,3 +82,4 @@ class EventBase {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/FFTPlanCache.hpp b/src/backend/common/FFTPlanCache.hpp
index bd341032a2..8ae853480d 100644
--- a/src/backend/common/FFTPlanCache.hpp
+++ b/src/backend/common/FFTPlanCache.hpp
@@ -13,6 +13,7 @@
 #include <string>
 #include <utility>
 
+namespace arrayfire {
 namespace common {
 // FFTPlanCache caches backend specific fft plans in FIFO order
 //
@@ -70,3 +71,4 @@ class FFTPlanCache {
     plan_cache_t mCache;
 };
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/HandleBase.hpp b/src/backend/common/HandleBase.hpp
index 4ffaf4dca1..713ae6f71f 100644
--- a/src/backend/common/HandleBase.hpp
+++ b/src/backend/common/HandleBase.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace common {
 template<typename T, typename H>
 class HandleBase {
@@ -28,6 +29,7 @@ class HandleBase {
     HandleBase& operator=(HandleBase&& h) = default;
 };
 }  // namespace common
+}  // namespace arrayfire
 
 #define CREATE_HANDLE(NAME, TYPE, CREATE_FUNCTION, DESTROY_FUNCTION,  \
                       CHECK_FUNCTION)                                 \
diff --git a/src/backend/common/InteropManager.hpp b/src/backend/common/InteropManager.hpp
index c784ae94aa..efdc76adb6 100644
--- a/src/backend/common/InteropManager.hpp
+++ b/src/backend/common/InteropManager.hpp
@@ -18,6 +18,7 @@
 #include <memory>
 #include <vector>
 
+namespace arrayfire {
 namespace common {
 template<class T, typename R>
 class InteropManager {
@@ -42,8 +43,7 @@ class InteropManager {
     res_vec_t getImageResources(const fg_window image) {
         if (mInteropMap.find(image) == mInteropMap.end()) {
             uint32_t buffer;
-            FG_CHECK(
-                graphics::forgePlugin().fg_get_pixel_buffer(&buffer, image));
+            FG_CHECK(common::forgePlugin().fg_get_pixel_buffer(&buffer, image));
             mInteropMap[image] =
                 static_cast<T *>(this)->registerResources({buffer});
         }
@@ -53,8 +53,8 @@ class InteropManager {
     res_vec_t getPlotResources(const fg_plot plot) {
         if (mInteropMap.find(plot) == mInteropMap.end()) {
             uint32_t buffer;
-            FG_CHECK(graphics::forgePlugin().fg_get_plot_vertex_buffer(&buffer,
-                                                                       plot));
+            FG_CHECK(
+                common::forgePlugin().fg_get_plot_vertex_buffer(&buffer, plot));
             mInteropMap[plot] =
                 static_cast<T *>(this)->registerResources({buffer});
         }
@@ -64,7 +64,7 @@ class InteropManager {
     res_vec_t getHistogramResources(const fg_histogram histogram) {
         if (mInteropMap.find(histogram) == mInteropMap.end()) {
             uint32_t buffer;
-            FG_CHECK(graphics::forgePlugin().fg_get_histogram_vertex_buffer(
+            FG_CHECK(common::forgePlugin().fg_get_histogram_vertex_buffer(
                 &buffer, histogram));
             mInteropMap[histogram] =
                 static_cast<T *>(this)->registerResources({buffer});
@@ -75,7 +75,7 @@ class InteropManager {
     res_vec_t getSurfaceResources(const fg_surface surface) {
         if (mInteropMap.find(surface) == mInteropMap.end()) {
             uint32_t buffer;
-            FG_CHECK(graphics::forgePlugin().fg_get_surface_vertex_buffer(
+            FG_CHECK(common::forgePlugin().fg_get_surface_vertex_buffer(
                 &buffer, surface));
             mInteropMap[surface] =
                 static_cast<T *>(this)->registerResources({buffer});
@@ -86,11 +86,10 @@ class InteropManager {
     res_vec_t getVectorFieldResources(const fg_vector_field field) {
         if (mInteropMap.find(field) == mInteropMap.end()) {
             uint32_t verts, dirs;
-            FG_CHECK(graphics::forgePlugin().fg_get_vector_field_vertex_buffer(
+            FG_CHECK(common::forgePlugin().fg_get_vector_field_vertex_buffer(
                 &verts, field));
-            FG_CHECK(
-                graphics::forgePlugin().fg_get_vector_field_direction_buffer(
-                    &dirs, field));
+            FG_CHECK(common::forgePlugin().fg_get_vector_field_direction_buffer(
+                &dirs, field));
             mInteropMap[field] =
                 static_cast<T *>(this)->registerResources({verts, dirs});
         }
@@ -108,3 +107,4 @@ class InteropManager {
     res_map_t mInteropMap;
 };
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/KernelInterface.hpp b/src/backend/common/KernelInterface.hpp
index 537c2a7a86..5eeb8710fd 100644
--- a/src/backend/common/KernelInterface.hpp
+++ b/src/backend/common/KernelInterface.hpp
@@ -12,6 +12,7 @@
 #include <cstddef>
 #include <string>
 
+namespace arrayfire {
 namespace common {
 
 /// Kernel Interface that should be implemented by each backend
@@ -101,3 +102,4 @@ class KernelInterface {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/Logger.cpp b/src/backend/common/Logger.cpp
index ac488cd40b..3081eab672 100644
--- a/src/backend/common/Logger.cpp
+++ b/src/backend/common/Logger.cpp
@@ -29,6 +29,7 @@ using spdlog::get;
 using spdlog::logger;
 using spdlog::stdout_logger_mt;
 
+namespace arrayfire {
 namespace common {
 
 shared_ptr<logger> loggerFactory(const string& name) {
@@ -62,3 +63,4 @@ string bytesToString(size_t bytes) {
     return fmt::format("{:.3g} {}", fbytes, units[count]);
 }
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/Logger.hpp b/src/backend/common/Logger.hpp
index 4b7b4d419e..a004e773fb 100644
--- a/src/backend/common/Logger.hpp
+++ b/src/backend/common/Logger.hpp
@@ -47,10 +47,12 @@
 /* Other */
 #endif
 
+namespace arrayfire {
 namespace common {
 std::shared_ptr<spdlog::logger> loggerFactory(const std::string& name);
 std::string bytesToString(size_t bytes);
 }  // namespace common
+}  // namespace arrayfire
 
 #ifdef AF_WITH_LOGGING
 #define AF_STR_H(x) #x
diff --git a/src/backend/common/MemoryManagerBase.hpp b/src/backend/common/MemoryManagerBase.hpp
index c338db1020..569154695e 100644
--- a/src/backend/common/MemoryManagerBase.hpp
+++ b/src/backend/common/MemoryManagerBase.hpp
@@ -19,8 +19,8 @@ namespace spdlog {
 class logger;
 }
 
+namespace arrayfire {
 namespace common {
-namespace memory {
 /**
  * A internal base interface for a memory manager which is exposed to AF
  * internals. Externally, both the default AF memory manager implementation and
@@ -89,5 +89,5 @@ class MemoryManagerBase {
     std::unique_ptr<AllocatorInterface> nmi_;
 };
 
-}  // namespace memory
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/MersenneTwister.hpp b/src/backend/common/MersenneTwister.hpp
index 2810a1da0c..a96e271a01 100644
--- a/src/backend/common/MersenneTwister.hpp
+++ b/src/backend/common/MersenneTwister.hpp
@@ -51,6 +51,7 @@
 
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace common {
 const dim_t MaxBlocks     = 32;
 const dim_t TableLength   = 16 * MaxBlocks;
@@ -261,3 +262,4 @@ static unsigned temper_tbl[] = {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/ModuleInterface.hpp b/src/backend/common/ModuleInterface.hpp
index 167c3b2304..2c3127abb2 100644
--- a/src/backend/common/ModuleInterface.hpp
+++ b/src/backend/common/ModuleInterface.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace common {
 
 /// Instances of this object are stored in jit kernel cache
@@ -44,3 +45,4 @@ class ModuleInterface {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/Source.hpp b/src/backend/common/Source.hpp
index 000c2809d2..2199b389da 100644
--- a/src/backend/common/Source.hpp
+++ b/src/backend/common/Source.hpp
@@ -8,6 +8,7 @@
  ********************************************************/
 #pragma once
 
+namespace arrayfire {
 namespace common {
 struct Source {
     const char* ptr;           // Pointer to the kernel source
@@ -15,3 +16,4 @@ struct Source {
     const std::size_t hash;    // hash value for the source *ptr;
 };
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/SparseArray.cpp b/src/backend/common/SparseArray.cpp
index 06156ad3f6..ac91a29f31 100644
--- a/src/backend/common/SparseArray.cpp
+++ b/src/backend/common/SparseArray.cpp
@@ -27,6 +27,7 @@ using detail::getActiveDeviceId;
 using detail::scalar;
 using detail::writeDeviceDataArray;
 
+namespace arrayfire {
 namespace common {
 ////////////////////////////////////////////////////////////////////////////
 // Sparse Array Base Implementations
@@ -260,3 +261,4 @@ INSTANTIATE(cdouble);
 #undef INSTANTIATE
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/SparseArray.hpp b/src/backend/common/SparseArray.hpp
index 2dbcdbd3e0..860f7814ac 100644
--- a/src/backend/common/SparseArray.hpp
+++ b/src/backend/common/SparseArray.hpp
@@ -16,6 +16,7 @@
 #include <cstddef>
 #include <vector>
 
+namespace arrayfire {
 namespace common {
 
 template<typename T>
@@ -248,3 +249,4 @@ class SparseArray {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/TemplateArg.hpp b/src/backend/common/TemplateArg.hpp
index a26df012ca..238c912de2 100644
--- a/src/backend/common/TemplateArg.hpp
+++ b/src/backend/common/TemplateArg.hpp
@@ -28,7 +28,7 @@ struct TemplateArg {
 
     template<typename T>
     constexpr TemplateArg(T value) noexcept
-        : _tparam(common::toString(value)) {}
+        : _tparam(arrayfire::common::toString(value)) {}
 };
 
 template<typename... Targs>
@@ -38,6 +38,7 @@ std::array<TemplateArg, sizeof...(Targs)> TemplateArgs(Targs &&...args) {
 }
 
 #define DefineKey(arg) " -D " #arg
-#define DefineValue(arg) " -D " #arg "=" + common::toString(arg)
-#define DefineKeyValue(key, arg) " -D " #key "=" + common::toString(arg)
+#define DefineValue(arg) " -D " #arg "=" + arrayfire::common::toString(arg)
+#define DefineKeyValue(key, arg) \
+    " -D " #key "=" + arrayfire::common::toString(arg)
 #define DefineKeyFromStr(arg) " -D " + std::string(arg)
diff --git a/src/backend/common/TemplateTypename.hpp b/src/backend/common/TemplateTypename.hpp
index 682070510a..47286af899 100644
--- a/src/backend/common/TemplateTypename.hpp
+++ b/src/backend/common/TemplateTypename.hpp
@@ -17,10 +17,10 @@
 template<typename T>
 struct TemplateTypename {
     operator TemplateArg() const noexcept {
-        return {std::string(dtype_traits<T>::getName())};
+        return {std::string(af::dtype_traits<T>::getName())};
     }
     operator std::string() const noexcept {
-        return {std::string(dtype_traits<T>::getName())};
+        return {std::string(af::dtype_traits<T>::getName())};
     }
 };
 
diff --git a/src/backend/common/Transform.hpp b/src/backend/common/Transform.hpp
index 4fb2a127f1..3d56cf0209 100644
--- a/src/backend/common/Transform.hpp
+++ b/src/backend/common/Transform.hpp
@@ -19,6 +19,7 @@
 
 #include "optypes.hpp"
 
+namespace arrayfire {
 namespace common {
 
 using namespace detail;  // NOLINT
@@ -61,3 +62,4 @@ struct Transform<Ti, To, af_notzero_t> {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/cast.cpp b/src/backend/common/cast.cpp
index f02267ecd0..cc98f0504f 100644
--- a/src/backend/common/cast.cpp
+++ b/src/backend/common/cast.cpp
@@ -10,7 +10,7 @@
 #include <common/cast.hpp>
 #include <handle.hpp>
 
-using common::half;
+using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
@@ -19,6 +19,9 @@ using detail::uint;
 using detail::uintl;
 using detail::ushort;
 
+namespace arrayfire {
+namespace common {
+
 template<typename To>
 detail::Array<To> castArray(const af_array &in) {
     const ArrayInfo &info = getInfo(in);
@@ -60,3 +63,6 @@ template detail::Array<uintl> castArray(const af_array &in);
 template detail::Array<short> castArray(const af_array &in);
 template detail::Array<ushort> castArray(const af_array &in);
 template detail::Array<half> castArray(const af_array &in);
+
+}  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/cast.hpp b/src/backend/common/cast.hpp
index d80caacfe6..4186a03914 100644
--- a/src/backend/common/cast.hpp
+++ b/src/backend/common/cast.hpp
@@ -17,6 +17,7 @@
 #include <jit/UnaryNode.hpp>
 #endif
 
+namespace arrayfire {
 namespace common {
 /// This function determines if consecutive cast operations should be
 /// removed from a JIT AST.
@@ -71,7 +72,7 @@ struct CastWrapper {
     }
 
     detail::Array<To> operator()(const detail::Array<Ti> &in) {
-        using cpu::jit::UnaryNode;
+        using detail::jit::UnaryNode;
 
         common::Node_ptr in_node = in.getNode();
         constexpr af::dtype to_dtype =
@@ -118,11 +119,11 @@ struct CastWrapper {
     }
 
     detail::Array<To> operator()(const detail::Array<Ti> &in) {
-        using common::UnaryNode;
+        using arrayfire::common::UnaryNode;
         detail::CastOp<To, Ti> cop;
         common::Node_ptr in_node = in.getNode();
         constexpr af::dtype to_dtype =
-            static_cast<af::dtype>(dtype_traits<To>::af_type);
+            static_cast<af::dtype>(af::dtype_traits<To>::af_type);
         constexpr af::dtype in_dtype =
             static_cast<af::dtype>(af::dtype_traits<Ti>::af_type);
 
@@ -137,7 +138,7 @@ struct CastWrapper {
             if (in_node_unary && in_node_unary->getOp() == af_cast_t) {
                 // child child's output type is the input type of the child
                 AF_TRACE("Cast optimiztion performed by removing cast to {}",
-                         dtype_traits<Ti>::getName());
+                         af::dtype_traits<Ti>::getName());
                 auto in_child_node = in_node_unary->getChildren()[0];
                 if (in_child_node->getType() == to_dtype) {
                     // ignore the input node and simply connect a noop node from
@@ -182,3 +183,4 @@ auto cast(const detail::Array<Ti> &in)
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/compile_module.hpp b/src/backend/common/compile_module.hpp
index c2abe76ecd..2f12f6386b 100644
--- a/src/backend/common/compile_module.hpp
+++ b/src/backend/common/compile_module.hpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace common {
 
 /// \brief Backend specific source compilation implementation
@@ -63,5 +64,6 @@ detail::Module loadModuleFromDisk(const int device,
                                   const bool isJIT);
 
 }  // namespace common
+}  // namespace arrayfire
 
 #endif
diff --git a/src/backend/common/complex.hpp b/src/backend/common/complex.hpp
index cb5a4cdabf..b7663580dc 100644
--- a/src/backend/common/complex.hpp
+++ b/src/backend/common/complex.hpp
@@ -13,6 +13,7 @@
 
 #include <type_traits>
 
+namespace arrayfire {
 namespace common {
 
 // The value returns true if the type is a complex type. False otherwise
@@ -39,3 +40,4 @@ using if_real =
     typename std::enable_if<is_complex<T>::value == false, TYPE>::type;
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/defines.hpp b/src/backend/common/defines.hpp
index c72c7b1b32..5c7eadc6ce 100644
--- a/src/backend/common/defines.hpp
+++ b/src/backend/common/defines.hpp
@@ -63,7 +63,9 @@ using LibHandle = void*;
 #define AF_MEM_DEBUG 0
 #endif
 
+namespace arrayfire {
 namespace common {
 using mutex_t      = std::mutex;
 using lock_guard_t = std::lock_guard<mutex_t>;
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/deterministicHash.cpp b/src/backend/common/deterministicHash.cpp
index 0529f7c58b..2280d4cbbb 100644
--- a/src/backend/common/deterministicHash.cpp
+++ b/src/backend/common/deterministicHash.cpp
@@ -36,7 +36,7 @@ size_t deterministicHash(span<const string> list, const size_t prevHash) {
     return hash;
 }
 
-size_t deterministicHash(span<const common::Source> list) {
+size_t deterministicHash(span<const arrayfire::common::Source> list) {
     // Combine the different source codes, via their hashes
     size_t hash = FNV1A_BASE_OFFSET;
     for (auto s : list) {
diff --git a/src/backend/common/deterministicHash.hpp b/src/backend/common/deterministicHash.hpp
index 25b43a8893..fa950bc2a5 100644
--- a/src/backend/common/deterministicHash.hpp
+++ b/src/backend/common/deterministicHash.hpp
@@ -33,4 +33,5 @@ std::size_t deterministicHash(nonstd::span<const std::string> list,
                               const std::size_t prevHash = FNV1A_BASE_OFFSET);
 
 // This concatenates hashes of multiple sources
-std::size_t deterministicHash(nonstd::span<const common::Source> list);
+std::size_t deterministicHash(
+    nonstd::span<const arrayfire::common::Source> list);
diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index 58bc0a9ced..68514bac29 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -31,9 +31,9 @@ using std::move;
 using std::string;
 using std::stringstream;
 
-using common::getEnvVar;
-using common::getName;
-using common::is_stacktrace_enabled;
+using arrayfire::common::getEnvVar;
+using arrayfire::common::getName;
+using arrayfire::common::is_stacktrace_enabled;
 
 AfError::AfError(const char *const func, const char *const file, const int line,
                  const char *const message, af_err err, stacktrace st)
@@ -222,6 +222,7 @@ const char *af_err_to_string(const af_err err) {
            "case in af_err_to_string.";
 }
 
+namespace arrayfire {
 namespace common {
 
 bool &is_stacktrace_enabled() noexcept {
@@ -230,3 +231,4 @@ bool &is_stacktrace_enabled() noexcept {
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/err_common.hpp b/src/backend/common/err_common.hpp
index 6adf600cf6..a2c55742e0 100644
--- a/src/backend/common/err_common.hpp
+++ b/src/backend/common/err_common.hpp
@@ -210,8 +210,10 @@ af_err set_global_error_string(const std::string& msg,
 static const int MAX_ERR_SIZE = 1024;
 std::string& get_global_error_string() noexcept;
 
+namespace arrayfire {
 namespace common {
 
 bool& is_stacktrace_enabled() noexcept;
 
-}  // namespace common
+}
+}  // namespace arrayfire
diff --git a/src/backend/common/forge_loader.hpp b/src/backend/common/forge_loader.hpp
index c87e98690c..6fcdd625ef 100644
--- a/src/backend/common/forge_loader.hpp
+++ b/src/backend/common/forge_loader.hpp
@@ -43,7 +43,10 @@
 /* Other */
 #endif
 
-class ForgeModule : public common::DependencyModule {
+namespace arrayfire {
+namespace common {
+
+class ForgeModule : public DependencyModule {
    public:
     ForgeModule();
 
@@ -117,9 +120,7 @@ class ForgeModule : public common::DependencyModule {
     MODULE_MEMBER(fg_err_to_string);
 };
 
-namespace graphics {
 ForgeModule& forgePlugin();
-}
 
 #define FG_CHECK(fn)                                        \
     do {                                                    \
@@ -128,3 +129,6 @@ ForgeModule& forgePlugin();
             AF_ERROR("forge call failed", AF_ERR_INTERNAL); \
         }                                                   \
     } while (0);
+
+}  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/graphics_common.cpp b/src/backend/common/graphics_common.cpp
index 75fe4c002c..07084c43b2 100644
--- a/src/backend/common/graphics_common.cpp
+++ b/src/backend/common/graphics_common.cpp
@@ -15,10 +15,13 @@
 #include <mutex>
 #include <utility>
 
-using common::getEnvVar;
+using arrayfire::common::getEnvVar;
 using std::make_pair;
 using std::string;
 
+namespace arrayfire {
+namespace common {
+
 /// Dynamically loads forge function pointer at runtime
 #define FG_MODULE_FUNCTION_INIT(NAME) \
     NAME = DependencyModule::getSymbol<decltype(&::NAME)>(#NAME)
@@ -175,7 +178,7 @@ size_t getTypeSize(GLenum type) {
 }
 
 void makeContextCurrent(fg_window window) {
-    FG_CHECK(graphics::forgePlugin().fg_make_window_current(window));
+    FG_CHECK(common::forgePlugin().fg_make_window_current(window));
     CheckGL("End makeContextCurrent");
 }
 
@@ -235,8 +238,6 @@ double step_round(const double in, const bool dir) {
     return mag * mult;
 }
 
-namespace graphics {
-
 ForgeModule& forgePlugin() { return detail::forgeManager().plugin(); }
 
 ForgeManager::ForgeManager() : mPlugin(new ForgeModule()) {}
@@ -519,4 +520,6 @@ void ForgeManager::setChartAxesOverride(const fg_chart chart, bool flag) {
     }
     mChartAxesOverrideMap[chart] = flag;
 }
-}  // namespace graphics
+
+}  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/graphics_common.hpp b/src/backend/common/graphics_common.hpp
index 6db366f323..ec59033fcb 100644
--- a/src/backend/common/graphics_common.hpp
+++ b/src/backend/common/graphics_common.hpp
@@ -17,6 +17,9 @@
 #include <utility>
 #include <vector>
 
+namespace arrayfire {
+namespace common {
+
 // default to f32(float) type
 template<typename T>
 fg_dtype getGLType();
@@ -25,7 +28,8 @@ fg_dtype getGLType();
 // Returns 1 if an OpenGL error occurred, 0 otherwise.
 GLenum glErrorCheck(const char* msg, const char* file, int line);
 
-#define CheckGL(msg) glErrorCheck(msg, __AF_FILENAME__, __LINE__)
+#define CheckGL(msg) \
+    arrayfire::common::glErrorCheck(msg, __AF_FILENAME__, __LINE__)
 
 fg_marker_type getFGMarker(const af_marker_type af_marker);
 
@@ -33,8 +37,6 @@ void makeContextCurrent(fg_window window);
 
 double step_round(const double in, const bool dir);
 
-namespace graphics {
-
 /// \brief The singleton manager class for Forge resources
 ///
 /// Only device manager class can create objects of this class.
@@ -59,7 +61,7 @@ class ForgeManager {
     ForgeManager& operator=(ForgeManager&&)      = delete;
 
     /// \brief Module used to invoke forge API calls
-    ForgeModule& plugin();
+    common::ForgeModule& plugin();
 
     /// \brief The main window with which all other windows share GL context
     fg_window getMainWindow();
@@ -294,7 +296,7 @@ class ForgeManager {
     using SurfaceMapIterator   = std::map<ChartKey, SurfacePtr>::iterator;
     using VecFieldMapIterator  = std::map<ChartKey, VectorFieldPtr>::iterator;
 
-    std::unique_ptr<ForgeModule> mPlugin;
+    std::unique_ptr<common::ForgeModule> mPlugin;
     std::unique_ptr<Window, Window::Deleter> mMainWindow;
 
     std::map<fg_window, ChartList> mChartMap;
@@ -307,4 +309,5 @@ class ForgeManager {
     std::map<fg_chart, bool> mChartAxesOverrideMap;
 };
 
-}  // namespace graphics
+}  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/half.cpp b/src/backend/common/half.cpp
index 3e41699c72..249346b038 100644
--- a/src/backend/common/half.cpp
+++ b/src/backend/common/half.cpp
@@ -2,6 +2,7 @@
 #include <common/half.hpp>
 #include <common/util.hpp>
 
+namespace arrayfire {
 namespace common {
 std::ostream &operator<<(std::ostream &os, const half &val) {
     os << float(val);
@@ -13,3 +14,4 @@ std::string toString(const half val) {
     return common::toString(static_cast<float>(val));
 }
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index bd5f143c28..8080dcffa1 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -50,6 +50,7 @@ using uint16_t = unsigned short;
 
 #endif
 
+namespace arrayfire {
 namespace common {
 
 #if defined(__CUDA_ARCH__)
@@ -807,20 +808,22 @@ static constexpr binary_t binary = binary_t{};
 
 class half;
 
-AF_CONSTEXPR __DH__ static inline bool operator==(common::half lhs,
-                                                  common::half rhs) noexcept;
-AF_CONSTEXPR __DH__ static inline bool operator!=(common::half lhs,
-                                                  common::half rhs) noexcept;
-__DH__ static inline bool operator<(common::half lhs,
-                                    common::half rhs) noexcept;
-__DH__ static inline bool operator<(common::half lhs, float rhs) noexcept;
+AF_CONSTEXPR __DH__ static inline bool operator==(
+    arrayfire::common::half lhs, arrayfire::common::half rhs) noexcept;
+AF_CONSTEXPR __DH__ static inline bool operator!=(
+    arrayfire::common::half lhs, arrayfire::common::half rhs) noexcept;
+__DH__ static inline bool operator<(arrayfire::common::half lhs,
+                                    arrayfire::common::half rhs) noexcept;
+__DH__ static inline bool operator<(arrayfire::common::half lhs,
+                                    float rhs) noexcept;
 AF_CONSTEXPR __DH__ static inline bool isinf(half val) noexcept;
 
 /// Classification implementation.
 /// \param arg value to classify
 /// \retval true if not a number
 /// \retval false else
-AF_CONSTEXPR __DH__ static inline bool isnan(common::half val) noexcept;
+AF_CONSTEXPR __DH__ static inline bool isnan(
+    arrayfire::common::half val) noexcept;
 
 class alignas(2) half {
     native_half_t data_ = native_half_t();
@@ -970,22 +973,26 @@ class alignas(2) half {
 
     friend AF_CONSTEXPR __DH__ bool operator==(half lhs, half rhs) noexcept;
     friend AF_CONSTEXPR __DH__ bool operator!=(half lhs, half rhs) noexcept;
-    friend __DH__ bool operator<(common::half lhs, common::half rhs) noexcept;
-    friend __DH__ bool operator<(common::half lhs, float rhs) noexcept;
+    friend __DH__ bool operator<(arrayfire::common::half lhs,
+                                 arrayfire::common::half rhs) noexcept;
+    friend __DH__ bool operator<(arrayfire::common::half lhs,
+                                 float rhs) noexcept;
     friend AF_CONSTEXPR __DH__ bool isinf(half val) noexcept;
     friend AF_CONSTEXPR __DH__ inline bool isnan(half val) noexcept;
 
-    AF_CONSTEXPR __DH__ common::half operator-() const {
+    AF_CONSTEXPR __DH__ arrayfire::common::half operator-() const {
 #if __CUDA_ARCH__ >= 530
-        return common::half(__hneg(data_));
+        return arrayfire::common::half(__hneg(data_));
 #elif defined(__CUDA_ARCH__)
-        return common::half(-(__half2float(data_)));
+        return arrayfire::common::half(-(__half2float(data_)));
 #else
-        return common::half(internal::binary, data_ ^ 0x8000);
+        return arrayfire::common::half(internal::binary, data_ ^ 0x8000);
 #endif
     }
 
-    AF_CONSTEXPR __DH__ common::half operator+() const { return *this; }
+    AF_CONSTEXPR __DH__ arrayfire::common::half operator+() const {
+        return *this;
+    }
 
     AF_CONSTEXPR static half infinity() {
         half out;
@@ -998,8 +1005,8 @@ class alignas(2) half {
     }
 };
 
-AF_CONSTEXPR __DH__ static inline bool operator==(common::half lhs,
-                                                  common::half rhs) noexcept {
+AF_CONSTEXPR __DH__ static inline bool operator==(
+    arrayfire::common::half lhs, arrayfire::common::half rhs) noexcept {
 #if __CUDA_ARCH__ >= 530
     return __heq(lhs.data_, rhs.data_);
 #elif defined(__CUDA_ARCH__)
@@ -1010,8 +1017,8 @@ AF_CONSTEXPR __DH__ static inline bool operator==(common::half lhs,
 #endif
 }
 
-AF_CONSTEXPR __DH__ static inline bool operator!=(common::half lhs,
-                                                  common::half rhs) noexcept {
+AF_CONSTEXPR __DH__ static inline bool operator!=(
+    arrayfire::common::half lhs, arrayfire::common::half rhs) noexcept {
 #if __CUDA_ARCH__ >= 530
     return __hne(lhs.data_, rhs.data_);
 #else
@@ -1019,8 +1026,8 @@ AF_CONSTEXPR __DH__ static inline bool operator!=(common::half lhs,
 #endif
 }
 
-__DH__ static inline bool operator<(common::half lhs,
-                                    common::half rhs) noexcept {
+__DH__ static inline bool operator<(arrayfire::common::half lhs,
+                                    arrayfire::common::half rhs) noexcept {
 #if __CUDA_ARCH__ >= 530
     return __hlt(lhs.data_, rhs.data_);
 #elif defined(__CUDA_ARCH__)
@@ -1033,7 +1040,8 @@ __DH__ static inline bool operator<(common::half lhs,
 #endif
 }
 
-__DH__ static inline bool operator<(common::half lhs, float rhs) noexcept {
+__DH__ static inline bool operator<(arrayfire::common::half lhs,
+                                    float rhs) noexcept {
 #if defined(__CUDA_ARCH__)
     return __half2float(lhs.data_) < rhs;
 #else
@@ -1054,6 +1062,7 @@ static inline std::string to_string(const half&& val) {
 #endif
 
 }  // namespace common
+}  // namespace arrayfire
 
 #if !defined(__NVCC__) && !defined(__CUDACC_RTC__)
 //#endif
@@ -1063,7 +1072,7 @@ namespace std {
 /// Because of the underlying single-precision implementation of many
 /// operations, it inherits some properties from `std::numeric_limits<float>`.
 template<>
-class numeric_limits<common::half> : public numeric_limits<float> {
+class numeric_limits<arrayfire::common::half> : public numeric_limits<float> {
    public:
     /// Supports signed values.
     static constexpr bool is_signed = true;
@@ -1120,60 +1129,70 @@ class numeric_limits<common::half> : public numeric_limits<float> {
     static constexpr int max_exponent10 = 4;
 
     /// Smallest positive normal value.
-    static AF_CONSTEXPR __DH__ common::half min() noexcept {
-        return common::half(common::internal::binary, 0x0400);
+    static AF_CONSTEXPR __DH__ arrayfire::common::half min() noexcept {
+        return arrayfire::common::half(arrayfire::common::internal::binary,
+                                       0x0400);
     }
 
     /// Smallest finite value.
-    static AF_CONSTEXPR __DH__ common::half lowest() noexcept {
-        return common::half(common::internal::binary, 0xFBFF);
+    static AF_CONSTEXPR __DH__ arrayfire::common::half lowest() noexcept {
+        return arrayfire::common::half(arrayfire::common::internal::binary,
+                                       0xFBFF);
     }
 
     /// Largest finite value.
-    static AF_CONSTEXPR __DH__ common::half max() noexcept {
-        return common::half(common::internal::binary, 0x7BFF);
+    static AF_CONSTEXPR __DH__ arrayfire::common::half max() noexcept {
+        return arrayfire::common::half(arrayfire::common::internal::binary,
+                                       0x7BFF);
     }
 
     /// Difference between one and next representable value.
-    static AF_CONSTEXPR __DH__ common::half epsilon() noexcept {
-        return common::half(common::internal::binary, 0x1400);
+    static AF_CONSTEXPR __DH__ arrayfire::common::half epsilon() noexcept {
+        return arrayfire::common::half(arrayfire::common::internal::binary,
+                                       0x1400);
     }
 
     /// Maximum rounding error.
-    static AF_CONSTEXPR __DH__ common::half round_error() noexcept {
-        return common::half(
-            common::internal::binary,
+    static AF_CONSTEXPR __DH__ arrayfire::common::half round_error() noexcept {
+        return arrayfire::common::half(
+            arrayfire::common::internal::binary,
             (round_style == std::round_to_nearest) ? 0x3800 : 0x3C00);
     }
 
     /// Positive infinity.
-    static AF_CONSTEXPR __DH__ common::half infinity() noexcept {
-        return common::half(common::internal::binary, 0x7C00);
+    static AF_CONSTEXPR __DH__ arrayfire::common::half infinity() noexcept {
+        return arrayfire::common::half(arrayfire::common::internal::binary,
+                                       0x7C00);
     }
 
     /// Quiet NaN.
-    static AF_CONSTEXPR __DH__ common::half quiet_NaN() noexcept {
-        return common::half(common::internal::binary, 0x7FFF);
+    static AF_CONSTEXPR __DH__ arrayfire::common::half quiet_NaN() noexcept {
+        return arrayfire::common::half(arrayfire::common::internal::binary,
+                                       0x7FFF);
     }
 
     /// Signalling NaN.
-    static AF_CONSTEXPR __DH__ common::half signaling_NaN() noexcept {
-        return common::half(common::internal::binary, 0x7DFF);
+    static AF_CONSTEXPR __DH__ arrayfire::common::half
+    signaling_NaN() noexcept {
+        return arrayfire::common::half(arrayfire::common::internal::binary,
+                                       0x7DFF);
     }
 
     /// Smallest positive subnormal value.
-    static AF_CONSTEXPR __DH__ common::half denorm_min() noexcept {
-        return common::half(common::internal::binary, 0x0001);
+    static AF_CONSTEXPR __DH__ arrayfire::common::half denorm_min() noexcept {
+        return arrayfire::common::half(arrayfire::common::internal::binary,
+                                       0x0001);
     }
 };
 
 /// Hash function for half-precision floats.
 /// This is only defined if C++11 `std::hash` is supported and enabled.
 template<>
-struct hash<common::half>  //: unary_function<common::half,size_t>
+struct hash<
+    arrayfire::common::half>  //: unary_function<arrayfire::common::half,size_t>
 {
     /// Type of function argument.
-    typedef common::half argument_type;
+    typedef arrayfire::common::half argument_type;
 
     /// Function return type.
     typedef size_t result_type;
@@ -1191,6 +1210,7 @@ struct hash<common::half>  //: unary_function<common::half,size_t>
 }  // namespace std
 #endif
 
+namespace arrayfire {
 namespace common {
 AF_CONSTEXPR __DH__ static bool isinf(half val) noexcept {
 #if __CUDA_ARCH__ >= 530
@@ -1213,3 +1233,4 @@ AF_CONSTEXPR __DH__ static inline bool isnan(half val) noexcept {
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/host_memory.cpp b/src/backend/common/host_memory.cpp
index 51a01e2164..0e213cb7e5 100644
--- a/src/backend/common/host_memory.cpp
+++ b/src/backend/common/host_memory.cpp
@@ -26,6 +26,7 @@
 #define NOMEMORYSIZE
 #endif
 
+namespace arrayfire {
 namespace common {
 
 #ifdef NOMEMORYSIZE
@@ -109,3 +110,4 @@ size_t getHostMemorySize() {
 
 #endif  // NOMEMORYSIZE
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/host_memory.hpp b/src/backend/common/host_memory.hpp
index 69557fb576..ead8a8c54e 100644
--- a/src/backend/common/host_memory.hpp
+++ b/src/backend/common/host_memory.hpp
@@ -10,8 +10,10 @@
 #pragma once
 #include <cstddef>
 
+namespace arrayfire {
 namespace common {
 
 size_t getHostMemorySize();
 
-}
+}  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/indexing_helpers.hpp b/src/backend/common/indexing_helpers.hpp
index 46e33492bb..9482fa639c 100644
--- a/src/backend/common/indexing_helpers.hpp
+++ b/src/backend/common/indexing_helpers.hpp
@@ -15,6 +15,7 @@
 
 #include <array>
 
+namespace arrayfire {
 namespace common {
 
 // will generate indexes to flip input array
@@ -34,3 +35,4 @@ static detail::Array<T> flip(const detail::Array<T>& in,
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/BinaryNode.cpp b/src/backend/common/jit/BinaryNode.cpp
index 1277aa10be..84c5597e31 100644
--- a/src/backend/common/jit/BinaryNode.cpp
+++ b/src/backend/common/jit/BinaryNode.cpp
@@ -18,6 +18,7 @@ using detail::createNodeArray;
 
 using std::make_shared;
 
+namespace arrayfire {
 namespace common {
 #ifdef AF_CPU
 template<typename To, typename Ti, af_op_t op>
@@ -152,3 +153,4 @@ INSTANTIATE_LOGIC(af_ge_t);
 #undef INSTANTIATE
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/BinaryNode.hpp b/src/backend/common/jit/BinaryNode.hpp
index bfc68bd8ea..e250382745 100644
--- a/src/backend/common/jit/BinaryNode.hpp
+++ b/src/backend/common/jit/BinaryNode.hpp
@@ -13,6 +13,7 @@
 
 #include <cmath>
 
+namespace arrayfire {
 namespace common {
 class BinaryNode : public NaryNode {
    public:
@@ -28,3 +29,4 @@ detail::Array<To> createBinaryNode(const detail::Array<Ti> &lhs,
                                    const af::dim4 &odims);
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index 9633b2a867..5af3a216d0 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -15,6 +15,7 @@
 #include <cstring>
 #include <sstream>
 
+namespace arrayfire {
 namespace common {
 
 template<typename DataType, typename ParamType>
@@ -118,3 +119,4 @@ class BufferNodeBase : public common::Node {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/ModdimNode.hpp b/src/backend/common/jit/ModdimNode.hpp
index 209593df5c..b0f7d927a6 100644
--- a/src/backend/common/jit/ModdimNode.hpp
+++ b/src/backend/common/jit/ModdimNode.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <common/jit/NaryNode.hpp>
 
+namespace arrayfire {
 namespace common {
 
 class ModdimNode : public NaryNode {
@@ -30,3 +31,4 @@ class ModdimNode : public NaryNode {
     }
 };
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/NaryNode.hpp b/src/backend/common/jit/NaryNode.hpp
index 5e97e249dd..0d78b9e86c 100644
--- a/src/backend/common/jit/NaryNode.hpp
+++ b/src/backend/common/jit/NaryNode.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <utility>
 
+namespace arrayfire {
 namespace common {
 
 class NaryNode : public Node {
@@ -136,3 +137,4 @@ common::Node_ptr createNaryNode(
     return ptr;
 }
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/Node.cpp b/src/backend/common/jit/Node.cpp
index ed24b9c1f8..0e67228f91 100644
--- a/src/backend/common/jit/Node.cpp
+++ b/src/backend/common/jit/Node.cpp
@@ -19,6 +19,7 @@
 
 using std::vector;
 
+namespace arrayfire {
 namespace common {
 
 int Node::getNodesMap(Node_map_t &node_map, vector<Node *> &full_nodes,
@@ -76,9 +77,11 @@ auto isScalar(const Node &ptr) -> bool { return ptr.isScalar(); }
 bool Node::isLinear(const dim_t dims[4]) const { return true; }
 
 }  // namespace common
+}  // namespace arrayfire
 
-size_t std::hash<common::Node *>::operator()(
-    common::Node *const node) const noexcept {
-    common::Node *const node_ptr = static_cast<common::Node *const>(node);
+size_t std::hash<arrayfire::common::Node *>::operator()(
+    arrayfire::common::Node *const node) const noexcept {
+    arrayfire::common::Node *const node_ptr =
+        static_cast<arrayfire::common::Node *const>(node);
     return node_ptr->getHash();
 }
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index bbe3fcb859..9ed090fbaa 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -31,29 +31,34 @@ enum class kJITHeuristics {
     MemoryPressure      = 3  /* eval due to memory pressure */
 };
 
+namespace arrayfire {
 namespace common {
 class Node;
-}
+}  // namespace common
+}  // namespace arrayfire
 
 #ifdef AF_CPU
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
 template<typename T>
 void evalMultiple(std::vector<Param<T>> arrays,
                   std::vector<std::shared_ptr<common::Node>> output_nodes_);
-}
+}  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
 #endif
 
 namespace std {
 template<>
-struct hash<common::Node *> {
+struct hash<arrayfire::common::Node *> {
     /// Calls the getHash function of the Node pointer
-    size_t operator()(common::Node *const n) const noexcept;
+    size_t operator()(arrayfire::common::Node *const n) const noexcept;
 };
 }  // namespace std
 
+namespace arrayfire {
 namespace common {
 class Node;
 struct Node_ids;
@@ -288,8 +293,8 @@ class Node {
 
 #ifdef AF_CPU
     template<typename U>
-    friend void cpu::kernel::evalMultiple(
-        std::vector<cpu::Param<U>> arrays,
+    friend void arrayfire::cpu::kernel::evalMultiple(
+        std::vector<arrayfire::cpu::Param<U>> arrays,
         std::vector<common::Node_ptr> output_nodes_);
 
     virtual void setShape(af::dim4 new_shape) { UNUSED(new_shape); }
@@ -313,3 +318,4 @@ auto isBuffer(const Node &ptr) -> bool;
 auto isScalar(const Node &ptr) -> bool;
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/NodeIO.hpp b/src/backend/common/jit/NodeIO.hpp
index bd4346f465..ac149d98d9 100644
--- a/src/backend/common/jit/NodeIO.hpp
+++ b/src/backend/common/jit/NodeIO.hpp
@@ -17,13 +17,13 @@ template<>
 struct fmt::formatter<af::dtype> : fmt::formatter<char> {
     template<typename FormatContext>
     auto format(const af::dtype& p, FormatContext& ctx) -> decltype(ctx.out()) {
-        format_to(ctx.out(), "{}", getName(p));
+        format_to(ctx.out(), "{}", arrayfire::common::getName(p));
         return ctx.out();
     }
 };
 
 template<>
-struct fmt::formatter<common::Node> {
+struct fmt::formatter<arrayfire::common::Node> {
     // Presentation format: 'p' - pointer, 't' - type.
     // char presentation;
     bool pointer;
@@ -58,7 +58,7 @@ struct fmt::formatter<common::Node> {
     // Formats the point p using the parsed format specification (presentation)
     // stored in this formatter.
     template<typename FormatContext>
-    auto format(const common::Node& node, FormatContext& ctx)
+    auto format(const arrayfire::common::Node& node, FormatContext& ctx)
         -> decltype(ctx.out()) {
         // ctx.out() is an output iterator to write to.
 
@@ -68,15 +68,17 @@ struct fmt::formatter<common::Node> {
             if (isBuffer(node)) {
                 format_to(ctx.out(), "buffer ");
             } else if (isScalar(node)) {
-                format_to(ctx.out(), "scalar ", common::toString(node.getOp()));
+                format_to(ctx.out(), "scalar ",
+                          arrayfire::common::toString(node.getOp()));
             } else {
-                format_to(ctx.out(), "{} ", common::toString(node.getOp()));
+                format_to(ctx.out(), "{} ",
+                          arrayfire::common::toString(node.getOp()));
             }
         }
         if (type) format_to(ctx.out(), "{} ", node.getType());
         if (children) {
             int count;
-            for (count = 0; count < common::Node::kMaxChildren &&
+            for (count = 0; count < arrayfire::common::Node::kMaxChildren &&
                             node.m_children[count].get() != nullptr;
                  count++) {}
             if (count > 0) {
diff --git a/src/backend/common/jit/NodeIterator.hpp b/src/backend/common/jit/NodeIterator.hpp
index e2883079a1..82e916c7ef 100644
--- a/src/backend/common/jit/NodeIterator.hpp
+++ b/src/backend/common/jit/NodeIterator.hpp
@@ -14,6 +14,7 @@
 #include <iterator>
 #include <vector>
 
+namespace arrayfire {
 namespace common {
 
 /// A node iterator that performs a breadth first traversal of the node tree
@@ -28,7 +29,7 @@ class NodeIterator {
 
    private:
     std::vector<pointer> tree;
-    size_t index;
+    size_t index = 0;
 
     /// Copies the children of the \p n Node to the end of the tree vector
     void copy_children_to_end(Node* n) {
@@ -101,3 +102,4 @@ class NodeIterator {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/ScalarNode.hpp b/src/backend/common/jit/ScalarNode.hpp
index 126e8860f7..3a530a6911 100644
--- a/src/backend/common/jit/ScalarNode.hpp
+++ b/src/backend/common/jit/ScalarNode.hpp
@@ -16,6 +16,7 @@
 #include <types.hpp>
 #include <iomanip>
 
+namespace arrayfire {
 namespace common {
 
 template<typename T>
@@ -94,3 +95,4 @@ class ScalarNode : public common::Node {
 };
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/ShiftNodeBase.hpp b/src/backend/common/jit/ShiftNodeBase.hpp
index df42002576..bbc0f5863f 100644
--- a/src/backend/common/jit/ShiftNodeBase.hpp
+++ b/src/backend/common/jit/ShiftNodeBase.hpp
@@ -20,6 +20,7 @@
 #include <sstream>
 #include <string>
 
+namespace arrayfire {
 namespace common {
 
 template<typename BufferNode>
@@ -115,3 +116,4 @@ class ShiftNodeBase : public Node {
     }
 };
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/jit/UnaryNode.hpp b/src/backend/common/jit/UnaryNode.hpp
index d7470a3378..c847bd9f91 100644
--- a/src/backend/common/jit/UnaryNode.hpp
+++ b/src/backend/common/jit/UnaryNode.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <common/jit/NaryNode.hpp>
 
+namespace arrayfire {
 namespace common {
 
 class UnaryNode : public NaryNode {
@@ -24,3 +25,4 @@ class UnaryNode : public NaryNode {
     }
 };
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/kernel_cache.cpp b/src/backend/common/kernel_cache.cpp
index 1fb81ad293..423204ba6b 100644
--- a/src/backend/common/kernel_cache.cpp
+++ b/src/backend/common/kernel_cache.cpp
@@ -36,6 +36,7 @@ using std::unique_lock;
 using std::unordered_map;
 using std::vector;
 
+namespace arrayfire {
 namespace common {
 
 using ModuleMap = unordered_map<size_t, Module>;
@@ -140,5 +141,6 @@ Kernel getKernel(const string& kernelName, span<const common::Source> sources,
 }
 
 }  // namespace common
+}  // namespace arrayfire
 
 #endif
diff --git a/src/backend/common/kernel_cache.hpp b/src/backend/common/kernel_cache.hpp
index eb1b90f47b..bef3b6b577 100644
--- a/src/backend/common/kernel_cache.hpp
+++ b/src/backend/common/kernel_cache.hpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace common {
 
 /// \brief Find/Create-Cache a Kernel that fits the given criteria
@@ -48,7 +49,8 @@ namespace common {
 /// Example Usage: transpose
 ///
 /// \code
-/// auto transpose = getKernel("cuda::transpose", std::array{transpase_cuh_src},
+/// auto transpose = getKernel("arrayfire::cuda::transpose",
+/// std::array{transpase_cuh_src},
 ///         {
 ///           TemplateTypename<T>(),
 ///           TemplateArg(conjugate),
@@ -103,5 +105,6 @@ detail::Kernel getKernel(const detail::Module& mod, const std::string& name,
                          const bool sourceWasJIT);
 
 }  // namespace common
+}  // namespace arrayfire
 
 #endif
diff --git a/src/backend/common/kernel_type.hpp b/src/backend/common/kernel_type.hpp
index d61f796f67..9d833b7e4b 100644
--- a/src/backend/common/kernel_type.hpp
+++ b/src/backend/common/kernel_type.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace common {
 
 /// \brief Maps a type between its data representation and the type used
@@ -33,3 +34,4 @@ struct kernel_type {
     using native = compute;
 };
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/moddims.cpp b/src/backend/common/moddims.cpp
index 50f9fc6846..6fbd99650e 100644
--- a/src/backend/common/moddims.cpp
+++ b/src/backend/common/moddims.cpp
@@ -22,11 +22,12 @@ using std::make_shared;
 using std::shared_ptr;
 using std::vector;
 
+namespace arrayfire {
 namespace common {
 template<typename T>
 Array<T> moddimOp(const Array<T> &in, af::dim4 outDim) {
-    using common::Node;
-    using common::Node_ptr;
+    using arrayfire::common::Node;
+    using arrayfire::common::Node_ptr;
     using std::array;
 
     auto createModdim = [outDim](array<Node_ptr, 1> &operands) {
@@ -80,18 +81,19 @@ detail::Array<T> flat(const detail::Array<T> &in) {
 }
 
 }  // namespace common
+}  // namespace arrayfire
 
-#define INSTANTIATE(TYPE)                                        \
-    template detail::Array<TYPE> common::modDims<TYPE>(          \
-        const detail::Array<TYPE> &in, const af::dim4 &newDims); \
-    template detail::Array<TYPE> common::flat<TYPE>(             \
+#define INSTANTIATE(TYPE)                                          \
+    template detail::Array<TYPE> arrayfire::common::modDims<TYPE>( \
+        const detail::Array<TYPE> &in, const af::dim4 &newDims);   \
+    template detail::Array<TYPE> arrayfire::common::flat<TYPE>(    \
         const detail::Array<TYPE> &in)
 
 INSTANTIATE(float);
 INSTANTIATE(double);
 INSTANTIATE(detail::cfloat);
 INSTANTIATE(detail::cdouble);
-INSTANTIATE(common::half);
+INSTANTIATE(arrayfire::common::half);
 INSTANTIATE(unsigned char);
 INSTANTIATE(char);
 INSTANTIATE(unsigned short);
diff --git a/src/backend/common/moddims.hpp b/src/backend/common/moddims.hpp
index a132db018c..c127407753 100644
--- a/src/backend/common/moddims.hpp
+++ b/src/backend/common/moddims.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace common {
 
 /// Modifies the shape of the Array<T> object to \p newDims
@@ -39,3 +40,4 @@ template<typename T>
 detail::Array<T> flat(const detail::Array<T> &in);
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/module_loading.hpp b/src/backend/common/module_loading.hpp
index 5a28c5bb9e..c64231a49a 100644
--- a/src/backend/common/module_loading.hpp
+++ b/src/backend/common/module_loading.hpp
@@ -9,6 +9,7 @@
 
 #include <common/defines.hpp>
 
+namespace arrayfire {
 namespace common {
 
 void* getFunctionPointer(LibHandle handle, const char* symbolName);
@@ -20,3 +21,4 @@ void unloadLibrary(LibHandle handle);
 std::string getErrorMessage();
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/module_loading_unix.cpp b/src/backend/common/module_loading_unix.cpp
index 81dc4e391c..8380cdf3b1 100644
--- a/src/backend/common/module_loading_unix.cpp
+++ b/src/backend/common/module_loading_unix.cpp
@@ -15,6 +15,7 @@
 #include <string>
 using std::string;
 
+namespace arrayfire {
 namespace common {
 
 void* getFunctionPointer(LibHandle handle, const char* symbolName) {
@@ -35,3 +36,4 @@ string getErrorMessage() {
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/module_loading_windows.cpp b/src/backend/common/module_loading_windows.cpp
index 7415792951..bccf1e9bbc 100644
--- a/src/backend/common/module_loading_windows.cpp
+++ b/src/backend/common/module_loading_windows.cpp
@@ -15,6 +15,7 @@
 
 using std::string;
 
+namespace arrayfire {
 namespace common {
 
 void* getFunctionPointer(LibHandle handle, const char* symbolName) {
@@ -40,3 +41,4 @@ string getErrorMessage() {
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/sparse_helpers.hpp b/src/backend/common/sparse_helpers.hpp
index 7a370bc38c..daec047eb3 100644
--- a/src/backend/common/sparse_helpers.hpp
+++ b/src/backend/common/sparse_helpers.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace common {
 
 class SparseArrayBase;
@@ -60,3 +61,4 @@ template<typename T>
 SparseArray<T> copySparseArray(const SparseArray<T> &other);
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/tile.hpp b/src/backend/common/tile.hpp
index 512d14b62b..b6ccdd2f60 100644
--- a/src/backend/common/tile.hpp
+++ b/src/backend/common/tile.hpp
@@ -17,6 +17,7 @@
 
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace common {
 
 /// duplicates the elements of an Array<T> array.
@@ -46,3 +47,4 @@ detail::Array<T> tile(const detail::Array<T> &in, const af::dim4 tileDims) {
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/traits.hpp b/src/backend/common/traits.hpp
index cfd07b8a0e..2b9090727c 100644
--- a/src/backend/common/traits.hpp
+++ b/src/backend/common/traits.hpp
@@ -16,6 +16,7 @@ template<typename T>
 struct dtype_traits;
 }
 
+namespace arrayfire {
 namespace common {
 class half;
 
@@ -69,12 +70,13 @@ constexpr bool isFloating(af::dtype type) {
 
 }  // namespace
 }  // namespace common
+}  // namespace arrayfire
 
 namespace af {
 template<>
-struct dtype_traits<common::half> {
+struct dtype_traits<arrayfire::common::half> {
     enum { af_type = f16, ctype = f16 };
-    typedef common::half base_type;
+    typedef arrayfire::common::half base_type;
     static const char *getName() { return "half"; }
 };
 }  // namespace af
diff --git a/src/backend/common/unique_handle.hpp b/src/backend/common/unique_handle.hpp
index 0c3fe8fe6f..c55e2ddf81 100644
--- a/src/backend/common/unique_handle.hpp
+++ b/src/backend/common/unique_handle.hpp
@@ -12,6 +12,7 @@
 
 #include <utility>
 
+namespace arrayfire {
 namespace common {
 
 template<typename T>
@@ -117,8 +118,10 @@ unique_handle<T> make_handle(Args... args) {
 }
 
 }  // namespace common
+}  // namespace arrayfire
 
 #define DEFINE_HANDLER(HANDLE_TYPE, HCREATOR, HDESTROYER)            \
+    namespace arrayfire {                                            \
     namespace common {                                               \
     template<>                                                       \
     class ResourceHandler<HANDLE_TYPE> {                             \
@@ -131,4 +134,5 @@ unique_handle<T> make_handle(Args... args) {
             return HDESTROYER(handle);                               \
         }                                                            \
     };                                                               \
-    }  // namespace common
+    }                                                                \
+    }
diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp
index f6d39a864e..a4cc1e2421 100644
--- a/src/backend/common/util.cpp
+++ b/src/backend/common/util.cpp
@@ -61,6 +61,7 @@ using std::to_string;
 using std::uint8_t;
 using std::vector;
 
+namespace arrayfire {
 namespace common {
 // http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring/217605#217605
 // trim from start
@@ -521,3 +522,4 @@ string toString(af_homography_type val) {
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/util.hpp b/src/backend/common/util.hpp
index 896223e140..ce154775f9 100644
--- a/src/backend/common/util.hpp
+++ b/src/backend/common/util.hpp
@@ -15,6 +15,7 @@
 
 #include <string>
 
+namespace arrayfire {
 namespace common {
 /// The environment variable that determines where the runtime kernels
 /// will be stored on the file system
@@ -59,3 +60,4 @@ template<typename T>
 std::string toString(T value);
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index 159fd2aa7c..9498fa36aa 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -38,12 +38,12 @@
 #include <utility>
 
 using af::dim4;
-using common::half;
-using common::Node;
-using common::Node_map_t;
-using common::Node_ptr;
-using common::NodeIterator;
-using cpu::jit::BufferNode;
+using arrayfire::common::half;
+using arrayfire::common::Node;
+using arrayfire::common::Node_map_t;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::NodeIterator;
+using arrayfire::cpu::jit::BufferNode;
 
 using nonstd::span;
 using std::accumulate;
@@ -55,6 +55,7 @@ using std::make_shared;
 using std::move;
 using std::vector;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -368,3 +369,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/Array.hpp b/src/backend/cpu/Array.hpp
index 8db2ee7e44..120d24b373 100644
--- a/src/backend/cpu/Array.hpp
+++ b/src/backend/cpu/Array.hpp
@@ -28,6 +28,7 @@
 #include <memory>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 
 namespace jit {
@@ -291,3 +292,4 @@ class Array {
 };
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/Event.cpp b/src/backend/cpu/Event.cpp
index e0c67519d9..8cdf94338c 100644
--- a/src/backend/cpu/Event.cpp
+++ b/src/backend/cpu/Event.cpp
@@ -18,6 +18,7 @@
 
 using std::make_unique;
 
+namespace arrayfire {
 namespace cpu {
 /// \brief Creates a new event and marks it in the queue
 Event makeEvent(cpu::queue& queue) {
@@ -68,3 +69,4 @@ af_event createAndMarkEvent() {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/Event.hpp b/src/backend/cpu/Event.hpp
index 2d15039cfb..103bc3e9ee 100644
--- a/src/backend/cpu/Event.hpp
+++ b/src/backend/cpu/Event.hpp
@@ -14,6 +14,7 @@
 
 #include <type_traits>
 
+namespace arrayfire {
 namespace cpu {
 
 class CPUEventPolicy {
@@ -58,3 +59,4 @@ void block(af_event eventHandle);
 af_event createAndMarkEvent();
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/Param.hpp b/src/backend/cpu/Param.hpp
index 20686c4430..55b507876a 100644
--- a/src/backend/cpu/Param.hpp
+++ b/src/backend/cpu/Param.hpp
@@ -12,6 +12,7 @@
 #include <af/defines.h>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 /// \brief Constant parameter object who's memory cannot be modified. Params
@@ -153,3 +154,4 @@ CParam<T> toParam(const Array<T> &val) noexcept {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/ParamIterator.hpp b/src/backend/cpu/ParamIterator.hpp
index ba2189bdeb..3d6427853e 100644
--- a/src/backend/cpu/ParamIterator.hpp
+++ b/src/backend/cpu/ParamIterator.hpp
@@ -16,6 +16,7 @@
 #include <iterator>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 
 /// A Param iterator that iterates through a Param object
@@ -137,3 +138,4 @@ ParamIterator<const T> end(CParam<T>& param) {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/anisotropic_diffusion.cpp b/src/backend/cpu/anisotropic_diffusion.cpp
index 97818aea50..7d38cbe5ab 100644
--- a/src/backend/cpu/anisotropic_diffusion.cpp
+++ b/src/backend/cpu/anisotropic_diffusion.cpp
@@ -11,6 +11,7 @@
 #include <kernel/anisotropic_diffusion.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
@@ -33,3 +34,4 @@ void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
 INSTANTIATE(double)
 INSTANTIATE(float)
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/anisotropic_diffusion.hpp b/src/backend/cpu/anisotropic_diffusion.hpp
index bf82cbde46..76d1f9ddcf 100644
--- a/src/backend/cpu/anisotropic_diffusion.hpp
+++ b/src/backend/cpu/anisotropic_diffusion.hpp
@@ -9,6 +9,7 @@
 
 #include "af/defines.h"
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 class Array;
@@ -18,3 +19,4 @@ void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
                           const af::fluxFunction fftype,
                           const af::diffusionEq eq);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/approx.cpp b/src/backend/cpu/approx.cpp
index 1d027eba2c..f65cd18961 100644
--- a/src/backend/cpu/approx.cpp
+++ b/src/backend/cpu/approx.cpp
@@ -12,6 +12,7 @@
 #include <platform.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename Ty, typename Tp>
@@ -88,3 +89,4 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/approx.hpp b/src/backend/cpu/approx.hpp
index 21a79bcb54..893250a824 100644
--- a/src/backend/cpu/approx.hpp
+++ b/src/backend/cpu/approx.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
@@ -23,3 +24,4 @@ void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
              const Tp &yi_step, const af_interp_type method,
              const float offGrid);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/arith.hpp b/src/backend/cpu/arith.hpp
index 7a8e5a2402..131f9ae64a 100644
--- a/src/backend/cpu/arith.hpp
+++ b/src/backend/cpu/arith.hpp
@@ -13,6 +13,7 @@
 #include <common/jit/BinaryNode.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, af_op_t op>
@@ -28,3 +29,4 @@ Array<T> arithOp(const Array<T> &lhs, const Array<T> &rhs,
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/assign.cpp b/src/backend/cpu/assign.cpp
index 0f32fab35d..cfeb5e168e 100644
--- a/src/backend/cpu/assign.cpp
+++ b/src/backend/cpu/assign.cpp
@@ -28,6 +28,7 @@
 using af::dim4;
 using std::vector;
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs) {
@@ -69,6 +70,7 @@ INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
 INSTANTIATE(short)
-INSTANTIATE(common::half)
+INSTANTIATE(arrayfire::common::half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/assign.hpp b/src/backend/cpu/assign.hpp
index 8a9536c14d..ccbdec5ddf 100644
--- a/src/backend/cpu/assign.hpp
+++ b/src/backend/cpu/assign.hpp
@@ -9,6 +9,7 @@
 
 #include <af/index.h>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 class Array;
@@ -17,3 +18,4 @@ template<typename T>
 void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs);
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/backend.hpp b/src/backend/cpu/backend.hpp
index 744fa8f290..ba9f9677d3 100644
--- a/src/backend/cpu/backend.hpp
+++ b/src/backend/cpu/backend.hpp
@@ -21,4 +21,4 @@
 
 #include "types.hpp"
 
-namespace detail = cpu;
+namespace detail = arrayfire::cpu;
diff --git a/src/backend/cpu/bilateral.cpp b/src/backend/cpu/bilateral.cpp
index 995e464302..027afb2c3b 100644
--- a/src/backend/cpu/bilateral.cpp
+++ b/src/backend/cpu/bilateral.cpp
@@ -17,6 +17,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename inType, typename outType>
@@ -42,3 +43,4 @@ INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/bilateral.hpp b/src/backend/cpu/bilateral.hpp
index 543f7eeff0..1cb6edb1e1 100644
--- a/src/backend/cpu/bilateral.hpp
+++ b/src/backend/cpu/bilateral.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename inType, typename outType>
 Array<outType> bilateral(const Array<inType> &in, const float &spatialSigma,
                          const float &chromaticSigma);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/binary.hpp b/src/backend/cpu/binary.hpp
index 1af89bd3a6..3d130ba520 100644
--- a/src/backend/cpu/binary.hpp
+++ b/src/backend/cpu/binary.hpp
@@ -14,6 +14,7 @@
 #include <types.hpp>
 #include <cmath>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename To, typename Ti, af_op_t op>
@@ -151,3 +152,4 @@ NUMERIC_FN(af_atan2_t, atan2)
 NUMERIC_FN(af_hypot_t, hypot)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/blas.cpp b/src/backend/cpu/blas.cpp
index 463c3e8fe1..b7d158eb21 100644
--- a/src/backend/cpu/blas.cpp
+++ b/src/backend/cpu/blas.cpp
@@ -34,12 +34,13 @@
 #include <vector>
 
 using af::dtype_traits;
-using common::cast;
-using common::half;
-using common::is_complex;
+using arrayfire::common::cast;
+using arrayfire::common::half;
+using arrayfire::common::is_complex;
 using std::conditional;
 using std::vector;
 
+namespace arrayfire {
 namespace cpu {
 
 // clang-format off
@@ -392,3 +393,4 @@ INSTANTIATE_DOT(cfloat);
 INSTANTIATE_DOT(cdouble);
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/blas.hpp b/src/backend/cpu/blas.hpp
index 956ba6a963..1043a567e9 100644
--- a/src/backend/cpu/blas.hpp
+++ b/src/backend/cpu/blas.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -34,3 +35,4 @@ Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
              af_mat_prop optRhs);
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/canny.cpp b/src/backend/cpu/canny.cpp
index 55ac39049a..17f242c0fc 100644
--- a/src/backend/cpu/canny.cpp
+++ b/src/backend/cpu/canny.cpp
@@ -15,6 +15,7 @@
 #include <platform.hpp>
 #include <queue.hpp>
 
+namespace arrayfire {
 namespace cpu {
 Array<float> nonMaximumSuppression(const Array<float>& mag,
                                    const Array<float>& gx,
@@ -35,3 +36,4 @@ Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
     return out;
 }
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/canny.hpp b/src/backend/cpu/canny.hpp
index e2910fd2a1..7f21d89fe5 100644
--- a/src/backend/cpu/canny.hpp
+++ b/src/backend/cpu/canny.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 Array<float> nonMaximumSuppression(const Array<float>& mag,
                                    const Array<float>& gx,
@@ -17,3 +18,4 @@ Array<float> nonMaximumSuppression(const Array<float>& mag,
 Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
                                      const Array<char>& weak);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/cast.hpp b/src/backend/cpu/cast.hpp
index 992030407a..dd756eb2b3 100644
--- a/src/backend/cpu/cast.hpp
+++ b/src/backend/cpu/cast.hpp
@@ -17,6 +17,7 @@
 #include <af/dim4.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename To, typename Ti>
@@ -33,8 +34,8 @@ struct UnOp<To, Ti, af_cast_t> {
 /// TODO(umar): make a macro to reduce repeat code
 
 template<typename To>
-struct UnOp<To, common::half, af_cast_t> {
-    typedef common::half Ti;
+struct UnOp<To, arrayfire::common::half, af_cast_t> {
+    typedef arrayfire::common::half Ti;
 
     void eval(jit::array<To> &out, const jit::array<Ti> &in, int lim) {
         for (int i = 0; i < lim; i++) {
@@ -49,8 +50,8 @@ struct UnOp<To, common::half, af_cast_t> {
 };
 
 template<typename Ti>
-struct UnOp<common::half, Ti, af_cast_t> {
-    typedef common::half To;
+struct UnOp<arrayfire::common::half, Ti, af_cast_t> {
+    typedef arrayfire::common::half To;
 
     void eval(jit::array<To> &out, const jit::array<Ti> &in, int lim) {
         for (int i = 0; i < lim; i++) {
@@ -65,8 +66,8 @@ struct UnOp<common::half, Ti, af_cast_t> {
 };
 
 template<>
-struct UnOp<common::half, std::complex<float>, af_cast_t> {
-    typedef common::half To;
+struct UnOp<arrayfire::common::half, std::complex<float>, af_cast_t> {
+    typedef arrayfire::common::half To;
     typedef std::complex<float> Ti;
 
     void eval(jit::array<To> &out, const jit::array<Ti> &in, int lim) {
@@ -82,8 +83,8 @@ struct UnOp<common::half, std::complex<float>, af_cast_t> {
 };
 
 template<>
-struct UnOp<common::half, std::complex<double>, af_cast_t> {
-    typedef common::half To;
+struct UnOp<arrayfire::common::half, std::complex<double>, af_cast_t> {
+    typedef arrayfire::common::half To;
     typedef std::complex<double> Ti;
 
     void eval(jit::array<To> &out, const jit::array<Ti> &in, int lim) {
@@ -153,3 +154,4 @@ CAST_B8(uchar)
 CAST_B8(char)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/cholesky.cpp b/src/backend/cpu/cholesky.cpp
index c4588d3b3e..cd478ad75e 100644
--- a/src/backend/cpu/cholesky.cpp
+++ b/src/backend/cpu/cholesky.cpp
@@ -24,6 +24,7 @@
 #include <triangle.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -87,9 +88,11 @@ INSTANTIATE_CH(double)
 INSTANTIATE_CH(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -113,5 +116,6 @@ INSTANTIATE_CH(double)
 INSTANTIATE_CH(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/cpu/cholesky.hpp b/src/backend/cpu/cholesky.hpp
index 9317718d72..5b1247be4d 100644
--- a/src/backend/cpu/cholesky.hpp
+++ b/src/backend/cpu/cholesky.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
@@ -16,3 +17,4 @@ Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
 template<typename T>
 int cholesky_inplace(Array<T> &in, const bool is_upper);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/complex.hpp b/src/backend/cpu/complex.hpp
index 4d262f7565..44dc574377 100644
--- a/src/backend/cpu/complex.hpp
+++ b/src/backend/cpu/complex.hpp
@@ -15,6 +15,7 @@
 #include <af/dim4.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename To, typename Ti>
@@ -83,3 +84,4 @@ Array<T> conj(const Array<T> &in) {
     return createNodeArray<T>(in.dims(), move(node));
 }
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/convolve.cpp b/src/backend/cpu/convolve.cpp
index d760b724b9..a57ace15f6 100644
--- a/src/backend/cpu/convolve.cpp
+++ b/src/backend/cpu/convolve.cpp
@@ -28,10 +28,11 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::flip;
-using common::half;
-using common::modDims;
+using arrayfire::common::flip;
+using arrayfire::common::half;
+using arrayfire::common::modDims;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename accT>
@@ -256,3 +257,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/convolve.hpp b/src/backend/cpu/convolve.hpp
index e2490e9c96..66963a1d58 100644
--- a/src/backend/cpu/convolve.hpp
+++ b/src/backend/cpu/convolve.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <common/defines.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename accT>
@@ -38,3 +39,4 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
                              const Array<T> &convolved_output, af::dim4 stride,
                              af::dim4 padding, af::dim4 dilation);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/copy.cpp b/src/backend/cpu/copy.cpp
index 0790454957..b1d0985680 100644
--- a/src/backend/cpu/copy.cpp
+++ b/src/backend/cpu/copy.cpp
@@ -23,9 +23,11 @@
 #include <cstdio>
 #include <cstring>
 
-using common::half;  // NOLINT(misc-unused-using-decls) bug in clang-tidy
-using common::is_complex;
+using arrayfire::common::half;  // NOLINT(misc-unused-using-decls) bug in
+                                // clang-tidy
+using arrayfire::common::is_complex;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -150,3 +152,4 @@ INSTANTIATE_GETSCALAR(short)
 INSTANTIATE_GETSCALAR(ushort)
 INSTANTIATE_GETSCALAR(half)
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/copy.hpp b/src/backend/cpu/copy.hpp
index 8aade1fe04..6e68bff2b7 100644
--- a/src/backend/cpu/copy.hpp
+++ b/src/backend/cpu/copy.hpp
@@ -17,6 +17,7 @@ namespace af {
 class dim4;
 }
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -73,3 +74,4 @@ void multiply_inplace(Array<T> &in, double val);
 template<typename T>
 T getScalar(const Array<T> &in);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/device_manager.cpp b/src/backend/cpu/device_manager.cpp
index a95d9f5a5c..e2d5ed6f68 100644
--- a/src/backend/cpu/device_manager.cpp
+++ b/src/backend/cpu/device_manager.cpp
@@ -17,7 +17,7 @@
 #include <cctype>
 #include <sstream>
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 using std::string;
 
 #ifdef CPUID_CAPABLE
@@ -119,11 +119,12 @@ CPUInfo::CPUInfo()
 
 #endif
 
+namespace arrayfire {
 namespace cpu {
 
 DeviceManager::DeviceManager()
     : queues(MAX_QUEUES)
-    , fgMngr(new graphics::ForgeManager())
+    , fgMngr(new common::ForgeManager())
     , memManager(new common::DefaultMemoryManager(
           getDeviceCount(), common::MAX_BUFFERS,
           AF_MEM_DEBUG || AF_CPU_MEM_DEBUG)) {
@@ -180,3 +181,4 @@ void DeviceManager::resetMemoryManagerPinned() {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/device_manager.hpp b/src/backend/cpu/device_manager.hpp
index 3015ae05f6..a67c611d24 100644
--- a/src/backend/cpu/device_manager.hpp
+++ b/src/backend/cpu/device_manager.hpp
@@ -15,7 +15,7 @@
 #include <mutex>
 #include <string>
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 
 #ifndef AF_CPU_MEM_DEBUG
 #define AF_CPU_MEM_DEBUG 0
@@ -86,6 +86,7 @@ class CPUInfo {
     bool mIsHTT;
 };
 
+namespace arrayfire {
 namespace cpu {
 
 class DeviceManager {
@@ -117,7 +118,7 @@ class DeviceManager {
 
     void resetMemoryManagerPinned();
 
-    friend graphics::ForgeManager& forgeManager();
+    friend arrayfire::common::ForgeManager& forgeManager();
 
     void setMemoryManager(std::unique_ptr<MemoryManagerBase> mgr);
 
@@ -136,10 +137,11 @@ class DeviceManager {
 
     // Attributes
     std::vector<queue> queues;
-    std::unique_ptr<graphics::ForgeManager> fgMngr;
+    std::unique_ptr<arrayfire::common::ForgeManager> fgMngr;
     const CPUInfo cinfo;
     std::unique_ptr<MemoryManagerBase> memManager;
     std::mutex mutex;
 };
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/diagonal.cpp b/src/backend/cpu/diagonal.cpp
index 9a8c61fc48..eddd8c0a49 100644
--- a/src/backend/cpu/diagonal.cpp
+++ b/src/backend/cpu/diagonal.cpp
@@ -19,10 +19,12 @@
 #include <algorithm>
 #include <cstdlib>
 
-using common::half;  // NOLINT(misc-unused-using-decls) bug in clang-tidy
-using std::abs;      // NOLINT(misc-unused-using-decls) bug in clang-tidy
-using std::min;      // NOLINT(misc-unused-using-decls) bug in clang-tidy
+using arrayfire::common::half;  // NOLINT(misc-unused-using-decls) bug in
+                                // clang-tidy
+using std::abs;  // NOLINT(misc-unused-using-decls) bug in clang-tidy
+using std::min;  // NOLINT(misc-unused-using-decls) bug in clang-tidy
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -66,3 +68,4 @@ INSTANTIATE_DIAGONAL(ushort)
 INSTANTIATE_DIAGONAL(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/diagonal.hpp b/src/backend/cpu/diagonal.hpp
index f58ce6fcdb..8a3807b913 100644
--- a/src/backend/cpu/diagonal.hpp
+++ b/src/backend/cpu/diagonal.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> diagCreate(const Array<T> &in, const int num);
@@ -16,3 +17,4 @@ Array<T> diagCreate(const Array<T> &in, const int num);
 template<typename T>
 Array<T> diagExtract(const Array<T> &in, const int num);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/diff.cpp b/src/backend/cpu/diff.cpp
index a64b7dbe3c..8e9c67cae1 100644
--- a/src/backend/cpu/diff.cpp
+++ b/src/backend/cpu/diff.cpp
@@ -15,6 +15,7 @@
 
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -61,3 +62,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/diff.hpp b/src/backend/cpu/diff.hpp
index 32913b9391..7a50aec7c2 100644
--- a/src/backend/cpu/diff.hpp
+++ b/src/backend/cpu/diff.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> diff1(const Array<T> &in, const int dim);
@@ -16,3 +17,4 @@ Array<T> diff1(const Array<T> &in, const int dim);
 template<typename T>
 Array<T> diff2(const Array<T> &in, const int dim);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/exampleFunction.cpp b/src/backend/cpu/exampleFunction.cpp
index f912cf7d66..ee7b847524 100644
--- a/src/backend/cpu/exampleFunction.cpp
+++ b/src/backend/cpu/exampleFunction.cpp
@@ -21,6 +21,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -61,3 +62,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/exampleFunction.hpp b/src/backend/cpu/exampleFunction.hpp
index 822ad57186..19a3d151ef 100644
--- a/src/backend/cpu/exampleFunction.hpp
+++ b/src/backend/cpu/exampleFunction.hpp
@@ -10,8 +10,10 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> exampleFunction(const Array<T> &a, const Array<T> &b,
                          const af_someenum_t method);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/fast.cpp b/src/backend/cpu/fast.cpp
index 057cf96552..b8ac38eeaf 100644
--- a/src/backend/cpu/fast.cpp
+++ b/src/backend/cpu/fast.cpp
@@ -23,6 +23,7 @@
 using af::dim4;
 using std::ceil;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -124,3 +125,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/fast.hpp b/src/backend/cpu/fast.hpp
index d588246916..7d22621bb4 100644
--- a/src/backend/cpu/fast.hpp
+++ b/src/backend/cpu/fast.hpp
@@ -7,6 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 class Array;
@@ -18,3 +19,4 @@ unsigned fast(Array<float> &x_out, Array<float> &y_out, Array<float> &score_out,
               const unsigned edge);
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/fft.cpp b/src/backend/cpu/fft.cpp
index fafc178c29..31515d0f99 100644
--- a/src/backend/cpu/fft.cpp
+++ b/src/backend/cpu/fft.cpp
@@ -22,6 +22,7 @@
 using af::dim4;
 using std::array;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -229,3 +230,4 @@ INSTANTIATE_REAL(float, cfloat)
 INSTANTIATE_REAL(double, cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/fft.hpp b/src/backend/cpu/fft.hpp
index fbdf7af339..383690ca21 100644
--- a/src/backend/cpu/fft.hpp
+++ b/src/backend/cpu/fft.hpp
@@ -15,6 +15,7 @@ namespace af {
 class dim4;
 }
 
+namespace arrayfire {
 namespace cpu {
 
 void setFFTPlanCacheSize(size_t numPlans);
@@ -28,3 +29,4 @@ Array<Tc> fft_r2c(const Array<Tr> &in, const int rank);
 template<typename Tr, typename Tc>
 Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/fftconvolve.cpp b/src/backend/cpu/fftconvolve.cpp
index 20047cf5b9..728238c1ef 100644
--- a/src/backend/cpu/fftconvolve.cpp
+++ b/src/backend/cpu/fftconvolve.cpp
@@ -25,6 +25,7 @@ using af::dim4;
 using std::array;
 using std::ceil;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename convT>
@@ -214,3 +215,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/fftconvolve.hpp b/src/backend/cpu/fftconvolve.hpp
index a2b9845dfd..8a21fbe958 100644
--- a/src/backend/cpu/fftconvolve.hpp
+++ b/src/backend/cpu/fftconvolve.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
                      const bool expand, AF_BATCH_KIND kind, const int rank);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/flood_fill.cpp b/src/backend/cpu/flood_fill.cpp
index 7a08663ef3..2ea32df803 100644
--- a/src/backend/cpu/flood_fill.cpp
+++ b/src/backend/cpu/flood_fill.cpp
@@ -14,6 +14,7 @@
 
 using af::connectivity;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -38,3 +39,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/flood_fill.hpp b/src/backend/cpu/flood_fill.hpp
index 8bd4623328..8ac52fbec1 100644
--- a/src/backend/cpu/flood_fill.hpp
+++ b/src/backend/cpu/flood_fill.hpp
@@ -12,6 +12,7 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
@@ -19,3 +20,4 @@ Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
                    const T lowValue, const T highValue,
                    const af::connectivity nlookup = AF_CONNECTIVITY_8);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/gradient.cpp b/src/backend/cpu/gradient.cpp
index 711cd72c49..d328e9f7e4 100644
--- a/src/backend/cpu/gradient.cpp
+++ b/src/backend/cpu/gradient.cpp
@@ -16,6 +16,7 @@
 #include <queue.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -33,3 +34,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/gradient.hpp b/src/backend/cpu/gradient.hpp
index cc18462ba1..d73ecafccf 100644
--- a/src/backend/cpu/gradient.hpp
+++ b/src/backend/cpu/gradient.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/harris.cpp b/src/backend/cpu/harris.cpp
index 29fddc5417..cf7f41ecbf 100644
--- a/src/backend/cpu/harris.cpp
+++ b/src/backend/cpu/harris.cpp
@@ -21,6 +21,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename convAccT>
@@ -148,3 +149,4 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/harris.hpp b/src/backend/cpu/harris.hpp
index c2f587b18d..b42f8cd4f8 100644
--- a/src/backend/cpu/harris.hpp
+++ b/src/backend/cpu/harris.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename convAccT>
@@ -21,4 +22,5 @@ unsigned harris(Array<float> &x_out, Array<float> &y_out,
                 const float sigma, const unsigned filter_len,
                 const float k_thr);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/hist_graphics.cpp b/src/backend/cpu/hist_graphics.cpp
index 4c68d6858e..7635004c91 100644
--- a/src/backend/cpu/hist_graphics.cpp
+++ b/src/backend/cpu/hist_graphics.cpp
@@ -12,11 +12,16 @@
 #include <platform.hpp>
 #include <queue.hpp>
 
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_histogram(const Array<T> &data, fg_histogram hist) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = forgePlugin();
     data.eval();
     getQueue().sync();
 
@@ -43,3 +48,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/hist_graphics.hpp b/src/backend/cpu/hist_graphics.hpp
index 1fd68a1adb..8971645496 100644
--- a/src/backend/cpu/hist_graphics.hpp
+++ b/src/backend/cpu/hist_graphics.hpp
@@ -12,9 +12,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_histogram(const Array<T> &data, fg_histogram hist);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/histogram.cpp b/src/backend/cpu/histogram.cpp
index 2b044efd02..e2f8e15433 100644
--- a/src/backend/cpu/histogram.cpp
+++ b/src/backend/cpu/histogram.cpp
@@ -16,8 +16,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -55,3 +56,4 @@ INSTANTIATE(uintl)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/histogram.hpp b/src/backend/cpu/histogram.hpp
index 650b59d621..086baf50f0 100644
--- a/src/backend/cpu/histogram.hpp
+++ b/src/backend/cpu/histogram.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<uint> histogram(const Array<T> &in, const unsigned &nbins,
                       const double &minval, const double &maxval,
                       const bool isLinear);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/homography.cpp b/src/backend/cpu/homography.cpp
index 9fbdf9fead..9be88a2e02 100644
--- a/src/backend/cpu/homography.cpp
+++ b/src/backend/cpu/homography.cpp
@@ -33,6 +33,7 @@ using std::round;
 using std::sqrt;
 using std::vector;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -420,3 +421,4 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/homography.hpp b/src/backend/cpu/homography.hpp
index 25acd7cb23..76ac8bbf86 100644
--- a/src/backend/cpu/homography.hpp
+++ b/src/backend/cpu/homography.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -18,4 +19,5 @@ int homography(Array<T> &H, const Array<float> &x_src,
                const af_homography_type htype, const float inlier_thr,
                const unsigned iterations);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/hsv_rgb.cpp b/src/backend/cpu/hsv_rgb.cpp
index da3cf25e54..cf278862d0 100644
--- a/src/backend/cpu/hsv_rgb.cpp
+++ b/src/backend/cpu/hsv_rgb.cpp
@@ -14,6 +14,7 @@
 #include <queue.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -42,3 +43,4 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/hsv_rgb.hpp b/src/backend/cpu/hsv_rgb.hpp
index eac988b035..3d0929c22b 100644
--- a/src/backend/cpu/hsv_rgb.hpp
+++ b/src/backend/cpu/hsv_rgb.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> rgb2hsv(const Array<T>& in);
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/identity.cpp b/src/backend/cpu/identity.cpp
index ded01b348e..05695d7629 100644
--- a/src/backend/cpu/identity.cpp
+++ b/src/backend/cpu/identity.cpp
@@ -15,8 +15,10 @@
 #include <queue.hpp>
 #include <af/dim4.hpp>
 
-using common::half;  // NOLINT(misc-unused-using-decls) bug in clang-tidy
+using arrayfire::common::half;  // NOLINT(misc-unused-using-decls) bug in
+                                // clang-tidy
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -46,3 +48,4 @@ INSTANTIATE_IDENTITY(ushort)
 INSTANTIATE_IDENTITY(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/identity.hpp b/src/backend/cpu/identity.hpp
index 805214585c..5a77fa2d9a 100644
--- a/src/backend/cpu/identity.hpp
+++ b/src/backend/cpu/identity.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> identity(const dim4& dim);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/iir.cpp b/src/backend/cpu/iir.cpp
index e1f6c0e4e4..9d3fcfc966 100644
--- a/src/backend/cpu/iir.cpp
+++ b/src/backend/cpu/iir.cpp
@@ -17,6 +17,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -49,3 +50,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/iir.hpp b/src/backend/cpu/iir.hpp
index 2286fd91e6..4075c48b43 100644
--- a/src/backend/cpu/iir.hpp
+++ b/src/backend/cpu/iir.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/image.cpp b/src/backend/cpu/image.cpp
index 4b5e3cd486..f11a2db4ca 100644
--- a/src/backend/cpu/image.cpp
+++ b/src/backend/cpu/image.cpp
@@ -17,11 +17,16 @@
 #include <platform.hpp>
 #include <queue.hpp>
 
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_image(const Array<T> &in, fg_image image) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = forgePlugin();
 
     CheckGL("Before CopyArrayToImage");
     const T *d_X = in.get();
@@ -50,3 +55,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/image.hpp b/src/backend/cpu/image.hpp
index 06493f6850..2dd41e585e 100644
--- a/src/backend/cpu/image.hpp
+++ b/src/backend/cpu/image.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_image(const Array<T> &in, fg_image image);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/index.cpp b/src/backend/cpu/index.cpp
index 9a2172569e..315406b46d 100644
--- a/src/backend/cpu/index.cpp
+++ b/src/backend/cpu/index.cpp
@@ -21,9 +21,11 @@
 #include <vector>
 
 using af::dim4;
-using common::half;  // NOLINT(misc-unused-using-decls) bug in clang-tidy
+using arrayfire::common::half;  // NOLINT(misc-unused-using-decls) bug in
+                                // clang-tidy
 using std::vector;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -77,3 +79,4 @@ INSTANTIATE(short)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/index.hpp b/src/backend/cpu/index.hpp
index d397db3ed7..14a6692db1 100644
--- a/src/backend/cpu/index.hpp
+++ b/src/backend/cpu/index.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/index.h>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 Array<T> index(const Array<T>& in, const af_index_t idxrs[]);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/inverse.cpp b/src/backend/cpu/inverse.cpp
index 47230f21d3..20543d027c 100644
--- a/src/backend/cpu/inverse.cpp
+++ b/src/backend/cpu/inverse.cpp
@@ -25,6 +25,7 @@
 #include <queue.hpp>
 #include <solve.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -76,9 +77,11 @@ INSTANTIATE(double)
 INSTANTIATE(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -94,5 +97,6 @@ INSTANTIATE(double)
 INSTANTIATE(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/cpu/inverse.hpp b/src/backend/cpu/inverse.hpp
index 460b2fd954..476388cb68 100644
--- a/src/backend/cpu/inverse.hpp
+++ b/src/backend/cpu/inverse.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> inverse(const Array<T> &in);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/iota.cpp b/src/backend/cpu/iota.cpp
index 38fb1c292b..1e7155bcd9 100644
--- a/src/backend/cpu/iota.cpp
+++ b/src/backend/cpu/iota.cpp
@@ -15,8 +15,10 @@
 #include <platform.hpp>
 #include <queue.hpp>
 
-using common::half;  // NOLINT(misc-unused-using-decls) bug in clang-tidy
+using arrayfire::common::half;  // NOLINT(misc-unused-using-decls) bug in
+                                // clang-tidy
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -45,3 +47,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/iota.hpp b/src/backend/cpu/iota.hpp
index c8551a14c4..9921933cbf 100644
--- a/src/backend/cpu/iota.hpp
+++ b/src/backend/cpu/iota.hpp
@@ -10,7 +10,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> iota(const dim4 &dim, const dim4 &tile_dims = dim4(1));
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/ireduce.cpp b/src/backend/cpu/ireduce.cpp
index 44b4b302be..435d6ea44d 100644
--- a/src/backend/cpu/ireduce.cpp
+++ b/src/backend/cpu/ireduce.cpp
@@ -18,8 +18,9 @@
 #include <complex>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<af_op_t op, typename T>
@@ -125,3 +126,4 @@ INSTANTIATE(af_max_t, ushort)
 INSTANTIATE(af_max_t, half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/ireduce.hpp b/src/backend/cpu/ireduce.hpp
index 39258a284e..301ee65e53 100644
--- a/src/backend/cpu/ireduce.hpp
+++ b/src/backend/cpu/ireduce.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<af_op_t op, typename T>
 void ireduce(Array<T> &out, Array<uint> &loc, const Array<T> &in,
@@ -22,3 +23,4 @@ void rreduce(Array<T> &out, Array<uint> &loc, const Array<T> &in, const int dim,
 template<af_op_t op, typename T>
 T ireduce_all(unsigned *loc, const Array<T> &in);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/jit/BinaryNode.hpp b/src/backend/cpu/jit/BinaryNode.hpp
index 0ce7e348f4..8c1cc39d68 100644
--- a/src/backend/cpu/jit/BinaryNode.hpp
+++ b/src/backend/cpu/jit/BinaryNode.hpp
@@ -17,6 +17,7 @@
 #include <array>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 
 namespace jit {
@@ -92,5 +93,5 @@ class BinaryNode : public TNode<compute_t<To>> {
 };
 
 }  // namespace jit
-
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/jit/BufferNode.hpp b/src/backend/cpu/jit/BufferNode.hpp
index ac789dc2ee..e6be492b7f 100644
--- a/src/backend/cpu/jit/BufferNode.hpp
+++ b/src/backend/cpu/jit/BufferNode.hpp
@@ -18,6 +18,7 @@
 #include <sstream>
 #include <string>
 
+namespace arrayfire {
 namespace cpu {
 
 namespace jit {
@@ -179,3 +180,4 @@ class BufferNode : public TNode<T> {
 
 }  // namespace jit
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/jit/Node.hpp b/src/backend/cpu/jit/Node.hpp
index 51ec0646ae..b3914cbc70 100644
--- a/src/backend/cpu/jit/Node.hpp
+++ b/src/backend/cpu/jit/Node.hpp
@@ -24,6 +24,7 @@ template<typename T>
 class NodeIterator;
 }
 
+namespace arrayfire {
 namespace cpu {
 
 namespace jit {
@@ -38,7 +39,7 @@ template<typename T>
 class TNode : public common::Node {
    public:
     alignas(16) jit::array<compute_t<T>> m_val;
-    using common::Node::m_children;
+    using arrayfire::common::Node::m_children;
 
    public:
     TNode(T val, const int height,
@@ -53,3 +54,4 @@ class TNode : public common::Node {
 };
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/jit/ScalarNode.hpp b/src/backend/cpu/jit/ScalarNode.hpp
index 79a9f40f22..a6d7eff5df 100644
--- a/src/backend/cpu/jit/ScalarNode.hpp
+++ b/src/backend/cpu/jit/ScalarNode.hpp
@@ -12,6 +12,7 @@
 #include <vector>
 #include "Node.hpp"
 
+namespace arrayfire {
 namespace cpu {
 
 namespace jit {
@@ -62,5 +63,5 @@ class ScalarNode : public TNode<T> {
     bool isScalar() const final { return true; }
 };
 }  // namespace jit
-
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/jit/UnaryNode.hpp b/src/backend/cpu/jit/UnaryNode.hpp
index 527d078dcc..9ae8e0aa94 100644
--- a/src/backend/cpu/jit/UnaryNode.hpp
+++ b/src/backend/cpu/jit/UnaryNode.hpp
@@ -16,6 +16,7 @@
 #include <jit/BufferNode.hpp>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 template<typename To, typename Ti, af_op_t op>
 struct UnOp {
@@ -28,7 +29,7 @@ namespace jit {
 template<typename To, typename Ti, af_op_t op>
 class UnaryNode : public TNode<To> {
    protected:
-    using common::Node::m_children;
+    using arrayfire::common::Node::m_children;
     UnOp<To, Ti, op> m_op;
 
    public:
@@ -70,5 +71,5 @@ class UnaryNode : public TNode<To> {
 };
 
 }  // namespace jit
-
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/join.cpp b/src/backend/cpu/join.cpp
index 52f73747e2..e9fed65df1 100644
--- a/src/backend/cpu/join.cpp
+++ b/src/backend/cpu/join.cpp
@@ -16,8 +16,9 @@
 
 #include <algorithm>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -97,3 +98,4 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/join.hpp b/src/backend/cpu/join.hpp
index efabe9c8a5..f13bea2fed 100644
--- a/src/backend/cpu/join.hpp
+++ b/src/backend/cpu/join.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
@@ -17,3 +18,4 @@ Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
 template<typename T>
 void join(Array<T> &output, const int dim, const std::vector<Array<T>> &inputs);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/Array.hpp b/src/backend/cpu/kernel/Array.hpp
index e13548aa60..7af4e35555 100644
--- a/src/backend/cpu/kernel/Array.hpp
+++ b/src/backend/cpu/kernel/Array.hpp
@@ -18,6 +18,7 @@
 #include <platform.hpp>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -25,7 +26,7 @@ namespace kernel {
 std::vector<std::shared_ptr<common::Node>> cloneNodes(
     const std::vector<common::Node *> &node_index_map,
     const std::vector<common::Node_ids> &ids) {
-    using common::Node;
+    using arrayfire::common::Node;
     // find all moddims in the tree
     std::vector<std::shared_ptr<Node>> node_clones;
     node_clones.reserve(node_index_map.size());
@@ -45,7 +46,7 @@ std::vector<std::shared_ptr<common::Node>> cloneNodes(
 /// new shape
 void propagateModdimsShape(
     std::vector<std::shared_ptr<common::Node>> &node_clones) {
-    using common::NodeIterator;
+    using arrayfire::common::NodeIterator;
     for (auto &node : node_clones) {
         if (node->getOp() == af_moddims_t) {
             common::ModdimNode *mn =
@@ -67,7 +68,7 @@ void propagateModdimsShape(
 /// Removes node_index_map whos operation matchs a unary operation \p op.
 void removeNodeOfOperation(
     std::vector<std::shared_ptr<common::Node>> &node_index_map, af_op_t op) {
-    using common::Node;
+    using arrayfire::common::Node;
 
     for (size_t nid = 0; nid < node_index_map.size(); nid++) {
         auto &node = node_index_map[nid];
@@ -124,10 +125,10 @@ std::vector<TNode<T> *> getClonedOutputNodes(
 template<typename T>
 void evalMultiple(std::vector<Param<T>> arrays,
                   std::vector<common::Node_ptr> output_nodes_) {
-    using common::ModdimNode;
-    using common::Node;
-    using common::Node_map_t;
-    using common::NodeIterator;
+    using arrayfire::common::ModdimNode;
+    using arrayfire::common::Node;
+    using arrayfire::common::Node_map_t;
+    using arrayfire::common::NodeIterator;
 
     af::dim4 odims = arrays[0].dims();
     af::dim4 ostrs = arrays[0].strides();
@@ -205,3 +206,4 @@ void evalMultiple(std::vector<Param<T>> arrays,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/anisotropic_diffusion.hpp b/src/backend/cpu/kernel/anisotropic_diffusion.hpp
index 0a8e773f00..1acad4857c 100644
--- a/src/backend/cpu/kernel/anisotropic_diffusion.hpp
+++ b/src/backend/cpu/kernel/anisotropic_diffusion.hpp
@@ -20,6 +20,7 @@ using std::exp;
 using std::pow;
 using std::sqrt;
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -188,3 +189,4 @@ void anisotropicDiffusion(Param<T> inout, const float dt, const float mct,
 }
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/approx.hpp b/src/backend/cpu/kernel/approx.hpp
index 35f3a2bd78..826b124fdb 100644
--- a/src/backend/cpu/kernel/approx.hpp
+++ b/src/backend/cpu/kernel/approx.hpp
@@ -12,6 +12,7 @@
 #include <math.hpp>
 #include "interp.hpp"
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -137,3 +138,4 @@ void approx2(Param<InT> zo, CParam<InT> zi, CParam<LocT> xo, const int xdim,
 }
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/assign.hpp b/src/backend/cpu/kernel/assign.hpp
index 8a055db0c5..4605f5d000 100644
--- a/src/backend/cpu/kernel/assign.hpp
+++ b/src/backend/cpu/kernel/assign.hpp
@@ -19,6 +19,7 @@
 
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -81,3 +82,4 @@ void assign(Param<T> out, af::dim4 dDims, CParam<T> rhs,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/bilateral.hpp b/src/backend/cpu/kernel/bilateral.hpp
index a2f316d15f..72d8edd12c 100644
--- a/src/backend/cpu/kernel/bilateral.hpp
+++ b/src/backend/cpu/kernel/bilateral.hpp
@@ -13,6 +13,7 @@
 #include <utility.hpp>
 #include <cmath>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -86,3 +87,4 @@ void bilateral(Param<OutT> out, CParam<InT> in, float const s_sigma,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/canny.hpp b/src/backend/cpu/kernel/canny.hpp
index ebf3474cf8..e68b73cfb6 100644
--- a/src/backend/cpu/kernel/canny.hpp
+++ b/src/backend/cpu/kernel/canny.hpp
@@ -13,6 +13,7 @@
 #include <cassert>
 #include <list>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 template<typename T>
@@ -182,3 +183,4 @@ void edgeTrackingHysteresis(Param<T> out, CParam<T> strong, CParam<T> weak) {
 }
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/convolve.hpp b/src/backend/cpu/kernel/convolve.hpp
index 1bb67b569f..62381dd749 100644
--- a/src/backend/cpu/kernel/convolve.hpp
+++ b/src/backend/cpu/kernel/convolve.hpp
@@ -12,6 +12,7 @@
 #include <math.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -289,3 +290,4 @@ void convolve2(Param<InT> out, CParam<InT> signal, CParam<AccT> c_filter,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/copy.hpp b/src/backend/cpu/kernel/copy.hpp
index 618d5deb22..9506ed7d70 100644
--- a/src/backend/cpu/kernel/copy.hpp
+++ b/src/backend/cpu/kernel/copy.hpp
@@ -15,6 +15,7 @@
 
 #include <cstring>  //memcpy
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -160,3 +161,4 @@ void copy(Param<OutT> dst, CParam<InT> src) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/diagonal.hpp b/src/backend/cpu/kernel/diagonal.hpp
index e5de90f41d..388bd4c459 100644
--- a/src/backend/cpu/kernel/diagonal.hpp
+++ b/src/backend/cpu/kernel/diagonal.hpp
@@ -13,6 +13,7 @@
 
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -62,3 +63,4 @@ void diagExtract(Param<T> out, CParam<T> in, int const num) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/diff.hpp b/src/backend/cpu/kernel/diff.hpp
index 9e2e8a4e21..b1ed5642b6 100644
--- a/src/backend/cpu/kernel/diff.hpp
+++ b/src/backend/cpu/kernel/diff.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -80,3 +81,4 @@ void diff2(Param<T> out, CParam<T> in, int const dim) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/dot.hpp b/src/backend/cpu/kernel/dot.hpp
index 8946534bb8..74ea9087c3 100644
--- a/src/backend/cpu/kernel/dot.hpp
+++ b/src/backend/cpu/kernel/dot.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -49,3 +50,4 @@ void dot(Param<T> output, CParam<T> lhs, CParam<T> rhs, af_mat_prop optLhs,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/exampleFunction.hpp b/src/backend/cpu/kernel/exampleFunction.hpp
index 853f96e60c..6b263830ab 100644
--- a/src/backend/cpu/kernel/exampleFunction.hpp
+++ b/src/backend/cpu/kernel/exampleFunction.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -47,3 +48,4 @@ void exampleFunction(Param<T> out, CParam<T> a, CParam<T> b,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/fast.hpp b/src/backend/cpu/kernel/fast.hpp
index b168021903..341ddbe701 100644
--- a/src/backend/cpu/kernel/fast.hpp
+++ b/src/backend/cpu/kernel/fast.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -215,3 +216,4 @@ void non_maximal(CParam<float> score, CParam<float> x_in, CParam<float> y_in,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/fftconvolve.hpp b/src/backend/cpu/kernel/fftconvolve.hpp
index d6c6f8493e..13109502c7 100644
--- a/src/backend/cpu/kernel/fftconvolve.hpp
+++ b/src/backend/cpu/kernel/fftconvolve.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -251,3 +252,4 @@ void reorder(Param<T> out, Param<convT> packed, CParam<T> filter,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/flood_fill.hpp b/src/backend/cpu/kernel/flood_fill.hpp
index 045564ef44..121adc87e6 100644
--- a/src/backend/cpu/kernel/flood_fill.hpp
+++ b/src/backend/cpu/kernel/flood_fill.hpp
@@ -15,6 +15,7 @@
 #include <queue>
 #include <utility>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -119,3 +120,4 @@ void floodFill(Param<T> out, CParam<T> in, CParam<uint> x, CParam<uint> y,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/gradient.hpp b/src/backend/cpu/kernel/gradient.hpp
index 35f1fa8248..407f4fc6da 100644
--- a/src/backend/cpu/kernel/gradient.hpp
+++ b/src/backend/cpu/kernel/gradient.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -84,3 +85,4 @@ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/harris.hpp b/src/backend/cpu/kernel/harris.hpp
index 7ea9350642..4b717c6187 100644
--- a/src/backend/cpu/kernel/harris.hpp
+++ b/src/backend/cpu/kernel/harris.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -118,3 +119,4 @@ static void keep_corners(Param<float> xOut, Param<float> yOut,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/histogram.hpp b/src/backend/cpu/kernel/histogram.hpp
index 4b18f94b5b..fb90631c52 100644
--- a/src/backend/cpu/kernel/histogram.hpp
+++ b/src/backend/cpu/kernel/histogram.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <types.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -47,3 +48,4 @@ void histogram(Param<uint> out, CParam<T> in, const unsigned nbins,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/hsv_rgb.hpp b/src/backend/cpu/kernel/hsv_rgb.hpp
index dd75815be2..1bf4c387bc 100644
--- a/src/backend/cpu/kernel/hsv_rgb.hpp
+++ b/src/backend/cpu/kernel/hsv_rgb.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <cmath>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -117,3 +118,4 @@ void rgb2hsv(Param<T> out, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/identity.hpp b/src/backend/cpu/kernel/identity.hpp
index 1c3b1cf12e..a00a2cc83c 100644
--- a/src/backend/cpu/kernel/identity.hpp
+++ b/src/backend/cpu/kernel/identity.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -32,3 +33,4 @@ void identity(Param<T> out) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/iir.hpp b/src/backend/cpu/kernel/iir.hpp
index b355c7dcbb..515d778f5d 100644
--- a/src/backend/cpu/kernel/iir.hpp
+++ b/src/backend/cpu/kernel/iir.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -52,3 +53,4 @@ void iir(Param<T> y, Param<T> c, CParam<T> a) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/index.hpp b/src/backend/cpu/kernel/index.hpp
index 605d1009d9..2a6a6d9bc4 100644
--- a/src/backend/cpu/kernel/index.hpp
+++ b/src/backend/cpu/kernel/index.hpp
@@ -12,6 +12,7 @@
 #include <utility.hpp>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -64,3 +65,4 @@ void index(Param<T> out, CParam<T> in, const af::dim4 dDims,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/interp.hpp b/src/backend/cpu/kernel/interp.hpp
index b0a9c18f5e..d316b22f19 100644
--- a/src/backend/cpu/kernel/interp.hpp
+++ b/src/backend/cpu/kernel/interp.hpp
@@ -13,6 +13,7 @@
 #include <af/constants.h>
 #include <type_traits>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -349,3 +350,4 @@ struct Interp2<InT, LocT, 3> {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/iota.hpp b/src/backend/cpu/kernel/iota.hpp
index e59151b82b..ef575a8166 100644
--- a/src/backend/cpu/kernel/iota.hpp
+++ b/src/backend/cpu/kernel/iota.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -39,3 +40,4 @@ void iota(Param<T> output, const af::dim4& sdims) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/ireduce.hpp b/src/backend/cpu/kernel/ireduce.hpp
index c04cbc7409..9c371498c7 100644
--- a/src/backend/cpu/kernel/ireduce.hpp
+++ b/src/backend/cpu/kernel/ireduce.hpp
@@ -12,6 +12,7 @@
 #include <common/Binary.hpp>
 #include <algorithm>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -104,3 +105,4 @@ struct ireduce_dim<op, T, 0> {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/join.hpp b/src/backend/cpu/kernel/join.hpp
index a81f8801fa..800ded1270 100644
--- a/src/backend/cpu/kernel/join.hpp
+++ b/src/backend/cpu/kernel/join.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -60,3 +61,4 @@ void join(const int dim, Param<T> out, const std::vector<CParam<T>> inputs,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/lookup.hpp b/src/backend/cpu/kernel/lookup.hpp
index fe333eb8cd..f968e48ff8 100644
--- a/src/backend/cpu/kernel/lookup.hpp
+++ b/src/backend/cpu/kernel/lookup.hpp
@@ -12,6 +12,7 @@
 #include <utility.hpp>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -60,3 +61,4 @@ void lookup(Param<InT> out, CParam<InT> input, CParam<IndexT> indices,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/lu.hpp b/src/backend/cpu/kernel/lu.hpp
index c1473a7918..170289919c 100644
--- a/src/backend/cpu/kernel/lu.hpp
+++ b/src/backend/cpu/kernel/lu.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -73,3 +74,4 @@ void convertPivot(Param<int> p, Param<int> pivot) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/match_template.hpp b/src/backend/cpu/kernel/match_template.hpp
index d2463bf3b0..bed6ef5354 100644
--- a/src/backend/cpu/kernel/match_template.hpp
+++ b/src/backend/cpu/kernel/match_template.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -140,3 +141,4 @@ void matchTemplate(Param<OutT> out, CParam<InT> sImg, CParam<InT> tImg) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/mean.hpp b/src/backend/cpu/kernel/mean.hpp
index 86f30e515c..c15773687e 100644
--- a/src/backend/cpu/kernel/mean.hpp
+++ b/src/backend/cpu/kernel/mean.hpp
@@ -11,6 +11,7 @@
 #include <Array.hpp>
 #include <common/Transform.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -123,3 +124,4 @@ struct mean_dim<Ti, Tw, To, 0> {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/meanshift.hpp b/src/backend/cpu/kernel/meanshift.hpp
index 141153bb75..490fb93af6 100644
--- a/src/backend/cpu/kernel/meanshift.hpp
+++ b/src/backend/cpu/kernel/meanshift.hpp
@@ -13,6 +13,7 @@
 #include <type_traits>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 template<typename T, bool IsColor>
@@ -139,3 +140,4 @@ void meanShift(Param<T> out, CParam<T> in, const float spatialSigma,
 }
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/medfilt.hpp b/src/backend/cpu/kernel/medfilt.hpp
index 269348cee5..cd998adf05 100644
--- a/src/backend/cpu/kernel/medfilt.hpp
+++ b/src/backend/cpu/kernel/medfilt.hpp
@@ -14,6 +14,7 @@
 #include <algorithm>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -202,3 +203,4 @@ void medfilt2(Param<T> out, CParam<T> in, dim_t w_len, dim_t w_wid) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/moments.hpp b/src/backend/cpu/kernel/moments.hpp
index f67b2deb48..0f3e6611eb 100644
--- a/src/backend/cpu/kernel/moments.hpp
+++ b/src/backend/cpu/kernel/moments.hpp
@@ -13,6 +13,7 @@
 #include <utility.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -58,3 +59,4 @@ void moments(Param<float> output, CParam<T> input, af_moment_type moment) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/morph.hpp b/src/backend/cpu/kernel/morph.hpp
index 1142940ba6..563420e57f 100644
--- a/src/backend/cpu/kernel/morph.hpp
+++ b/src/backend/cpu/kernel/morph.hpp
@@ -13,6 +13,7 @@
 #include <utility.hpp>
 #include <limits>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 template<typename T>
@@ -143,3 +144,4 @@ void morph3d(Param<T> out, CParam<T> in, CParam<T> mask) {
 }
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/nearest_neighbour.hpp b/src/backend/cpu/kernel/nearest_neighbour.hpp
index 39b005c4ed..af94d03ec4 100644
--- a/src/backend/cpu/kernel/nearest_neighbour.hpp
+++ b/src/backend/cpu/kernel/nearest_neighbour.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -98,3 +99,4 @@ void nearest_neighbour(Param<To> dists, CParam<T> query, CParam<T> train,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/orb.hpp b/src/backend/cpu/kernel/orb.hpp
index df36f3655b..385f71abb6 100644
--- a/src/backend/cpu/kernel/orb.hpp
+++ b/src/backend/cpu/kernel/orb.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -281,3 +282,4 @@ void extract_orb(unsigned* desc_out, const unsigned n_feat, float* x_in_out,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/pad_array_borders.hpp b/src/backend/cpu/kernel/pad_array_borders.hpp
index 5d9ea155a3..8b44c9d425 100644
--- a/src/backend/cpu/kernel/pad_array_borders.hpp
+++ b/src/backend/cpu/kernel/pad_array_borders.hpp
@@ -14,6 +14,7 @@
 
 #include <algorithm>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 namespace {
@@ -130,3 +131,4 @@ void padBorders(Param<T> out, CParam<T> in, const dim4 lBoundPadSize,
 }
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/random_engine.hpp b/src/backend/cpu/kernel/random_engine.hpp
index 6f55f69719..09c2bff20c 100644
--- a/src/backend/cpu/kernel/random_engine.hpp
+++ b/src/backend/cpu/kernel/random_engine.hpp
@@ -25,6 +25,7 @@
 using std::array;
 using std::memcpy;
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 // Utils
@@ -70,21 +71,21 @@ static float getFloatNegative11(uint *val, uint index) {
 }
 
 // Generates rationals in [0, 1)
-common::half getHalf01(uint *val, uint index) {
+arrayfire::common::half getHalf01(uint *val, uint index) {
     float v = val[index >> 1U] >> (16U * (index & 1U)) & 0x0000ffff;
-    return static_cast<common::half>(
+    return static_cast<arrayfire::common::half>(
         fmaf(v, unsigned_half_factor, unsigned_half_half_factor));
 }
 
 // Generates rationals in (-1, 1]
-static common::half getHalfNegative11(uint *val, uint index) {
+static arrayfire::common::half getHalfNegative11(uint *val, uint index) {
     float v = val[index >> 1U] >> (16U * (index & 1U)) & 0x0000ffff;
     // Conversion to half adapted from Random123
     constexpr float factor =
         ((1.0f) / (std::numeric_limits<short>::max() + (1.0f)));
     constexpr float half_factor = ((0.5f) * factor);
 
-    return static_cast<common::half>(fmaf(v, factor, half_factor));
+    return static_cast<arrayfire::common::half>(fmaf(v, factor, half_factor));
 }
 
 // Generates rationals in [0, 1)
@@ -154,9 +155,10 @@ double transform<double>(uint *val, uint index) {
 }
 
 template<>
-common::half transform<common::half>(uint *val, uint index) {
+arrayfire::common::half transform<arrayfire::common::half>(uint *val,
+                                                           uint index) {
     float v = val[index >> 1U] >> (16U * (index & 1U)) & 0x0000ffff;
-    return static_cast<common::half>(
+    return static_cast<arrayfire::common::half>(
         1.f - fmaf(v, unsigned_half_factor, unsigned_half_half_factor));
 }
 
@@ -274,8 +276,8 @@ void boxMullerTransform(uint val[4], float *temp) {
                               getFloat01(val, 3));
 }
 
-void boxMullerTransform(uint val[4], common::half *temp) {
-    using common::half;
+void boxMullerTransform(uint val[4], arrayfire::common::half *temp) {
+    using arrayfire::common::half;
     boxMullerTransform<half>(&temp[0], &temp[1], getHalfNegative11(val, 0),
                              getHalf01(val, 1));
     boxMullerTransform<half>(&temp[2], &temp[3], getHalfNegative11(val, 2),
@@ -416,3 +418,4 @@ void normalDistributionCBRNG(T *out, size_t elements,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/random_engine_mersenne.hpp b/src/backend/cpu/kernel/random_engine_mersenne.hpp
index ada96f231e..5087621b26 100644
--- a/src/backend/cpu/kernel/random_engine_mersenne.hpp
+++ b/src/backend/cpu/kernel/random_engine_mersenne.hpp
@@ -44,6 +44,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -117,3 +118,4 @@ void initMersenneState(uint* const state, const uint* const tbl,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/random_engine_philox.hpp b/src/backend/cpu/kernel/random_engine_philox.hpp
index 7b2efd45f9..f1a82014df 100644
--- a/src/backend/cpu/kernel/random_engine_philox.hpp
+++ b/src/backend/cpu/kernel/random_engine_philox.hpp
@@ -47,6 +47,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 // Utils
@@ -103,3 +104,4 @@ void philox(uint* const key, uint* const ctr) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/random_engine_threefry.hpp b/src/backend/cpu/kernel/random_engine_threefry.hpp
index 8affc5bcaa..df728c9a81 100644
--- a/src/backend/cpu/kernel/random_engine_threefry.hpp
+++ b/src/backend/cpu/kernel/random_engine_threefry.hpp
@@ -46,6 +46,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 // Utils
@@ -156,3 +157,4 @@ static inline void threefry(uint k[2], uint c[2], uint X[2]) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/range.hpp b/src/backend/cpu/kernel/range.hpp
index dd6995386f..8d93d384be 100644
--- a/src/backend/cpu/kernel/range.hpp
+++ b/src/backend/cpu/kernel/range.hpp
@@ -13,6 +13,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -48,3 +49,4 @@ void range(Param<T> output) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/reduce.hpp b/src/backend/cpu/kernel/reduce.hpp
index db39dbc8b8..de685b426a 100644
--- a/src/backend/cpu/kernel/reduce.hpp
+++ b/src/backend/cpu/kernel/reduce.hpp
@@ -13,6 +13,7 @@
 #include <common/Transform.hpp>
 #include <common/half.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -200,3 +201,4 @@ struct reduce_all {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/regions.hpp b/src/backend/cpu/kernel/regions.hpp
index 40aa507b74..fab7398720 100644
--- a/src/backend/cpu/kernel/regions.hpp
+++ b/src/backend/cpu/kernel/regions.hpp
@@ -13,6 +13,7 @@
 #include <map>
 #include <set>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -167,3 +168,4 @@ void regions(Param<T> out, CParam<char> in, af_connectivity connectivity) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/reorder.hpp b/src/backend/cpu/kernel/reorder.hpp
index b038d4920b..ccaf8efc72 100644
--- a/src/backend/cpu/kernel/reorder.hpp
+++ b/src/backend/cpu/kernel/reorder.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -48,3 +49,4 @@ void reorder(Param<T> out, CParam<T> in, const af::dim4 oDims,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/resize.hpp b/src/backend/cpu/kernel/resize.hpp
index 0a3d3a0e33..d5e1a3f6b9 100644
--- a/src/backend/cpu/kernel/resize.hpp
+++ b/src/backend/cpu/kernel/resize.hpp
@@ -13,6 +13,7 @@
 #include <math.hpp>
 #include <af/traits.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -173,3 +174,4 @@ void resize(Param<T> out, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/rotate.hpp b/src/backend/cpu/kernel/rotate.hpp
index af2e21f31d..67a34a9e71 100644
--- a/src/backend/cpu/kernel/rotate.hpp
+++ b/src/backend/cpu/kernel/rotate.hpp
@@ -16,6 +16,7 @@
 
 using af::dtype_traits;
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -89,3 +90,4 @@ void rotate(Param<T> output, CParam<T> input, const float theta,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/scan.hpp b/src/backend/cpu/kernel/scan.hpp
index 6e6cc84d54..3ad4e04688 100644
--- a/src/backend/cpu/kernel/scan.hpp
+++ b/src/backend/cpu/kernel/scan.hpp
@@ -12,6 +12,7 @@
 #include <common/Binary.hpp>
 #include <common/Transform.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -72,3 +73,4 @@ struct scan_dim<op, Ti, To, 0, inclusive_scan> {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/scan_by_key.hpp b/src/backend/cpu/kernel/scan_by_key.hpp
index d4546377e0..4639dfcda7 100644
--- a/src/backend/cpu/kernel/scan_by_key.hpp
+++ b/src/backend/cpu/kernel/scan_by_key.hpp
@@ -12,6 +12,7 @@
 #include <common/Binary.hpp>
 #include <common/Transform.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -86,3 +87,4 @@ struct scan_dim_by_key<op, Ti, Tk, To, 0> {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/select.hpp b/src/backend/cpu/kernel/select.hpp
index 88a95fd5bc..dcc3c8855c 100644
--- a/src/backend/cpu/kernel/select.hpp
+++ b/src/backend/cpu/kernel/select.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -120,3 +121,4 @@ void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const T b) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/shift.hpp b/src/backend/cpu/kernel/shift.hpp
index ea844439e9..223c3081a0 100644
--- a/src/backend/cpu/kernel/shift.hpp
+++ b/src/backend/cpu/kernel/shift.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <cassert>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -63,3 +64,4 @@ void shift(Param<T> out, CParam<T> in, const af::dim4 sdims) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sift.hpp b/src/backend/cpu/kernel/sift.hpp
index e7d4821e37..ee1eb046a7 100644
--- a/src/backend/cpu/kernel/sift.hpp
+++ b/src/backend/cpu/kernel/sift.hpp
@@ -26,6 +26,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 static const float PI_VAL = 3.14159265358979323846f;
@@ -1053,3 +1054,4 @@ unsigned sift_impl(Array<float>& x, Array<float>& y, Array<float>& score,
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sobel.hpp b/src/backend/cpu/kernel/sobel.hpp
index 1bf3203874..54315203d4 100644
--- a/src/backend/cpu/kernel/sobel.hpp
+++ b/src/backend/cpu/kernel/sobel.hpp
@@ -14,6 +14,7 @@
 
 #include <cassert>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -73,3 +74,4 @@ void derivative(Param<To> output, CParam<Ti> input) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sort.hpp b/src/backend/cpu/kernel/sort.hpp
index 5c0bf21a99..0e4c91aa56 100644
--- a/src/backend/cpu/kernel/sort.hpp
+++ b/src/backend/cpu/kernel/sort.hpp
@@ -15,6 +15,7 @@
 #include <functional>
 #include <numeric>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -45,3 +46,4 @@ void sort0Iterative(Param<T> val, bool isAscending) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sort_by_key.hpp b/src/backend/cpu/kernel/sort_by_key.hpp
index 9f67a570c0..785a25b378 100644
--- a/src/backend/cpu/kernel/sort_by_key.hpp
+++ b/src/backend/cpu/kernel/sort_by_key.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -25,3 +26,4 @@ void sort0ByKey(Param<Tk> okey, Param<Tv> oval, bool isAscending);
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp
index c1ae75110e..6ac6875f3e 100644
--- a/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp
+++ b/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp
@@ -11,8 +11,10 @@
 
 // SBK_TYPES:float double int uint intl uintl short ushort char uchar
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 INSTANTIATE1(TYPE)
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sort_by_key_impl.hpp b/src/backend/cpu/kernel/sort_by_key_impl.hpp
index c10ac89747..acd7524a9b 100644
--- a/src/backend/cpu/kernel/sort_by_key_impl.hpp
+++ b/src/backend/cpu/kernel/sort_by_key_impl.hpp
@@ -20,6 +20,7 @@
 #include <tuple>
 #include <utility>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -171,5 +172,7 @@ void sort0ByKey(Param<Tk> okey, Param<Tv> oval, bool isAscending) {
     INSTANTIATE(Tk, uchar)   \
     INSTANTIATE(Tk, intl)    \
     INSTANTIATE(Tk, uintl)
+
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sort_helper.hpp b/src/backend/cpu/kernel/sort_helper.hpp
index 955460bf86..ff301c0e0a 100644
--- a/src/backend/cpu/kernel/sort_helper.hpp
+++ b/src/backend/cpu/kernel/sort_helper.hpp
@@ -10,6 +10,7 @@
 #include <err_cpu.hpp>
 #include <tuple>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 template<typename Tk, typename Tv>
@@ -60,3 +61,4 @@ struct KIPCompareK {
 };
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sparse.hpp b/src/backend/cpu/kernel/sparse.hpp
index a8b796a702..9cf8074d80 100644
--- a/src/backend/cpu/kernel/sparse.hpp
+++ b/src/backend/cpu/kernel/sparse.hpp
@@ -15,6 +15,7 @@
 #include <algorithm>
 #include <tuple>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -173,3 +174,4 @@ void coo2csr(Param<T> ovalues, Param<int> orowIdx, Param<int> ocolIdx,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/sparse_arith.hpp b/src/backend/cpu/kernel/sparse_arith.hpp
index 2c4afcfb8f..07eae80aca 100644
--- a/src/backend/cpu/kernel/sparse_arith.hpp
+++ b/src/backend/cpu/kernel/sparse_arith.hpp
@@ -13,6 +13,7 @@
 
 #include <cmath>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -223,3 +224,4 @@ void sparseArithOp(Param<T> oVals, Param<int> oColIdx, CParam<int> oRowIdx,
 }
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/susan.hpp b/src/backend/cpu/kernel/susan.hpp
index 13dee51519..161f185f8b 100644
--- a/src/backend/cpu/kernel/susan.hpp
+++ b/src/backend/cpu/kernel/susan.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -94,3 +95,4 @@ void non_maximal(Param<float> xcoords, Param<float> ycoords,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/tile.hpp b/src/backend/cpu/kernel/tile.hpp
index 5fdaba9db7..bb533889ac 100644
--- a/src/backend/cpu/kernel/tile.hpp
+++ b/src/backend/cpu/kernel/tile.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -48,3 +49,4 @@ void tile(Param<T> out, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/transform.hpp b/src/backend/cpu/kernel/transform.hpp
index f0e388cbe7..bfa1485629 100644
--- a/src/backend/cpu/kernel/transform.hpp
+++ b/src/backend/cpu/kernel/transform.hpp
@@ -14,6 +14,7 @@
 #include <type_traits>
 #include "interp.hpp"
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -140,3 +141,4 @@ void transform(Param<T> output, CParam<T> input, CParam<float> transform,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/transpose.hpp b/src/backend/cpu/kernel/transpose.hpp
index 6ea41b65df..5c9a254401 100644
--- a/src/backend/cpu/kernel/transpose.hpp
+++ b/src/backend/cpu/kernel/transpose.hpp
@@ -12,6 +12,7 @@
 #include <err_cpu.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -178,3 +179,4 @@ void transpose_inplace(Param<T> in, const bool conjugate) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/triangle.hpp b/src/backend/cpu/kernel/triangle.hpp
index 40ba7e4591..3c6051ce0b 100644
--- a/src/backend/cpu/kernel/triangle.hpp
+++ b/src/backend/cpu/kernel/triangle.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -56,3 +57,4 @@ void triangle(Param<T> out, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/unwrap.hpp b/src/backend/cpu/kernel/unwrap.hpp
index 2b4e4f662d..e9cd6675a3 100644
--- a/src/backend/cpu/kernel/unwrap.hpp
+++ b/src/backend/cpu/kernel/unwrap.hpp
@@ -12,6 +12,7 @@
 #include <err_cpu.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -80,3 +81,4 @@ void unwrap_dim(Param<T> out, CParam<T> in, const dim_t wx, const dim_t wy,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/kernel/wrap.hpp b/src/backend/cpu/kernel/wrap.hpp
index 6b574ee158..0a6eb63a5d 100644
--- a/src/backend/cpu/kernel/wrap.hpp
+++ b/src/backend/cpu/kernel/wrap.hpp
@@ -14,6 +14,7 @@
 
 #include <algorithm>
 
+namespace arrayfire {
 namespace cpu {
 namespace kernel {
 
@@ -144,3 +145,4 @@ void wrap_dim_dilated(Param<T> out, CParam<T> in, const dim_t wx,
 
 }  // namespace kernel
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/logic.hpp b/src/backend/cpu/logic.hpp
index b5ed91f615..40a90e0167 100644
--- a/src/backend/cpu/logic.hpp
+++ b/src/backend/cpu/logic.hpp
@@ -14,6 +14,7 @@
 #include <types.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, af_op_t op>
@@ -28,3 +29,4 @@ Array<T> bitOp(const Array<T> &lhs, const Array<T> &rhs,
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/lookup.cpp b/src/backend/cpu/lookup.cpp
index 9eda1f9253..8a5c40d55c 100644
--- a/src/backend/cpu/lookup.cpp
+++ b/src/backend/cpu/lookup.cpp
@@ -14,8 +14,9 @@
 #include <queue.hpp>
 #include <cstdlib>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
@@ -69,3 +70,4 @@ INSTANTIATE(ushort);
 INSTANTIATE(short);
 INSTANTIATE(half);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/lookup.hpp b/src/backend/cpu/lookup.hpp
index cd5f72a78d..c21a757d10 100644
--- a/src/backend/cpu/lookup.hpp
+++ b/src/backend/cpu/lookup.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                    const unsigned dim);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/lu.cpp b/src/backend/cpu/lu.cpp
index 22a3a25d57..43df22e90c 100644
--- a/src/backend/cpu/lu.cpp
+++ b/src/backend/cpu/lu.cpp
@@ -23,6 +23,7 @@
 #include <cassert>
 #include <iostream>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -88,9 +89,11 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
 bool isLAPACKAvailable() { return true; }
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -107,9 +110,11 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
 bool isLAPACKAvailable() { return false; }
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 #define INSTANTIATE_LU(T)                                        \
@@ -124,3 +129,4 @@ INSTANTIATE_LU(double)
 INSTANTIATE_LU(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/lu.hpp b/src/backend/cpu/lu.hpp
index 4092d4445c..d114d4f2b4 100644
--- a/src/backend/cpu/lu.hpp
+++ b/src/backend/cpu/lu.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
@@ -19,3 +20,4 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot = true);
 
 bool isLAPACKAvailable();
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/match_template.cpp b/src/backend/cpu/match_template.cpp
index 5b609ad0a7..d3cfb26b4a 100644
--- a/src/backend/cpu/match_template.cpp
+++ b/src/backend/cpu/match_template.cpp
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename To, typename Ti>
@@ -55,3 +56,4 @@ INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/match_template.hpp b/src/backend/cpu/match_template.hpp
index ebe78e6023..6fbbec0a9e 100644
--- a/src/backend/cpu/match_template.hpp
+++ b/src/backend/cpu/match_template.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename inType, typename outType>
 Array<outType> match_template(const Array<inType> &sImg,
                               const Array<inType> &tImg,
                               const af::matchType mType);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/math.cpp b/src/backend/cpu/math.cpp
index 04e426e48a..07b037a30a 100644
--- a/src/backend/cpu/math.cpp
+++ b/src/backend/cpu/math.cpp
@@ -10,6 +10,7 @@
 #include <math.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace cpu {
 
 uint abs(uint val) { return val; }
@@ -39,3 +40,4 @@ cdouble max(cdouble lhs, cdouble rhs) {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/math.hpp b/src/backend/cpu/math.hpp
index f55530f531..d2735acd2a 100644
--- a/src/backend/cpu/math.hpp
+++ b/src/backend/cpu/math.hpp
@@ -18,6 +18,7 @@
 #include <limits>
 #include <numeric>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 static inline T abs(T val) {
@@ -76,8 +77,8 @@ inline double maxval() {
     return std::numeric_limits<double>::infinity();
 }
 template<>
-inline common::half maxval() {
-    return std::numeric_limits<common::half>::infinity();
+inline arrayfire::common::half maxval() {
+    return std::numeric_limits<arrayfire::common::half>::infinity();
 }
 template<>
 inline float minval() {
@@ -88,8 +89,8 @@ inline double minval() {
     return -std::numeric_limits<double>::infinity();
 }
 template<>
-inline common::half minval() {
-    return -std::numeric_limits<common::half>::infinity();
+inline arrayfire::common::half minval() {
+    return -std::numeric_limits<arrayfire::common::half>::infinity();
 }
 
 template<typename T>
@@ -113,3 +114,4 @@ inline double imag(cdouble in) noexcept { return std::imag(in); }
 inline float imag(cfloat in) noexcept { return std::imag(in); }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/mean.cpp b/src/backend/cpu/mean.cpp
index 6da92b98e2..6a256113f7 100644
--- a/src/backend/cpu/mean.cpp
+++ b/src/backend/cpu/mean.cpp
@@ -19,8 +19,9 @@
 #include <complex>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename Ti, typename Tw, typename To>
@@ -159,3 +160,4 @@ INSTANTIATE_WGT(cdouble, double);
 INSTANTIATE_WGT(half, float);
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/mean.hpp b/src/backend/cpu/mean.hpp
index ecc481c203..7079a91528 100644
--- a/src/backend/cpu/mean.hpp
+++ b/src/backend/cpu/mean.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename Ti, typename Tw, typename To>
 Array<To> mean(const Array<Ti>& in, const int dim);
@@ -22,3 +23,4 @@ T mean(const Array<T>& in, const Array<Tw>& wts);
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/meanshift.cpp b/src/backend/cpu/meanshift.cpp
index e8a0f55ba4..d52b56a99e 100644
--- a/src/backend/cpu/meanshift.cpp
+++ b/src/backend/cpu/meanshift.cpp
@@ -21,6 +21,7 @@
 using af::dim4;
 using std::vector;
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
@@ -55,3 +56,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/meanshift.hpp b/src/backend/cpu/meanshift.hpp
index b8ba8d2c24..c17d922414 100644
--- a/src/backend/cpu/meanshift.hpp
+++ b/src/backend/cpu/meanshift.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
                    const float &chromaticSigma, const unsigned &numIterations,
                    const bool &isColor);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/medfilt.cpp b/src/backend/cpu/medfilt.cpp
index cb24b81c43..53497be8c9 100644
--- a/src/backend/cpu/medfilt.cpp
+++ b/src/backend/cpu/medfilt.cpp
@@ -19,6 +19,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -67,3 +68,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/medfilt.hpp b/src/backend/cpu/medfilt.hpp
index 25f3ff2fe6..5d9f8e688c 100644
--- a/src/backend/cpu/medfilt.hpp
+++ b/src/backend/cpu/medfilt.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -20,3 +21,4 @@ Array<T> medfilt2(const Array<T> &in, const int w_len, const int w_wid,
                   const af::borderType edge_pad);
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/memory.cpp b/src/backend/cpu/memory.cpp
index f64bed56ff..440680b48d 100644
--- a/src/backend/cpu/memory.cpp
+++ b/src/backend/cpu/memory.cpp
@@ -22,12 +22,13 @@
 #include <utility>
 
 using af::dim4;
-using common::bytesToString;
-using common::half;
+using arrayfire::common::bytesToString;
+using arrayfire::common::half;
 using std::function;
 using std::move;
 using std::unique_ptr;
 
+namespace arrayfire {
 namespace cpu {
 float getMemoryPressure() { return memoryManager().getMemoryPressure(); }
 float getMemoryPressureThreshold() {
@@ -156,3 +157,4 @@ void Allocator::nativeFree(void *ptr) {
     free(ptr);  // NOLINT(hicpp-no-malloc)
 }
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/memory.hpp b/src/backend/cpu/memory.hpp
index bdd7365559..a45ca06ec1 100644
--- a/src/backend/cpu/memory.hpp
+++ b/src/backend/cpu/memory.hpp
@@ -14,6 +14,7 @@
 #include <functional>
 #include <memory>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 using uptr = std::unique_ptr<T[], std::function<void(T[])>>;
@@ -52,7 +53,7 @@ bool jitTreeExceedsMemoryPressure(size_t bytes);
 void setMemStepSize(size_t step_bytes);
 size_t getMemStepSize(void);
 
-class Allocator final : public common::memory::AllocatorInterface {
+class Allocator final : public common::AllocatorInterface {
    public:
     Allocator();
     ~Allocator() = default;
@@ -64,3 +65,4 @@ class Allocator final : public common::memory::AllocatorInterface {
 };
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/moments.cpp b/src/backend/cpu/moments.cpp
index aedb9bc214..bd5c520eac 100644
--- a/src/backend/cpu/moments.cpp
+++ b/src/backend/cpu/moments.cpp
@@ -14,6 +14,7 @@
 #include <queue.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 
 static inline unsigned bitCount(unsigned v) {
@@ -54,3 +55,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/moments.hpp b/src/backend/cpu/moments.hpp
index 20a4ff4ed0..43793307da 100644
--- a/src/backend/cpu/moments.hpp
+++ b/src/backend/cpu/moments.hpp
@@ -10,7 +10,9 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<float> moments(const Array<T> &in, const af_moment_type moment);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/morph.cpp b/src/backend/cpu/morph.cpp
index eca2424cb5..add13de416 100644
--- a/src/backend/cpu/morph.cpp
+++ b/src/backend/cpu/morph.cpp
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation) {
@@ -70,3 +71,4 @@ INSTANTIATE(uchar)
 INSTANTIATE(ushort)
 INSTANTIATE(short)
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/morph.hpp b/src/backend/cpu/morph.hpp
index cf9e46bd9f..d1fabb47f7 100644
--- a/src/backend/cpu/morph.hpp
+++ b/src/backend/cpu/morph.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
@@ -16,3 +17,4 @@ Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
 template<typename T>
 Array<T> morph3d(const Array<T> &in, const Array<T> &mask, bool isDilation);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/nearest_neighbour.cpp b/src/backend/cpu/nearest_neighbour.cpp
index 916d43d416..2979090dd9 100644
--- a/src/backend/cpu/nearest_neighbour.cpp
+++ b/src/backend/cpu/nearest_neighbour.cpp
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename To>
@@ -73,3 +74,4 @@ INSTANTIATE(short, int)
 INSTANTIATE(uintl, uint)  // For Hamming
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/nearest_neighbour.hpp b/src/backend/cpu/nearest_neighbour.hpp
index 22e190cb16..0c5bd401d9 100644
--- a/src/backend/cpu/nearest_neighbour.hpp
+++ b/src/backend/cpu/nearest_neighbour.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename To>
@@ -17,4 +18,5 @@ void nearest_neighbour(Array<uint>& idx, Array<To>& dist, const Array<T>& query,
                        const uint n_dist,
                        const af_match_type dist_type = AF_SSD);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/orb.cpp b/src/backend/cpu/orb.cpp
index 0a415c5cee..f03eb6427b 100644
--- a/src/backend/cpu/orb.cpp
+++ b/src/backend/cpu/orb.cpp
@@ -37,6 +37,7 @@ using std::sqrt;
 using std::unique_ptr;
 using std::vector;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename convAccT>
@@ -292,3 +293,4 @@ INSTANTIATE(float, float)
 INSTANTIATE(double, double)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/orb.hpp b/src/backend/cpu/orb.hpp
index cfb5904935..8bdd7a92c0 100644
--- a/src/backend/cpu/orb.hpp
+++ b/src/backend/cpu/orb.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename convAccT>
@@ -21,4 +22,5 @@ unsigned orb(Array<float> &x, Array<float> &y, Array<float> &score,
              const unsigned max_feat, const float scl_fctr,
              const unsigned levels, const bool blur_img);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index 8676054136..dc73e76f17 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -21,15 +21,17 @@
 #include <sstream>
 #include <string>
 
-using common::getEnvVar;
-using common::ltrim;
-using common::memory::MemoryManagerBase;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::getEnvVar;
+using arrayfire::common::ltrim;
+using arrayfire::common::MemoryManagerBase;
 using std::endl;
 using std::ostringstream;
 using std::stoi;
 using std::string;
 using std::unique_ptr;
 
+namespace arrayfire {
 namespace cpu {
 
 static string get_system() {
@@ -174,8 +176,7 @@ void resetMemoryManagerPinned() {
     return DeviceManager::getInstance().resetMemoryManagerPinned();
 }
 
-graphics::ForgeManager& forgeManager() {
-    return *(DeviceManager::getInstance().fgMngr);
-}
+ForgeManager& forgeManager() { return *(DeviceManager::getInstance().fgMngr); }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/platform.hpp b/src/backend/cpu/platform.hpp
index f50e16461b..b02a1ca118 100644
--- a/src/backend/cpu/platform.hpp
+++ b/src/backend/cpu/platform.hpp
@@ -12,18 +12,16 @@
 #include <queue.hpp>
 #include <string>
 
-namespace graphics {
-class ForgeManager;
-}
-
+namespace arrayfire {
 namespace common {
-namespace memory {
+class ForgeManager;
 class MemoryManagerBase;
-}
 }  // namespace common
+}  // namespace arrayfire
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 
+namespace arrayfire {
 namespace cpu {
 
 int getBackend();
@@ -67,6 +65,7 @@ void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
 
 void resetMemoryManagerPinned();
 
-graphics::ForgeManager& forgeManager();
+arrayfire::common::ForgeManager& forgeManager();
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/plot.cpp b/src/backend/cpu/plot.cpp
index bc4afa5059..abf1a7b397 100644
--- a/src/backend/cpu/plot.cpp
+++ b/src/backend/cpu/plot.cpp
@@ -15,12 +15,16 @@
 #include <queue.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_plot(const Array<T> &P, fg_plot plot) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = forgePlugin();
     P.eval();
     getQueue().sync();
 
@@ -47,3 +51,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/plot.hpp b/src/backend/cpu/plot.hpp
index f64ec8966c..11063e22f4 100644
--- a/src/backend/cpu/plot.hpp
+++ b/src/backend/cpu/plot.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_plot(const Array<T> &P, fg_plot plot);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/print.hpp b/src/backend/cpu/print.hpp
index 9d9d8da4f1..52e3e62877 100644
--- a/src/backend/cpu/print.hpp
+++ b/src/backend/cpu/print.hpp
@@ -7,6 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+namespace arrayfire {
 namespace cpu {
 // Nothing here
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/qr.cpp b/src/backend/cpu/qr.cpp
index 7cf0595eff..61d6305438 100644
--- a/src/backend/cpu/qr.cpp
+++ b/src/backend/cpu/qr.cpp
@@ -22,6 +22,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -108,9 +109,11 @@ Array<T> qr_inplace(Array<T> &in) {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -124,9 +127,11 @@ Array<T> qr_inplace(Array<T> &in) {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 #define INSTANTIATE_QR(T)                                         \
@@ -140,3 +145,4 @@ INSTANTIATE_QR(double)
 INSTANTIATE_QR(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/qr.hpp b/src/backend/cpu/qr.hpp
index b8a43d4d02..4a3290e61c 100644
--- a/src/backend/cpu/qr.hpp
+++ b/src/backend/cpu/qr.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in);
@@ -16,3 +17,4 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in);
 template<typename T>
 Array<T> qr_inplace(Array<T> &in);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/queue.hpp b/src/backend/cpu/queue.hpp
index 97142f4f1a..594396a78e 100644
--- a/src/backend/cpu/queue.hpp
+++ b/src/backend/cpu/queue.hpp
@@ -48,6 +48,7 @@ using event_impl = threads::event;
 
 #endif
 
+namespace arrayfire {
 namespace cpu {
 
 /// Wraps the async_queue class
@@ -108,3 +109,4 @@ class queue_event {
     operator bool() const noexcept { return event_; }
 };
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/random_engine.cpp b/src/backend/cpu/random_engine.cpp
index d6f6e7c792..3e1c8745c8 100644
--- a/src/backend/cpu/random_engine.cpp
+++ b/src/backend/cpu/random_engine.cpp
@@ -12,8 +12,9 @@
 #include <kernel/random_engine.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl) {
@@ -164,3 +165,4 @@ COMPLEX_NORMAL_DISTRIBUTION(cdouble, double)  // NOLINT
 COMPLEX_NORMAL_DISTRIBUTION(cfloat, float)    // NOLINT
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/random_engine.hpp b/src/backend/cpu/random_engine.hpp
index e2e490167d..adfa7b9fc6 100644
--- a/src/backend/cpu/random_engine.hpp
+++ b/src/backend/cpu/random_engine.hpp
@@ -13,6 +13,7 @@
 #include <backend.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cpu {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl);
@@ -41,3 +42,4 @@ Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
                             Array<uint> recursion_table,
                             Array<uint> temper_table, Array<uint> state);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/range.cpp b/src/backend/cpu/range.cpp
index b2fc132547..3b782837e0 100644
--- a/src/backend/cpu/range.cpp
+++ b/src/backend/cpu/range.cpp
@@ -19,8 +19,9 @@
 #include <numeric>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -59,3 +60,4 @@ INSTANTIATE(short)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/range.hpp b/src/backend/cpu/range.hpp
index 9b30f261f7..b6d0f58bd9 100644
--- a/src/backend/cpu/range.hpp
+++ b/src/backend/cpu/range.hpp
@@ -10,7 +10,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> range(const dim4& dim, const int seq_dim = -1);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/reduce.cpp b/src/backend/cpu/reduce.cpp
index e1baf5daea..6ce141b316 100644
--- a/src/backend/cpu/reduce.cpp
+++ b/src/backend/cpu/reduce.cpp
@@ -21,11 +21,12 @@
 #include <functional>
 
 using af::dim4;
-using common::Binary;
-using common::half;
-using common::Transform;
-using cpu::cdouble;
+using arrayfire::common::Binary;
+using arrayfire::common::half;
+using arrayfire::common::Transform;
+using arrayfire::cpu::cdouble;
 
+namespace arrayfire {
 namespace common {
 
 template<>
@@ -38,7 +39,6 @@ struct Binary<cdouble, af_add_t> {
 };
 
 }  // namespace common
-
 namespace cpu {
 
 template<af_op_t op, typename Ti, typename To>
@@ -250,3 +250,4 @@ INSTANTIATE(af_and_t, ushort, char)
 INSTANTIATE(af_and_t, half, char)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/reduce.hpp b/src/backend/cpu/reduce.hpp
index 3db9b0cc8a..8ff97c51a6 100644
--- a/src/backend/cpu/reduce.hpp
+++ b/src/backend/cpu/reduce.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<af_op_t op, typename Ti, typename To>
 Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan = false,
@@ -24,3 +25,4 @@ template<af_op_t op, typename Ti, typename To>
 Array<To> reduce_all(const Array<Ti> &in, bool change_nan = false,
                      double nanval = 0);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/regions.cpp b/src/backend/cpu/regions.cpp
index 0f6612768d..821a5285c3 100644
--- a/src/backend/cpu/regions.cpp
+++ b/src/backend/cpu/regions.cpp
@@ -21,6 +21,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -43,3 +44,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/regions.hpp b/src/backend/cpu/regions.hpp
index 0e2ce0f319..b1c06b1911 100644
--- a/src/backend/cpu/regions.hpp
+++ b/src/backend/cpu/regions.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 Array<T> regions(const Array<char> &in, af_connectivity connectivity);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/reorder.cpp b/src/backend/cpu/reorder.cpp
index 83d2038f38..67233542bd 100644
--- a/src/backend/cpu/reorder.cpp
+++ b/src/backend/cpu/reorder.cpp
@@ -14,8 +14,9 @@
 #include <platform.hpp>
 #include <queue.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -47,3 +48,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/reorder.hpp b/src/backend/cpu/reorder.hpp
index bc689f74c2..5dee87f401 100644
--- a/src/backend/cpu/reorder.hpp
+++ b/src/backend/cpu/reorder.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> reorder(const Array<T> &in, const af::dim4 &rdims);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/reshape.cpp b/src/backend/cpu/reshape.cpp
index 7844f3a596..b2d46eb066 100644
--- a/src/backend/cpu/reshape.cpp
+++ b/src/backend/cpu/reshape.cpp
@@ -14,6 +14,7 @@
 #include <platform.hpp>
 #include <queue.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void multiply_inplace(Array<T> &in, double val) {
@@ -82,7 +83,7 @@ INSTANTIATE_PAD_ARRAY(uchar)
 INSTANTIATE_PAD_ARRAY(char)
 INSTANTIATE_PAD_ARRAY(ushort)
 INSTANTIATE_PAD_ARRAY(short)
-INSTANTIATE_PAD_ARRAY(common::half)
+INSTANTIATE_PAD_ARRAY(arrayfire::common::half)
 
 #define INSTANTIATE_PAD_ARRAY_COMPLEX(SRC_T)                 \
     template Array<cfloat> reshape<SRC_T, cfloat>(           \
@@ -93,3 +94,4 @@ INSTANTIATE_PAD_ARRAY(common::half)
 INSTANTIATE_PAD_ARRAY_COMPLEX(cfloat)
 INSTANTIATE_PAD_ARRAY_COMPLEX(cdouble)
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/resize.cpp b/src/backend/cpu/resize.cpp
index f5850bb106..4f899d89d8 100644
--- a/src/backend/cpu/resize.cpp
+++ b/src/backend/cpu/resize.cpp
@@ -14,6 +14,7 @@
 #include <queue.hpp>
 #include <resize.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -58,3 +59,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/resize.hpp b/src/backend/cpu/resize.hpp
index 83852f1e29..d31290daf5 100644
--- a/src/backend/cpu/resize.hpp
+++ b/src/backend/cpu/resize.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
                 const af_interp_type method);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/rotate.cpp b/src/backend/cpu/rotate.cpp
index 7a0fada05f..0e9806a2af 100644
--- a/src/backend/cpu/rotate.cpp
+++ b/src/backend/cpu/rotate.cpp
@@ -13,6 +13,7 @@
 #include <queue.hpp>
 #include <rotate.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -58,3 +59,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/rotate.hpp b/src/backend/cpu/rotate.hpp
index 094bc24f92..cf18a7df56 100644
--- a/src/backend/cpu/rotate.hpp
+++ b/src/backend/cpu/rotate.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
                 const af_interp_type method);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/scan.cpp b/src/backend/cpu/scan.cpp
index f4412168d1..af5c4d9efe 100644
--- a/src/backend/cpu/scan.cpp
+++ b/src/backend/cpu/scan.cpp
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<af_op_t op, typename Ti, typename To>
@@ -93,3 +94,4 @@ INSTANTIATE_SCAN_ALL(af_mul_t)
 INSTANTIATE_SCAN_ALL(af_min_t)
 INSTANTIATE_SCAN_ALL(af_max_t)
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/scan.hpp b/src/backend/cpu/scan.hpp
index 431c46b1f9..45cd171092 100644
--- a/src/backend/cpu/scan.hpp
+++ b/src/backend/cpu/scan.hpp
@@ -10,7 +10,9 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusive_scan = true);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/scan_by_key.cpp b/src/backend/cpu/scan_by_key.cpp
index ef7a9d3036..f869098ffd 100644
--- a/src/backend/cpu/scan_by_key.cpp
+++ b/src/backend/cpu/scan_by_key.cpp
@@ -17,6 +17,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
@@ -64,3 +65,4 @@ INSTANTIATE_SCAN_BY_KEY_ALL_OP(af_mul_t)
 INSTANTIATE_SCAN_BY_KEY_ALL_OP(af_min_t)
 INSTANTIATE_SCAN_BY_KEY_ALL_OP(af_max_t)
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/scan_by_key.hpp b/src/backend/cpu/scan_by_key.hpp
index 3bc934d529..414840dc35 100644
--- a/src/backend/cpu/scan_by_key.hpp
+++ b/src/backend/cpu/scan_by_key.hpp
@@ -10,8 +10,10 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
                bool inclusive_scan = true);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/select.cpp b/src/backend/cpu/select.cpp
index a801bb5e86..96849cecd1 100644
--- a/src/backend/cpu/select.cpp
+++ b/src/backend/cpu/select.cpp
@@ -15,8 +15,9 @@
 #include <queue.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -56,3 +57,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/select.hpp b/src/backend/cpu/select.hpp
index b92a8d36c5..1ed5d3969b 100644
--- a/src/backend/cpu/select.hpp
+++ b/src/backend/cpu/select.hpp
@@ -9,6 +9,7 @@
 #pragma once
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
@@ -34,3 +35,4 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
     return out;
 }
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/set.cpp b/src/backend/cpu/set.cpp
index d4bb1612e3..838ad7675e 100644
--- a/src/backend/cpu/set.cpp
+++ b/src/backend/cpu/set.cpp
@@ -19,6 +19,7 @@
 #include <complex>
 #include <vector>
 
+namespace arrayfire {
 namespace cpu {
 
 using af::dim4;
@@ -126,3 +127,4 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/set.hpp b/src/backend/cpu/set.hpp
index 762a7329db..086fcc6866 100644
--- a/src/backend/cpu/set.hpp
+++ b/src/backend/cpu/set.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> setUnique(const Array<T> &in, const bool is_sorted);
@@ -22,3 +23,4 @@ template<typename T>
 Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
                       const bool is_unique);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/shift.cpp b/src/backend/cpu/shift.cpp
index 5126cda592..f8942f641f 100644
--- a/src/backend/cpu/shift.cpp
+++ b/src/backend/cpu/shift.cpp
@@ -13,6 +13,7 @@
 #include <queue.hpp>
 #include <shift.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -42,3 +43,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/shift.hpp b/src/backend/cpu/shift.hpp
index 4f992e7fb0..0e298f16ae 100644
--- a/src/backend/cpu/shift.hpp
+++ b/src/backend/cpu/shift.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> shift(const Array<T> &in, const int sdims[4]);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sift.cpp b/src/backend/cpu/sift.cpp
index 3b7e6b554c..246505a206 100644
--- a/src/backend/cpu/sift.cpp
+++ b/src/backend/cpu/sift.cpp
@@ -13,6 +13,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename convAccT>
@@ -41,3 +42,4 @@ INSTANTIATE(float, float)
 INSTANTIATE(double, double)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sift.hpp b/src/backend/cpu/sift.hpp
index 66f0d191bb..804e52eb27 100644
--- a/src/backend/cpu/sift.hpp
+++ b/src/backend/cpu/sift.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename convAccT>
@@ -23,4 +24,5 @@ unsigned sift(Array<float>& x, Array<float>& y, Array<float>& score,
               const float img_scale, const float feature_ratio,
               const bool compute_GLOH);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sobel.cpp b/src/backend/cpu/sobel.cpp
index 76ecf17dc6..68bddee784 100644
--- a/src/backend/cpu/sobel.cpp
+++ b/src/backend/cpu/sobel.cpp
@@ -17,6 +17,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename Ti, typename To>
@@ -48,3 +49,4 @@ INSTANTIATE(short, int)
 INSTANTIATE(ushort, int)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sobel.hpp b/src/backend/cpu/sobel.hpp
index dcd41b9366..ad1082d18e 100644
--- a/src/backend/cpu/sobel.hpp
+++ b/src/backend/cpu/sobel.hpp
@@ -10,10 +10,12 @@
 #include <Array.hpp>
 #include <utility>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename Ti, typename To>
 std::pair<Array<To>, Array<To>> sobelDerivatives(const Array<Ti> &img,
                                                  const unsigned &ker_size);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/solve.cpp b/src/backend/cpu/solve.cpp
index 52843d2fae..0e8d863817 100644
--- a/src/backend/cpu/solve.cpp
+++ b/src/backend/cpu/solve.cpp
@@ -26,6 +26,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -322,9 +323,11 @@ Array<T> solve(const Array<T> &a, const Array<T> &b,
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -344,9 +347,11 @@ Array<T> solve(const Array<T> &a, const Array<T> &b,
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 #define INSTANTIATE_SOLVE(T)                                                 \
@@ -362,3 +367,4 @@ INSTANTIATE_SOLVE(double)
 INSTANTIATE_SOLVE(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/solve.hpp b/src/backend/cpu/solve.hpp
index 2469a39451..c63ec1252b 100644
--- a/src/backend/cpu/solve.hpp
+++ b/src/backend/cpu/solve.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> solve(const Array<T> &a, const Array<T> &b,
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> solveLU(const Array<T> &a, const Array<int> &pivot, const Array<T> &b,
                  const af_mat_prop options = AF_MAT_NONE);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sort.cpp b/src/backend/cpu/sort.cpp
index 50f44dcae9..e5067a8dba 100644
--- a/src/backend/cpu/sort.cpp
+++ b/src/backend/cpu/sort.cpp
@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <functional>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, int dim>
@@ -104,3 +105,4 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sort.hpp b/src/backend/cpu/sort.hpp
index 4ec954685c..c22dab7c7d 100644
--- a/src/backend/cpu/sort.hpp
+++ b/src/backend/cpu/sort.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sort_by_key.cpp b/src/backend/cpu/sort_by_key.cpp
index e69672e6a4..169b598558 100644
--- a/src/backend/cpu/sort_by_key.cpp
+++ b/src/backend/cpu/sort_by_key.cpp
@@ -17,6 +17,7 @@
 #include <reorder.hpp>
 #include <sort_by_key.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename Tk, typename Tv>
@@ -88,3 +89,4 @@ INSTANTIATE1(intl)
 INSTANTIATE1(uintl)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sort_by_key.hpp b/src/backend/cpu/sort_by_key.hpp
index a8c6fc2078..8ed3bb63f4 100644
--- a/src/backend/cpu/sort_by_key.hpp
+++ b/src/backend/cpu/sort_by_key.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename Tk, typename Tv>
 void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
                  const Array<Tv> &ival, const unsigned dim, bool isAscending);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sort_index.cpp b/src/backend/cpu/sort_index.cpp
index c7ec0b8c05..cec724c85d 100644
--- a/src/backend/cpu/sort_index.cpp
+++ b/src/backend/cpu/sort_index.cpp
@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <numeric>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -81,3 +82,4 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sort_index.hpp b/src/backend/cpu/sort_index.hpp
index e4a3cbf775..b0b50fbf87 100644
--- a/src/backend/cpu/sort_index.hpp
+++ b/src/backend/cpu/sort_index.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void sort_index(Array<T> &okey, Array<unsigned> &oval, const Array<T> &in,
                 const unsigned dim, bool isAscending);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sparse.cpp b/src/backend/cpu/sparse.cpp
index 30c7475292..3641c96a90 100644
--- a/src/backend/cpu/sparse.cpp
+++ b/src/backend/cpu/sparse.cpp
@@ -28,14 +28,15 @@
 
 #include <functional>
 
-using common::cast;
+using arrayfire::common::cast;
 using std::function;
 
+namespace arrayfire {
 namespace cpu {
 
-using common::createArrayDataSparseArray;
-using common::createEmptySparseArray;
-using common::SparseArray;
+using arrayfire::common::createArrayDataSparseArray;
+using arrayfire::common::createEmptySparseArray;
+using arrayfire::common::SparseArray;
 
 template<typename T, af_storage stype>
 SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
@@ -161,3 +162,4 @@ INSTANTIATE_SPARSE(cdouble)
 #undef INSTANTIATE_SPARSE
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sparse.hpp b/src/backend/cpu/sparse.hpp
index 9246a529a1..8709fe199d 100644
--- a/src/backend/cpu/sparse.hpp
+++ b/src/backend/cpu/sparse.hpp
@@ -12,6 +12,7 @@
 #include <Array.hpp>
 #include <common/SparseArray.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T, af_storage stype>
 common::SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in);
@@ -23,3 +24,4 @@ template<typename T, af_storage dest, af_storage src>
 common::SparseArray<T> sparseConvertStorageToStorage(
     const common::SparseArray<T> &in);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sparse_arith.cpp b/src/backend/cpu/sparse_arith.cpp
index f07d9c57c4..d6d7e5391e 100644
--- a/src/backend/cpu/sparse_arith.cpp
+++ b/src/backend/cpu/sparse_arith.cpp
@@ -27,11 +27,12 @@
 #include <string>
 #include <vector>
 
-using common::createArrayDataSparseArray;
-using common::createEmptySparseArray;
-using common::SparseArray;
+using arrayfire::common::createArrayDataSparseArray;
+using arrayfire::common::createEmptySparseArray;
+using arrayfire::common::SparseArray;
 using std::numeric_limits;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -166,3 +167,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sparse_arith.hpp b/src/backend/cpu/sparse_arith.hpp
index f37f55a42d..2563802c4d 100644
--- a/src/backend/cpu/sparse_arith.hpp
+++ b/src/backend/cpu/sparse_arith.hpp
@@ -14,6 +14,7 @@
 #include <optypes.hpp>
 #include <sparse.hpp>
 
+namespace arrayfire {
 namespace cpu {
 // These two functions cannot be overloaded by return type.
 // So have to give them separate names.
@@ -29,3 +30,4 @@ template<typename T, af_op_t op>
 common::SparseArray<T> arithOp(const common::SparseArray<T> &lhs,
                                const common::SparseArray<T> &rhs);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sparse_blas.cpp b/src/backend/cpu/sparse_blas.cpp
index dcb8158d9a..d6bd338575 100644
--- a/src/backend/cpu/sparse_blas.cpp
+++ b/src/backend/cpu/sparse_blas.cpp
@@ -26,6 +26,7 @@
 #include <stdexcept>
 #include <string>
 
+namespace arrayfire {
 namespace cpu {
 
 #ifdef USE_MKL
@@ -462,3 +463,4 @@ INSTANTIATE_SPARSE(cfloat)
 INSTANTIATE_SPARSE(cdouble)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/sparse_blas.hpp b/src/backend/cpu/sparse_blas.hpp
index 54da96c282..f59ef83d60 100644
--- a/src/backend/cpu/sparse_blas.hpp
+++ b/src/backend/cpu/sparse_blas.hpp
@@ -11,10 +11,12 @@
 #include <common/SparseArray.hpp>
 #include <sparse.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhs,
                 af_mat_prop optLhs, af_mat_prop optRhs);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/surface.cpp b/src/backend/cpu/surface.cpp
index 7eb1034d49..e861dbeac7 100644
--- a/src/backend/cpu/surface.cpp
+++ b/src/backend/cpu/surface.cpp
@@ -15,12 +15,16 @@
 #include <surface.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_surface(const Array<T> &P, fg_surface surface) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = common::forgePlugin();
     P.eval();
     getQueue().sync();
 
@@ -48,3 +52,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/surface.hpp b/src/backend/cpu/surface.hpp
index 8437d45e18..1bcf57fac3 100644
--- a/src/backend/cpu/surface.hpp
+++ b/src/backend/cpu/surface.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_surface(const Array<T> &P, fg_surface surface);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/susan.cpp b/src/backend/cpu/susan.cpp
index 7f69925b16..0d79078988 100644
--- a/src/backend/cpu/susan.cpp
+++ b/src/backend/cpu/susan.cpp
@@ -19,6 +19,7 @@
 using af::features;
 using std::shared_ptr;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -77,3 +78,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/susan.hpp b/src/backend/cpu/susan.hpp
index 29504b8f2b..af6640e195 100644
--- a/src/backend/cpu/susan.hpp
+++ b/src/backend/cpu/susan.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -21,4 +22,5 @@ unsigned susan(Array<float> &x_out, Array<float> &y_out,
                const float geom_thr, const float feature_ratio,
                const unsigned edge);
 
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/svd.cpp b/src/backend/cpu/svd.cpp
index 7093689812..75804d240b 100644
--- a/src/backend/cpu/svd.cpp
+++ b/src/backend/cpu/svd.cpp
@@ -18,6 +18,7 @@
 #include <platform.hpp>
 #include <queue.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 #define SVD_FUNC_DEF(FUNC)            \
@@ -85,9 +86,11 @@ void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in) {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T, typename Tr>
@@ -101,9 +104,11 @@ void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in) {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace cpu {
 
 #define INSTANTIATE_SVD(T, Tr)                                           \
@@ -118,3 +123,4 @@ INSTANTIATE_SVD(cfloat, float)
 INSTANTIATE_SVD(cdouble, double)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/svd.hpp b/src/backend/cpu/svd.hpp
index 2019ea57c5..ba667d2032 100644
--- a/src/backend/cpu/svd.hpp
+++ b/src/backend/cpu/svd.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T, typename Tr>
 void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
@@ -16,3 +17,4 @@ void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
 template<typename T, typename Tr>
 void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/tile.cpp b/src/backend/cpu/tile.cpp
index 9d951badf8..d2a8d3ab7c 100644
--- a/src/backend/cpu/tile.cpp
+++ b/src/backend/cpu/tile.cpp
@@ -14,8 +14,9 @@
 #include <common/half.hpp>
 #include <platform.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -53,3 +54,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/tile.hpp b/src/backend/cpu/tile.hpp
index 4e71919789..eee387cb87 100644
--- a/src/backend/cpu/tile.hpp
+++ b/src/backend/cpu/tile.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> tile(const Array<T> &in, const af::dim4 &tileDims);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/topk.cpp b/src/backend/cpu/topk.cpp
index a87d257a8c..0103c3586b 100644
--- a/src/backend/cpu/topk.cpp
+++ b/src/backend/cpu/topk.cpp
@@ -18,12 +18,13 @@
 #include <numeric>
 #include <vector>
 
-using common::half;
+using arrayfire::common::half;
 using std::iota;
 using std::min;
 using std::partial_sort_copy;
 using std::vector;
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
@@ -130,3 +131,4 @@ INSTANTIATE(long long)
 INSTANTIATE(unsigned long long)
 INSTANTIATE(half)
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/topk.hpp b/src/backend/cpu/topk.hpp
index 75cb5e7cfe..0383e13fcf 100644
--- a/src/backend/cpu/topk.hpp
+++ b/src/backend/cpu/topk.hpp
@@ -7,8 +7,10 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void topk(Array<T>& keys, Array<unsigned>& vals, const Array<T>& in,
           const int k, const int dim, const af::topkFunction order);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/transform.cpp b/src/backend/cpu/transform.cpp
index f03dd57919..9a57424250 100644
--- a/src/backend/cpu/transform.cpp
+++ b/src/backend/cpu/transform.cpp
@@ -13,6 +13,7 @@
 #include <platform.hpp>
 #include <transform.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -63,3 +64,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/transform.hpp b/src/backend/cpu/transform.hpp
index e00284980a..1df2b38934 100644
--- a/src/backend/cpu/transform.hpp
+++ b/src/backend/cpu/transform.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const af_interp_type method, const bool inverse,
                const bool perspective);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/transpose.cpp b/src/backend/cpu/transpose.cpp
index 4617f19b97..7cd713afd6 100644
--- a/src/backend/cpu/transpose.cpp
+++ b/src/backend/cpu/transpose.cpp
@@ -18,8 +18,9 @@
 #include <utility>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -58,3 +59,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/transpose.hpp b/src/backend/cpu/transpose.hpp
index 27337bd0fb..565f89cc6c 100644
--- a/src/backend/cpu/transpose.hpp
+++ b/src/backend/cpu/transpose.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 void transpose_inplace(Array<T> &in, const bool conjugate);
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/triangle.cpp b/src/backend/cpu/triangle.cpp
index 6440a286b4..8e3b0569b2 100644
--- a/src/backend/cpu/triangle.cpp
+++ b/src/backend/cpu/triangle.cpp
@@ -15,8 +15,9 @@
 
 #include <functional>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -63,3 +64,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/triangle.hpp b/src/backend/cpu/triangle.hpp
index 8178767b45..01e55f7c0b 100644
--- a/src/backend/cpu/triangle.hpp
+++ b/src/backend/cpu/triangle.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 void triangle(Array<T> &out, const Array<T> &in, const bool is_upper,
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> triangle(const Array<T> &in, const bool is_upper,
                   const bool is_unit_diag);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/types.hpp b/src/backend/cpu/types.hpp
index d0263fbf0b..27a678af82 100644
--- a/src/backend/cpu/types.hpp
+++ b/src/backend/cpu/types.hpp
@@ -11,6 +11,7 @@
 #include <common/kernel_type.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace cpu {
 
 namespace {
@@ -49,8 +50,8 @@ struct kernel_type;
 class half;
 
 template<>
-struct kernel_type<common::half> {
-    using data = common::half;
+struct kernel_type<arrayfire::common::half> {
+    using data = arrayfire::common::half;
 
     // These are the types within a kernel
     using native = float;
@@ -58,3 +59,5 @@ struct kernel_type<common::half> {
     using compute = float;
 };
 }  // namespace common
+
+}  // namespace arrayfire
diff --git a/src/backend/cpu/unary.hpp b/src/backend/cpu/unary.hpp
index 3a1c7677dd..620ed26e8c 100644
--- a/src/backend/cpu/unary.hpp
+++ b/src/backend/cpu/unary.hpp
@@ -14,6 +14,7 @@
 #include <optypes.hpp>
 #include <cmath>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -120,3 +121,4 @@ Array<char> checkOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
 }
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/unwrap.cpp b/src/backend/cpu/unwrap.cpp
index ce062b6b8a..49086fad49 100644
--- a/src/backend/cpu/unwrap.cpp
+++ b/src/backend/cpu/unwrap.cpp
@@ -15,8 +15,9 @@
 #include <platform.hpp>
 #include <unwrap.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> unwrap(const Array<T> &in, const dim_t wx, const dim_t wy,
@@ -62,3 +63,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/unwrap.hpp b/src/backend/cpu/unwrap.hpp
index 260605734d..fcfad88f6f 100644
--- a/src/backend/cpu/unwrap.hpp
+++ b/src/backend/cpu/unwrap.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<T> unwrap(const Array<T> &in, const dim_t wx, const dim_t wy,
                 const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
                 const dim_t dx, const dim_t dy, const bool is_column);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/utility.hpp b/src/backend/cpu/utility.hpp
index f7d74f9162..9cd3de96f0 100644
--- a/src/backend/cpu/utility.hpp
+++ b/src/backend/cpu/utility.hpp
@@ -13,6 +13,7 @@
 #include <cmath>
 #include "backend.hpp"
 
+namespace arrayfire {
 namespace cpu {
 static inline dim_t trimIndex(int const& idx, dim_t const& len) {
     int ret_val = idx;
@@ -47,3 +48,4 @@ void gaussian1D(T* out, int const dim, double sigma = 0.0) {
     for (int k = 0; k < dim; k++) out[k] /= sum;
 }
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/vector_field.cpp b/src/backend/cpu/vector_field.cpp
index 2f9f2d34e4..2a7549de81 100644
--- a/src/backend/cpu/vector_field.cpp
+++ b/src/backend/cpu/vector_field.cpp
@@ -15,13 +15,17 @@
 #include <vector_field.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        fg_vector_field vfield) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = forgePlugin();
     points.eval();
     directions.eval();
     getQueue().sync();
@@ -59,3 +63,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/vector_field.hpp b/src/backend/cpu/vector_field.hpp
index c25a1501e4..a64414e781 100644
--- a/src/backend/cpu/vector_field.hpp
+++ b/src/backend/cpu/vector_field.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        fg_vector_field vfield);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/where.cpp b/src/backend/cpu/where.cpp
index 14dbdddfa5..3eb65015f0 100644
--- a/src/backend/cpu/where.cpp
+++ b/src/backend/cpu/where.cpp
@@ -21,6 +21,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -77,3 +78,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/where.hpp b/src/backend/cpu/where.hpp
index 8ec35b1526..35c671c2b0 100644
--- a/src/backend/cpu/where.hpp
+++ b/src/backend/cpu/where.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 template<typename T>
 Array<uint> where(const Array<T>& in);
-}
+}  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/wrap.cpp b/src/backend/cpu/wrap.cpp
index 6a6c887faa..d502bc85ad 100644
--- a/src/backend/cpu/wrap.cpp
+++ b/src/backend/cpu/wrap.cpp
@@ -15,8 +15,9 @@
 #include <platform.hpp>
 #include <wrap.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -84,3 +85,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cpu/wrap.hpp b/src/backend/cpu/wrap.hpp
index bcfe18ef5e..0bec7c8727 100644
--- a/src/backend/cpu/wrap.hpp
+++ b/src/backend/cpu/wrap.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cpu {
 
 template<typename T>
@@ -22,3 +23,4 @@ Array<T> wrap_dilated(const Array<T> &in, const dim_t ox, const dim_t oy,
                       const dim_t sy, const dim_t px, const dim_t py,
                       const dim_t dx, const dim_t dy, const bool is_column);
 }  // namespace cpu
+}  // namespace arrayfire
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index c6347d1bbe..ea5a7e971a 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -24,11 +24,11 @@
 #include <vector>
 
 using af::dim4;
-using common::half;
-using common::Node;
-using common::Node_ptr;
-using common::NodeIterator;
-using cuda::jit::BufferNode;
+using arrayfire::common::half;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::NodeIterator;
+using arrayfire::cuda::jit::BufferNode;
 
 using nonstd::span;
 using std::accumulate;
@@ -36,6 +36,7 @@ using std::move;
 using std::shared_ptr;
 using std::vector;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -87,14 +88,14 @@ Array<T>::Array(const af::dim4 &dims, const T *const in_data, bool is_device,
         offsetof(Array<T>, info) == 0,
         "Array<T>::info must be the first member variable of Array<T>");
     if (!is_device) {
-        CUDA_CHECK(
-            cudaMemcpyAsync(data.get(), in_data, dims.elements() * sizeof(T),
-                            cudaMemcpyHostToDevice, cuda::getActiveStream()));
+        CUDA_CHECK(cudaMemcpyAsync(data.get(), in_data,
+                                   dims.elements() * sizeof(T),
+                                   cudaMemcpyHostToDevice, getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
     } else if (copy_device) {
         CUDA_CHECK(
             cudaMemcpyAsync(data.get(), in_data, dims.elements() * sizeof(T),
-                            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+                            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
     }
 }
@@ -407,7 +408,7 @@ void writeHostDataArray(Array<T> &arr, const T *const data,
     T *ptr = arr.get();
 
     CUDA_CHECK(cudaMemcpyAsync(ptr, data, bytes, cudaMemcpyHostToDevice,
-                               cuda::getActiveStream()));
+                               getActiveStream()));
     CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 }
 
@@ -419,7 +420,7 @@ void writeDeviceDataArray(Array<T> &arr, const void *const data,
     T *ptr = arr.get();
 
     CUDA_CHECK(cudaMemcpyAsync(ptr, data, bytes, cudaMemcpyDeviceToDevice,
-                               cuda::getActiveStream()));
+                               getActiveStream()));
 }
 
 template<typename T>
@@ -473,3 +474,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp
index 52dbed7aeb..07e06f0681 100644
--- a/src/backend/cuda/Array.hpp
+++ b/src/backend/cuda/Array.hpp
@@ -25,7 +25,9 @@
 #include <nonstd/span.hpp>
 #include <vector>
 
+namespace arrayfire {
 namespace cuda {
+
 using af::dim4;
 
 template<typename T>
@@ -287,3 +289,4 @@ class Array {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index ece17d962f..5e0119d93d 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -129,7 +129,7 @@ file_to_string(
     EXTENSION "hpp"
     OUTPUT_DIR "kernel_headers"
     TARGETS jit_kernel_targets
-    NAMESPACE "cuda"
+    NAMESPACE "arrayfire cuda"
     WITH_EXTENSION
     )
 
@@ -222,7 +222,7 @@ file_to_string(
     EXTENSION "hpp"
     OUTPUT_DIR "nvrtc_kernel_headers"
     TARGETS nvrtc_kernel_targets
-    NAMESPACE "cuda"
+    NAMESPACE "arrayfire cuda"
     WITH_EXTENSION
     NULLTERM
     )
diff --git a/src/backend/cuda/EnqueueArgs.hpp b/src/backend/cuda/EnqueueArgs.hpp
index 9dbac7eaa7..f3fb608b4c 100644
--- a/src/backend/cuda/EnqueueArgs.hpp
+++ b/src/backend/cuda/EnqueueArgs.hpp
@@ -14,6 +14,7 @@
 
 #include <vector>
 
+namespace arrayfire {
 namespace cuda {
 
 ///
@@ -51,3 +52,4 @@ struct EnqueueArgs {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/Event.cpp b/src/backend/cuda/Event.cpp
index 0b0d9618e8..fb5fbff170 100644
--- a/src/backend/cuda/Event.cpp
+++ b/src/backend/cuda/Event.cpp
@@ -17,6 +17,7 @@
 
 #include <memory>
 
+namespace arrayfire {
 namespace cuda {
 /// \brief Creates a new event and marks it in the queue
 Event makeEvent(cudaStream_t queue) {
@@ -69,3 +70,4 @@ af_event createAndMarkEvent() {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/Event.hpp b/src/backend/cuda/Event.hpp
index b6600934e4..2db9679aca 100644
--- a/src/backend/cuda/Event.hpp
+++ b/src/backend/cuda/Event.hpp
@@ -13,6 +13,7 @@
 #include <cuda_runtime_api.h>
 #include <af/event.h>
 
+namespace arrayfire {
 namespace cuda {
 
 class CUDARuntimeEventPolicy {
@@ -64,3 +65,4 @@ void block(af_event eventHandle);
 af_event createAndMarkEvent();
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/GraphicsResourceManager.cpp b/src/backend/cuda/GraphicsResourceManager.cpp
index 5778f72658..cca78f286f 100644
--- a/src/backend/cuda/GraphicsResourceManager.cpp
+++ b/src/backend/cuda/GraphicsResourceManager.cpp
@@ -16,6 +16,7 @@
 #include <err_cuda.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace cuda {
 GraphicsResourceManager::ShrdResVector
 GraphicsResourceManager::registerResources(
@@ -43,3 +44,4 @@ GraphicsResourceManager::registerResources(
     return output;
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/GraphicsResourceManager.hpp b/src/backend/cuda/GraphicsResourceManager.hpp
index ba05c2dbe3..dde6a30ab5 100644
--- a/src/backend/cuda/GraphicsResourceManager.hpp
+++ b/src/backend/cuda/GraphicsResourceManager.hpp
@@ -15,6 +15,7 @@
 #include <map>
 #include <vector>
 
+namespace arrayfire {
 namespace cuda {
 class GraphicsResourceManager
     : public common::InteropManager<GraphicsResourceManager,
@@ -31,3 +32,4 @@ class GraphicsResourceManager
     void operator=(GraphicsResourceManager const &);
 };
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/Kernel.cpp b/src/backend/cuda/Kernel.cpp
index f2f64bdeb0..d72672a1fc 100644
--- a/src/backend/cuda/Kernel.cpp
+++ b/src/backend/cuda/Kernel.cpp
@@ -11,6 +11,7 @@
 
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 Kernel::DevPtrType Kernel::getDevPtr(const char* name) {
@@ -22,22 +23,22 @@ Kernel::DevPtrType Kernel::getDevPtr(const char* name) {
 
 void Kernel::copyToReadOnly(Kernel::DevPtrType dst, Kernel::DevPtrType src,
                             size_t bytes) {
-    CU_CHECK(cuMemcpyDtoDAsync(dst, src, bytes, cuda::getActiveStream()));
+    CU_CHECK(cuMemcpyDtoDAsync(dst, src, bytes, getActiveStream()));
 }
 
 void Kernel::setFlag(Kernel::DevPtrType dst, int* scalarValPtr,
                      const bool syncCopy) {
-    CU_CHECK(cuMemcpyHtoDAsync(dst, scalarValPtr, sizeof(int),
-                               cuda::getActiveStream()));
-    if (syncCopy) { CU_CHECK(cuStreamSynchronize(cuda::getActiveStream())); }
+    CU_CHECK(
+        cuMemcpyHtoDAsync(dst, scalarValPtr, sizeof(int), getActiveStream()));
+    if (syncCopy) { CU_CHECK(cuStreamSynchronize(getActiveStream())); }
 }
 
 int Kernel::getFlag(Kernel::DevPtrType src) {
     int retVal = 0;
-    CU_CHECK(
-        cuMemcpyDtoHAsync(&retVal, src, sizeof(int), cuda::getActiveStream()));
-    CU_CHECK(cuStreamSynchronize(cuda::getActiveStream()));
+    CU_CHECK(cuMemcpyDtoHAsync(&retVal, src, sizeof(int), getActiveStream()));
+    CU_CHECK(cuStreamSynchronize(getActiveStream()));
     return retVal;
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/Kernel.hpp b/src/backend/cuda/Kernel.hpp
index a728940d97..b5375f6ad2 100644
--- a/src/backend/cuda/Kernel.hpp
+++ b/src/backend/cuda/Kernel.hpp
@@ -18,6 +18,7 @@
 #include <cstdlib>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 
 struct Enqueuer {
@@ -72,3 +73,4 @@ class Kernel
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/LookupTable1D.hpp b/src/backend/cuda/LookupTable1D.hpp
index ffbfb0f4c8..f688ac4b7e 100644
--- a/src/backend/cuda/LookupTable1D.hpp
+++ b/src/backend/cuda/LookupTable1D.hpp
@@ -14,6 +14,7 @@
 
 #include <type_traits>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -64,3 +65,4 @@ class LookupTable1D {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/Module.hpp b/src/backend/cuda/Module.hpp
index ceefd2f94e..b5eb028765 100644
--- a/src/backend/cuda/Module.hpp
+++ b/src/backend/cuda/Module.hpp
@@ -17,6 +17,7 @@
 #include <string>
 #include <unordered_map>
 
+namespace arrayfire {
 namespace cuda {
 
 /// CUDA backend wrapper for CUmodule
@@ -57,3 +58,4 @@ class Module : public common::ModuleInterface<CUmodule> {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/Param.hpp b/src/backend/cuda/Param.hpp
index cd1651cae5..817d601eaa 100644
--- a/src/backend/cuda/Param.hpp
+++ b/src/backend/cuda/Param.hpp
@@ -13,6 +13,7 @@
 #include <types.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -77,3 +78,4 @@ class CParam {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/ThrustAllocator.cuh b/src/backend/cuda/ThrustAllocator.cuh
index 917cc5e9ba..21152e6059 100644
--- a/src/backend/cuda/ThrustAllocator.cuh
+++ b/src/backend/cuda/ThrustAllocator.cuh
@@ -16,7 +16,9 @@
 // Below Class definition is found at the following URL
 // http://stackoverflow.com/questions/9007343/mix-custom-memory-managment-and-thrust-in-cuda
 
+namespace arrayfire {
 namespace cuda {
+
 template<typename T>
 struct ThrustAllocator : thrust::device_malloc_allocator<T> {
     // shorthand for the name of the base class
@@ -41,3 +43,4 @@ struct ThrustAllocator : thrust::device_malloc_allocator<T> {
     }
 };
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/ThrustArrayFirePolicy.hpp b/src/backend/cuda/ThrustArrayFirePolicy.hpp
index 6787d405de..189ee558b3 100644
--- a/src/backend/cuda/ThrustArrayFirePolicy.hpp
+++ b/src/backend/cuda/ThrustArrayFirePolicy.hpp
@@ -12,8 +12,10 @@
 #include <backend.hpp>
 #include <memory.hpp>
 #include <platform.hpp>
+#include <thrust/memory.h>
 #include <thrust/system/cuda/execution_policy.h>
 
+namespace arrayfire {
 namespace cuda {
 struct ThrustArrayFirePolicy
     : thrust::cuda::execution_policy<ThrustArrayFirePolicy> {};
@@ -22,7 +24,7 @@ template<typename T>
 thrust::pair<thrust::pointer<T, ThrustArrayFirePolicy>, std::ptrdiff_t>
 get_temporary_buffer(ThrustArrayFirePolicy, std::ptrdiff_t n) {
     thrust::pointer<T, ThrustArrayFirePolicy> result(
-        cuda::memAlloc<T>(n / sizeof(T)).release());
+        arrayfire::cuda::memAlloc<T>(n / sizeof(T)).release());
 
     return thrust::make_pair(result, n);
 }
@@ -33,25 +35,27 @@ inline void return_temporary_buffer(ThrustArrayFirePolicy, Pointer p) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
 
 namespace thrust {
 namespace cuda_cub {
 template<>
-__DH__ inline cudaStream_t get_stream<::cuda::ThrustArrayFirePolicy>(
-    execution_policy<::cuda::ThrustArrayFirePolicy> &) {
+__DH__ inline cudaStream_t get_stream<arrayfire::cuda::ThrustArrayFirePolicy>(
+    execution_policy<arrayfire::cuda::ThrustArrayFirePolicy> &) {
 #if defined(__CUDA_ARCH__)
     return 0;
 #else
-    return ::cuda::getActiveStream();
+    return arrayfire::cuda::getActiveStream();
 #endif
 }
 
 __DH__
-inline cudaError_t synchronize_stream(const ::cuda::ThrustArrayFirePolicy &) {
+inline cudaError_t synchronize_stream(
+    const arrayfire::cuda::ThrustArrayFirePolicy &) {
 #if defined(__CUDA_ARCH__)
     return cudaSuccess;
 #else
-    return cudaStreamSynchronize(::cuda::getActiveStream());
+    return cudaStreamSynchronize(arrayfire::cuda::getActiveStream());
 #endif
 }
 
diff --git a/src/backend/cuda/all.cu b/src/backend/cuda/all.cu
index b681a87384..3ff42ad599 100644
--- a/src/backend/cuda/all.cu
+++ b/src/backend/cuda/all.cu
@@ -7,11 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include "reduce_impl.hpp"
 #include <common/half.hpp>
+#include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 // alltrue
 INSTANTIATE(af_and_t, float, char)
@@ -28,3 +29,4 @@ INSTANTIATE(af_and_t, short, char)
 INSTANTIATE(af_and_t, ushort, char)
 INSTANTIATE(af_and_t, half, char)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/anisotropic_diffusion.cpp b/src/backend/cuda/anisotropic_diffusion.cpp
index 3d6294ed46..45b84b8b6f 100644
--- a/src/backend/cuda/anisotropic_diffusion.cpp
+++ b/src/backend/cuda/anisotropic_diffusion.cpp
@@ -12,6 +12,7 @@
 #include <kernel/anisotropic_diffusion.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
@@ -29,3 +30,4 @@ void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
 INSTANTIATE(double)
 INSTANTIATE(float)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/anisotropic_diffusion.hpp b/src/backend/cuda/anisotropic_diffusion.hpp
index 4dca3740f2..6e9c2e4c1c 100644
--- a/src/backend/cuda/anisotropic_diffusion.hpp
+++ b/src/backend/cuda/anisotropic_diffusion.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
                           const af::fluxFunction fftype,
                           const af::diffusionEq eq);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/any.cu b/src/backend/cuda/any.cu
index 2da5d3349f..34092c94d3 100644
--- a/src/backend/cuda/any.cu
+++ b/src/backend/cuda/any.cu
@@ -7,11 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include "reduce_impl.hpp"
 #include <common/half.hpp>
+#include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 // anytrue
 INSTANTIATE(af_or_t, float, char)
@@ -28,3 +29,4 @@ INSTANTIATE(af_or_t, short, char)
 INSTANTIATE(af_or_t, ushort, char)
 INSTANTIATE(af_or_t, half, char)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/approx.cpp b/src/backend/cuda/approx.cpp
index 0c1bc0bb1f..b9bd55e78d 100644
--- a/src/backend/cuda/approx.cpp
+++ b/src/backend/cuda/approx.cpp
@@ -13,6 +13,7 @@
 #include <kernel/approx.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
@@ -49,3 +50,4 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/approx.hpp b/src/backend/cuda/approx.hpp
index 0d459970f1..c72d2cbe9b 100644
--- a/src/backend/cuda/approx.hpp
+++ b/src/backend/cuda/approx.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
@@ -22,3 +23,4 @@ void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
              const Tp &yi_step, const af_interp_type method,
              const float offGrid);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/arith.hpp b/src/backend/cuda/arith.hpp
index f478ecf6c0..67e39f54f4 100644
--- a/src/backend/cuda/arith.hpp
+++ b/src/backend/cuda/arith.hpp
@@ -13,6 +13,7 @@
 #include <common/jit/BinaryNode.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, af_op_t op>
@@ -27,3 +28,4 @@ Array<T> arithOp(const Array<T> &lhs, const Array<T> &rhs,
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/assign.cpp b/src/backend/cuda/assign.cpp
index 8c910fceb6..67bcbd1291 100644
--- a/src/backend/cuda/assign.cpp
+++ b/src/backend/cuda/assign.cpp
@@ -17,8 +17,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -78,3 +79,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/assign.hpp b/src/backend/cuda/assign.hpp
index 1e2eff86bf..be2f725e90 100644
--- a/src/backend/cuda/assign.hpp
+++ b/src/backend/cuda/assign.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/index.h>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/assign_kernel_param.hpp b/src/backend/cuda/assign_kernel_param.hpp
index 6587465ce2..0591ca80ad 100644
--- a/src/backend/cuda/assign_kernel_param.hpp
+++ b/src/backend/cuda/assign_kernel_param.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cuda {
 
 typedef struct {
@@ -21,3 +22,4 @@ typedef struct {
 using IndexKernelParam = AssignKernelParam;
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/backend.hpp b/src/backend/cuda/backend.hpp
index 33ce38d384..149353ca21 100644
--- a/src/backend/cuda/backend.hpp
+++ b/src/backend/cuda/backend.hpp
@@ -24,6 +24,8 @@
 #endif
 #endif
 
-namespace cuda {}
+namespace arrayfire {
+namespace cuda {}  // namespace cuda
+}  // namespace arrayfire
 
-namespace detail = cuda;
+namespace detail = arrayfire::cuda;
diff --git a/src/backend/cuda/bilateral.cpp b/src/backend/cuda/bilateral.cpp
index 12b2907b4f..f9f828018d 100644
--- a/src/backend/cuda/bilateral.cpp
+++ b/src/backend/cuda/bilateral.cpp
@@ -14,6 +14,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename inType, typename outType>
@@ -38,3 +39,4 @@ INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/bilateral.hpp b/src/backend/cuda/bilateral.hpp
index 35fa575500..63cdaee7af 100644
--- a/src/backend/cuda/bilateral.hpp
+++ b/src/backend/cuda/bilateral.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename inType, typename outType>
 Array<outType> bilateral(const Array<inType> &in, const float &spatialSigma,
                          const float &chromaticSigma);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/binary.hpp b/src/backend/cuda/binary.hpp
index ad3b95bb89..20f2bea9a6 100644
--- a/src/backend/cuda/binary.hpp
+++ b/src/backend/cuda/binary.hpp
@@ -11,6 +11,7 @@
 #include <math.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename To, typename Ti, af_op_t op>
@@ -125,3 +126,4 @@ struct BinOp<To, Ti, af_hypot_t> {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/blas.cu b/src/backend/cuda/blas.cu
index bb88c60feb..6c88ea002a 100644
--- a/src/backend/cuda/blas.cu
+++ b/src/backend/cuda/blas.cu
@@ -33,11 +33,12 @@
 #include <string>
 #include <vector>
 
-using common::half;
-using common::kernel_type;
+using arrayfire::common::half;
+using arrayfire::common::kernel_type;
 using std::is_same;
 using std::vector;
 
+namespace arrayfire {
 namespace cuda {
 
 cublasOperation_t toCblasTranspose(af_mat_prop opt) {
@@ -373,3 +374,4 @@ INSTANTIATE_TRSM(double)
 INSTANTIATE_TRSM(cdouble)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/blas.hpp b/src/backend/cuda/blas.hpp
index ce1aac1f3a..dc4382d013 100644
--- a/src/backend/cuda/blas.hpp
+++ b/src/backend/cuda/blas.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
@@ -36,3 +37,4 @@ void trsm(const Array<T> &lhs, Array<T> &rhs, af_mat_prop trans = AF_MAT_NONE,
           bool is_upper = false, bool is_left = true, bool is_unit = false);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/canny.cpp b/src/backend/cuda/canny.cpp
index a967aaf3ee..ebf8ba2e04 100644
--- a/src/backend/cuda/canny.cpp
+++ b/src/backend/cuda/canny.cpp
@@ -14,6 +14,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 Array<float> nonMaximumSuppression(const Array<float>& mag,
                                    const Array<float>& gx,
@@ -30,3 +31,4 @@ Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
     return out;
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/canny.hpp b/src/backend/cuda/canny.hpp
index bbd90a9ca2..7f8142493b 100644
--- a/src/backend/cuda/canny.hpp
+++ b/src/backend/cuda/canny.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 Array<float> nonMaximumSuppression(const Array<float>& mag,
                                    const Array<float>& gx,
@@ -17,3 +18,4 @@ Array<float> nonMaximumSuppression(const Array<float>& mag,
 Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
                                      const Array<char>& weak);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cast.hpp b/src/backend/cuda/cast.hpp
index cfcc9a8042..9328dd5052 100644
--- a/src/backend/cuda/cast.hpp
+++ b/src/backend/cuda/cast.hpp
@@ -17,6 +17,7 @@
 #include <types.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename To, typename Ti>
@@ -84,3 +85,4 @@ struct CastOp<unsigned char, common::half> {
 #undef CAST_CFN
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cholesky.cpp b/src/backend/cuda/cholesky.cpp
index 2757d50e26..7c48dbb40c 100644
--- a/src/backend/cuda/cholesky.cpp
+++ b/src/backend/cuda/cholesky.cpp
@@ -21,6 +21,7 @@
 #include <common/err_common.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 // cusolverStatus_t cusolverDn<>potrf_bufferSize(
@@ -124,3 +125,4 @@ INSTANTIATE_CH(cfloat)
 INSTANTIATE_CH(double)
 INSTANTIATE_CH(cdouble)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cholesky.hpp b/src/backend/cuda/cholesky.hpp
index 82bfcc3580..4a97aab757 100644
--- a/src/backend/cuda/cholesky.hpp
+++ b/src/backend/cuda/cholesky.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
@@ -16,3 +17,4 @@ Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
 template<typename T>
 int cholesky_inplace(Array<T> &in, const bool is_upper);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp
index de22e8c493..3fddb93d95 100644
--- a/src/backend/cuda/compile_module.cpp
+++ b/src/backend/cuda/compile_module.cpp
@@ -62,8 +62,12 @@
 #include <utility>
 #include <vector>
 
-using namespace cuda;
-
+using arrayfire::common::getCacheDirectory;
+using arrayfire::common::makeTempFilename;
+using arrayfire::common::removeFile;
+using arrayfire::common::renameFile;
+using arrayfire::cuda::getComputeCapability;
+using arrayfire::cuda::getDeviceProp;
 using detail::Module;
 using nonstd::span;
 using std::accumulate;
@@ -127,7 +131,8 @@ constexpr size_t linkLogSize = 2048;
     } while (0)
 
 spdlog::logger *getLogger() {
-    static std::shared_ptr<spdlog::logger> logger(common::loggerFactory("jit"));
+    static std::shared_ptr<spdlog::logger> logger(
+        arrayfire::common::loggerFactory("jit"));
     return logger.get();
 }
 
@@ -140,12 +145,14 @@ string getKernelCacheFilename(const int device, const string &key) {
            to_string(AF_API_VERSION_CURRENT) + ".bin";
 }
 
+namespace arrayfire {
 namespace common {
 
 Module compileModule(const string &moduleKey, span<const string> sources,
                      span<const string> opts, span<const string> kInstances,
                      const bool sourceIsJIT) {
     nvrtcProgram prog;
+    using namespace arrayfire::cuda;
     if (sourceIsJIT) {
         constexpr const char *header_names[] = {
             "utility",
@@ -252,8 +259,8 @@ Module compileModule(const string &moduleKey, span<const string> sources,
                                        includeNames));
     }
 
-    int device       = cuda::getActiveDeviceId();
-    auto computeFlag = cuda::getComputeCapability(device);
+    int device       = getActiveDeviceId();
+    auto computeFlag = getComputeCapability(device);
     array<char, 32> arch;
     snprintf(arch.data(), arch.size(), "--gpu-architecture=compute_%d%d",
              computeFlag.first, computeFlag.second);
@@ -482,8 +489,8 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
     return retVal;
 }
 
-Kernel getKernel(const Module &mod, const string &nameExpr,
-                 const bool sourceWasJIT) {
+arrayfire::cuda::Kernel getKernel(const Module &mod, const string &nameExpr,
+                                  const bool sourceWasJIT) {
     std::string name  = (sourceWasJIT ? nameExpr : mod.mangledName(nameExpr));
     CUfunction kernel = nullptr;
     CU_CHECK(cuModuleGetFunction(&kernel, mod.get(), name.c_str()));
@@ -491,3 +498,4 @@ Kernel getKernel(const Module &mod, const string &nameExpr,
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/cuda/complex.hpp b/src/backend/cuda/complex.hpp
index 68b5313150..d9d143ddbf 100644
--- a/src/backend/cuda/complex.hpp
+++ b/src/backend/cuda/complex.hpp
@@ -14,6 +14,7 @@
 #include <optypes.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename To, typename Ti>
 Array<To> cplx(const Array<Ti> &lhs, const Array<Ti> &rhs,
@@ -87,3 +88,4 @@ Array<T> conj(const Array<T> &in) {
     return createNodeArray<T>(in.dims(), common::Node_ptr(node));
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/convolve.cpp b/src/backend/cuda/convolve.cpp
index 2fe0b8d653..3a33c6f64f 100644
--- a/src/backend/cuda/convolve.cpp
+++ b/src/backend/cuda/convolve.cpp
@@ -18,10 +18,11 @@
 #include <type_traits>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using std::conditional;
 using std::is_same;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename accT>
@@ -103,3 +104,4 @@ INSTANTIATE(intl, float)
 #undef INSTANTIATE
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/convolve.hpp b/src/backend/cuda/convolve.hpp
index 636031b30d..b7faa73f00 100644
--- a/src/backend/cuda/convolve.hpp
+++ b/src/backend/cuda/convolve.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename accT>
@@ -37,3 +38,4 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
                              const Array<T> &convolved_output, af::dim4 stride,
                              af::dim4 padding, af::dim4 dilation);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/convolveNN.cpp b/src/backend/cuda/convolveNN.cpp
index 075817925e..47dbe634cb 100644
--- a/src/backend/cuda/convolveNN.cpp
+++ b/src/backend/cuda/convolveNN.cpp
@@ -33,16 +33,17 @@
 #include <vector>
 
 using af::dim4;
-using common::flip;
-using common::half;
-using common::make_handle;
-using common::modDims;
+using arrayfire::common::flip;
+using arrayfire::common::half;
+using arrayfire::common::make_handle;
+using arrayfire::common::modDims;
 using std::conditional;
 using std::is_same;
 using std::pair;
 using std::tie;
 using std::vector;
 
+namespace arrayfire {
 namespace cuda {
 
 #ifdef WITH_CUDNN
@@ -536,3 +537,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/copy.cpp b/src/backend/cuda/copy.cpp
index dbcf1284fe..f8472a7dfb 100644
--- a/src/backend/cuda/copy.cpp
+++ b/src/backend/cuda/copy.cpp
@@ -16,9 +16,10 @@
 #include <kernel/memcopy.hpp>
 #include <math.hpp>
 
-using common::half;
-using common::is_complex;
+using arrayfire::common::half;
+using arrayfire::common::is_complex;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -26,7 +27,7 @@ void copyData(T *data, const Array<T> &src) {
     if (src.elements() > 0) {
         Array<T> lin = src.isReady() && src.isLinear() ? src : copyArray(src);
         // out is now guaranteed linear
-        auto stream = cuda::getActiveStream();
+        auto stream = getActiveStream();
         CUDA_CHECK(cudaMemcpyAsync(data, lin.get(), lin.elements() * sizeof(T),
                                    cudaMemcpyDeviceToHost, stream));
         CUDA_CHECK(cudaStreamSynchronize(stream));
@@ -76,7 +77,7 @@ struct copyWrapper<T, T> {
                     if (dst.isLinear() && src.isLinear()) {
                         CUDA_CHECK(cudaMemcpyAsync(
                             dst.get(), src.get(), src.elements() * sizeof(T),
-                            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+                            cudaMemcpyDeviceToDevice, getActiveStream()));
                     } else {
                         kernel::memcopy<T>(dst, src, src.ndims());
                     }
@@ -173,9 +174,8 @@ template<typename T>
 T getScalar(const Array<T> &src) {
     T retVal{};
     CUDA_CHECK(cudaMemcpyAsync(&retVal, src.get(), sizeof(T),
-                               cudaMemcpyDeviceToHost,
-                               cuda::getActiveStream()));
-    CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
+                               cudaMemcpyDeviceToHost, getActiveStream()));
+    CUDA_CHECK(cudaStreamSynchronize(getActiveStream()));
     return retVal;
 }
 
@@ -196,3 +196,4 @@ INSTANTIATE_GETSCALAR(ushort)
 INSTANTIATE_GETSCALAR(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/copy.hpp b/src/backend/cuda/copy.hpp
index 143e6f0888..454e50679e 100644
--- a/src/backend/cuda/copy.hpp
+++ b/src/backend/cuda/copy.hpp
@@ -10,6 +10,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 // Copies(blocking) data from an Array<T> object to a contiguous host side
 // pointer.
@@ -60,3 +61,4 @@ void multiply_inplace(Array<T> &in, double val);
 template<typename T>
 T getScalar(const Array<T> &in);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/count.cu b/src/backend/cuda/count.cu
index c15c543cdb..373def999c 100644
--- a/src/backend/cuda/count.cu
+++ b/src/backend/cuda/count.cu
@@ -7,11 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include "reduce_impl.hpp"
 #include <common/half.hpp>
+#include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 // count
 INSTANTIATE(af_notzero_t, float, uint)
@@ -28,3 +29,4 @@ INSTANTIATE(af_notzero_t, char, uint)
 INSTANTIATE(af_notzero_t, uchar, uint)
 INSTANTIATE(af_notzero_t, half, uint)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cublas.cpp b/src/backend/cuda/cublas.cpp
index 4f024b8117..31111deda4 100644
--- a/src/backend/cuda/cublas.cpp
+++ b/src/backend/cuda/cublas.cpp
@@ -12,6 +12,7 @@
 #include <common/err_common.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace cuda {
 const char* errorString(cublasStatus_t err) {
     switch (err) {
@@ -32,3 +33,4 @@ const char* errorString(cublasStatus_t err) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cublas.hpp b/src/backend/cuda/cublas.hpp
index da93d41791..d0611263d8 100644
--- a/src/backend/cuda/cublas.hpp
+++ b/src/backend/cuda/cublas.hpp
@@ -15,6 +15,7 @@
 
 DEFINE_HANDLER(cublasHandle_t, cublasCreate, cublasDestroy);
 
+namespace arrayfire {
 namespace cuda {
 
 const char* errorString(cublasStatus_t err);
@@ -25,9 +26,10 @@ const char* errorString(cublasStatus_t err);
         if (_error != CUBLAS_STATUS_SUCCESS) {                              \
             char _err_msg[1024];                                            \
             snprintf(_err_msg, sizeof(_err_msg), "CUBLAS Error (%d): %s\n", \
-                     (int)(_error), cuda::errorString(_error));             \
+                     (int)(_error), arrayfire::cuda::errorString(_error));  \
             AF_ERROR(_err_msg, AF_ERR_INTERNAL);                            \
         }                                                                   \
     } while (0)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cudaDataType.hpp b/src/backend/cuda/cudaDataType.hpp
index 4e1d874e97..1da3429e60 100644
--- a/src/backend/cuda/cudaDataType.hpp
+++ b/src/backend/cuda/cudaDataType.hpp
@@ -13,6 +13,7 @@
 #include <library_types.h>  // cudaDataType enum
 #include <types.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -66,3 +67,4 @@ inline cudaDataType_t getComputeType<common::half>() {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cudnn.cpp b/src/backend/cuda/cudnn.cpp
index f75769d8f6..aa5ffd2db4 100644
--- a/src/backend/cuda/cudnn.cpp
+++ b/src/backend/cuda/cudnn.cpp
@@ -12,6 +12,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 const char *errorString(cudnnStatus_t err) {
@@ -297,3 +298,4 @@ cudnnStatus_t cudnnConvolutionBackwardFilter(
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cudnn.hpp b/src/backend/cuda/cudnn.hpp
index 4fae40692e..5cd8f5f7e6 100644
--- a/src/backend/cuda/cudnn.hpp
+++ b/src/backend/cuda/cudnn.hpp
@@ -16,15 +16,16 @@
 #include <af/dim4.hpp>
 
 // clang-format off
-DEFINE_HANDLER(cudnnHandle_t, cuda::getCudnnPlugin().cudnnCreate, cuda::getCudnnPlugin().cudnnDestroy);
+DEFINE_HANDLER(cudnnHandle_t, arrayfire::cuda::getCudnnPlugin().cudnnCreate, arrayfire::cuda::getCudnnPlugin().cudnnDestroy);
 
-DEFINE_HANDLER(cudnnTensorDescriptor_t, cuda::getCudnnPlugin().cudnnCreateTensorDescriptor, cuda::getCudnnPlugin().cudnnDestroyTensorDescriptor);
+DEFINE_HANDLER(cudnnTensorDescriptor_t, arrayfire::cuda::getCudnnPlugin().cudnnCreateTensorDescriptor, arrayfire::cuda::getCudnnPlugin().cudnnDestroyTensorDescriptor);
 
-DEFINE_HANDLER(cudnnFilterDescriptor_t, cuda::getCudnnPlugin().cudnnCreateFilterDescriptor, cuda::getCudnnPlugin().cudnnDestroyFilterDescriptor);
+DEFINE_HANDLER(cudnnFilterDescriptor_t, arrayfire::cuda::getCudnnPlugin().cudnnCreateFilterDescriptor, arrayfire::cuda::getCudnnPlugin().cudnnDestroyFilterDescriptor);
 
-DEFINE_HANDLER(cudnnConvolutionDescriptor_t, cuda::getCudnnPlugin().cudnnCreateConvolutionDescriptor, cuda::getCudnnPlugin().cudnnDestroyConvolutionDescriptor);
+DEFINE_HANDLER(cudnnConvolutionDescriptor_t, arrayfire::cuda::getCudnnPlugin().cudnnCreateConvolutionDescriptor, arrayfire::cuda::getCudnnPlugin().cudnnDestroyConvolutionDescriptor);
 // clang-format on
 
+namespace arrayfire {
 namespace cuda {
 
 const char *errorString(cudnnStatus_t err);
@@ -184,3 +185,4 @@ cudnnStatus_t cudnnConvolutionBackwardFilter(
     const cudnnFilterDescriptor_t dwDesc, void *dw);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cudnnModule.cpp b/src/backend/cuda/cudnnModule.cpp
index 4a2f3e792c..596516bbe5 100644
--- a/src/backend/cuda/cudnnModule.cpp
+++ b/src/backend/cuda/cudnnModule.cpp
@@ -18,11 +18,12 @@
 #include <string>
 #include <tuple>
 
-using common::int_version_to_string;
-using common::Version;
+using arrayfire::common::int_version_to_string;
+using arrayfire::common::Version;
 using std::make_tuple;
 using std::string;
 
+namespace arrayfire {
 namespace cuda {
 
 // clang-format off
@@ -165,3 +166,4 @@ cudnnModule& getCudnnPlugin() noexcept {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cudnnModule.hpp b/src/backend/cuda/cudnnModule.hpp
index aafefa6b84..54c4b3b708 100644
--- a/src/backend/cuda/cudnnModule.hpp
+++ b/src/backend/cuda/cudnnModule.hpp
@@ -61,6 +61,7 @@ cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm(
     cudnnConvolutionBwdFilterAlgo_t* algo);
 #endif
 
+namespace arrayfire {
 namespace cuda {
 
 class cudnnModule {
@@ -111,3 +112,4 @@ class cudnnModule {
 cudnnModule& getCudnnPlugin() noexcept;
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cufft.cu b/src/backend/cuda/cufft.cu
index 9dd976e9fe..69d7229b6b 100644
--- a/src/backend/cuda/cufft.cu
+++ b/src/backend/cuda/cufft.cu
@@ -12,6 +12,7 @@
 #include <memory.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace cuda {
 const char *_cufftGetResultString(cufftResult res) {
     switch (res) {
@@ -94,7 +95,7 @@ SharedPlan findPlan(int rank, int *n, int *inembed, int istride, int idist,
     sprintf(key_str_temp, "%d:%d", (int)type, batch);
     key_string.append(std::string(key_str_temp));
 
-    PlanCache &planner = cuda::fftManager();
+    PlanCache &planner = arrayfire::cuda::fftManager();
     SharedPlan retVal  = planner.find(key_string);
 
     if (retVal) return retVal;
@@ -105,7 +106,7 @@ SharedPlan findPlan(int rank, int *n, int *inembed, int istride, int idist,
 
     // If plan creation fails, clean up the memory we hold on to and try again
     if (res != CUFFT_SUCCESS) {
-        cuda::signalMemoryCleanup();
+        arrayfire::cuda::signalMemoryCleanup();
         CUFFT_CHECK(cufftPlanMany(temp, rank, n, inembed, istride, idist,
                                   onembed, ostride, odist, type, batch));
     }
@@ -120,3 +121,4 @@ SharedPlan findPlan(int rank, int *n, int *inembed, int istride, int idist,
     return retVal;
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cufft.hpp b/src/backend/cuda/cufft.hpp
index 937af94759..80ba06c8f5 100644
--- a/src/backend/cuda/cufft.hpp
+++ b/src/backend/cuda/cufft.hpp
@@ -17,6 +17,7 @@
 
 DEFINE_HANDLER(cufftHandle, cufftCreate, cufftDestroy);
 
+namespace arrayfire {
 namespace cuda {
 
 typedef cufftHandle PlanType;
@@ -35,16 +36,17 @@ class PlanCache : public common::FFTPlanCache<PlanCache, PlanType> {
 };
 
 }  // namespace cuda
-
-#define CUFFT_CHECK(fn)                                           \
-    do {                                                          \
-        cufftResult _cufft_res = fn;                              \
-        if (_cufft_res != CUFFT_SUCCESS) {                        \
-            char cufft_res_msg[1024];                             \
-            snprintf(cufft_res_msg, sizeof(cufft_res_msg),        \
-                     "cuFFT Error (%d): %s\n", (int)(_cufft_res), \
-                     cuda::_cufftGetResultString(_cufft_res));    \
-                                                                  \
-            AF_ERROR(cufft_res_msg, AF_ERR_INTERNAL);             \
-        }                                                         \
+}  // namespace arrayfire
+
+#define CUFFT_CHECK(fn)                                                   \
+    do {                                                                  \
+        cufftResult _cufft_res = fn;                                      \
+        if (_cufft_res != CUFFT_SUCCESS) {                                \
+            char cufft_res_msg[1024];                                     \
+            snprintf(cufft_res_msg, sizeof(cufft_res_msg),                \
+                     "cuFFT Error (%d): %s\n", (int)(_cufft_res),         \
+                     arrayfire::cuda::_cufftGetResultString(_cufft_res)); \
+                                                                          \
+            AF_ERROR(cufft_res_msg, AF_ERR_INTERNAL);                     \
+        }                                                                 \
     } while (0)
diff --git a/src/backend/cuda/cusolverDn.cpp b/src/backend/cuda/cusolverDn.cpp
index afe88d3374..3cbfec6898 100644
--- a/src/backend/cuda/cusolverDn.cpp
+++ b/src/backend/cuda/cusolverDn.cpp
@@ -13,6 +13,7 @@
 #include <stdexcept>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 const char *errorString(cusolverStatus_t err) {
     switch (err) {
@@ -42,3 +43,4 @@ const char *errorString(cusolverStatus_t err) {
     }
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cusolverDn.hpp b/src/backend/cuda/cusolverDn.hpp
index e643934930..e9edab58b5 100644
--- a/src/backend/cuda/cusolverDn.hpp
+++ b/src/backend/cuda/cusolverDn.hpp
@@ -14,6 +14,7 @@
 
 DEFINE_HANDLER(cusolverDnHandle_t, cusolverDnCreate, cusolverDnDestroy);
 
+namespace arrayfire {
 namespace cuda {
 
 const char* errorString(cusolverStatus_t err);
@@ -24,10 +25,11 @@ const char* errorString(cusolverStatus_t err);
         if (_error != CUSOLVER_STATUS_SUCCESS) {                              \
             char _err_msg[1024];                                              \
             snprintf(_err_msg, sizeof(_err_msg), "CUSOLVER Error (%d): %s\n", \
-                     (int)(_error), cuda::errorString(_error));               \
+                     (int)(_error), arrayfire::cuda::errorString(_error));    \
                                                                               \
             AF_ERROR(_err_msg, AF_ERR_INTERNAL);                              \
         }                                                                     \
     } while (0)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cusparse.cpp b/src/backend/cuda/cusparse.cpp
index a2471d6267..224d798327 100644
--- a/src/backend/cuda/cusparse.cpp
+++ b/src/backend/cuda/cusparse.cpp
@@ -12,6 +12,7 @@
 #include <stdexcept>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 const char* errorString(cusparseStatus_t err) {
     switch (err) {
@@ -38,3 +39,4 @@ const char* errorString(cusparseStatus_t err) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cusparse.hpp b/src/backend/cuda/cusparse.hpp
index b7a332a856..467b2a82ec 100644
--- a/src/backend/cuda/cusparse.hpp
+++ b/src/backend/cuda/cusparse.hpp
@@ -16,15 +16,16 @@
 #include <cusparse_v2.h>
 
 // clang-format off
-DEFINE_HANDLER(cusparseHandle_t, cuda::getCusparsePlugin().cusparseCreate, cuda::getCusparsePlugin().cusparseDestroy);
-DEFINE_HANDLER(cusparseMatDescr_t, cuda::getCusparsePlugin().cusparseCreateMatDescr, cuda::getCusparsePlugin().cusparseDestroyMatDescr);
+DEFINE_HANDLER(cusparseHandle_t, arrayfire::cuda::getCusparsePlugin().cusparseCreate, arrayfire::cuda::getCusparsePlugin().cusparseDestroy);
+DEFINE_HANDLER(cusparseMatDescr_t, arrayfire::cuda::getCusparsePlugin().cusparseCreateMatDescr, arrayfire::cuda::getCusparsePlugin().cusparseDestroyMatDescr);
 #if defined(AF_USE_NEW_CUSPARSE_API)
-DEFINE_HANDLER(cusparseSpMatDescr_t, cuda::getCusparsePlugin().cusparseCreateCsr, cuda::getCusparsePlugin().cusparseDestroySpMat);
-DEFINE_HANDLER(cusparseDnVecDescr_t, cuda::getCusparsePlugin().cusparseCreateDnVec, cuda::getCusparsePlugin().cusparseDestroyDnVec);
-DEFINE_HANDLER(cusparseDnMatDescr_t, cuda::getCusparsePlugin().cusparseCreateDnMat, cuda::getCusparsePlugin().cusparseDestroyDnMat);
+DEFINE_HANDLER(cusparseSpMatDescr_t, arrayfire::cuda::getCusparsePlugin().cusparseCreateCsr, arrayfire::cuda::getCusparsePlugin().cusparseDestroySpMat);
+DEFINE_HANDLER(cusparseDnVecDescr_t, arrayfire::cuda::getCusparsePlugin().cusparseCreateDnVec, arrayfire::cuda::getCusparsePlugin().cusparseDestroyDnVec);
+DEFINE_HANDLER(cusparseDnMatDescr_t, arrayfire::cuda::getCusparsePlugin().cusparseCreateDnMat, arrayfire::cuda::getCusparsePlugin().cusparseDestroyDnMat);
 #endif
 // clang-format on
 
+namespace arrayfire {
 namespace cuda {
 
 const char* errorString(cusparseStatus_t err);
@@ -35,10 +36,11 @@ const char* errorString(cusparseStatus_t err);
         if (_error != CUSPARSE_STATUS_SUCCESS) {                              \
             char _err_msg[1024];                                              \
             snprintf(_err_msg, sizeof(_err_msg), "CUSPARSE Error (%d): %s\n", \
-                     (int)(_error), cuda::errorString(_error));               \
+                     (int)(_error), arrayfire::cuda::errorString(_error));    \
                                                                               \
             AF_ERROR(_err_msg, AF_ERR_INTERNAL);                              \
         }                                                                     \
     } while (0)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cusparseModule.cpp b/src/backend/cuda/cusparseModule.cpp
index e7b8105221..bc049fcb01 100644
--- a/src/backend/cuda/cusparseModule.cpp
+++ b/src/backend/cuda/cusparseModule.cpp
@@ -15,6 +15,7 @@
 #include <cuda.h>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 
 cusparseModule::cusparseModule()
@@ -133,3 +134,4 @@ cusparseModule& getCusparsePlugin() noexcept {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cusparseModule.hpp b/src/backend/cuda/cusparseModule.hpp
index 57878c2cf8..ac7e826a13 100644
--- a/src/backend/cuda/cusparseModule.hpp
+++ b/src/backend/cuda/cusparseModule.hpp
@@ -13,9 +13,10 @@
 #include <cuda.h>
 #include <cusparse_v2.h>
 
+namespace arrayfire {
 namespace cuda {
 class cusparseModule {
-    common::DependencyModule module;
+    arrayfire::common::DependencyModule module;
 
    public:
     cusparseModule();
@@ -94,3 +95,4 @@ class cusparseModule {
 cusparseModule& getCusparsePlugin() noexcept;
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/cusparse_descriptor_helpers.hpp b/src/backend/cuda/cusparse_descriptor_helpers.hpp
index 3e94f89f47..41e369b0d8 100644
--- a/src/backend/cuda/cusparse_descriptor_helpers.hpp
+++ b/src/backend/cuda/cusparse_descriptor_helpers.hpp
@@ -17,6 +17,7 @@
 
 #include <utility>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -44,5 +45,6 @@ auto denMatDescriptor(const Array<T> &in) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
 
 #endif
diff --git a/src/backend/cuda/debug_cuda.hpp b/src/backend/cuda/debug_cuda.hpp
index 25f266c268..555944a5ed 100644
--- a/src/backend/cuda/debug_cuda.hpp
+++ b/src/backend/cuda/debug_cuda.hpp
@@ -13,6 +13,7 @@
 #include <platform.hpp>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel_logger {
 
@@ -22,6 +23,7 @@ inline auto getLogger() {
 }
 }  // namespace kernel_logger
 }  // namespace cuda
+}  // namespace arrayfire
 
 template<>
 struct fmt::formatter<dim3> : fmt::formatter<std::string> {
@@ -33,16 +35,17 @@ struct fmt::formatter<dim3> : fmt::formatter<std::string> {
     }
 };
 
-#define CUDA_LAUNCH_SMEM(fn, blks, thrds, smem_size, ...)                     \
-    do {                                                                      \
-        {                                                                     \
-            using namespace cuda::kernel_logger;                              \
-            AF_TRACE(                                                         \
-                "Launching {}: Blocks: [{}] Threads: [{}] "                   \
-                "Shared Memory: {}",                                          \
-                #fn, blks, thrds, smem_size);                                 \
-        }                                                                     \
-        fn<<<blks, thrds, smem_size, cuda::getActiveStream()>>>(__VA_ARGS__); \
+#define CUDA_LAUNCH_SMEM(fn, blks, thrds, smem_size, ...)                   \
+    do {                                                                    \
+        {                                                                   \
+            using namespace arrayfire::cuda::kernel_logger;                 \
+            AF_TRACE(                                                       \
+                "Launching {}: Blocks: [{}] Threads: [{}] "                 \
+                "Shared Memory: {}",                                        \
+                #fn, blks, thrds, smem_size);                               \
+        }                                                                   \
+        fn<<<blks, thrds, smem_size, arrayfire::cuda::getActiveStream()>>>( \
+            __VA_ARGS__);                                                   \
     } while (false)
 
 #define CUDA_LAUNCH(fn, blks, thrds, ...) \
@@ -51,18 +54,21 @@ struct fmt::formatter<dim3> : fmt::formatter<std::string> {
 // FIXME: Add a special flag for debug
 #ifndef NDEBUG
 
-#define POST_LAUNCH_CHECK() \
-    do { CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream())); } while (0)
+#define POST_LAUNCH_CHECK()                                                    \
+    do {                                                                       \
+        CUDA_CHECK(cudaStreamSynchronize(arrayfire::cuda::getActiveStream())); \
+    } while (0)
 
 #else
 
-#define POST_LAUNCH_CHECK()                                             \
-    do {                                                                \
-        if (cuda::synchronize_calls()) {                                \
-            CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream())); \
-        } else {                                                        \
-            CUDA_CHECK(cudaPeekAtLastError());                          \
-        }                                                               \
+#define POST_LAUNCH_CHECK()                                                 \
+    do {                                                                    \
+        if (arrayfire::cuda::synchronize_calls()) {                         \
+            CUDA_CHECK(                                                     \
+                cudaStreamSynchronize(arrayfire::cuda::getActiveStream())); \
+        } else {                                                            \
+            CUDA_CHECK(cudaPeekAtLastError());                              \
+        }                                                                   \
     } while (0)
 
 #endif
diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 4b946a7fee..5f79b00abf 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -46,8 +46,8 @@
 #include <thread>
 #include <utility>
 
-using common::getEnvVar;
-using common::int_version_to_string;
+using arrayfire::common::getEnvVar;
+using arrayfire::common::int_version_to_string;
 using std::begin;
 using std::end;
 using std::find;
@@ -57,6 +57,7 @@ using std::pair;
 using std::string;
 using std::stringstream;
 
+namespace arrayfire {
 namespace cuda {
 
 struct cuNVRTCcompute {
@@ -380,7 +381,7 @@ void DeviceManager::setMemoryManager(
     memManager = std::move(newMgr);
     // Set the backend memory manager for this new manager to register native
     // functions correctly.
-    std::unique_ptr<cuda::Allocator> deviceMemoryManager(new cuda::Allocator());
+    std::unique_ptr<cuda::Allocator> deviceMemoryManager(new Allocator());
     memManager->setAllocator(std::move(deviceMemoryManager));
     memManager->initialize();
 }
@@ -407,7 +408,7 @@ void DeviceManager::setMemoryManagerPinned(
     // functions correctly.
     pinnedMemManager = std::move(newMgr);
     std::unique_ptr<cuda::AllocatorPinned> deviceMemoryManager(
-        new cuda::AllocatorPinned());
+        new AllocatorPinned());
     pinnedMemManager->setAllocator(std::move(deviceMemoryManager));
     pinnedMemManager->initialize();
 }
@@ -547,7 +548,7 @@ DeviceManager::DeviceManager()
     : logger(common::loggerFactory("platform"))
     , cuDevices(0)
     , nDevices(0)
-    , fgMngr(new graphics::ForgeManager()) {
+    , fgMngr(new arrayfire::common::ForgeManager()) {
     try {
         checkCudaVsDriverVersion();
 
@@ -726,3 +727,4 @@ int DeviceManager::setActiveDevice(int device, int nId) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/device_manager.hpp b/src/backend/cuda/device_manager.hpp
index 5ea6d3a2f6..9275386011 100644
--- a/src/backend/cuda/device_manager.hpp
+++ b/src/backend/cuda/device_manager.hpp
@@ -17,12 +17,13 @@
 #include <utility>
 #include <vector>
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 
 #ifndef AF_CUDA_MEM_DEBUG
 #define AF_CUDA_MEM_DEBUG 0
 #endif
 
+namespace arrayfire {
 namespace cuda {
 
 struct cudaDevice_t {
@@ -66,7 +67,7 @@ class DeviceManager {
 
     void resetMemoryManagerPinned();
 
-    friend graphics::ForgeManager& forgeManager();
+    friend arrayfire::common::ForgeManager& forgeManager();
 
     friend GraphicsResourceManager& interopManager();
 
@@ -122,7 +123,7 @@ class DeviceManager {
     int nDevices;
     cudaStream_t streams[MAX_DEVICES]{};
 
-    std::unique_ptr<graphics::ForgeManager> fgMngr;
+    std::unique_ptr<arrayfire::common::ForgeManager> fgMngr;
 
     std::unique_ptr<MemoryManagerBase> memManager;
 
@@ -134,3 +135,4 @@ class DeviceManager {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/diagonal.cpp b/src/backend/cuda/diagonal.cpp
index 2a2f07b594..cbf3180a70 100644
--- a/src/backend/cuda/diagonal.cpp
+++ b/src/backend/cuda/diagonal.cpp
@@ -15,8 +15,9 @@
 #include <math.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> diagCreate(const Array<T> &in, const int num) {
@@ -59,3 +60,4 @@ INSTANTIATE_DIAGONAL(ushort)
 INSTANTIATE_DIAGONAL(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/diagonal.hpp b/src/backend/cuda/diagonal.hpp
index c6e2aff5fd..a1a9828a2a 100644
--- a/src/backend/cuda/diagonal.hpp
+++ b/src/backend/cuda/diagonal.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> diagCreate(const Array<T> &in, const int num);
@@ -16,3 +17,4 @@ Array<T> diagCreate(const Array<T> &in, const int num);
 template<typename T>
 Array<T> diagExtract(const Array<T> &in, const int num);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/diff.cpp b/src/backend/cuda/diff.cpp
index f67a0eabda..55bb68ece0 100644
--- a/src/backend/cuda/diff.cpp
+++ b/src/backend/cuda/diff.cpp
@@ -13,6 +13,7 @@
 #include <kernel/diff.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -60,3 +61,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/diff.hpp b/src/backend/cuda/diff.hpp
index 30ac6661e9..c2b4900862 100644
--- a/src/backend/cuda/diff.hpp
+++ b/src/backend/cuda/diff.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> diff1(const Array<T> &in, const int dim);
@@ -16,3 +17,4 @@ Array<T> diff1(const Array<T> &in, const int dim);
 template<typename T>
 Array<T> diff2(const Array<T> &in, const int dim);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/dims_param.hpp b/src/backend/cuda/dims_param.hpp
index 3692a68838..273eaf13cb 100644
--- a/src/backend/cuda/dims_param.hpp
+++ b/src/backend/cuda/dims_param.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cuda {
 
 typedef struct {
@@ -16,3 +17,4 @@ typedef struct {
 } dims_t;
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/exampleFunction.cpp b/src/backend/cuda/exampleFunction.cpp
index f4b7a7fc8f..b94f9f8e54 100644
--- a/src/backend/cuda/exampleFunction.cpp
+++ b/src/backend/cuda/exampleFunction.cpp
@@ -26,6 +26,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -65,3 +66,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/exampleFunction.hpp b/src/backend/cuda/exampleFunction.hpp
index b0c20927ab..d0e9938dda 100644
--- a/src/backend/cuda/exampleFunction.hpp
+++ b/src/backend/cuda/exampleFunction.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> exampleFunction(const Array<T> &a, const Array<T> &b,
                          const af_someenum_t method);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/fast.cu b/src/backend/cuda/fast.cu
index d4f00274bc..7744d4b6d6 100644
--- a/src/backend/cuda/fast.cu
+++ b/src/backend/cuda/fast.cu
@@ -19,6 +19,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -66,3 +67,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/fast.hpp b/src/backend/cuda/fast.hpp
index 84f509c5aa..d60c671634 100644
--- a/src/backend/cuda/fast.hpp
+++ b/src/backend/cuda/fast.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -20,4 +21,5 @@ unsigned fast(Array<float> &x_out, Array<float> &y_out, Array<float> &score_out,
               const bool non_max, const float feature_ratio,
               const unsigned edge);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/fast_pyramid.cpp b/src/backend/cuda/fast_pyramid.cpp
index 8d14cf752c..97228af248 100644
--- a/src/backend/cuda/fast_pyramid.cpp
+++ b/src/backend/cuda/fast_pyramid.cpp
@@ -18,6 +18,7 @@
 using af::dim4;
 using std::vector;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -124,3 +125,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/fast_pyramid.hpp b/src/backend/cuda/fast_pyramid.hpp
index ceac076d95..af8e902ea2 100644
--- a/src/backend/cuda/fast_pyramid.hpp
+++ b/src/backend/cuda/fast_pyramid.hpp
@@ -13,6 +13,7 @@
 
 #include <vector>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void fast_pyramid(std::vector<unsigned> &feat_pyr,
@@ -23,4 +24,5 @@ void fast_pyramid(std::vector<unsigned> &feat_pyr,
                   const float fast_thr, const unsigned max_feat,
                   const float scl_fctr, const unsigned levels,
                   const unsigned patch_size);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/fft.cu b/src/backend/cuda/fft.cu
index 4254b719bf..800e6571d2 100644
--- a/src/backend/cuda/fft.cu
+++ b/src/backend/cuda/fft.cu
@@ -23,6 +23,7 @@ using af::dim4;
 using std::array;
 using std::string;
 
+namespace arrayfire {
 namespace cuda {
 void setFFTPlanCacheSize(size_t numPlans) {
     fftManager().setMaxCacheSize(numPlans);
@@ -84,7 +85,7 @@ void fft_inplace(Array<T> &in, const int rank, const bool direction) {
                  (cufftType)cufft_transform<T>::type, batch);
 
     cufft_transform<T> transform;
-    CUFFT_CHECK(cufftSetStream(*plan.get(), cuda::getActiveStream()));
+    CUFFT_CHECK(cufftSetStream(*plan.get(), getActiveStream()));
     CUFFT_CHECK(transform(*plan.get(), (T *)in.get(), in.get(),
                           direction ? CUFFT_FORWARD : CUFFT_INVERSE));
 }
@@ -114,7 +115,7 @@ Array<Tc> fft_r2c(const Array<Tr> &in, const int rank) {
                  (cufftType)cufft_real_transform<Tc, Tr>::type, batch);
 
     cufft_real_transform<Tc, Tr> transform;
-    CUFFT_CHECK(cufftSetStream(*plan.get(), cuda::getActiveStream()));
+    CUFFT_CHECK(cufftSetStream(*plan.get(), getActiveStream()));
     CUFFT_CHECK(transform(*plan.get(), (Tr *)in.get(), out.get()));
     return out;
 }
@@ -140,7 +141,7 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank) {
                  istrides[rank], out_embed.data(), ostrides[0], ostrides[rank],
                  (cufftType)cufft_real_transform<Tr, Tc>::type, batch);
 
-    CUFFT_CHECK(cufftSetStream(*plan.get(), cuda::getActiveStream()));
+    CUFFT_CHECK(cufftSetStream(*plan.get(), getActiveStream()));
     CUFFT_CHECK(transform(*plan.get(), (Tc *)in.get(), out.get()));
     return out;
 }
@@ -159,3 +160,4 @@ INSTANTIATE(cdouble)
 INSTANTIATE_REAL(float, cfloat)
 INSTANTIATE_REAL(double, cdouble)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/fft.hpp b/src/backend/cuda/fft.hpp
index c9ff79877a..5cc2bf42e4 100644
--- a/src/backend/cuda/fft.hpp
+++ b/src/backend/cuda/fft.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 void setFFTPlanCacheSize(size_t numPlans);
@@ -23,3 +24,4 @@ template<typename Tr, typename Tc>
 Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/fftconvolve.cpp b/src/backend/cuda/fftconvolve.cpp
index 36a449256a..7c50c0838c 100644
--- a/src/backend/cuda/fftconvolve.cpp
+++ b/src/backend/cuda/fftconvolve.cpp
@@ -21,6 +21,7 @@ using std::conditional;
 using std::is_integral;
 using std::is_same;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -117,3 +118,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/fftconvolve.hpp b/src/backend/cuda/fftconvolve.hpp
index f7cf19a199..c158bdaa3d 100644
--- a/src/backend/cuda/fftconvolve.hpp
+++ b/src/backend/cuda/fftconvolve.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
                      const bool expand, AF_BATCH_KIND kind, const int rank);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/flood_fill.cpp b/src/backend/cuda/flood_fill.cpp
index 1442ba2619..2165f8a6c8 100644
--- a/src/backend/cuda/flood_fill.cpp
+++ b/src/backend/cuda/flood_fill.cpp
@@ -12,6 +12,7 @@
 #include <err_cuda.hpp>
 #include <kernel/flood_fill.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -36,3 +37,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/flood_fill.hpp b/src/backend/cuda/flood_fill.hpp
index b4d432feec..6716abeae7 100644
--- a/src/backend/cuda/flood_fill.hpp
+++ b/src/backend/cuda/flood_fill.hpp
@@ -12,6 +12,7 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
@@ -19,3 +20,4 @@ Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
                    const T lowValue, const T highValue,
                    const af::connectivity nlookup = AF_CONNECTIVITY_8);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/gradient.cpp b/src/backend/cuda/gradient.cpp
index 0fdd4941ee..b7274a736f 100644
--- a/src/backend/cuda/gradient.cpp
+++ b/src/backend/cuda/gradient.cpp
@@ -16,6 +16,7 @@
 
 #include <stdexcept>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in) {
@@ -31,3 +32,4 @@ INSTANTIATE(double)
 INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/gradient.hpp b/src/backend/cuda/gradient.hpp
index 1378fba097..46ff6db000 100644
--- a/src/backend/cuda/gradient.hpp
+++ b/src/backend/cuda/gradient.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/harris.cu b/src/backend/cuda/harris.cu
index 375b9e1570..1c9c9a482c 100644
--- a/src/backend/cuda/harris.cu
+++ b/src/backend/cuda/harris.cu
@@ -16,6 +16,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename convAccT>
@@ -55,3 +56,4 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/harris.hpp b/src/backend/cuda/harris.hpp
index ce51eaf3de..4cf4fc8084 100644
--- a/src/backend/cuda/harris.hpp
+++ b/src/backend/cuda/harris.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename convAccT>
@@ -21,4 +22,5 @@ unsigned harris(Array<float> &x_out, Array<float> &y_out,
                 const float sigma, const unsigned filter_len,
                 const float k_thr);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp
index d415a12aad..6678281db6 100644
--- a/src/backend/cuda/hist_graphics.cpp
+++ b/src/backend/cuda/hist_graphics.cpp
@@ -14,11 +14,16 @@
 #include <err_cuda.hpp>
 #include <hist_graphics.hpp>
 
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_histogram(const Array<T> &data, fg_histogram hist) {
-    auto stream = cuda::getActiveStream();
+    auto stream = getActiveStream();
     if (DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = data.get();
 
@@ -36,7 +41,7 @@ void copy_histogram(const Array<T> &data, fg_histogram hist) {
 
         POST_LAUNCH_CHECK();
     } else {
-        ForgeModule &_ = graphics::forgePlugin();
+        ForgeModule &_ = common::forgePlugin();
         unsigned bytes = 0, buffer = 0;
         FG_CHECK(_.fg_get_histogram_vertex_buffer(&buffer, hist));
         FG_CHECK(_.fg_get_histogram_vertex_buffer_size(&bytes, hist));
@@ -67,3 +72,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/hist_graphics.hpp b/src/backend/cuda/hist_graphics.hpp
index 10cae9ae94..348d84ba3c 100644
--- a/src/backend/cuda/hist_graphics.hpp
+++ b/src/backend/cuda/hist_graphics.hpp
@@ -12,9 +12,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_histogram(const Array<T> &data, fg_histogram hist);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/histogram.cpp b/src/backend/cuda/histogram.cpp
index a2680de686..ca7e6ced86 100644
--- a/src/backend/cuda/histogram.cpp
+++ b/src/backend/cuda/histogram.cpp
@@ -15,8 +15,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -48,3 +49,4 @@ INSTANTIATE(uintl)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/histogram.hpp b/src/backend/cuda/histogram.hpp
index b07453f083..f9498d422c 100644
--- a/src/backend/cuda/histogram.hpp
+++ b/src/backend/cuda/histogram.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<uint> histogram(const Array<T> &in, const unsigned &nbins,
                       const double &minval, const double &maxval,
                       const bool isLinear);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/homography.cu b/src/backend/cuda/homography.cu
index b8525dee8e..7b70064902 100644
--- a/src/backend/cuda/homography.cu
+++ b/src/backend/cuda/homography.cu
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 #define RANSACConfidence 0.99f
@@ -64,3 +65,4 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/homography.hpp b/src/backend/cuda/homography.hpp
index 38ad486e93..95c4bdf853 100644
--- a/src/backend/cuda/homography.hpp
+++ b/src/backend/cuda/homography.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -18,4 +19,5 @@ int homography(Array<T> &H, const Array<float> &x_src,
                const af_homography_type htype, const float inlier_thr,
                const unsigned iterations);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/hsv_rgb.cpp b/src/backend/cuda/hsv_rgb.cpp
index 13d1a95187..d4eda7ef58 100644
--- a/src/backend/cuda/hsv_rgb.cpp
+++ b/src/backend/cuda/hsv_rgb.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -39,3 +40,4 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/hsv_rgb.hpp b/src/backend/cuda/hsv_rgb.hpp
index 7758ce5181..26288245e6 100644
--- a/src/backend/cuda/hsv_rgb.hpp
+++ b/src/backend/cuda/hsv_rgb.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> rgb2hsv(const Array<T>& in);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/identity.cpp b/src/backend/cuda/identity.cpp
index 293489c216..995b09a9d9 100644
--- a/src/backend/cuda/identity.cpp
+++ b/src/backend/cuda/identity.cpp
@@ -14,8 +14,9 @@
 #include <debug_cuda.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> identity(const dim4& dims) {
@@ -42,3 +43,4 @@ INSTANTIATE_IDENTITY(ushort)
 INSTANTIATE_IDENTITY(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/identity.hpp b/src/backend/cuda/identity.hpp
index 77b58f6ab7..f03d9f6199 100644
--- a/src/backend/cuda/identity.hpp
+++ b/src/backend/cuda/identity.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> identity(const dim4& dim);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/iir.cpp b/src/backend/cuda/iir.cpp
index 616411805a..63a662b885 100644
--- a/src/backend/cuda/iir.cpp
+++ b/src/backend/cuda/iir.cpp
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x) {
@@ -56,3 +57,4 @@ INSTANTIATE(double)
 INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/iir.hpp b/src/backend/cuda/iir.hpp
index f2ff082d2a..1ad18333f3 100644
--- a/src/backend/cuda/iir.hpp
+++ b/src/backend/cuda/iir.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp
index d247322201..810d36d968 100644
--- a/src/backend/cuda/image.cpp
+++ b/src/backend/cuda/image.cpp
@@ -18,12 +18,16 @@
 #include <image.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_image(const Array<T> &in, fg_image image) {
-    auto stream = cuda::getActiveStream();
+    auto stream = getActiveStream();
     if (DeviceManager::checkGraphicsInteropCapability()) {
         auto res = interopManager().getImageResources(image);
 
@@ -39,7 +43,7 @@ void copy_image(const Array<T> &in, fg_image image) {
         POST_LAUNCH_CHECK();
         CheckGL("After cuda resource copy");
     } else {
-        ForgeModule &_ = graphics::forgePlugin();
+        ForgeModule &_ = common::forgePlugin();
         CheckGL("Begin CUDA fallback-resource copy");
         unsigned data_size = 0, buffer = 0;
         FG_CHECK(_.fg_get_image_size(&data_size, image));
@@ -72,3 +76,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/image.hpp b/src/backend/cuda/image.hpp
index e97d78aaa7..2a98743dd4 100644
--- a/src/backend/cuda/image.hpp
+++ b/src/backend/cuda/image.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_image(const Array<T> &in, fg_image image);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/index.cpp b/src/backend/cuda/index.cpp
index 0974e71dbb..88a95da73b 100644
--- a/src/backend/cuda/index.cpp
+++ b/src/backend/cuda/index.cpp
@@ -18,8 +18,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -85,3 +86,4 @@ INSTANTIATE(short)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/index.hpp b/src/backend/cuda/index.hpp
index 3a439c9941..5966078eaf 100644
--- a/src/backend/cuda/index.hpp
+++ b/src/backend/cuda/index.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/index.h>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 Array<T> index(const Array<T>& in, const af_index_t idxrs[]);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/inverse.cpp b/src/backend/cuda/inverse.cpp
index 22c1ae88b3..db7059d4a9 100644
--- a/src/backend/cuda/inverse.cpp
+++ b/src/backend/cuda/inverse.cpp
@@ -13,6 +13,7 @@
 #include <identity.hpp>
 #include <solve.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -29,3 +30,4 @@ INSTANTIATE(double)
 INSTANTIATE(cdouble)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/inverse.hpp b/src/backend/cuda/inverse.hpp
index 27ba153175..7c662b8cda 100644
--- a/src/backend/cuda/inverse.hpp
+++ b/src/backend/cuda/inverse.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> inverse(const Array<T> &in);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/iota.cpp b/src/backend/cuda/iota.cpp
index f79cb6c492..d9afef41c5 100644
--- a/src/backend/cuda/iota.cpp
+++ b/src/backend/cuda/iota.cpp
@@ -15,8 +15,9 @@
 #include <math.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> iota(const dim4 &dims, const dim4 &tile_dims) {
@@ -42,3 +43,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/iota.hpp b/src/backend/cuda/iota.hpp
index bbc01a94e8..5232fdddbc 100644
--- a/src/backend/cuda/iota.hpp
+++ b/src/backend/cuda/iota.hpp
@@ -10,7 +10,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> iota(const dim4 &dim, const dim4 &tile_dims = dim4(1));
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/ireduce.cpp b/src/backend/cuda/ireduce.cpp
index abbea5514d..94cd340a66 100644
--- a/src/backend/cuda/ireduce.cpp
+++ b/src/backend/cuda/ireduce.cpp
@@ -19,8 +19,9 @@
 #include <complex>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<af_op_t op, typename T>
@@ -79,3 +80,4 @@ INSTANTIATE(af_max_t, char)
 INSTANTIATE(af_max_t, uchar)
 INSTANTIATE(af_max_t, half)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/ireduce.hpp b/src/backend/cuda/ireduce.hpp
index 69f25be476..f65eb863a4 100644
--- a/src/backend/cuda/ireduce.hpp
+++ b/src/backend/cuda/ireduce.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<af_op_t op, typename T>
 void ireduce(Array<T> &out, Array<uint> &loc, const Array<T> &in,
@@ -22,3 +23,4 @@ void rreduce(Array<T> &out, Array<uint> &loc, const Array<T> &in, const int dim,
 template<af_op_t op, typename T>
 T ireduce_all(unsigned *loc, const Array<T> &in);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 4dab53a877..2ffc2f72cf 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -34,17 +34,17 @@
 #include <string>
 #include <vector>
 
-using common::findModule;
-using common::getEnvVar;
-using common::getFuncName;
-using common::half;
-using common::ModdimNode;
-using common::Node;
-using common::Node_ids;
-using common::Node_map_t;
-using common::Node_ptr;
-using common::NodeIterator;
-using common::saveKernel;
+using arrayfire::common::findModule;
+using arrayfire::common::getEnvVar;
+using arrayfire::common::getFuncName;
+using arrayfire::common::half;
+using arrayfire::common::ModdimNode;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ids;
+using arrayfire::common::Node_map_t;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::NodeIterator;
+using arrayfire::common::saveKernel;
 
 using std::array;
 using std::equal;
@@ -56,6 +56,7 @@ using std::stringstream;
 using std::to_string;
 using std::vector;
 
+namespace arrayfire {
 namespace cuda {
 using jit::BufferNode;
 
@@ -498,7 +499,7 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
     for (auto& out : outputs) { args.push_back(static_cast<void*>(&out)); }
 
     {
-        using namespace cuda::kernel_logger;
+        using namespace arrayfire::cuda::kernel_logger;
         AF_TRACE(
             "Launching : Dims: [{},{},{},{}] Blocks: [{}] "
             "Threads: [{}] threads: {}",
@@ -564,3 +565,4 @@ template void evalNodes<ushort>(vector<Param<ushort>>& out,
 template void evalNodes<half>(vector<Param<half>>& out,
                               const vector<Node*>& node);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/jit/BufferNode.hpp b/src/backend/cuda/jit/BufferNode.hpp
index 21601f2a03..195353fdd8 100644
--- a/src/backend/cuda/jit/BufferNode.hpp
+++ b/src/backend/cuda/jit/BufferNode.hpp
@@ -11,12 +11,12 @@
 #include <common/jit/BufferNodeBase.hpp>
 #include "../Param.hpp"
 
+namespace arrayfire {
 namespace cuda {
 namespace jit {
 template<typename T>
 using BufferNode = common::BufferNodeBase<std::shared_ptr<T>, Param<T>>;
-}
-
+}  // namespace jit
 }  // namespace cuda
 
 namespace common {
@@ -32,3 +32,5 @@ bool BufferNodeBase<DataType, ParamType>::operator==(
 }
 
 }  // namespace common
+
+}  // namespace arrayfire
diff --git a/src/backend/cuda/jit/kernel_generators.hpp b/src/backend/cuda/jit/kernel_generators.hpp
index cc67ac6996..f675faf4b4 100644
--- a/src/backend/cuda/jit/kernel_generators.hpp
+++ b/src/backend/cuda/jit/kernel_generators.hpp
@@ -16,6 +16,7 @@
 #include <sstream>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 
 namespace {
@@ -104,3 +105,4 @@ inline void generateShiftNodeRead(std::stringstream& kerStream, int id,
 
 }  // namespace
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/join.cpp b/src/backend/cuda/join.cpp
index 7f65773d0a..3eed6f7fb5 100644
--- a/src/backend/cuda/join.cpp
+++ b/src/backend/cuda/join.cpp
@@ -19,11 +19,12 @@
 #include <vector>
 
 using af::dim4;
-using common::half;
-using common::Node;
-using common::Node_ptr;
+using arrayfire::common::half;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ptr;
 using std::vector;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -234,3 +235,4 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/join.hpp b/src/backend/cuda/join.hpp
index cf74076b8a..18767feae9 100644
--- a/src/backend/cuda/join.hpp
+++ b/src/backend/cuda/join.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
@@ -16,3 +17,4 @@ Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
 template<typename T>
 void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/anisotropic_diffusion.cuh b/src/backend/cuda/kernel/anisotropic_diffusion.cuh
index cdb5c59121..cd393474aa 100644
--- a/src/backend/cuda/kernel/anisotropic_diffusion.cuh
+++ b/src/backend/cuda/kernel/anisotropic_diffusion.cuh
@@ -10,24 +10,22 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
-__forceinline__ __device__
-int index(const int x, const int y, const int dim0,
-          const int dim1, const int stride0, const int stride1) {
+__forceinline__ __device__ int index(const int x, const int y, const int dim0,
+                                     const int dim1, const int stride0,
+                                     const int stride1) {
     return clamp(x, 0, dim0 - 1) * stride0 + clamp(y, 0, dim1 - 1) * stride1;
 }
 
-__device__
-float quadratic(const float value) { return 1.0 / (1.0 + value); }
+__device__ float quadratic(const float value) { return 1.0 / (1.0 + value); }
 
 template<af_flux_function FluxEnum>
-__device__
-float gradientUpdate(const float mct, const float C,
-                     const float S, const float N,
-                     const float W, const float E,
-                     const float SE, const float SW,
-                     const float NE, const float NW) {
+__device__ float gradientUpdate(const float mct, const float C, const float S,
+                                const float N, const float W, const float E,
+                                const float SE, const float SW, const float NE,
+                                const float NW) {
     float delta = 0;
 
     float dx, dy, df, db, cx, cxd;
@@ -69,11 +67,10 @@ float gradientUpdate(const float mct, const float C,
     return delta;
 }
 
-__device__
-float curvatureUpdate(const float mct, const float C, const float S,
-                      const float N, const float W, const float E,
-                      const float SE, const float SW, const float NE,
-                      const float NW) {
+__device__ float curvatureUpdate(const float mct, const float C, const float S,
+                                 const float N, const float W, const float E,
+                                 const float SE, const float SW, const float NE,
+                                 const float NW) {
     float delta     = 0;
     float prop_grad = 0;
 
@@ -131,11 +128,10 @@ float curvatureUpdate(const float mct, const float C, const float S,
 }
 
 template<typename T, af_flux_function FluxEnum, bool isMCDE>
-__global__
-void diffUpdate(Param<T> inout, const float dt, const float mct,
-                const unsigned blkX, const unsigned blkY) {
-    const unsigned RADIUS = 1;
-    const unsigned SHRD_MEM_WIDTH = THREADS_X + 2 * RADIUS;
+__global__ void diffUpdate(Param<T> inout, const float dt, const float mct,
+                           const unsigned blkX, const unsigned blkY) {
+    const unsigned RADIUS          = 1;
+    const unsigned SHRD_MEM_WIDTH  = THREADS_X + 2 * RADIUS;
     const unsigned SHRD_MEM_HEIGHT = THREADS_Y * YDIM_LOAD + 2 * RADIUS;
 
     __shared__ float shrdMem[SHRD_MEM_HEIGHT][SHRD_MEM_WIDTH];
@@ -152,7 +148,7 @@ void diffUpdate(Param<T> inout, const float dt, const float mct,
     const int b3 = blockIdx.y / blkY;
 
     const int gx = blockDim.x * (blockIdx.x - b2 * blkX) + lx;
-          int gy = blockDim.y * (blockIdx.y - b3 * blkY) + ly;
+    int gy       = blockDim.y * (blockIdx.y - b3 * blkY) + ly;
 
     T* img = (T*)inout.ptr + (b3 * inout.strides[3] + b2 * inout.strides[2]);
 
@@ -162,7 +158,7 @@ void diffUpdate(Param<T> inout, const float dt, const float mct,
 #pragma unroll
         for (int a = lx, gx2 = gx - RADIUS; a < SHRD_MEM_WIDTH;
              a += blockDim.x, gx2 += blockDim.x) {
-            shrdMem[b][a] = img[ index(gx2, gy2, l0, l1, s0, s1) ];
+            shrdMem[b][a] = img[index(gx2, gy2, l0, l1, s0, s1)];
         }
     }
     __syncthreads();
@@ -171,19 +167,19 @@ void diffUpdate(Param<T> inout, const float dt, const float mct,
     int j = ly + RADIUS;
 
 #pragma unroll
-    for (int ld = 0; ld < YDIM_LOAD; ++ld, j+= blockDim.y, gy += blockDim.y) {
-        float C = shrdMem[j][i];
+    for (int ld = 0; ld < YDIM_LOAD; ++ld, j += blockDim.y, gy += blockDim.y) {
+        float C     = shrdMem[j][i];
         float delta = 0.0f;
         if (isMCDE) {
             delta = curvatureUpdate(
-                    mct, C, shrdMem[j][i + 1], shrdMem[j][i - 1], shrdMem[j - 1][i],
-                    shrdMem[j + 1][i], shrdMem[j + 1][i + 1], shrdMem[j - 1][i + 1],
-                    shrdMem[j + 1][i - 1], shrdMem[j - 1][i - 1]);
+                mct, C, shrdMem[j][i + 1], shrdMem[j][i - 1], shrdMem[j - 1][i],
+                shrdMem[j + 1][i], shrdMem[j + 1][i + 1], shrdMem[j - 1][i + 1],
+                shrdMem[j + 1][i - 1], shrdMem[j - 1][i - 1]);
         } else {
             delta = gradientUpdate<FluxEnum>(
-                    mct, C, shrdMem[j][i + 1], shrdMem[j][i - 1], shrdMem[j - 1][i],
-                    shrdMem[j + 1][i], shrdMem[j + 1][i + 1], shrdMem[j - 1][i + 1],
-                    shrdMem[j + 1][i - 1], shrdMem[j - 1][i - 1]);
+                mct, C, shrdMem[j][i + 1], shrdMem[j][i - 1], shrdMem[j - 1][i],
+                shrdMem[j + 1][i], shrdMem[j + 1][i + 1], shrdMem[j - 1][i + 1],
+                shrdMem[j + 1][i - 1], shrdMem[j - 1][i - 1]);
         }
         if (gy < l1 && gx < l0) {
             img[gx * s0 + gy * s1] = (T)(C + delta * dt);
@@ -191,4 +187,5 @@ void diffUpdate(Param<T> inout, const float dt, const float mct,
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/anisotropic_diffusion.hpp b/src/backend/cuda/kernel/anisotropic_diffusion.hpp
index 1c247bb499..e727d7ca4c 100644
--- a/src/backend/cuda/kernel/anisotropic_diffusion.hpp
+++ b/src/backend/cuda/kernel/anisotropic_diffusion.hpp
@@ -16,6 +16,7 @@
 #include <nvrtc_kernel_headers/anisotropic_diffusion_cuh.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -27,7 +28,8 @@ template<typename T>
 void anisotropicDiffusion(Param<T> inout, const float dt, const float mct,
                           const af::fluxFunction fftype, bool isMCDE) {
     auto diffUpdate = common::getKernel(
-        "cuda::diffUpdate", std::array{anisotropic_diffusion_cuh_src},
+        "arrayfire::cuda::diffUpdate",
+        std::array{anisotropic_diffusion_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(fftype),
                      TemplateArg(isMCDE)),
         std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y),
@@ -40,9 +42,8 @@ void anisotropicDiffusion(Param<T> inout, const float dt, const float mct,
 
     dim3 blocks(blkX * inout.dims[2], blkY * inout.dims[3], 1);
 
-    const int maxBlkY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    const int blkZ = divup(blocks.y, maxBlkY);
+    const int maxBlkY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    const int blkZ    = divup(blocks.y, maxBlkY);
 
     if (blkZ > 1) {
         blocks.y = maxBlkY;
@@ -58,3 +59,4 @@ void anisotropicDiffusion(Param<T> inout, const float dt, const float mct,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/approx.hpp b/src/backend/cuda/kernel/approx.hpp
index 66dea16fe6..db705da687 100644
--- a/src/backend/cuda/kernel/approx.hpp
+++ b/src/backend/cuda/kernel/approx.hpp
@@ -15,6 +15,7 @@
 #include <nvrtc_kernel_headers/approx2_cuh.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -28,7 +29,7 @@ void approx1(Param<Ty> yo, CParam<Ty> yi, CParam<Tp> xo, const int xdim,
              const Tp &xi_beg, const Tp &xi_step, const float offGrid,
              const af::interpType method, const int order) {
     auto approx1 = common::getKernel(
-        "cuda::approx1", std::array{approx1_cuh_src},
+        "arrayfire::cuda::approx1", std::array{approx1_cuh_src},
         TemplateArgs(TemplateTypename<Ty>(), TemplateTypename<Tp>(),
                      TemplateArg(xdim), TemplateArg(order)));
 
@@ -38,10 +39,9 @@ void approx1(Param<Ty> yo, CParam<Ty> yi, CParam<Tp> xo, const int xdim,
 
     bool batch = !(xo.dims[1] == 1 && xo.dims[2] == 1 && xo.dims[3] == 1);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -57,7 +57,7 @@ void approx2(Param<Ty> zo, CParam<Ty> zi, CParam<Tp> xo, const int xdim,
              const Tp &yi_beg, const Tp &yi_step, const float offGrid,
              const af::interpType method, const int order) {
     auto approx2 = common::getKernel(
-        "cuda::approx2", std::array{approx2_cuh_src},
+        "arrayfire::cuda::approx2", std::array{approx2_cuh_src},
         TemplateArgs(TemplateTypename<Ty>(), TemplateTypename<Tp>(),
                      TemplateArg(xdim), TemplateArg(ydim), TemplateArg(order)));
 
@@ -68,10 +68,9 @@ void approx2(Param<Ty> zo, CParam<Ty> zi, CParam<Tp> xo, const int xdim,
 
     bool batch = !(xo.dims[2] == 1 && xo.dims[3] == 1);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -83,3 +82,4 @@ void approx2(Param<Ty> zo, CParam<Ty> zi, CParam<Tp> xo, const int xdim,
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/approx1.cuh b/src/backend/cuda/kernel/approx1.cuh
index 6ef6a837a4..9ccf95e504 100644
--- a/src/backend/cuda/kernel/approx1.cuh
+++ b/src/backend/cuda/kernel/approx1.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <interp.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename Ty, typename Tp, int xdim, int order>
@@ -69,3 +70,4 @@ __global__ void approx1(Param<Ty> yo, CParam<Ty> yi, CParam<Tp> xo,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/approx2.cuh b/src/backend/cuda/kernel/approx2.cuh
index 191a4e8919..7d4179643e 100644
--- a/src/backend/cuda/kernel/approx2.cuh
+++ b/src/backend/cuda/kernel/approx2.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <interp.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename Ty, typename Tp, int xdim, int ydim, int order>
@@ -74,3 +75,4 @@ __global__ void approx2(Param<Ty> zo, CParam<Ty> zi, CParam<Tp> xo,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/assign.cuh b/src/backend/cuda/kernel/assign.cuh
index 102d42ec99..ddf159288b 100644
--- a/src/backend/cuda/kernel/assign.cuh
+++ b/src/backend/cuda/kernel/assign.cuh
@@ -13,12 +13,12 @@
 #include <assign_kernel_param.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
-__global__ void assign(Param<T> out, CParam<T> in,
-                       const cuda::AssignKernelParam p, const int nBBS0,
-                       const int nBBS1) {
+__global__ void assign(Param<T> out, CParam<T> in, const AssignKernelParam p,
+                       const int nBBS0, const int nBBS1) {
     // retrieve index pointers
     // these can be 0 where af_array index is not used
     const uint* ptr0 = p.ptr[0];
@@ -60,3 +60,4 @@ __global__ void assign(Param<T> out, CParam<T> in,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/assign.hpp b/src/backend/cuda/kernel/assign.hpp
index 523dad2505..75c24e874c 100644
--- a/src/backend/cuda/kernel/assign.hpp
+++ b/src/backend/cuda/kernel/assign.hpp
@@ -14,6 +14,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/assign_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -23,7 +24,7 @@ void assign(Param<T> out, CParam<T> in, const AssignKernelParam& p) {
     constexpr int THREADS_Y = 8;
 
     auto assignKer =
-        common::getKernel("cuda::assign", std::array{assign_cuh_src},
+        common::getKernel("arrayfire::cuda::assign", std::array{assign_cuh_src},
                           TemplateArgs(TemplateTypename<T>()));
 
     const dim3 threads(THREADS_X, THREADS_Y);
@@ -33,10 +34,9 @@ void assign(Param<T> out, CParam<T> in, const AssignKernelParam& p) {
 
     dim3 blocks(blks_x * in.dims[2], blks_y * in.dims[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -47,3 +47,4 @@ void assign(Param<T> out, CParam<T> in, const AssignKernelParam& p) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/atomics.hpp b/src/backend/cuda/kernel/atomics.hpp
index 47ed2f4747..cea1678e59 100644
--- a/src/backend/cuda/kernel/atomics.hpp
+++ b/src/backend/cuda/kernel/atomics.hpp
@@ -7,6 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 template<typename T>
@@ -49,3 +50,4 @@ __device__ cdouble atomicAdd<cdouble>(cdouble *ptr, cdouble val) {
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/bilateral.cuh b/src/backend/cuda/kernel/bilateral.cuh
index fb618005ac..6fdfbd1a3d 100644
--- a/src/backend/cuda/kernel/bilateral.cuh
+++ b/src/backend/cuda/kernel/bilateral.cuh
@@ -11,28 +11,26 @@
 #include <math.hpp>
 #include <shared.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
-inline __device__
-int lIdx(int x, int y, int stride1, int stride0) {
+inline __device__ int lIdx(int x, int y, int stride1, int stride0) {
     return (y * stride1 + x * stride0);
 }
 
 template<typename inType, typename outType>
-inline __device__
-void load2ShrdMem(outType *shrd, const inType *const in,
-                  int lx, int ly, int shrdStride, int dim0,
-                  int dim1, int gx, int gy, int inStride1,
-                  int inStride0) {
+inline __device__ void load2ShrdMem(outType *shrd, const inType *const in,
+                                    int lx, int ly, int shrdStride, int dim0,
+                                    int dim1, int gx, int gy, int inStride1,
+                                    int inStride0) {
     shrd[ly * shrdStride + lx] = in[lIdx(
         clamp(gx, 0, dim0 - 1), clamp(gy, 0, dim1 - 1), inStride1, inStride0)];
 }
 
 template<typename inType, typename outType>
-__global__
-void bilateral(Param<outType> out, CParam<inType> in,
-               float sigma_space, float sigma_color,
-               int gaussOff, int nBBS0, int nBBS1) {
+__global__ void bilateral(Param<outType> out, CParam<inType> in,
+                          float sigma_space, float sigma_color, int gaussOff,
+                          int nBBS0, int nBBS1) {
     SharedMemory<outType> shared;
     outType *localMem = shared.getPointer();
     outType *gauss2d  = localMem + gaussOff;
@@ -110,4 +108,5 @@ void bilateral(Param<outType> out, CParam<inType> in,
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/bilateral.hpp b/src/backend/cuda/kernel/bilateral.hpp
index 357b57a8bc..cf19eeb97c 100644
--- a/src/backend/cuda/kernel/bilateral.hpp
+++ b/src/backend/cuda/kernel/bilateral.hpp
@@ -13,6 +13,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/bilateral_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -23,7 +24,7 @@ template<typename inType, typename outType>
 void bilateral(Param<outType> out, CParam<inType> in, float s_sigma,
                float c_sigma) {
     auto bilateral = common::getKernel(
-        "cuda::bilateral", std::array{bilateral_cuh_src},
+        "arrayfire::cuda::bilateral", std::array{bilateral_cuh_src},
         TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>()),
         std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
 
@@ -41,8 +42,7 @@ void bilateral(Param<outType> out, CParam<inType> in, float s_sigma,
     size_t total_shrd_size =
         sizeof(outType) * (num_shrd_elems + num_gauss_elems);
 
-    size_t MAX_SHRD_SIZE =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).sharedMemPerBlock;
+    size_t MAX_SHRD_SIZE = getDeviceProp(getActiveDeviceId()).sharedMemPerBlock;
     if (total_shrd_size > MAX_SHRD_SIZE) {
         char errMessage[256];
         snprintf(errMessage, sizeof(errMessage),
@@ -60,3 +60,4 @@ void bilateral(Param<outType> out, CParam<inType> in, float s_sigma,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/canny.cuh b/src/backend/cuda/kernel/canny.cuh
index 27c758d1c4..bdd9ac2217 100644
--- a/src/backend/cuda/kernel/canny.cuh
+++ b/src/backend/cuda/kernel/canny.cuh
@@ -15,17 +15,17 @@
 // the breath first search algorithm
 __device__ int hasChanged = 0;
 
+namespace arrayfire {
 namespace cuda {
 
-__forceinline__ __device__
-int lIdx(int x, int y, int stride0, int stride1) {
+__forceinline__ __device__ int lIdx(int x, int y, int stride0, int stride1) {
     return (x * stride0 + y * stride1);
 }
 
 template<typename T>
-__global__
-void nonMaxSuppression(Param<float> output, CParam<T> in, CParam<T> dx,
-                       CParam<T> dy, unsigned nBBS0, unsigned nBBS1) {
+__global__ void nonMaxSuppression(Param<float> output, CParam<T> in,
+                                  CParam<T> dx, CParam<T> dy, unsigned nBBS0,
+                                  unsigned nBBS1) {
     const unsigned SHRD_MEM_WIDTH  = THREADS_X + 2;  // Coloumns
     const unsigned SHRD_MEM_HEIGHT = THREADS_Y + 2;  // Rows
 
@@ -46,8 +46,7 @@ void nonMaxSuppression(Param<float> output, CParam<T> in, CParam<T> dx,
 
     // Offset input and output pointers to second pixel of second coloumn/row
     // to skip the border
-    const T* mag = (const T*)in.ptr +
-                   (b2 * in.strides[2] + b3 * in.strides[3]);
+    const T* mag = (const T*)in.ptr + (b2 * in.strides[2] + b3 * in.strides[3]);
     const T* dX = (const T*)dx.ptr + (b2 * dx.strides[2] + b3 * dx.strides[3]) +
                   dx.strides[1] + 1;
     const T* dY = (const T*)dy.ptr + (b2 * dy.strides[2] + b3 * dy.strides[3]) +
@@ -63,8 +62,7 @@ void nonMaxSuppression(Param<float> output, CParam<T> in, CParam<T> dx,
 #pragma unroll
         for (int a = lx, gx2 = gx; a < SHRD_MEM_WIDTH && gx2 < in.dims[0];
              a += blockDim.x, gx2 += blockDim.x)
-            shrdMem[b][a] =
-                mag[lIdx(gx2, gy2, in.strides[0], in.strides[1])];
+            shrdMem[b][a] = mag[lIdx(gx2, gy2, in.strides[0], in.strides[1])];
 
     int i = lx + 1;
     int j = ly + 1;
@@ -143,9 +141,8 @@ void nonMaxSuppression(Param<float> output, CParam<T> in, CParam<T> dx,
 }
 
 template<typename T>
-__global__
-void initEdgeOut(Param<T> output, CParam<T> strong, CParam<T> weak,
-                 unsigned nBBS0, unsigned nBBS1) {
+__global__ void initEdgeOut(Param<T> output, CParam<T> strong, CParam<T> weak,
+                            unsigned nBBS0, unsigned nBBS1) {
     // batch offsets for 3rd and 4th dimension
     const unsigned b2 = blockIdx.x / nBBS0;
     const unsigned b3 = blockIdx.y / nBBS1;
@@ -175,8 +172,7 @@ void initEdgeOut(Param<T> output, CParam<T> strong, CParam<T> weak,
      (i) < (SHRD_MEM_WIDTH - 1))
 
 template<typename T>
-__global__
-void edgeTrack(Param<T> output, unsigned nBBS0, unsigned nBBS1) {
+__global__ void edgeTrack(Param<T> output, unsigned nBBS0, unsigned nBBS1) {
     const unsigned SHRD_MEM_WIDTH  = THREADS_X + 2;  // Cols
     const unsigned SHRD_MEM_HEIGHT = THREADS_Y + 2;  // Rows
 
@@ -226,25 +222,24 @@ void edgeTrack(Param<T> output, unsigned nBBS0, unsigned nBBS1) {
     int continueIter = 1;
 
     while (continueIter) {
-
-      int nw ,no ,ne ,we ,ea ,sw ,so ,se;
-
-      if(outMem[j][i] == WEAK) {
-        nw = outMem[j - 1][i - 1];
-        no = outMem[j - 1][i];
-        ne = outMem[j - 1][i + 1];
-        we = outMem[j    ][i - 1];
-        ea = outMem[j    ][i + 1];
-        sw = outMem[j + 1][i - 1];
-        so = outMem[j + 1][i];
-        se = outMem[j + 1][i + 1];
-
-        bool hasStrongNeighbour =
-            nw == STRONG || no == STRONG || ne == STRONG || ea == STRONG ||
-            se == STRONG || so == STRONG || sw == STRONG || we == STRONG;
-
-        if (hasStrongNeighbour) outMem[j][i] = STRONG;
-      }
+        int nw, no, ne, we, ea, sw, so, se;
+
+        if (outMem[j][i] == WEAK) {
+            nw = outMem[j - 1][i - 1];
+            no = outMem[j - 1][i];
+            ne = outMem[j - 1][i + 1];
+            we = outMem[j][i - 1];
+            ea = outMem[j][i + 1];
+            sw = outMem[j + 1][i - 1];
+            so = outMem[j + 1][i];
+            se = outMem[j + 1][i + 1];
+
+            bool hasStrongNeighbour =
+                nw == STRONG || no == STRONG || ne == STRONG || ea == STRONG ||
+                se == STRONG || so == STRONG || sw == STRONG || we == STRONG;
+
+            if (hasStrongNeighbour) outMem[j][i] = STRONG;
+        }
 
         __syncthreads();
 
@@ -252,17 +247,17 @@ void edgeTrack(Param<T> output, unsigned nBBS0, unsigned nBBS1) {
         // This search however ignores 1-pixel border encompassing the
         // shared memory tile region.
         bool hasWeakNeighbour = false;
-        if(outMem[j][i] == STRONG) {
-          nw = outMem[j - 1][i - 1] == WEAK && VALID_BLOCK_IDX(j - 1, i - 1);
-          no = outMem[j - 1][i    ] == WEAK && VALID_BLOCK_IDX(j - 1, i);
-          ne = outMem[j - 1][i + 1] == WEAK && VALID_BLOCK_IDX(j - 1, i + 1);
-          we = outMem[j    ][i - 1] == WEAK && VALID_BLOCK_IDX(j, i - 1);
-          ea = outMem[j    ][i + 1] == WEAK && VALID_BLOCK_IDX(j, i + 1);
-          sw = outMem[j + 1][i - 1] == WEAK && VALID_BLOCK_IDX(j + 1, i - 1);
-          so = outMem[j + 1][i    ] == WEAK && VALID_BLOCK_IDX(j + 1, i);
-          se = outMem[j + 1][i + 1] == WEAK && VALID_BLOCK_IDX(j + 1, i + 1);
-
-          hasWeakNeighbour = nw || no || ne || ea || se || so || sw || we;
+        if (outMem[j][i] == STRONG) {
+            nw = outMem[j - 1][i - 1] == WEAK && VALID_BLOCK_IDX(j - 1, i - 1);
+            no = outMem[j - 1][i] == WEAK && VALID_BLOCK_IDX(j - 1, i);
+            ne = outMem[j - 1][i + 1] == WEAK && VALID_BLOCK_IDX(j - 1, i + 1);
+            we = outMem[j][i - 1] == WEAK && VALID_BLOCK_IDX(j, i - 1);
+            ea = outMem[j][i + 1] == WEAK && VALID_BLOCK_IDX(j, i + 1);
+            sw = outMem[j + 1][i - 1] == WEAK && VALID_BLOCK_IDX(j + 1, i - 1);
+            so = outMem[j + 1][i] == WEAK && VALID_BLOCK_IDX(j + 1, i);
+            se = outMem[j + 1][i + 1] == WEAK && VALID_BLOCK_IDX(j + 1, i + 1);
+
+            hasWeakNeighbour = nw || no || ne || ea || se || so || sw || we;
         }
 
         continueIter = __syncthreads_or(hasWeakNeighbour);
@@ -291,12 +286,13 @@ void edgeTrack(Param<T> output, unsigned nBBS0, unsigned nBBS1) {
 
     // Update output with shared memory result
     if (gx < (output.dims[0] - 2) && gy < (output.dims[1] - 2))
-        oPtr[lIdx(gx, gy, output.strides[0], output.strides[1]) + output.strides[1] + 1] = outMem[j][i];
+        oPtr[lIdx(gx, gy, output.strides[0], output.strides[1]) +
+             output.strides[1] + 1] = outMem[j][i];
 }
 
 template<typename T>
-__global__
-void suppressLeftOver(Param<T> output, unsigned nBBS0, unsigned nBBS1) {
+__global__ void suppressLeftOver(Param<T> output, unsigned nBBS0,
+                                 unsigned nBBS1) {
     // batch offsets for 3rd and 4th dimension
     const unsigned b2 = blockIdx.x / nBBS0;
     const unsigned b3 = blockIdx.y / nBBS1;
@@ -317,4 +313,5 @@ void suppressLeftOver(Param<T> output, unsigned nBBS0, unsigned nBBS1) {
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/canny.hpp b/src/backend/cuda/kernel/canny.hpp
index cc63a029c4..61af04ba6c 100644
--- a/src/backend/cuda/kernel/canny.hpp
+++ b/src/backend/cuda/kernel/canny.hpp
@@ -13,6 +13,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/canny_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -27,7 +28,7 @@ template<typename T>
 void nonMaxSuppression(Param<T> output, CParam<T> magnitude, CParam<T> dx,
                        CParam<T> dy) {
     auto nonMaxSuppress = common::getKernel(
-        "cuda::nonMaxSuppression", std::array{canny_cuh_src},
+        "arrayfire::cuda::nonMaxSuppression", std::array{canny_cuh_src},
         TemplateArgs(TemplateTypename<T>()),
         std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
                    DefineValue(THREADS_X), DefineValue(THREADS_Y)});
@@ -49,17 +50,17 @@ void nonMaxSuppression(Param<T> output, CParam<T> magnitude, CParam<T> dx,
 template<typename T>
 void edgeTrackingHysteresis(Param<T> output, CParam<T> strong, CParam<T> weak) {
     auto initEdgeOut = common::getKernel(
-        "cuda::initEdgeOut", std::array{canny_cuh_src},
+        "arrayfire::cuda::initEdgeOut", std::array{canny_cuh_src},
         TemplateArgs(TemplateTypename<T>()),
         std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
                    DefineValue(THREADS_X), DefineValue(THREADS_Y)});
     auto edgeTrack = common::getKernel(
-        "cuda::edgeTrack", std::array{canny_cuh_src},
+        "arrayfire::cuda::edgeTrack", std::array{canny_cuh_src},
         TemplateArgs(TemplateTypename<T>()),
         std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
                    DefineValue(THREADS_X), DefineValue(THREADS_Y)});
     auto suppressLeftOver = common::getKernel(
-        "cuda::suppressLeftOver", std::array{canny_cuh_src},
+        "arrayfire::cuda::suppressLeftOver", std::array{canny_cuh_src},
         TemplateArgs(TemplateTypename<T>()),
         std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
                    DefineValue(THREADS_X), DefineValue(THREADS_Y)});
@@ -92,3 +93,4 @@ void edgeTrackingHysteresis(Param<T> output, CParam<T> strong, CParam<T> weak) {
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/config.hpp b/src/backend/cuda/kernel/config.hpp
index 975d6ff987..9bef1d7784 100644
--- a/src/backend/cuda/kernel/config.hpp
+++ b/src/backend/cuda/kernel/config.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -18,3 +19,4 @@ static const uint THREADS_Y         = THREADS_PER_BLOCK / THREADS_X;
 static const uint REPEAT            = 32;
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/convolve.hpp b/src/backend/cuda/kernel/convolve.hpp
index 7b105ef842..8183805e7c 100644
--- a/src/backend/cuda/kernel/convolve.hpp
+++ b/src/backend/cuda/kernel/convolve.hpp
@@ -20,6 +20,7 @@
 #include <nvrtc_kernel_headers/convolve_separable_cuh.hpp>
 #include <traits.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -63,8 +64,7 @@ void prepareKernelArgs(conv_kparam_t& params, dim_t oDims[], dim_t fDims[],
         batchDims[i] = (params.launchMoreBlocks ? 1 : oDims[i]);
     }
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
     if (baseDim == 1) {
         params.mThreads = dim3(CONV_THREADS, 1);
         params.mBlk_x   = divup(oDims[0], params.mThreads.x);
@@ -101,7 +101,7 @@ template<typename T, typename aT>
 void convolve_1d(conv_kparam_t& p, Param<T> out, CParam<T> sig, CParam<aT> filt,
                  const bool expand) {
     auto convolve1 = common::getKernel(
-        "cuda::convolve1", std::array{convolve1_cuh_src},
+        "arrayfire::cuda::convolve1", std::array{convolve1_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
                      TemplateArg(expand)),
         std::array{DefineValue(MAX_CONV1_FILTER_LEN),
@@ -158,7 +158,7 @@ void conv2Helper(const conv_kparam_t& p, Param<T> out, CParam<T> sig,
     }
 
     auto convolve2 = common::getKernel(
-        "cuda::convolve2", std::array{convolve2_cuh_src},
+        "arrayfire::cuda::convolve2", std::array{convolve2_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
                      TemplateArg(expand), TemplateArg(f0), TemplateArg(f1)),
         std::array{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
@@ -203,7 +203,7 @@ template<typename T, typename aT>
 void convolve_3d(conv_kparam_t& p, Param<T> out, CParam<T> sig, CParam<aT> filt,
                  const bool expand) {
     auto convolve3 = common::getKernel(
-        "cuda::convolve3", std::array{convolve3_cuh_src},
+        "arrayfire::cuda::convolve3", std::array{convolve3_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
                      TemplateArg(expand)),
         std::array{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
@@ -308,7 +308,8 @@ void convolve2(Param<T> out, CParam<T> signal, CParam<aT> filter, int conv_dim,
     }
 
     auto convolve2_separable = common::getKernel(
-        "cuda::convolve2_separable", std::array{convolve_separable_cuh_src},
+        "arrayfire::cuda::convolve2_separable",
+        std::array{convolve_separable_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
                      TemplateArg(conv_dim), TemplateArg(expand),
                      TemplateArg(fLen)),
@@ -335,3 +336,4 @@ void convolve2(Param<T> out, CParam<T> signal, CParam<aT> filter, int conv_dim,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/convolve1.cuh b/src/backend/cuda/kernel/convolve1.cuh
index 765703cf99..f82c85427c 100644
--- a/src/backend/cuda/kernel/convolve1.cuh
+++ b/src/backend/cuda/kernel/convolve1.cuh
@@ -11,17 +11,16 @@
 #include <math.hpp>
 #include <shared.hpp>
 
-__constant__ char
-    cFilter[2 * (2 * (MAX_CONV1_FILTER_LEN - 1) + CONV_THREADS) *
-            sizeof(double)];
+__constant__ char cFilter[2 * (2 * (MAX_CONV1_FILTER_LEN - 1) + CONV_THREADS) *
+                          sizeof(double)];
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename aT, bool expand>
-__global__
-void convolve1(Param<T> out, CParam<T> signal,
-               int fLen, int nBBS0, int nBBS1,
-               int o1, int o2, int o3, int s1, int s2, int s3) {
+__global__ void convolve1(Param<T> out, CParam<T> signal, int fLen, int nBBS0,
+                          int nBBS1, int o1, int o2, int o3, int s1, int s2,
+                          int s3) {
     SharedMemory<T> shared;
     T *shrdMem = shared.getPointer();
 
@@ -74,4 +73,5 @@ void convolve1(Param<T> out, CParam<T> signal,
     }
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/convolve2.cuh b/src/backend/cuda/kernel/convolve2.cuh
index 7bd8fa4375..3699cb9e51 100644
--- a/src/backend/cuda/kernel/convolve2.cuh
+++ b/src/backend/cuda/kernel/convolve2.cuh
@@ -10,16 +10,15 @@
 #include <Param.hpp>
 #include <math.hpp>
 
-__constant__ char
-    cFilter[2 * (2 * (MAX_CONV1_FILTER_LEN - 1) + CONV_THREADS) *
-            sizeof(double)];
+__constant__ char cFilter[2 * (2 * (MAX_CONV1_FILTER_LEN - 1) + CONV_THREADS) *
+                          sizeof(double)];
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename aT, bool expand, int fLen0, int fLen1>
-__global__
-void convolve2(Param<T> out, CParam<T> signal, int nBBS0, int nBBS1,
-               int o2, int o3, int s2, int s3) {
+__global__ void convolve2(Param<T> out, CParam<T> signal, int nBBS0, int nBBS1,
+                          int o2, int o3, int s2, int s3) {
     const size_t C_SIZE = (CONV2_THREADS_X + 2 * (fLen0 - 1)) *
                           (CONV2_THREADS_Y + 2 * (fLen1 - 1));
     __shared__ T shrdMem[C_SIZE];
@@ -51,8 +50,9 @@ void convolve2(Param<T> out, CParam<T> signal, int nBBS0, int nBBS1,
     int lx = threadIdx.x;
     int ly = threadIdx.y;
     int gx = CONV2_THREADS_X * (blockIdx.x - b0 * nBBS0) + lx;
-    int gy = CONV2_THREADS_Y *
-                 ((blockIdx.y + blockIdx.z * gridDim.y) - b1 * nBBS1) + ly;
+    int gy =
+        CONV2_THREADS_Y * ((blockIdx.y + blockIdx.z * gridDim.y) - b1 * nBBS1) +
+        ly;
 
     if (b1 >= out.dims[3]) return;
 
@@ -97,4 +97,5 @@ void convolve2(Param<T> out, CParam<T> signal, int nBBS0, int nBBS1,
     }
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/convolve3.cuh b/src/backend/cuda/kernel/convolve3.cuh
index 08e671692c..18ad939054 100644
--- a/src/backend/cuda/kernel/convolve3.cuh
+++ b/src/backend/cuda/kernel/convolve3.cuh
@@ -11,21 +11,19 @@
 #include <math.hpp>
 #include <shared.hpp>
 
-__constant__ char
-    cFilter[2 * (2 * (MAX_CONV1_FILTER_LEN - 1) + CONV_THREADS) *
-            sizeof(double)];
+__constant__ char cFilter[2 * (2 * (MAX_CONV1_FILTER_LEN - 1) + CONV_THREADS) *
+                          sizeof(double)];
 
+namespace arrayfire {
 namespace cuda {
 
-__inline__
-int index(int i, int j, int k, int jstride, int kstride) {
+__inline__ int index(int i, int j, int k, int jstride, int kstride) {
     return i + j * jstride + k * kstride;
 }
 
 template<typename T, typename aT, bool expand>
-__global__
-void convolve3(Param<T> out, CParam<T> signal, int fLen0, int fLen1,
-               int fLen2, int nBBS, int o3, int s3) {
+__global__ void convolve3(Param<T> out, CParam<T> signal, int fLen0, int fLen1,
+                          int fLen2, int nBBS, int o3, int s3) {
     SharedMemory<T> shared;
 
     T *shrdMem   = shared.getPointer();
@@ -109,4 +107,5 @@ void convolve3(Param<T> out, CParam<T> signal, int fLen0, int fLen1,
     }
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/convolve_separable.cpp b/src/backend/cuda/kernel/convolve_separable.cpp
index c95f48afeb..3c18a02240 100644
--- a/src/backend/cuda/kernel/convolve_separable.cpp
+++ b/src/backend/cuda/kernel/convolve_separable.cpp
@@ -8,6 +8,7 @@
  ********************************************************/
 #include <kernel/convolve.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -29,3 +30,4 @@ INSTANTIATE(intl, float)
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/convolve_separable.cuh b/src/backend/cuda/kernel/convolve_separable.cuh
index 8a2e076dec..ead157df92 100644
--- a/src/backend/cuda/kernel/convolve_separable.cuh
+++ b/src/backend/cuda/kernel/convolve_separable.cuh
@@ -14,11 +14,12 @@ __constant__ char sFilter[2 * SCONV_THREADS_Y *
                           (2 * (MAX_SCONV_FILTER_LEN - 1) + SCONV_THREADS_X) *
                           sizeof(double)];
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename accType, int conv_dim, bool expand, int fLen>
-__global__
-void convolve2_separable(Param<T> out, CParam<T> signal, int nBBS0, int nBBS1) {
+__global__ void convolve2_separable(Param<T> out, CParam<T> signal, int nBBS0,
+                                    int nBBS1) {
     const int smem_len =
         (conv_dim == 0 ? (SCONV_THREADS_X + 2 * (fLen - 1)) * SCONV_THREADS_Y
                        : (SCONV_THREADS_Y + 2 * (fLen - 1)) * SCONV_THREADS_X);
@@ -96,4 +97,5 @@ void convolve2_separable(Param<T> out, CParam<T> signal, int nBBS0, int nBBS1) {
     }
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/copy.cuh b/src/backend/cuda/kernel/copy.cuh
index 5c6b6e485a..9e771e8c52 100644
--- a/src/backend/cuda/kernel/copy.cuh
+++ b/src/backend/cuda/kernel/copy.cuh
@@ -14,6 +14,7 @@
 #include <dims_param.hpp>
 #include <types.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -49,15 +50,14 @@ convertType<char, compute_t<common::half>>(char value) {
 }
 
 template<>
-__inline__ __device__ cuda::uchar
-convertType<compute_t<common::half>, cuda::uchar>(
-    compute_t<common::half> value) {
-    return (cuda::uchar)((short)value);
+__inline__ __device__ uchar
+convertType<compute_t<common::half>, uchar>(compute_t<common::half> value) {
+    return (uchar)((short)value);
 }
 
 template<>
 __inline__ __device__ compute_t<common::half>
-convertType<cuda::uchar, compute_t<common::half>>(cuda::uchar value) {
+convertType<uchar, compute_t<common::half>>(uchar value) {
     return compute_t<common::half>(value);
 }
 
@@ -290,3 +290,4 @@ __global__ void scaledCopyLoop123(Param<outType> out, CParam<inType> in,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/diagonal.cuh b/src/backend/cuda/kernel/diagonal.cuh
index d337c8f2a1..6e47af5b22 100644
--- a/src/backend/cuda/kernel/diagonal.cuh
+++ b/src/backend/cuda/kernel/diagonal.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -53,3 +54,4 @@ __global__ void extractDiagonal(Param<T> out, CParam<T> in, int num,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/diagonal.hpp b/src/backend/cuda/kernel/diagonal.hpp
index 87ba53965b..4ffb6fa4ff 100644
--- a/src/backend/cuda/kernel/diagonal.hpp
+++ b/src/backend/cuda/kernel/diagonal.hpp
@@ -15,12 +15,13 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/diagonal_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
 template<typename T>
 void diagCreate(Param<T> out, CParam<T> in, int num) {
-    auto genDiagMat = common::getKernel("cuda::createDiagonalMat",
+    auto genDiagMat = common::getKernel("arrayfire::cuda::createDiagonalMat",
                                         std::array{diagonal_cuh_src},
                                         TemplateArgs(TemplateTypename<T>()));
 
@@ -29,8 +30,7 @@ void diagCreate(Param<T> out, CParam<T> in, int num) {
     int blocks_y = divup(out.dims[1], threads.y);
     dim3 blocks(blocks_x * out.dims[2], blocks_y);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
+    const int maxBlocksY    = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
     const int blocksPerMatZ = divup(blocks.y, maxBlocksY);
     if (blocksPerMatZ > 1) {
         blocks.y = maxBlocksY;
@@ -46,19 +46,18 @@ void diagCreate(Param<T> out, CParam<T> in, int num) {
 
 template<typename T>
 void diagExtract(Param<T> out, CParam<T> in, int num) {
-    auto extractDiag =
-        common::getKernel("cuda::extractDiagonal", std::array{diagonal_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto extractDiag = common::getKernel("arrayfire::cuda::extractDiagonal",
+                                         std::array{diagonal_cuh_src},
+                                         TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(256, 1);
     int blocks_x = divup(out.dims[0], threads.x);
     int blocks_z = out.dims[2];
     dim3 blocks(blocks_x, out.dims[3] * blocks_z);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -69,3 +68,4 @@ void diagExtract(Param<T> out, CParam<T> in, int num) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/diff.cuh b/src/backend/cuda/kernel/diff.cuh
index 2f6305eb0f..fc02296b5c 100644
--- a/src/backend/cuda/kernel/diff.cuh
+++ b/src/backend/cuda/kernel/diff.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool D>
@@ -58,3 +59,4 @@ __global__ void diff(Param<T> out, CParam<T> in, const unsigned oElem,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/diff.hpp b/src/backend/cuda/kernel/diff.hpp
index fb157af798..c547e0e933 100644
--- a/src/backend/cuda/kernel/diff.hpp
+++ b/src/backend/cuda/kernel/diff.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/diff_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -25,7 +26,7 @@ void diff(Param<T> out, CParam<T> in, const int indims, const unsigned dim,
     constexpr unsigned TY = 16;
 
     auto diff =
-        common::getKernel("cuda::diff", std::array{diff_cuh_src},
+        common::getKernel("arrayfire::cuda::diff", std::array{diff_cuh_src},
                           TemplateArgs(TemplateTypename<T>(), TemplateArg(dim),
                                        TemplateArg(isDiff2)));
 
@@ -39,10 +40,9 @@ void diff(Param<T> out, CParam<T> in, const int indims, const unsigned dim,
 
     const int oElem = out.dims[0] * out.dims[1] * out.dims[2] * out.dims[3];
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -52,3 +52,4 @@ void diff(Param<T> out, CParam<T> in, const int indims, const unsigned dim,
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/exampleFunction.cuh b/src/backend/cuda/kernel/exampleFunction.cuh
index 9670d89ef6..e0a4ddffd6 100644
--- a/src/backend/cuda/kernel/exampleFunction.cuh
+++ b/src/backend/cuda/kernel/exampleFunction.cuh
@@ -10,6 +10,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -34,4 +35,5 @@ __global__ void exampleFunc(Param<T> c, CParam<T> a, CParam<T> b,
     }
 }
 
-} //namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/exampleFunction.hpp b/src/backend/cuda/kernel/exampleFunction.hpp
index 019b8c9743..730c309a86 100644
--- a/src/backend/cuda/kernel/exampleFunction.hpp
+++ b/src/backend/cuda/kernel/exampleFunction.hpp
@@ -18,6 +18,7 @@
 
 #include <nvrtc_kernel_headers/exampleFunction_cuh.hpp>  //kernel generated by nvrtc
 
+namespace arrayfire {
 namespace cuda {
 
 namespace kernel {
@@ -27,7 +28,7 @@ static const unsigned TY = 16;  // Kernel Launch Config Values
 
 template<typename T>  // CUDA kernel wrapper function
 void exampleFunc(Param<T> c, CParam<T> a, CParam<T> b, const af_someenum_t p) {
-    auto exampleFunc = common::getKernel("cuda::exampleFunc",
+    auto exampleFunc = common::getKernel("arrayfire::cuda::exampleFunc",
                                          std::array{exampleFunction_cuh_src},
                                          TemplateArgs(TemplateTypename<T>()));
 
@@ -43,7 +44,7 @@ void exampleFunc(Param<T> c, CParam<T> a, CParam<T> b, const af_someenum_t p) {
     // on your CUDA kernels needs such as shared memory etc.
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
-    // Call the kernel functor retrieved using common::getKernel
+    // Call the kernel functor retrieved using arrayfire::common::getKernel
     exampleFunc(qArgs, c, a, b, p);
 
     POST_LAUNCH_CHECK();  // Macro for post kernel launch checks
@@ -52,3 +53,4 @@ void exampleFunc(Param<T> c, CParam<T> a, CParam<T> b, const af_someenum_t p) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/fast.hpp b/src/backend/cuda/kernel/fast.hpp
index 3521f8cfcb..7b54162b42 100644
--- a/src/backend/cuda/kernel/fast.hpp
+++ b/src/backend/cuda/kernel/fast.hpp
@@ -17,6 +17,7 @@
 #include <memory.hpp>
 #include <cub/block/block_reduce.cuh>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -397,7 +398,7 @@ void fast(unsigned *out_feat, float **x_out, float **y_out, float **score_out,
 
     unsigned *d_total = (unsigned *)(d_score.get() + (indims[0] * indims[1]));
     CUDA_CHECK(
-        cudaMemsetAsync(d_total, 0, sizeof(unsigned), cuda::getActiveStream()));
+        cudaMemsetAsync(d_total, 0, sizeof(unsigned), getActiveStream()));
     auto d_counts  = memAlloc<unsigned>(blocks.x * blocks.y);
     auto d_offsets = memAlloc<unsigned>(blocks.x * blocks.y);
 
@@ -415,9 +416,8 @@ void fast(unsigned *out_feat, float **x_out, float **y_out, float **score_out,
     // Dimensions of output array
     unsigned total;
     CUDA_CHECK(cudaMemcpyAsync(&total, d_total, sizeof(unsigned),
-                               cudaMemcpyDeviceToHost,
-                               cuda::getActiveStream()));
-    CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
+                               cudaMemcpyDeviceToHost, getActiveStream()));
+    CUDA_CHECK(cudaStreamSynchronize(getActiveStream()));
     total = total < max_feat ? total : max_feat;
 
     if (total > 0) {
@@ -444,3 +444,4 @@ void fast(unsigned *out_feat, float **x_out, float **y_out, float **score_out,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/fftconvolve.cuh b/src/backend/cuda/kernel/fftconvolve.cuh
index c5df6a1df4..350a7b299f 100644
--- a/src/backend/cuda/kernel/fftconvolve.cuh
+++ b/src/backend/cuda/kernel/fftconvolve.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/internal_enums.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename To, typename Ti>
@@ -218,3 +219,4 @@ __global__ void reorderOutput(Param<To> out, Param<Ti> in, CParam<To> filter,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/fftconvolve.hpp b/src/backend/cuda/kernel/fftconvolve.hpp
index 6ca9569206..cf45bc18a4 100644
--- a/src/backend/cuda/kernel/fftconvolve.hpp
+++ b/src/backend/cuda/kernel/fftconvolve.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/fftconvolve_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -24,10 +25,10 @@ template<typename convT, typename T>
 void packDataHelper(Param<convT> sig_packed, Param<convT> filter_packed,
                     CParam<T> sig, CParam<T> filter) {
     auto packData = common::getKernel(
-        "cuda::packData", std::array{fftconvolve_cuh_src},
+        "arrayfire::cuda::packData", std::array{fftconvolve_cuh_src},
         TemplateArgs(TemplateTypename<convT>(), TemplateTypename<T>()));
     auto padArray = common::getKernel(
-        "cuda::padArray", std::array{fftconvolve_cuh_src},
+        "arrayfire::cuda::padArray", std::array{fftconvolve_cuh_src},
         TemplateArgs(TemplateTypename<convT>(), TemplateTypename<T>()));
 
     dim_t *sd = sig.dims;
@@ -68,7 +69,7 @@ template<typename T, typename convT>
 void complexMultiplyHelper(Param<convT> sig_packed, Param<convT> filter_packed,
                            AF_BATCH_KIND kind) {
     auto cplxMul = common::getKernel(
-        "cuda::complexMultiply", std::array{fftconvolve_cuh_src},
+        "arrayfire::cuda::complexMultiply", std::array{fftconvolve_cuh_src},
         TemplateArgs(TemplateTypename<convT>(), TemplateArg(kind)));
 
     int sig_packed_elem    = 1;
@@ -101,7 +102,7 @@ void reorderOutputHelper(Param<T> out, Param<convT> packed, CParam<T> sig,
     constexpr bool RoundResult = std::is_integral<T>::value;
 
     auto reorderOut = common::getKernel(
-        "cuda::reorderOutput", std::array{fftconvolve_cuh_src},
+        "arrayfire::cuda::reorderOutput", std::array{fftconvolve_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<convT>(),
                      TemplateArg(expand), TemplateArg(RoundResult)));
 
@@ -125,3 +126,4 @@ void reorderOutputHelper(Param<T> out, Param<convT> packed, CParam<T> sig,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/flood_fill.cuh b/src/backend/cuda/kernel/flood_fill.cuh
index bab68916ec..ede793c0d3 100644
--- a/src/backend/cuda/kernel/flood_fill.cuh
+++ b/src/backend/cuda/kernel/flood_fill.cuh
@@ -8,14 +8,15 @@
  ********************************************************/
 
 #include <Param.hpp>
-#include <af/defines.h>
 #include <math.hpp>
+#include <af/defines.h>
 
 /// doAnotherLaunch is a variable in kernel space
 /// used to track the convergence of
 /// the breath first search algorithm
 __device__ int doAnotherLaunch = 0;
 
+namespace arrayfire {
 namespace cuda {
 
 /// Output array is set to the following values during the progression
@@ -27,24 +28,33 @@ namespace cuda {
 ///
 /// Once, the algorithm is finished, output is reset
 /// to either zero or \p newValue for all valid pixels.
-template<typename T> constexpr T VALID() { return T(2); }
-template<typename T> constexpr T INVALID() { return T(1); }
-template<typename T> constexpr T ZERO() { return T(0); }
+template<typename T>
+constexpr T VALID() {
+    return T(2);
+}
+template<typename T>
+constexpr T INVALID() {
+    return T(1);
+}
+template<typename T>
+constexpr T ZERO() {
+    return T(0);
+}
 
 template<typename T>
-__global__
-void initSeeds(Param<T> out, CParam<uint> seedsx, CParam<uint> seedsy) {
+__global__ void initSeeds(Param<T> out, CParam<uint> seedsx,
+                          CParam<uint> seedsy) {
     uint idx = blockDim.x * blockIdx.x + threadIdx.x;
     if (idx < seedsx.elements()) {
-        uint x = seedsx.ptr[ idx ];
-        uint y = seedsy.ptr[ idx ];
-        out.ptr[ x + y * out.dims[0] ] = VALID<T>();
+        uint x                       = seedsx.ptr[idx];
+        uint y                       = seedsy.ptr[idx];
+        out.ptr[x + y * out.dims[0]] = VALID<T>();
     }
 }
 
 template<typename T>
-__global__
-void floodStep(Param<T> out, CParam<T> img, T lowValue, T highValue) {
+__global__ void floodStep(Param<T> out, CParam<T> img, T lowValue,
+                          T highValue) {
     constexpr int RADIUS      = 1;
     constexpr int SMEM_WIDTH  = THREADS_X + 2 * RADIUS;
     constexpr int SMEM_HEIGHT = THREADS_Y + 2 * RADIUS;
@@ -61,7 +71,7 @@ void floodStep(Param<T> out, CParam<T> img, T lowValue, T highValue) {
     const int s1 = out.strides[1];
 
     const T *iptr = (const T *)img.ptr;
-          T *optr = (T *)out.ptr;
+    T *optr       = (T *)out.ptr;
 #pragma unroll
     for (int b = ly, gy2 = gy; b < SMEM_HEIGHT;
          b += blockDim.y, gy2 += blockDim.y) {
@@ -71,14 +81,14 @@ void floodStep(Param<T> out, CParam<T> img, T lowValue, T highValue) {
             int x      = gx2 - RADIUS;
             int y      = gy2 - RADIUS;
             bool inROI = (x >= 0 && x < d0 && y >= 0 && y < d1);
-            smem[b][a] = (inROI ? optr[ x*s0+y*s1 ] : INVALID<T>());
+            smem[b][a] = (inROI ? optr[x * s0 + y * s1] : INVALID<T>());
         }
     }
     int i = lx + RADIUS;
     int j = ly + RADIUS;
 
-    T tImgVal = iptr[(clamp(gx, 0, int(img.dims[0]-1)) * img.strides[0] +
-                      clamp(gy, 0, int(img.dims[1]-1)) * img.strides[1])];
+    T tImgVal = iptr[(clamp(gx, 0, int(img.dims[0] - 1)) * img.strides[0] +
+                      clamp(gy, 0, int(img.dims[1] - 1)) * img.strides[1])];
     const int isPxBtwnThresholds =
         (tImgVal >= lowValue && tImgVal <= highValue);
     __syncthreads();
@@ -86,7 +96,7 @@ void floodStep(Param<T> out, CParam<T> img, T lowValue, T highValue) {
     T origOutVal      = smem[j][i];
     bool blockChanged = false;
     bool isBorderPxl  = (lx == 0 || ly == 0 || lx == (blockDim.x - 1) ||
-                         ly == (blockDim.y - 1));
+                        ly == (blockDim.y - 1));
     do {
         int validNeighbors = 0;
 #pragma unroll
@@ -100,16 +110,14 @@ void floodStep(Param<T> out, CParam<T> img, T lowValue, T highValue) {
         __syncthreads();
 
         bool outChanged = (smem[j][i] == ZERO<T>() && (validNeighbors > 0));
-        if (outChanged) {
-            smem[j][i] = T(isPxBtwnThresholds + INVALID<T>());
-        }
+        if (outChanged) { smem[j][i] = T(isPxBtwnThresholds + INVALID<T>()); }
         blockChanged = __syncthreads_or(int(outChanged));
     } while (blockChanged);
 
     T newOutVal = smem[j][i];
 
-    bool borderChanged = (isBorderPxl &&
-                          newOutVal != origOutVal && newOutVal == VALID<T>());
+    bool borderChanged =
+        (isBorderPxl && newOutVal != origOutVal && newOutVal == VALID<T>());
 
     borderChanged = __syncthreads_or(int(borderChanged));
 
@@ -120,21 +128,19 @@ void floodStep(Param<T> out, CParam<T> img, T lowValue, T highValue) {
         doAnotherLaunch = 1;
     }
 
-    if (gx < d0 && gy < d1) {
-        optr[ (gx*s0 + gy*s1) ] = smem[j][i];
-    }
+    if (gx < d0 && gy < d1) { optr[(gx * s0 + gy * s1)] = smem[j][i]; }
 }
 
 template<typename T>
-__global__
-void finalizeOutput(Param<T> out, T newValue) {
+__global__ void finalizeOutput(Param<T> out, T newValue) {
     uint gx = blockDim.x * blockIdx.x + threadIdx.x;
     uint gy = blockDim.y * blockIdx.y + threadIdx.y;
     if (gx < out.dims[0] && gy < out.dims[1]) {
-        uint idx = gx * out.strides[0] + gy * out.strides[1];
-        T val = out.ptr[idx];
+        uint idx     = gx * out.strides[0] + gy * out.strides[1];
+        T val        = out.ptr[idx];
         out.ptr[idx] = (val == VALID<T>() ? newValue : ZERO<T>());
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/flood_fill.hpp b/src/backend/cuda/kernel/flood_fill.hpp
index ad6366a286..29f5741a04 100644
--- a/src/backend/cuda/kernel/flood_fill.hpp
+++ b/src/backend/cuda/kernel/flood_fill.hpp
@@ -16,6 +16,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/flood_fill_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -37,7 +38,7 @@ void floodFill(Param<T> out, CParam<T> image, CParam<uint> seedsx,
                const T highValue, const af::connectivity nlookup) {
     UNUSED(nlookup);
     if (sharedMemRequiredByFloodFill<T>() >
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).sharedMemPerBlock) {
+        getDeviceProp(getActiveDeviceId()).sharedMemPerBlock) {
         char errMessage[256];
         snprintf(errMessage, sizeof(errMessage),
                  "\nCurrent thread's CUDA device doesn't have sufficient "
@@ -45,15 +46,15 @@ void floodFill(Param<T> out, CParam<T> image, CParam<uint> seedsx,
         CUDA_NOT_SUPPORTED(errMessage);
     }
 
-    auto initSeeds =
-        common::getKernel("cuda::initSeeds", std::array{flood_fill_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto initSeeds = common::getKernel("arrayfire::cuda::initSeeds",
+                                       std::array{flood_fill_cuh_src},
+                                       TemplateArgs(TemplateTypename<T>()));
     auto floodStep = common::getKernel(
-        "cuda::floodStep", std::array{flood_fill_cuh_src},
+        "arrayfire::cuda::floodStep", std::array{flood_fill_cuh_src},
         TemplateArgs(TemplateTypename<T>()),
         std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
     auto finalizeOutput = common::getKernel(
-        "cuda::finalizeOutput", std::array{flood_fill_cuh_src},
+        "arrayfire::cuda::finalizeOutput", std::array{flood_fill_cuh_src},
         TemplateArgs(TemplateTypename<T>()));
 
     EnqueueArgs qArgs(dim3(divup(seedsx.elements(), THREADS)), dim3(THREADS),
@@ -81,3 +82,4 @@ void floodFill(Param<T> out, CParam<T> image, CParam<uint> seedsx,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/gradient.cuh b/src/backend/cuda/kernel/gradient.cuh
index 94051dc6a8..19ec419887 100644
--- a/src/backend/cuda/kernel/gradient.cuh
+++ b/src/backend/cuda/kernel/gradient.cuh
@@ -12,14 +12,14 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 #define sidx(y, x) scratch[y + 1][x + 1]
 
 template<typename T>
 __global__ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in,
-                         const int blocksPerMatX,
-                         const int blocksPerMatY) {
+                         const int blocksPerMatX, const int blocksPerMatY) {
     const int idz = blockIdx.x / blocksPerMatX;
     const int idw = (blockIdx.y + blockIdx.z * gridDim.y) / blocksPerMatY;
 
@@ -63,9 +63,9 @@ __global__ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in,
     // Cols
     if (threadIdx.y == 0) {
         // Y-1
-        sidx(-1, threadIdx.x) = (cond || idy == 0)
-                                    ? sidx(0, threadIdx.x)
-                                    : in.ptr[iIdx - in.strides[1]];
+        sidx(-1, threadIdx.x)   = (cond || idy == 0)
+                                      ? sidx(0, threadIdx.x)
+                                      : in.ptr[iIdx - in.strides[1]];
         sidx(ymax, threadIdx.x) = (cond || (idy + ymax) >= in.dims[1])
                                       ? sidx(ymax - 1, threadIdx.x)
                                       : in.ptr[iIdx + ymax * in.strides[1]];
@@ -90,3 +90,4 @@ __global__ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/gradient.hpp b/src/backend/cuda/kernel/gradient.hpp
index 8f1306e2b0..a6f2a8a6b9 100644
--- a/src/backend/cuda/kernel/gradient.hpp
+++ b/src/backend/cuda/kernel/gradient.hpp
@@ -17,6 +17,7 @@
 
 #include <array>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -25,10 +26,10 @@ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in) {
     constexpr unsigned TX = 32;
     constexpr unsigned TY = 8;
 
-    auto gradient =
-        common::getKernel("cuda::gradient", std::array{gradient_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()),
-                          std::array{DefineValue(TX), DefineValue(TY)});
+    auto gradient = common::getKernel(
+        "arrayfire::cuda::gradient", std::array{gradient_cuh_src},
+        TemplateArgs(TemplateTypename<T>()),
+        std::array{DefineValue(TX), DefineValue(TY)});
 
     dim3 threads(TX, TY, 1);
 
@@ -36,10 +37,9 @@ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in) {
     int blocksPerMatY = divup(in.dims[1], TY);
     dim3 blocks(blocksPerMatX * in.dims[2], blocksPerMatY * in.dims[3], 1);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -49,3 +49,4 @@ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/harris.hpp b/src/backend/cuda/kernel/harris.hpp
index e8fe490b52..e956f02441 100644
--- a/src/backend/cuda/kernel/harris.hpp
+++ b/src/backend/cuda/kernel/harris.hpp
@@ -23,6 +23,7 @@
 
 #include <vector>
 
+namespace arrayfire {
 namespace cuda {
 
 namespace kernel {
@@ -176,9 +177,9 @@ void harris(unsigned* corners_out, float** x_out, float** y_out,
     int filter_elem   = filter.strides[3] * filter.dims[3];
     auto filter_alloc = memAlloc<convAccT>(filter_elem);
     filter.ptr        = filter_alloc.get();
-    CUDA_CHECK(cudaMemcpyAsync(
-        filter.ptr, h_filter.data(), filter_elem * sizeof(convAccT),
-        cudaMemcpyHostToDevice, cuda::getActiveStream()));
+    CUDA_CHECK(cudaMemcpyAsync(filter.ptr, h_filter.data(),
+                               filter_elem * sizeof(convAccT),
+                               cudaMemcpyHostToDevice, getActiveStream()));
 
     const unsigned border_len = filter_len / 2 + 1;
 
@@ -238,7 +239,7 @@ void harris(unsigned* corners_out, float** x_out, float** y_out,
 
     auto d_corners_found = memAlloc<unsigned>(1);
     CUDA_CHECK(cudaMemsetAsync(d_corners_found.get(), 0, sizeof(unsigned),
-                               cuda::getActiveStream()));
+                               getActiveStream()));
 
     auto d_x_corners    = memAlloc<float>(corner_lim);
     auto d_y_corners    = memAlloc<float>(corner_lim);
@@ -265,7 +266,7 @@ void harris(unsigned* corners_out, float** x_out, float** y_out,
     unsigned corners_found = 0;
     CUDA_CHECK(cudaMemcpyAsync(&corners_found, d_corners_found.get(),
                                sizeof(unsigned), cudaMemcpyDeviceToHost,
-                               cuda::getActiveStream()));
+                               getActiveStream()));
     CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
     *corners_out =
@@ -327,13 +328,13 @@ void harris(unsigned* corners_out, float** x_out, float** y_out,
 
         CUDA_CHECK(cudaMemcpyAsync(
             *x_out, d_x_corners.get(), *corners_out * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *y_out, d_y_corners.get(), *corners_out * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *resp_out, d_resp_corners.get(), *corners_out * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
 
         x_out_alloc.release();
         y_out_alloc.release();
@@ -349,3 +350,4 @@ void harris(unsigned* corners_out, float** x_out, float** y_out,
 }  // namespace kernel
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/histogram.cuh b/src/backend/cuda/kernel/histogram.cuh
index 3cd68a1485..258dc6ff3c 100644
--- a/src/backend/cuda/kernel/histogram.cuh
+++ b/src/backend/cuda/kernel/histogram.cuh
@@ -12,6 +12,7 @@
 #include <shared.hpp>
 #include <types.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool isLinear>
@@ -21,9 +22,10 @@ __global__ void histogram(Param<uint> out, CParam<T> in, int len, int nbins,
     uint *shrdMem = shared.getPointer();
 
     // offset input and output to account for batch ops
-    unsigned b2   = blockIdx.x / nBBS;
-    const data_t<T> *iptr = in.ptr + b2 * in.strides[2] + blockIdx.y * in.strides[3];
-    uint *optr    = out.ptr + b2 * out.strides[2] + blockIdx.y * out.strides[3];
+    unsigned b2 = blockIdx.x / nBBS;
+    const data_t<T> *iptr =
+        in.ptr + b2 * in.strides[2] + blockIdx.y * in.strides[3];
+    uint *optr = out.ptr + b2 * out.strides[2] + blockIdx.y * out.strides[3];
 
     int start = (blockIdx.x - b2 * nBBS) * THRD_LOAD * blockDim.x + threadIdx.x;
     int end   = min((start + THRD_LOAD * blockDim.x), len);
@@ -45,9 +47,10 @@ __global__ void histogram(Param<uint> out, CParam<T> in, int len, int nbins,
             isLinear
                 ? row
                 : ((row % in.dims[0]) + (row / in.dims[0]) * in.strides[1]);
-        int bin = (int)(static_cast<float>(compute_t<T>(iptr[idx]) - minvalT) / step);
-        bin     = (bin < 0) ? 0 : bin;
-        bin     = (bin >= nbins) ? (nbins - 1) : bin;
+        int bin =
+            (int)(static_cast<float>(compute_t<T>(iptr[idx]) - minvalT) / step);
+        bin = (bin < 0) ? 0 : bin;
+        bin = (bin >= nbins) ? (nbins - 1) : bin;
 
         if (use_global) {
             atomicAdd((optr + bin), 1);
@@ -66,3 +69,4 @@ __global__ void histogram(Param<uint> out, CParam<T> in, int len, int nbins,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/histogram.hpp b/src/backend/cuda/kernel/histogram.hpp
index 4e4fe8c901..b9a9945c99 100644
--- a/src/backend/cuda/kernel/histogram.hpp
+++ b/src/backend/cuda/kernel/histogram.hpp
@@ -13,6 +13,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/histogram_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -24,7 +25,7 @@ template<typename T>
 void histogram(Param<uint> out, CParam<T> in, int nbins, float minval,
                float maxval, bool isLinear) {
     auto histogram = common::getKernel(
-        "cuda::histogram", std::array{histogram_cuh_src},
+        "arrayfire::cuda::histogram", std::array{histogram_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(isLinear)),
         std::array{DefineValue(MAX_BINS), DefineValue(THRD_LOAD)});
 
@@ -45,3 +46,4 @@ void histogram(Param<uint> out, CParam<T> in, int nbins, float minval,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/homography.hpp b/src/backend/cuda/kernel/homography.hpp
index aaad7af358..72627f84a8 100644
--- a/src/backend/cuda/kernel/homography.hpp
+++ b/src/backend/cuda/kernel/homography.hpp
@@ -17,6 +17,7 @@
 
 #include <cfloat>
 
+namespace arrayfire {
 namespace cuda {
 
 namespace kernel {
@@ -553,25 +554,25 @@ int computeH(Param<T> bestH, Param<T> H, Param<float> err, CParam<float> x_src,
 
             CUDA_CHECK(cudaMemcpyAsync(&minMedian, finalMedian.get(),
                                        sizeof(float), cudaMemcpyDeviceToHost,
-                                       cuda::getActiveStream()));
+                                       getActiveStream()));
             CUDA_CHECK(cudaMemcpyAsync(&minIdx, finalIdx.get(),
                                        sizeof(unsigned), cudaMemcpyDeviceToHost,
-                                       cuda::getActiveStream()));
+                                       getActiveStream()));
             CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         } else {
             CUDA_CHECK(cudaMemcpyAsync(&minMedian, median.get(), sizeof(float),
                                        cudaMemcpyDeviceToHost,
-                                       cuda::getActiveStream()));
+                                       getActiveStream()));
             CUDA_CHECK(cudaMemcpyAsync(&minIdx, idx.get(), sizeof(unsigned),
                                        cudaMemcpyDeviceToHost,
-                                       cuda::getActiveStream()));
+                                       getActiveStream()));
             CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         }
 
         // Copy best homography to output
         CUDA_CHECK(cudaMemcpyAsync(bestH.ptr, H.ptr + minIdx * 9, 9 * sizeof(T),
                                    cudaMemcpyDeviceToDevice,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
 
         blocks = dim3(divup(nsamples, threads.x));
         // sync stream for the device to host copies to be visible for
@@ -588,7 +589,7 @@ int computeH(Param<T> bestH, Param<T> H, Param<float> err, CParam<float> x_src,
 
         CUDA_CHECK(cudaMemcpyAsync(&inliersH, totalInliers.get(),
                                    sizeof(unsigned), cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
     } else if (htype == AF_HOMOGRAPHY_RANSAC) {
@@ -597,11 +598,11 @@ int computeH(Param<T> bestH, Param<T> H, Param<float> err, CParam<float> x_src,
         // Copies back index and number of inliers of best homography estimation
         CUDA_CHECK(cudaMemcpyAsync(&idxH, idx.get() + blockIdx,
                                    sizeof(unsigned), cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(bestH.ptr, H.ptr + idxH * 9, 9 * sizeof(T),
                                    cudaMemcpyDeviceToDevice,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
     }
 
     // sync stream for the device to host copies to be visible for
@@ -614,3 +615,4 @@ int computeH(Param<T> bestH, Param<T> H, Param<float> err, CParam<float> x_src,
 }  // namespace kernel
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/hsv_rgb.cuh b/src/backend/cuda/kernel/hsv_rgb.cuh
index ca7322777c..9ffcf0cc61 100644
--- a/src/backend/cuda/kernel/hsv_rgb.cuh
+++ b/src/backend/cuda/kernel/hsv_rgb.cuh
@@ -9,11 +9,11 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool isHSV2RGB>
-__global__
-void hsvrgbConverter(Param<T> out, CParam<T> in, int nBBS) {
+__global__ void hsvrgbConverter(Param<T> out, CParam<T> in, int nBBS) {
     // batch offsets
     unsigned batchId = blockIdx.x / nBBS;
     const T* src     = (const T*)in.ptr + (batchId * in.strides[3]);
@@ -81,4 +81,5 @@ void hsvrgbConverter(Param<T> out, CParam<T> in, int nBBS) {
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/hsv_rgb.hpp b/src/backend/cuda/kernel/hsv_rgb.hpp
index a10a6ade93..fe89bb34cb 100644
--- a/src/backend/cuda/kernel/hsv_rgb.hpp
+++ b/src/backend/cuda/kernel/hsv_rgb.hpp
@@ -13,6 +13,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/hsv_rgb_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -22,7 +23,7 @@ static const int THREADS_Y = 16;
 template<typename T>
 void hsv2rgb_convert(Param<T> out, CParam<T> in, bool isHSV2RGB) {
     auto hsvrgbConverter = common::getKernel(
-        "cuda::hsvrgbConverter", std::array{hsv_rgb_cuh_src},
+        "arrayfire::cuda::hsvrgbConverter", std::array{hsv_rgb_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(isHSV2RGB)));
 
     const dim3 threads(THREADS_X, THREADS_Y);
@@ -34,10 +35,9 @@ void hsv2rgb_convert(Param<T> out, CParam<T> in, bool isHSV2RGB) {
     // parameter would be along 4th dimension
     dim3 blocks(blk_x * in.dims[3], blk_y);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
     hsvrgbConverter(qArgs, out, in, blk_x);
@@ -46,3 +46,4 @@ void hsv2rgb_convert(Param<T> out, CParam<T> in, bool isHSV2RGB) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/identity.cuh b/src/backend/cuda/kernel/identity.cuh
index 22ba3709d6..e8868f0a9a 100644
--- a/src/backend/cuda/kernel/identity.cuh
+++ b/src/backend/cuda/kernel/identity.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -39,3 +40,4 @@ __global__ void identity(Param<T> out, int blocks_x, int blocks_y) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/identity.hpp b/src/backend/cuda/kernel/identity.hpp
index 58e369823b..42fe1707e8 100644
--- a/src/backend/cuda/kernel/identity.hpp
+++ b/src/backend/cuda/kernel/identity.hpp
@@ -15,24 +15,24 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/identity_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
 template<typename T>
 void identity(Param<T> out) {
-    auto identity =
-        common::getKernel("cuda::identity", std::array{identity_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto identity = common::getKernel("arrayfire::cuda::identity",
+                                      std::array{identity_cuh_src},
+                                      TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(32, 8);
     int blocks_x = divup(out.dims[0], threads.x);
     int blocks_y = divup(out.dims[1], threads.y);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -41,3 +41,4 @@ void identity(Param<T> out) {
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/iir.cuh b/src/backend/cuda/kernel/iir.cuh
index edd18062eb..e5b195f77a 100644
--- a/src/backend/cuda/kernel/iir.cuh
+++ b/src/backend/cuda/kernel/iir.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool batch_a>
@@ -67,3 +68,4 @@ __global__ void iir(Param<T> y, CParam<T> c, CParam<T> a, const int blocks_y) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/iir.hpp b/src/backend/cuda/kernel/iir.hpp
index 38b9ece04d..f0f58512d8 100644
--- a/src/backend/cuda/kernel/iir.hpp
+++ b/src/backend/cuda/kernel/iir.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/iir_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -23,7 +24,7 @@ void iir(Param<T> y, CParam<T> c, CParam<T> a) {
     constexpr int MAX_A_SIZE = 1024;
 
     auto iir = common::getKernel(
-        "cuda::iir", std::array{iir_cuh_src},
+        "arrayfire::cuda::iir", std::array{iir_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(batch_a)),
         std::array{DefineValue(MAX_A_SIZE)});
 
@@ -43,3 +44,4 @@ void iir(Param<T> y, CParam<T> c, CParam<T> a) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/index.cuh b/src/backend/cuda/kernel/index.cuh
index 643fe87837..37b6b63d46 100644
--- a/src/backend/cuda/kernel/index.cuh
+++ b/src/backend/cuda/kernel/index.cuh
@@ -13,12 +13,12 @@
 #include <assign_kernel_param.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
-__global__ void index(Param<T> out, CParam<T> in,
-                      const cuda::IndexKernelParam p, const int nBBS0,
-                      const int nBBS1) {
+__global__ void index(Param<T> out, CParam<T> in, const IndexKernelParam p,
+                      const int nBBS0, const int nBBS1) {
     // retrieve index pointers
     // these can be 0 where af_array index is not used
     const uint* ptr0 = p.ptr[0];
@@ -60,3 +60,4 @@ __global__ void index(Param<T> out, CParam<T> in,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/index.hpp b/src/backend/cuda/kernel/index.hpp
index 5a44f4be6f..63d318408e 100644
--- a/src/backend/cuda/kernel/index.hpp
+++ b/src/backend/cuda/kernel/index.hpp
@@ -16,13 +16,15 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/index_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
 template<typename T>
 void index(Param<T> out, CParam<T> in, const IndexKernelParam& p) {
-    auto index = common::getKernel("cuda::index", std::array{index_cuh_src},
-                                   TemplateArgs(TemplateTypename<T>()));
+    auto index =
+        common::getKernel("arrayfire::cuda::index", std::array{index_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
     dim3 threads;
     switch (out.dims[1]) {
         case 1: threads.y = 1; break;
@@ -38,10 +40,9 @@ void index(Param<T> out, CParam<T> in, const IndexKernelParam& p) {
 
     dim3 blocks(blks_x * out.dims[2], blks_y * out.dims[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -51,3 +52,4 @@ void index(Param<T> out, CParam<T> in, const IndexKernelParam& p) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/interp.hpp b/src/backend/cuda/kernel/interp.hpp
index 8101fba41e..39fb7a77ff 100644
--- a/src/backend/cuda/kernel/interp.hpp
+++ b/src/backend/cuda/kernel/interp.hpp
@@ -9,6 +9,7 @@
 
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -328,3 +329,4 @@ struct Interp2<Ty, Tp, xdim, ydim, 3> {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/iota.cuh b/src/backend/cuda/kernel/iota.cuh
index 1554e08096..ce0ec56168 100644
--- a/src/backend/cuda/kernel/iota.cuh
+++ b/src/backend/cuda/kernel/iota.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -51,3 +52,4 @@ __global__ void iota(Param<T> out, const int s0, const int s1, const int s2,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/iota.hpp b/src/backend/cuda/kernel/iota.hpp
index d108bc2a25..7624f68559 100644
--- a/src/backend/cuda/kernel/iota.hpp
+++ b/src/backend/cuda/kernel/iota.hpp
@@ -16,6 +16,7 @@
 #include <nvrtc_kernel_headers/iota_cuh.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -26,8 +27,9 @@ void iota(Param<T> out, const af::dim4 &sdims) {
     constexpr unsigned TILEX   = 512;
     constexpr unsigned TILEY   = 32;
 
-    auto iota = common::getKernel("cuda::iota", std::array{iota_cuh_src},
-                                  TemplateArgs(TemplateTypename<T>()));
+    auto iota =
+        common::getKernel("arrayfire::cuda::iota", std::array{iota_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(IOTA_TX, IOTA_TY, 1);
 
@@ -36,10 +38,9 @@ void iota(Param<T> out, const af::dim4 &sdims) {
 
     dim3 blocks(blocksPerMatX * out.dims[2], blocksPerMatY * out.dims[3], 1);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -50,3 +51,4 @@ void iota(Param<T> out, const af::dim4 &sdims) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/ireduce.cuh b/src/backend/cuda/kernel/ireduce.cuh
index 1c6cd63b60..6c59a360b1 100644
--- a/src/backend/cuda/kernel/ireduce.cuh
+++ b/src/backend/cuda/kernel/ireduce.cuh
@@ -13,6 +13,7 @@
 #include <common/Binary.hpp>
 #include <minmax_op.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, af_op_t op, uint dim, bool is_first, uint DIMY>
@@ -176,7 +177,7 @@ __global__ static void ireduceFirst(Param<T> out, uint *olptr, CParam<T> in,
     const uint *rlenptr   = (rlen.ptr) ? rlen.ptr + wid * rlen.strides[3] +
                                            zid * rlen.strides[2] +
                                            yid * rlen.strides[1]
-                                     : nullptr;
+                                       : nullptr;
 
     iptr += wid * in.strides[3] + zid * in.strides[2] + yid * in.strides[1];
     optr += wid * out.strides[3] + zid * out.strides[2] + yid * out.strides[1];
@@ -251,3 +252,4 @@ __global__ static void ireduceFirst(Param<T> out, uint *olptr, CParam<T> in,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/ireduce.hpp b/src/backend/cuda/kernel/ireduce.hpp
index b57ba5d29b..91539469eb 100644
--- a/src/backend/cuda/kernel/ireduce.hpp
+++ b/src/backend/cuda/kernel/ireduce.hpp
@@ -20,6 +20,7 @@
 
 #include <memory>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -31,13 +32,12 @@ void ireduce_dim_launcher(Param<T> out, uint *olptr, CParam<T> in,
 
     dim3 blocks(blocks_dim[0] * blocks_dim[2], blocks_dim[1] * blocks_dim[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     auto ireduceDim = common::getKernel(
-        "cuda::ireduceDim", std::array{ireduce_cuh_src},
+        "arrayfire::cuda::ireduceDim", std::array{ireduce_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(op), TemplateArg(dim),
                      TemplateArg(is_first), TemplateArg(threads_y)),
         std::array{DefineValue(THREADS_X)});
@@ -96,16 +96,15 @@ void ireduce_first_launcher(Param<T> out, uint *olptr, CParam<T> in,
                             CParam<uint> rlen) {
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * in.dims[2], blocks_y * in.dims[3]);
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     uint repeat = divup(in.dims[0], (blocks_x * threads_x));
 
     // threads_x can take values 32, 64, 128, 256
     auto ireduceFirst = common::getKernel(
-        "cuda::ireduceFirst", std::array{ireduce_cuh_src},
+        "arrayfire::cuda::ireduceFirst", std::array{ireduce_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(op),
                      TemplateArg(is_first), TemplateArg(threads_x)),
         std::array{DefineValue(THREADS_PER_BLOCK)});
@@ -218,12 +217,11 @@ T ireduce_all(uint *idx, CParam<T> in) {
         uint *h_lptr_raw = h_lptr.get();
 
         CUDA_CHECK(cudaMemcpyAsync(h_ptr_raw, tmp.ptr, tmp_elements * sizeof(T),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
-        CUDA_CHECK(
-            cudaMemcpyAsync(h_lptr_raw, tlptr, tmp_elements * sizeof(uint),
-                            cudaMemcpyDeviceToHost, cuda::getActiveStream()));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
+                                   cudaMemcpyDeviceToHost, getActiveStream()));
+        CUDA_CHECK(cudaMemcpyAsync(h_lptr_raw, tlptr,
+                                   tmp_elements * sizeof(uint),
+                                   cudaMemcpyDeviceToHost, getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(getActiveStream()));
 
         if (!is_linear) {
             // Converting n-d index into a linear index
@@ -248,9 +246,8 @@ T ireduce_all(uint *idx, CParam<T> in) {
         unique_ptr<T[]> h_ptr(new T[in_elements]);
         T *h_ptr_raw = h_ptr.get();
         CUDA_CHECK(cudaMemcpyAsync(h_ptr_raw, in.ptr, in_elements * sizeof(T),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
+                                   cudaMemcpyDeviceToHost, getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(getActiveStream()));
 
         MinMaxOp<op, T> Op(h_ptr_raw[0], 0);
         for (int i = 1; i < in_elements; i++) { Op(h_ptr_raw[i], i); }
@@ -262,3 +259,4 @@ T ireduce_all(uint *idx, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/jit.cuh b/src/backend/cuda/kernel/jit.cuh
index cf69146114..3d66c02f24 100644
--- a/src/backend/cuda/kernel/jit.cuh
+++ b/src/backend/cuda/kernel/jit.cuh
@@ -42,8 +42,8 @@ typedef cuDoubleComplex cdouble;
 #define __neq(lhs, rhs) (lhs) != (rhs)
 
 #define __conj(in) (in)
-#define __real(in)(in)
-#define __imag(in)(0)
+#define __real(in) (in)
+#define __imag(in) (0)
 #define __abs(in) abs(in)
 #define __sigmoid(in) (1.0 / (1 + exp(-(in))))
 
@@ -60,8 +60,9 @@ typedef cuDoubleComplex cdouble;
 #define __mod(lhs, rhs) ((lhs) % (rhs))
 
 #ifdef AF_WITH_FAST_MATH
-#define __pow(lhs, rhs) \
-    static_cast<double>(pow(static_cast<double>(lhs), static_cast<double>(rhs)));
+#define __pow(lhs, rhs)  \
+    static_cast<double>( \
+        pow(static_cast<double>(lhs), static_cast<double>(rhs)));
 #else
 #define __pow(lhs, rhs) \
     __float2int_rn(pow(__int2float_rn((int)lhs), __int2float_rn((int)rhs)))
@@ -185,7 +186,7 @@ __device__ cdouble __cdiv(cdouble lhs, cdouble rhs) {
     double rhs_x       = inv_rhs_abs * rhs.x;
     double rhs_y       = inv_rhs_abs * rhs.y;
     cdouble out        = {lhs.x * rhs_x + lhs.y * rhs_y,
-                   lhs.y * rhs_x - lhs.x * rhs_y};
+                          lhs.y * rhs_x - lhs.x * rhs_y};
     out.x *= inv_rhs_abs;
     out.y *= inv_rhs_abs;
     return out;
@@ -200,20 +201,17 @@ __device__ cdouble __cmax(cdouble lhs, cdouble rhs) {
 }
 
 template<typename T>
-static __device__ __inline__
-int iszero(T a) {
-  return a == T(0);
+static __device__ __inline__ int iszero(T a) {
+    return a == T(0);
 }
 
 template<typename T>
-static __device__ __inline__
-int __isinf(const T in) {
+static __device__ __inline__ int __isinf(const T in) {
     return isinf(in);
 }
 
 template<>
-__device__ __inline__
-int __isinf<__half>(const __half in) {
+__device__ __inline__ int __isinf<__half>(const __half in) {
 #if __CUDA_ARCH__ >= 530
     return __hisinf(in);
 #else
@@ -222,14 +220,12 @@ int __isinf<__half>(const __half in) {
 }
 
 template<typename T>
-static __device__ __inline__
-int __isnan(const T in) {
+static __device__ __inline__ int __isnan(const T in) {
     return isnan(in);
 }
 
 template<>
-__device__ __inline__
-int __isnan<__half>(const __half in) {
+__device__ __inline__ int __isnan<__half>(const __half in) {
 #if __CUDA_ARCH__ >= 530
     return __hisnan(in);
 #else
diff --git a/src/backend/cuda/kernel/lookup.cuh b/src/backend/cuda/kernel/lookup.cuh
index 6613095ae6..753ea8c6db 100644
--- a/src/backend/cuda/kernel/lookup.cuh
+++ b/src/backend/cuda/kernel/lookup.cuh
@@ -13,6 +13,7 @@
 #include <math.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename in_t, typename idx_t>
@@ -68,3 +69,4 @@ __global__ void lookupND(Param<in_t> out, CParam<in_t> in,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/lookup.hpp b/src/backend/cuda/kernel/lookup.hpp
index bca81cdebc..b4395980f0 100644
--- a/src/backend/cuda/kernel/lookup.hpp
+++ b/src/backend/cuda/kernel/lookup.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/lookup_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -43,7 +44,7 @@ void lookup(Param<in_t> out, CParam<in_t> in, CParam<idx_t> indices, int nDims,
         dim3 blocks(blks, 1);
 
         auto lookup1d = common::getKernel(
-            "cuda::lookup1D", std::array{lookup_cuh_src},
+            "arrayfire::cuda::lookup1D", std::array{lookup_cuh_src},
             TemplateArgs(TemplateTypename<in_t>(), TemplateTypename<idx_t>()),
             std::array{DefineValue(THREADS), DefineValue(THRD_LOAD)});
 
@@ -59,12 +60,12 @@ void lookup(Param<in_t> out, CParam<in_t> in, CParam<idx_t> indices, int nDims,
         dim3 blocks(blks_x * out.dims[2], blks_y * out.dims[3]);
 
         const int maxBlocksY =
-            cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
+            getDeviceProp(getActiveDeviceId()).maxGridSize[1];
         blocks.z = divup(blocks.y, maxBlocksY);
         blocks.y = divup(blocks.y, blocks.z);
 
         auto lookupnd = common::getKernel(
-            "cuda::lookupND", std::array{lookup_cuh_src},
+            "arrayfire::cuda::lookupND", std::array{lookup_cuh_src},
             TemplateArgs(TemplateTypename<in_t>(), TemplateTypename<idx_t>(),
                          TemplateArg(dim)));
         EnqueueArgs qArgs(blocks, threads, getActiveStream());
@@ -76,3 +77,4 @@ void lookup(Param<in_t> out, CParam<in_t> in, CParam<idx_t> indices, int nDims,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/lu_split.cuh b/src/backend/cuda/kernel/lu_split.cuh
index 4299419382..f2f892bbce 100644
--- a/src/backend/cuda/kernel/lu_split.cuh
+++ b/src/backend/cuda/kernel/lu_split.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool same_dims>
@@ -62,3 +63,4 @@ __global__ void luSplit(Param<T> lower, Param<T> upper, Param<T> in,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/lu_split.hpp b/src/backend/cuda/kernel/lu_split.hpp
index 8e74c6fbe5..1d2a185276 100644
--- a/src/backend/cuda/kernel/lu_split.hpp
+++ b/src/backend/cuda/kernel/lu_split.hpp
@@ -17,6 +17,7 @@
 
 #include <array>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -31,7 +32,7 @@ void lu_split(Param<T> lower, Param<T> upper, Param<T> in) {
         lower.dims[0] == in.dims[0] && lower.dims[1] == in.dims[1];
 
     auto luSplit = common::getKernel(
-        "cuda::luSplit", std::array{lu_split_cuh_src},
+        "arrayfire::cuda::luSplit", std::array{lu_split_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(sameDims)));
 
     dim3 threads(TX, TY, 1);
@@ -48,3 +49,4 @@ void lu_split(Param<T> lower, Param<T> upper, Param<T> in) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/match_template.cuh b/src/backend/cuda/kernel/match_template.cuh
index daffdb9ceb..16cf172e1b 100644
--- a/src/backend/cuda/kernel/match_template.cuh
+++ b/src/backend/cuda/kernel/match_template.cuh
@@ -9,12 +9,12 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename inType, typename outType, af::matchType mType, bool needMean>
-__global__
-void matchTemplate(Param<outType> out, CParam<inType> srch,
-                   CParam<inType> tmplt, int nBBS0, int nBBS1) {
+__global__ void matchTemplate(Param<outType> out, CParam<inType> srch,
+                              CParam<inType> tmplt, int nBBS0, int nBBS1) {
     unsigned b2 = blockIdx.x / nBBS0;
     unsigned b3 = blockIdx.y / nBBS1;
 
@@ -118,4 +118,5 @@ void matchTemplate(Param<outType> out, CParam<inType> srch,
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/match_template.hpp b/src/backend/cuda/kernel/match_template.hpp
index 3969bfd453..c9754473ae 100644
--- a/src/backend/cuda/kernel/match_template.hpp
+++ b/src/backend/cuda/kernel/match_template.hpp
@@ -14,6 +14,7 @@
 #include <nvrtc_kernel_headers/match_template_cuh.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -25,7 +26,7 @@ void matchTemplate(Param<outType> out, CParam<inType> srch,
                    CParam<inType> tmplt, const af::matchType mType,
                    bool needMean) {
     auto matchTemplate = common::getKernel(
-        "cuda::matchTemplate", std::array{match_template_cuh_src},
+        "arrayfire::cuda::matchTemplate", std::array{match_template_cuh_src},
         TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>(),
                      TemplateArg(mType), TemplateArg(needMean)));
 
@@ -43,3 +44,4 @@ void matchTemplate(Param<outType> out, CParam<inType> srch,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/mean.hpp b/src/backend/cuda/kernel/mean.hpp
index c981d59656..a26eeac7fd 100644
--- a/src/backend/cuda/kernel/mean.hpp
+++ b/src/backend/cuda/kernel/mean.hpp
@@ -24,6 +24,7 @@
 #include <memory>
 #include <vector>
 
+namespace arrayfire {
 namespace cuda {
 
 __device__ auto operator*(float lhs, __half rhs) -> __half {
@@ -476,16 +477,13 @@ T mean_all_weighted(CParam<T> in, CParam<Tw> iwt) {
         std::vector<T> h_ptr(tmp_elements);
         std::vector<Tw> h_wptr(tmp_elements);
 
-        CUDA_CHECK(cudaMemcpyAsync(h_ptr.data(), tmpOut.get(),
-                                   tmp_elements * sizeof(T),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaMemcpyAsync(h_wptr.data(), tmpWt.get(),
-                                   tmp_elements * sizeof(Tw),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(
-            cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(
+            h_ptr.data(), tmpOut.get(), tmp_elements * sizeof(T),
+            cudaMemcpyDeviceToHost, getStream(getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(
+            h_wptr.data(), tmpWt.get(), tmp_elements * sizeof(Tw),
+            cudaMemcpyDeviceToHost, getStream(getActiveDeviceId())));
+        CUDA_CHECK(cudaStreamSynchronize(getStream(getActiveDeviceId())));
 
         compute_t<T> val     = static_cast<compute_t<T>>(h_ptr[0]);
         compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_wptr[0]);
@@ -500,16 +498,13 @@ T mean_all_weighted(CParam<T> in, CParam<Tw> iwt) {
         std::vector<T> h_ptr(in_elements);
         std::vector<Tw> h_wptr(in_elements);
 
-        CUDA_CHECK(cudaMemcpyAsync(h_ptr.data(), in.ptr,
-                                   in_elements * sizeof(T),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaMemcpyAsync(h_wptr.data(), iwt.ptr,
-                                   in_elements * sizeof(Tw),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(
-            cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(
+            h_ptr.data(), in.ptr, in_elements * sizeof(T),
+            cudaMemcpyDeviceToHost, getStream(getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(
+            h_wptr.data(), iwt.ptr, in_elements * sizeof(Tw),
+            cudaMemcpyDeviceToHost, getStream(getActiveDeviceId())));
+        CUDA_CHECK(cudaStreamSynchronize(getStream(getActiveDeviceId())));
 
         compute_t<T> val     = static_cast<compute_t<T>>(h_ptr[0]);
         compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_wptr[0]);
@@ -561,16 +556,13 @@ To mean_all(CParam<Ti> in) {
         std::vector<To> h_ptr(tmp_elements);
         std::vector<Tw> h_cptr(tmp_elements);
 
-        CUDA_CHECK(cudaMemcpyAsync(h_ptr.data(), tmpOut.get(),
-                                   tmp_elements * sizeof(To),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(cudaMemcpyAsync(h_cptr.data(), tmpCt.get(),
-                                   tmp_elements * sizeof(Tw),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(
-            cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(
+            h_ptr.data(), tmpOut.get(), tmp_elements * sizeof(To),
+            cudaMemcpyDeviceToHost, getStream(getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(
+            h_cptr.data(), tmpCt.get(), tmp_elements * sizeof(Tw),
+            cudaMemcpyDeviceToHost, getStream(getActiveDeviceId())));
+        CUDA_CHECK(cudaStreamSynchronize(getStream(getActiveDeviceId())));
 
         compute_t<To> val    = static_cast<compute_t<To>>(h_ptr[0]);
         compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_cptr[0]);
@@ -584,12 +576,10 @@ To mean_all(CParam<Ti> in) {
     } else {
         std::vector<Ti> h_ptr(in_elements);
 
-        CUDA_CHECK(cudaMemcpyAsync(h_ptr.data(), in.ptr,
-                                   in_elements * sizeof(Ti),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getStream(cuda::getActiveDeviceId())));
-        CUDA_CHECK(
-            cudaStreamSynchronize(cuda::getStream(cuda::getActiveDeviceId())));
+        CUDA_CHECK(cudaMemcpyAsync(
+            h_ptr.data(), in.ptr, in_elements * sizeof(Ti),
+            cudaMemcpyDeviceToHost, getStream(getActiveDeviceId())));
+        CUDA_CHECK(cudaStreamSynchronize(getStream(getActiveDeviceId())));
 
         common::Transform<Ti, compute_t<To>, af_add_t> transform;
         compute_t<Tw> count = static_cast<compute_t<Tw>>(1);
@@ -606,3 +596,4 @@ To mean_all(CParam<Ti> in) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/meanshift.cuh b/src/backend/cuda/kernel/meanshift.cuh
index 4e599385e3..240c853f46 100644
--- a/src/backend/cuda/kernel/meanshift.cuh
+++ b/src/backend/cuda/kernel/meanshift.cuh
@@ -10,12 +10,12 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename AccType, typename T, int channels>
-__global__
-void meanshift(Param<T> out, CParam<T> in, int radius, float cvar,
-               uint numIters, int nBBS0, int nBBS1) {
+__global__ void meanshift(Param<T> out, CParam<T> in, int radius, float cvar,
+                          uint numIters, int nBBS0, int nBBS1) {
     unsigned b2 = blockIdx.x / nBBS0;
     unsigned b3 = blockIdx.y / nBBS1;
     const T* iptr =
@@ -126,4 +126,5 @@ void meanshift(Param<T> out, CParam<T> in, int radius, float cvar,
               ch * out.strides[2])] = currentCenterColors[ch];
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/meanshift.hpp b/src/backend/cuda/kernel/meanshift.hpp
index 530279fd1b..c1882c91fc 100644
--- a/src/backend/cuda/kernel/meanshift.hpp
+++ b/src/backend/cuda/kernel/meanshift.hpp
@@ -16,6 +16,7 @@
 #include <array>
 #include <type_traits>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -28,7 +29,7 @@ void meanshift(Param<T> out, CParam<T> in, const float spatialSigma,
     typedef typename std::conditional<std::is_same<T, double>::value, double,
                                       float>::type AccType;
     auto meanshift = common::getKernel(
-        "cuda::meanshift", std::array{meanshift_cuh_src},
+        "arrayfire::cuda::meanshift", std::array{meanshift_cuh_src},
         TemplateArgs(TemplateTypename<AccType>(), TemplateTypename<T>(),
                      TemplateArg((IsColor ? 3 : 1))  // channels
                      ));
@@ -52,3 +53,4 @@ void meanshift(Param<T> out, CParam<T> in, const float spatialSigma,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/medfilt.cuh b/src/backend/cuda/kernel/medfilt.cuh
index d04c9ec1db..e2d513cf95 100644
--- a/src/backend/cuda/kernel/medfilt.cuh
+++ b/src/backend/cuda/kernel/medfilt.cuh
@@ -10,6 +10,7 @@
 #include <Param.hpp>
 #include <shared.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 // Exchange trick: Morgan McGuire, ShaderX 2008
@@ -20,16 +21,14 @@ namespace cuda {
         b     = max(tmp, b); \
     }
 
-__forceinline__ __device__
-int lIdx(int x, int y, int stride1, int stride0) {
+__forceinline__ __device__ int lIdx(int x, int y, int stride1, int stride0) {
     return (y * stride1 + x * stride0);
 }
 
 template<typename T, af::borderType pad>
-__device__
-void load2ShrdMem(T* shrd, const T* in, int lx, int ly,
-                  int shrdStride, int dim0, int dim1, int gx, int gy,
-                  int inStride1, int inStride0) {
+__device__ void load2ShrdMem(T* shrd, const T* in, int lx, int ly,
+                             int shrdStride, int dim0, int dim1, int gx, int gy,
+                             int inStride1, int inStride0) {
     switch (pad) {
         case AF_PAD_ZERO: {
             if (gx < 0 || gx >= dim0 || gy < 0 || gy >= dim1)
@@ -51,9 +50,8 @@ void load2ShrdMem(T* shrd, const T* in, int lx, int ly,
 }
 
 template<typename T, af::borderType pad>
-__device__
-void load2ShrdMem_1d(T* shrd, const T* in, int lx, int dim0, int gx,
-                     int inStride0) {
+__device__ void load2ShrdMem_1d(T* shrd, const T* in, int lx, int dim0, int gx,
+                                int inStride0) {
     switch (pad) {
         case AF_PAD_ZERO: {
             if (gx < 0 || gx >= dim0)
@@ -71,8 +69,7 @@ void load2ShrdMem_1d(T* shrd, const T* in, int lx, int dim0, int gx,
 }
 
 template<typename T, af::borderType pad, unsigned w_len, unsigned w_wid>
-__global__
-void medfilt2(Param<T> out, CParam<T> in, int nBBS0, int nBBS1) {
+__global__ void medfilt2(Param<T> out, CParam<T> in, int nBBS0, int nBBS1) {
     __shared__ T shrdMem[(THREADS_X + w_len - 1) * (THREADS_Y + w_wid - 1)];
 
     // calculate necessary offset and window parameters
@@ -182,8 +179,8 @@ void medfilt2(Param<T> out, CParam<T> in, int nBBS0, int nBBS1) {
 }
 
 template<typename T, af::borderType pad, unsigned ARR_SIZE>
-__global__
-void medfilt1(Param<T> out, CParam<T> in, unsigned w_wid, int nBBS0) {
+__global__ void medfilt1(Param<T> out, CParam<T> in, unsigned w_wid,
+                         int nBBS0) {
     SharedMemory<T> shared;
     T* shrdMem = shared.getPointer();
 
@@ -285,4 +282,5 @@ void medfilt1(Param<T> out, CParam<T> in, unsigned w_wid, int nBBS0) {
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/medfilt.hpp b/src/backend/cuda/kernel/medfilt.hpp
index c0062ccc2f..69920b5ac0 100644
--- a/src/backend/cuda/kernel/medfilt.hpp
+++ b/src/backend/cuda/kernel/medfilt.hpp
@@ -14,6 +14,7 @@
 #include <nvrtc_kernel_headers/medfilt_cuh.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -27,7 +28,7 @@ void medfilt2(Param<T> out, CParam<T> in, const af::borderType pad, int w_len,
               int w_wid) {
     UNUSED(w_wid);
     auto medfilt2 = common::getKernel(
-        "cuda::medfilt2", std::array{medfilt_cuh_src},
+        "arrayfire::cuda::medfilt2", std::array{medfilt_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
                      TemplateArg(w_len), TemplateArg(w_wid)),
         std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
@@ -46,10 +47,10 @@ void medfilt2(Param<T> out, CParam<T> in, const af::borderType pad, int w_len,
 
 template<typename T>
 void medfilt1(Param<T> out, CParam<T> in, const af::borderType pad, int w_wid) {
-    auto medfilt1 =
-        common::getKernel("cuda::medfilt1", std::array{medfilt_cuh_src},
-                          TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
-                                       TemplateArg(w_wid)));
+    auto medfilt1 = common::getKernel(
+        "arrayfire::cuda::medfilt1", std::array{medfilt_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
+                     TemplateArg(w_wid)));
 
     const dim3 threads(THREADS_X);
 
@@ -66,3 +67,4 @@ void medfilt1(Param<T> out, CParam<T> in, const af::borderType pad, int w_wid) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/memcopy.cuh b/src/backend/cuda/kernel/memcopy.cuh
index ecef444cce..b078a48aea 100644
--- a/src/backend/cuda/kernel/memcopy.cuh
+++ b/src/backend/cuda/kernel/memcopy.cuh
@@ -11,6 +11,7 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 // memCopy without looping, so dim3 has to be 1.
@@ -223,3 +224,4 @@ __global__ void memCopyLoop123(Param<T> out, CParam<T> in) {
     }
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/memcopy.hpp b/src/backend/cuda/kernel/memcopy.hpp
index 1592d62ec9..b75cc39c86 100644
--- a/src/backend/cuda/kernel/memcopy.hpp
+++ b/src/backend/cuda/kernel/memcopy.hpp
@@ -20,6 +20,7 @@
 
 #include <algorithm>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -116,12 +117,13 @@ void memcopy(Param<T> out, CParam<T> in, dim_t indims) {
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
     // select the kernel with the necessary loopings
-    const char *kernelName{th.loop0   ? "cuda::memCopyLoop0"
-                           : th.loop2 ? "cuda::memCopyLoop123"
-                           : th.loop1 ? th.loop3 ? "cuda::memCopyLoop13"
-                                                 : "cuda::memCopyLoop1"
-                           : th.loop3 ? "cuda::memCopyLoop3"
-                                      : "cuda::memCopy"};
+    const char *kernelName{th.loop0   ? "arrayfire::cuda::memCopyLoop0"
+                           : th.loop2 ? "arrayfire::cuda::memCopyLoop123"
+                           : th.loop1 ? th.loop3
+                                            ? "arrayfire::cuda::memCopyLoop13"
+                                            : "arrayfire::cuda::memCopyLoop1"
+                           : th.loop3 ? "arrayfire::cuda::memCopyLoop3"
+                                      : "arrayfire::cuda::memCopy"};
 
     // Conversion to cuda base vector types.
     switch (sizeofNewT) {
@@ -194,10 +196,10 @@ void copy(Param<outType> dst, CParam<inType> src, dim_t ondims,
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
     auto copy{common::getKernel(
-        th.loop0                 ? "cuda::scaledCopyLoop0"
-        : (th.loop2 || th.loop3) ? "cuda::scaledCopyLoop123"
-        : th.loop1               ? "cuda::scaledCopyLoop1"
-                                 : "cuda::scaledCopy",
+        th.loop0                 ? "arrayfire::cuda::scaledCopyLoop0"
+        : (th.loop2 || th.loop3) ? "arrayfire::cuda::scaledCopyLoop123"
+        : th.loop1               ? "arrayfire::cuda::scaledCopyLoop1"
+                                 : "arrayfire::cuda::scaledCopy",
         std::array{copy_cuh_src},
         TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>(),
                      TemplateArg(same_dims), TemplateArg(factor != 1.0)))};
@@ -208,3 +210,4 @@ void copy(Param<outType> dst, CParam<inType> src, dim_t ondims,
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/moments.cuh b/src/backend/cuda/kernel/moments.cuh
index 765b15d2a8..12703a6343 100644
--- a/src/backend/cuda/kernel/moments.cuh
+++ b/src/backend/cuda/kernel/moments.cuh
@@ -9,11 +9,12 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
-__global__
-void moments(Param<float> out, CParam<T> in, af::momentType moment, const bool pBatch) {
+__global__ void moments(Param<float> out, CParam<T> in, af::momentType moment,
+                        const bool pBatch) {
     const dim_t idw = blockIdx.y / in.dims[2];
     const dim_t idz = blockIdx.y - idw * in.dims[2];
 
@@ -56,4 +57,5 @@ void moments(Param<float> out, CParam<T> in, af::momentType moment, const bool p
         atomicAdd(offset, blk_moment_sum[threadIdx.x]);
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/moments.hpp b/src/backend/cuda/kernel/moments.hpp
index 2af86afef6..ece6627c71 100644
--- a/src/backend/cuda/kernel/moments.hpp
+++ b/src/backend/cuda/kernel/moments.hpp
@@ -14,6 +14,7 @@
 #include <nvrtc_kernel_headers/moments_cuh.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -21,9 +22,9 @@ static const int THREADS = 128;
 
 template<typename T>
 void moments(Param<float> out, CParam<T> in, const af::momentType moment) {
-    auto moments =
-        common::getKernel("cuda::moments", std::array{moments_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto moments = common::getKernel("arrayfire::cuda::moments",
+                                     std::array{moments_cuh_src},
+                                     TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(THREADS, 1, 1);
     dim3 blocks(in.dims[1], in.dims[2] * in.dims[3]);
@@ -40,3 +41,4 @@ void moments(Param<float> out, CParam<T> in, const af::momentType moment) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/morph.cuh b/src/backend/cuda/kernel/morph.cuh
index 086c4508ea..34e7a10e1c 100644
--- a/src/backend/cuda/kernel/morph.cuh
+++ b/src/backend/cuda/kernel/morph.cuh
@@ -20,6 +20,7 @@
 __constant__ char
     cFilter[MAX_MORPH_FILTER_LEN * MAX_MORPH_FILTER_LEN * sizeof(double)];
 
+namespace arrayfire {
 namespace cuda {
 
 __forceinline__ __device__ int lIdx(int x, int y, int stride1, int stride0) {
@@ -101,7 +102,7 @@ __global__ void morph(Param<T> out, CParam<T> in, int nBBS0, int nBBS1,
 
     const T* d_filt = (const T*)cFilter;
     T acc           = isDilation ? common::Binary<T, af_max_t>::init()
-                       : common::Binary<T, af_min_t>::init();
+                                 : common::Binary<T, af_min_t>::init();
 #pragma unroll
     for (int wj = 0; wj < windLen; ++wj) {
         int joff   = wj * windLen;
@@ -197,7 +198,7 @@ __global__ void morph3D(Param<T> out, CParam<T> in, int nBBS) {
 
     const T* d_filt = (const T*)cFilter;
     T acc           = isDilation ? common::Binary<T, af_max_t>::init()
-                       : common::Binary<T, af_min_t>::init();
+                                 : common::Binary<T, af_min_t>::init();
 #pragma unroll
     for (int wk = 0; wk < windLen; ++wk) {
         int koff   = wk * se_area;
@@ -227,3 +228,4 @@ __global__ void morph3D(Param<T> out, CParam<T> in, int nBBS) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/morph.hpp b/src/backend/cuda/kernel/morph.hpp
index 1202850f40..4936d659b4 100644
--- a/src/backend/cuda/kernel/morph.hpp
+++ b/src/backend/cuda/kernel/morph.hpp
@@ -15,6 +15,7 @@
 
 #include <limits>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -31,7 +32,7 @@ void morph(Param<T> out, CParam<T> in, CParam<T> mask, bool isDilation) {
     const int SeLength = (windLen <= 10 ? windLen : 0);
 
     auto morph = common::getKernel(
-        "cuda::morph", std::array{morph_cuh_src},
+        "arrayfire::cuda::morph", std::array{morph_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(isDilation),
                      TemplateArg(SeLength)),
         std::array{DefineValue(MAX_MORPH_FILTER_LEN)});
@@ -67,7 +68,7 @@ void morph3d(Param<T> out, CParam<T> in, CParam<T> mask, bool isDilation) {
     }
 
     auto morph3D = common::getKernel(
-        "cuda::morph3D", std::array{morph_cuh_src},
+        "arrayfire::cuda::morph3D", std::array{morph_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(isDilation),
                      TemplateArg(windLen)),
         std::array{DefineValue(MAX_MORPH_FILTER_LEN)});
@@ -97,3 +98,4 @@ void morph3d(Param<T> out, CParam<T> in, CParam<T> mask, bool isDilation) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/nearest_neighbour.hpp b/src/backend/cuda/kernel/nearest_neighbour.hpp
index 170f81868a..a628c18a48 100644
--- a/src/backend/cuda/kernel/nearest_neighbour.hpp
+++ b/src/backend/cuda/kernel/nearest_neighbour.hpp
@@ -15,6 +15,7 @@
 #include <memory.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 namespace kernel {
@@ -188,3 +189,4 @@ void all_distances(Param<To> dist, CParam<T> query, CParam<T> train,
 }  // namespace kernel
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/orb.hpp b/src/backend/cuda/kernel/orb.hpp
index 672da31fc3..c1df7620f5 100644
--- a/src/backend/cuda/kernel/orb.hpp
+++ b/src/backend/cuda/kernel/orb.hpp
@@ -21,6 +21,7 @@
 using std::unique_ptr;
 using std::vector;
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -291,7 +292,7 @@ void orb(unsigned* out_feat, float** d_x, float** d_y, float** d_score,
     // distribution instead of using the reference one
     // CUDA_CHECK(cudaMemcpyToSymbolAsync(d_ref_pat, h_ref_pat, 256 * 4 *
     // sizeof(int), 0,
-    // cudaMemcpyHostToDevice, cuda::getActiveStream()));
+    // cudaMemcpyHostToDevice, getActiveStream()));
 
     vector<float*> d_score_pyr(max_levels);
     vector<float*> d_ori_pyr(max_levels);
@@ -311,8 +312,7 @@ void orb(unsigned* out_feat, float** d_x, float** d_y, float** d_score,
         gauss_filter = createHostDataArray<convAccT>(gauss_dim, h_gauss.data());
         CUDA_CHECK(cudaMemcpyAsync(gauss_filter.get(), h_gauss.data(),
                                    h_gauss.size() * sizeof(convAccT),
-                                   cudaMemcpyHostToDevice,
-                                   cuda::getActiveStream()));
+                                   cudaMemcpyHostToDevice, getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
     }
 
@@ -378,7 +378,7 @@ void orb(unsigned* out_feat, float** d_x, float** d_y, float** d_score,
         unsigned* d_desc_lvl = memAlloc<unsigned>(feat_pyr[i] * 8).release();
         CUDA_CHECK(cudaMemsetAsync(d_desc_lvl, 0,
                                    feat_pyr[i] * 8 * sizeof(unsigned),
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
 
         // Compute ORB descriptors
         threads = dim3(THREADS_X, THREADS_Y);
@@ -419,23 +419,23 @@ void orb(unsigned* out_feat, float** d_x, float** d_y, float** d_score,
 
         CUDA_CHECK(cudaMemcpyAsync(
             *d_x + offset, d_x_pyr[i], feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *d_y + offset, d_y_pyr[i], feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *d_score + offset, d_score_pyr[i], feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *d_ori + offset, d_ori_pyr[i], feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *d_size + offset, d_size_pyr[i], feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_desc + (offset * 8), d_desc_pyr[i],
                                    feat_pyr[i] * 8 * sizeof(unsigned),
                                    cudaMemcpyDeviceToDevice,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
 
         memFree(d_x_pyr[i]);
         memFree(d_y_pyr[i]);
@@ -451,3 +451,4 @@ void orb(unsigned* out_feat, float** d_x, float** d_y, float** d_score,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/orb_patch.hpp b/src/backend/cuda/kernel/orb_patch.hpp
index 6dfe3fb037..8a384c24ad 100644
--- a/src/backend/cuda/kernel/orb_patch.hpp
+++ b/src/backend/cuda/kernel/orb_patch.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cuda {
 
 // Reference pattern, generated for a patch size of 31x31, as suggested by
@@ -94,3 +95,4 @@ int d_ref_pat[REF_PAT_LENGTH] = {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/pad_array_borders.cuh b/src/backend/cuda/kernel/pad_array_borders.cuh
index 20e8ac6bc7..73df3261a7 100644
--- a/src/backend/cuda/kernel/pad_array_borders.cuh
+++ b/src/backend/cuda/kernel/pad_array_borders.cuh
@@ -11,30 +11,29 @@
 #include <math.hpp>
 #include <utility.hpp>
 
-namespace  cuda {
+namespace arrayfire {
+namespace cuda {
 
 template<af::borderType BType>
-__device__
-int idxByndEdge(const int i, const int lb, const int len) {
+__device__ int idxByndEdge(const int i, const int lb, const int len) {
     uint retVal;
     switch (BType) {
-        case AF_PAD_SYM: retVal = trimIndex(i-lb, len); break;
+        case AF_PAD_SYM: retVal = trimIndex(i - lb, len); break;
         case AF_PAD_CLAMP_TO_EDGE: retVal = clamp(i - lb, 0, len - 1); break;
         case AF_PAD_PERIODIC: {
             int rem   = (i - lb) % len;
             bool cond = rem < 0;
             retVal    = cond * (rem + len) + (1 - cond) * rem;
         } break;
-        default: retVal = 0; break; // AF_PAD_ZERO
+        default: retVal = 0; break;  // AF_PAD_ZERO
     }
     return retVal;
 }
 
 template<typename T, af::borderType BType>
-__global__
-void padBorders(Param<T> out, CParam<T> in, const int l0,
-               const int l1, const int l2, const int l3,
-               unsigned blk_x, unsigned blk_y) {
+__global__ void padBorders(Param<T> out, CParam<T> in, const int l0,
+                           const int l1, const int l2, const int l3,
+                           unsigned blk_x, unsigned blk_y) {
     const int lx = threadIdx.x;
     const int ly = threadIdx.y;
     const int k  = blockIdx.x / blk_x;
@@ -86,4 +85,5 @@ void padBorders(Param<T> out, CParam<T> in, const int l0,
     }
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/pad_array_borders.hpp b/src/backend/cuda/kernel/pad_array_borders.hpp
index b55bd419c5..85acaabb26 100644
--- a/src/backend/cuda/kernel/pad_array_borders.hpp
+++ b/src/backend/cuda/kernel/pad_array_borders.hpp
@@ -18,6 +18,7 @@
 
 #include <array>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -28,7 +29,7 @@ template<typename T>
 void padBorders(Param<T> out, CParam<T> in, dim4 const lBoundPadding,
                 const af::borderType btype) {
     auto padBorders = common::getKernel(
-        "cuda::padBorders", std::array{pad_array_borders_cuh_src},
+        "arrayfire::cuda::padBorders", std::array{pad_array_borders_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(btype)));
 
     dim3 threads(kernel::PADB_THREADS_X, kernel::PADB_THREADS_Y);
@@ -48,3 +49,4 @@ void padBorders(Param<T> out, CParam<T> in, dim4 const lBoundPadding,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/random_engine.hpp b/src/backend/cuda/kernel/random_engine.hpp
index 31f9a711ed..7fddcbfd20 100644
--- a/src/backend/cuda/kernel/random_engine.hpp
+++ b/src/backend/cuda/kernel/random_engine.hpp
@@ -21,6 +21,7 @@
 
 #include <limits>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 530
@@ -1101,3 +1102,4 @@ void normalDistributionCBRNG(T *out, size_t elements,
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/random_engine_mersenne.hpp b/src/backend/cuda/kernel/random_engine_mersenne.hpp
index 6e8862574e..5b288bc6b4 100644
--- a/src/backend/cuda/kernel/random_engine_mersenne.hpp
+++ b/src/backend/cuda/kernel/random_engine_mersenne.hpp
@@ -42,6 +42,7 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *******************************************************/
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -128,3 +129,4 @@ void initMersenneState(uint *state, const uint *tbl, uintl seed) {
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/random_engine_philox.hpp b/src/backend/cuda/kernel/random_engine_philox.hpp
index 4648617a8a..8124416e03 100644
--- a/src/backend/cuda/kernel/random_engine_philox.hpp
+++ b/src/backend/cuda/kernel/random_engine_philox.hpp
@@ -46,6 +46,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 // Utils
@@ -102,3 +103,4 @@ static inline __device__ void philox(uint key[2], uint ctr[4]) {
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/random_engine_threefry.hpp b/src/backend/cuda/kernel/random_engine_threefry.hpp
index dbafbfae44..a2bbbcaec1 100644
--- a/src/backend/cuda/kernel/random_engine_threefry.hpp
+++ b/src/backend/cuda/kernel/random_engine_threefry.hpp
@@ -46,6 +46,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 // Utils
@@ -160,3 +161,4 @@ __device__ void threefry(uint k[2], uint c[2], uint X[2]) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/range.cuh b/src/backend/cuda/kernel/range.cuh
index 8e703b356f..753bbad174 100644
--- a/src/backend/cuda/kernel/range.cuh
+++ b/src/backend/cuda/kernel/range.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -56,3 +57,4 @@ __global__ void range(Param<T> out, const int dim, const int blocksPerMatX,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/range.hpp b/src/backend/cuda/kernel/range.hpp
index cb1f8e13e4..2e222f6e21 100644
--- a/src/backend/cuda/kernel/range.hpp
+++ b/src/backend/cuda/kernel/range.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/range_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -25,8 +26,9 @@ void range(Param<T> out, const int dim) {
     constexpr unsigned RANGE_TILEX = 512;
     constexpr unsigned RANGE_TILEY = 32;
 
-    auto range = common::getKernel("cuda::range", std::array{range_cuh_src},
-                                   TemplateArgs(TemplateTypename<T>()));
+    auto range =
+        common::getKernel("arrayfire::cuda::range", std::array{range_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(RANGE_TX, RANGE_TY, 1);
 
@@ -34,10 +36,9 @@ void range(Param<T> out, const int dim) {
     int blocksPerMatY = divup(out.dims[1], RANGE_TILEY);
     dim3 blocks(blocksPerMatX * out.dims[2], blocksPerMatY * out.dims[3], 1);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -47,3 +48,4 @@ void range(Param<T> out, const int dim) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/reduce.hpp b/src/backend/cuda/kernel/reduce.hpp
index fb51a72851..c3cf279b39 100644
--- a/src/backend/cuda/kernel/reduce.hpp
+++ b/src/backend/cuda/kernel/reduce.hpp
@@ -26,6 +26,7 @@
 
 using std::unique_ptr;
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -117,10 +118,9 @@ void reduce_dim_launcher(Param<To> out, CParam<Ti> in, const uint threads_y,
 
     dim3 blocks(blocks_dim[0] * blocks_dim[2], blocks_dim[1] * blocks_dim[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     switch (threads_y) {
         case 8:
@@ -390,10 +390,9 @@ void reduce_all_launcher(Param<To> out, CParam<Ti> in, const uint blocks_x,
 
     uint repeat = divup(in.dims[0], (blocks_x * threads_x));
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     long tmp_elements = blocks.x * blocks.y * blocks.z;
     if (tmp_elements > UINT_MAX) {
@@ -438,10 +437,9 @@ void reduce_first_launcher(Param<To> out, CParam<Ti> in, const uint blocks_x,
 
     uint repeat = divup(in.dims[0], (blocks_x * threads_x));
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     switch (threads_x) {
         case 32:
@@ -546,3 +544,4 @@ void reduce_all(Param<To> out, CParam<Ti> in, bool change_nan, double nanval) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/reduce_by_key.hpp b/src/backend/cuda/kernel/reduce_by_key.hpp
index 72b5c7b146..ea015aaff2 100644
--- a/src/backend/cuda/kernel/reduce_by_key.hpp
+++ b/src/backend/cuda/kernel/reduce_by_key.hpp
@@ -27,6 +27,7 @@ using std::unique_ptr;
 
 const static unsigned int FULL_MASK = 0xFFFFFFFF;
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -637,3 +638,4 @@ __global__ static void reduce_blocks_dim_by_key(
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/regions.hpp b/src/backend/cuda/kernel/regions.hpp
index 7a459a6fb9..b1fe3f7c8d 100644
--- a/src/backend/cuda/kernel/regions.hpp
+++ b/src/backend/cuda/kernel/regions.hpp
@@ -34,14 +34,15 @@ __device__ static int continue_flag = 1;
 
 // Wrapper function for texture fetch
 template<typename T>
-static inline __device__ T fetch(const int n, cuda::Param<T> equiv_map,
+static inline __device__ T fetch(const int n,
+                                 arrayfire::cuda::Param<T> equiv_map,
                                  cudaTextureObject_t tex) {
     return tex1Dfetch<T>(tex, n);
 }
 
 template<>
 __device__ inline double fetch<double>(const int n,
-                                       cuda::Param<double> equiv_map,
+                                       arrayfire::cuda::Param<double> equiv_map,
                                        cudaTextureObject_t tex) {
     return equiv_map.ptr[n];
 }
@@ -49,8 +50,8 @@ __device__ inline double fetch<double>(const int n,
 // The initial label kernel distinguishes between valid (nonzero)
 // pixels and "background" (zero) pixels.
 template<typename T, int n_per_thread>
-__global__ static void initial_label(cuda::Param<T> equiv_map,
-                                     cuda::CParam<char> bin) {
+__global__ static void initial_label(arrayfire::cuda::Param<T> equiv_map,
+                                     arrayfire::cuda::CParam<char> bin) {
     const int base_x = (blockIdx.x * blockDim.x * n_per_thread) + threadIdx.x;
     const int base_y = (blockIdx.y * blockDim.y * n_per_thread) + threadIdx.y;
 
@@ -70,8 +71,9 @@ __global__ static void initial_label(cuda::Param<T> equiv_map,
 }
 
 template<typename T, int n_per_thread>
-__global__ static void final_relabel(cuda::Param<T> equiv_map,
-                                     cuda::CParam<char> bin, const T* d_tmp) {
+__global__ static void final_relabel(arrayfire::cuda::Param<T> equiv_map,
+                                     arrayfire::cuda::CParam<char> bin,
+                                     const T* d_tmp) {
     const int base_x = (blockIdx.x * blockDim.x * n_per_thread) + threadIdx.x;
     const int base_y = (blockIdx.y * blockDim.y * n_per_thread) + threadIdx.y;
 
@@ -96,8 +98,8 @@ __global__ static void final_relabel(cuda::Param<T> equiv_map,
 // do not choose zero, which indicates invalid.
 template<typename T>
 __device__ __inline__ static T relabel(const T a, const T b) {
-    T aa = (a == 0) ? cuda::maxval<T>() : a;
-    T bb = (b == 0) ? cuda::maxval<T>() : b;
+    T aa = (a == 0) ? arrayfire::cuda::maxval<T>() : a;
+    T bb = (b == 0) ? arrayfire::cuda::maxval<T>() : b;
     return min(aa, bb);
 }
 
@@ -120,7 +122,7 @@ struct warp_count {
 // Number of elements to handle per thread in each dimension
 // int n_per_thread = 2; // 2x2 per thread = 4 total elems per thread
 template<typename T, int block_dim, int n_per_thread, bool full_conn>
-__global__ static void update_equiv(cuda::Param<T> equiv_map,
+__global__ static void update_equiv(arrayfire::cuda::Param<T> equiv_map,
                                     const cudaTextureObject_t tex) {
     // Basic coordinates
     const int base_x = (blockIdx.x * blockDim.x * n_per_thread) + threadIdx.x;
@@ -346,8 +348,9 @@ struct clamp_to_one : public thrust::unary_function<T, T> {
 };
 
 template<typename T, bool full_conn, int n_per_thread>
-void regions(cuda::Param<T> out, cuda::CParam<char> in,
+void regions(arrayfire::cuda::Param<T> out, arrayfire::cuda::CParam<char> in,
              cudaTextureObject_t tex) {
+    using arrayfire::cuda::getActiveStream;
     const dim3 threads(THREADS_X, THREADS_Y);
 
     const int blk_x = divup(in.dims[0], threads.x * 2);
@@ -363,9 +366,9 @@ void regions(cuda::Param<T> out, cuda::CParam<char> in,
 
     while (h_continue) {
         h_continue = 0;
-        CUDA_CHECK(cudaMemcpyToSymbolAsync(
-            continue_flag, &h_continue, sizeof(int), 0, cudaMemcpyHostToDevice,
-            cuda::getActiveStream()));
+        CUDA_CHECK(
+            cudaMemcpyToSymbolAsync(continue_flag, &h_continue, sizeof(int), 0,
+                                    cudaMemcpyHostToDevice, getActiveStream()));
 
         CUDA_LAUNCH((update_equiv<T, 16, n_per_thread, full_conn>), blocks,
                     threads, out, tex);
@@ -374,8 +377,8 @@ void regions(cuda::Param<T> out, cuda::CParam<char> in,
 
         CUDA_CHECK(cudaMemcpyFromSymbolAsync(
             &h_continue, continue_flag, sizeof(int), 0, cudaMemcpyDeviceToHost,
-            cuda::getActiveStream()));
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
+            getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(getActiveStream()));
     }
 
     // Now, perform the final relabeling.  This converts the equivalency
@@ -383,10 +386,9 @@ void regions(cuda::Param<T> out, cuda::CParam<char> in,
     // component to being sequentially numbered components starting at
     // 1.
     int size = in.dims[0] * in.dims[1];
-    auto tmp = cuda::memAlloc<T>(size);
+    auto tmp = arrayfire::cuda::memAlloc<T>(size);
     CUDA_CHECK(cudaMemcpyAsync(tmp.get(), out.ptr, size * sizeof(T),
-                               cudaMemcpyDeviceToDevice,
-                               cuda::getActiveStream()));
+                               cudaMemcpyDeviceToDevice, getActiveStream()));
 
     // Wrap raw device ptr
     thrust::device_ptr<T> wrapped_tmp = thrust::device_pointer_cast(tmp.get());
@@ -405,7 +407,7 @@ void regions(cuda::Param<T> out, cuda::CParam<char> in,
     // post-processing of labels is required.
     if (num_bins <= 2) return;
 
-    cuda::ThrustVector<T> labels(num_bins);
+    arrayfire::cuda::ThrustVector<T> labels(num_bins);
 
     // Find the end of each section of values
     thrust::counting_iterator<T> search_begin(0);
diff --git a/src/backend/cuda/kernel/reorder.cuh b/src/backend/cuda/kernel/reorder.cuh
index 617943cc87..4f1db7bf3a 100644
--- a/src/backend/cuda/kernel/reorder.cuh
+++ b/src/backend/cuda/kernel/reorder.cuh
@@ -11,6 +11,7 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -56,3 +57,4 @@ __global__ void reorder(Param<T> out, CParam<T> in, const int d0, const int d1,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/reorder.hpp b/src/backend/cuda/kernel/reorder.hpp
index cb10ad3cb0..e2b83e4ab8 100644
--- a/src/backend/cuda/kernel/reorder.hpp
+++ b/src/backend/cuda/kernel/reorder.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/reorder_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -25,9 +26,9 @@ void reorder(Param<T> out, CParam<T> in, const dim_t *rdims) {
     constexpr unsigned TILEX = 512;
     constexpr unsigned TILEY = 32;
 
-    auto reorder =
-        common::getKernel("cuda::reorder", std::array{reorder_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto reorder = common::getKernel("arrayfire::cuda::reorder",
+                                     std::array{reorder_cuh_src},
+                                     TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(TX, TY, 1);
 
@@ -35,10 +36,9 @@ void reorder(Param<T> out, CParam<T> in, const dim_t *rdims) {
     int blocksPerMatY = divup(out.dims[1], TILEY);
     dim3 blocks(blocksPerMatX * out.dims[2], blocksPerMatY * out.dims[3], 1);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -49,3 +49,4 @@ void reorder(Param<T> out, CParam<T> in, const dim_t *rdims) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/resize.cuh b/src/backend/cuda/kernel/resize.cuh
index 22a0d1d159..8186804dae 100644
--- a/src/backend/cuda/kernel/resize.cuh
+++ b/src/backend/cuda/kernel/resize.cuh
@@ -10,15 +10,15 @@
 #include <Param.hpp>
 #include <interp.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 // nearest-neighbor resampling
 template<typename T>
-__host__ __device__
-void resize_n(Param<T> out, CParam<T> in, const int o_off,
-              const int i_off, const int blockIdx_x,
-              const int blockIdx_y, const float xf,
-              const float yf) {
+__host__ __device__ void resize_n(Param<T> out, CParam<T> in, const int o_off,
+                                  const int i_off, const int blockIdx_x,
+                                  const int blockIdx_y, const float xf,
+                                  const float yf) {
     const int ox = threadIdx.x + blockIdx_x * blockDim.x;
     const int oy = threadIdx.y + blockIdx_y * blockDim.y;
 
@@ -35,11 +35,10 @@ void resize_n(Param<T> out, CParam<T> in, const int o_off,
 
 // bilinear resampling
 template<typename T>
-__host__ __device__
-void resize_b(Param<T> out, CParam<T> in, const int o_off,
-              const int i_off, const int blockIdx_x,
-              const int blockIdx_y, const float xf_,
-              const float yf_) {
+__host__ __device__ void resize_b(Param<T> out, CParam<T> in, const int o_off,
+                                  const int i_off, const int blockIdx_x,
+                                  const int blockIdx_y, const float xf_,
+                                  const float yf_) {
     const int ox = threadIdx.x + blockIdx_x * blockDim.x;
     const int oy = threadIdx.y + blockIdx_y * blockDim.y;
 
@@ -78,11 +77,10 @@ void resize_b(Param<T> out, CParam<T> in, const int o_off,
 
 // lower resampling
 template<typename T>
-__host__ __device__
-void resize_l(Param<T> out, CParam<T> in, const int o_off,
-              const int i_off, const int blockIdx_x,
-              const int blockIdx_y, const float xf,
-              const float yf) {
+__host__ __device__ void resize_l(Param<T> out, CParam<T> in, const int o_off,
+                                  const int i_off, const int blockIdx_x,
+                                  const int blockIdx_y, const float xf,
+                                  const float yf) {
     const int ox = threadIdx.x + blockIdx_x * blockDim.x;
     const int oy = threadIdx.y + blockIdx_y * blockDim.y;
 
@@ -98,9 +96,8 @@ void resize_l(Param<T> out, CParam<T> in, const int o_off,
 }
 
 template<typename T, af::interpType method>
-__global__
-void resize(Param<T> out, CParam<T> in, const int b0,
-            const int b1, const float xf, const float yf) {
+__global__ void resize(Param<T> out, CParam<T> in, const int b0, const int b1,
+                       const float xf, const float yf) {
     const int bIdx = blockIdx.x / b0;
     const int bIdy = blockIdx.y / b1;
     // channel adjustment
@@ -119,4 +116,5 @@ void resize(Param<T> out, CParam<T> in, const int b0,
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/resize.hpp b/src/backend/cuda/kernel/resize.hpp
index 231dab781b..254e23e7d3 100644
--- a/src/backend/cuda/kernel/resize.hpp
+++ b/src/backend/cuda/kernel/resize.hpp
@@ -14,6 +14,7 @@
 #include <nvrtc_kernel_headers/resize_cuh.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -24,7 +25,7 @@ static const unsigned TY = 16;
 template<typename T>
 void resize(Param<T> out, CParam<T> in, af_interp_type method) {
     auto resize = common::getKernel(
-        "cuda::resize", std::array{resize_cuh_src},
+        "arrayfire::cuda::resize", std::array{resize_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(method)));
 
     dim3 threads(TX, TY, 1);
@@ -46,3 +47,4 @@ void resize(Param<T> out, CParam<T> in, af_interp_type method) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/rotate.cuh b/src/backend/cuda/kernel/rotate.cuh
index bd76c490e6..f6fa755ac2 100644
--- a/src/backend/cuda/kernel/rotate.cuh
+++ b/src/backend/cuda/kernel/rotate.cuh
@@ -10,6 +10,7 @@
 #include <Param.hpp>
 #include <interp.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 typedef struct {
@@ -68,4 +69,5 @@ __global__ void rotate(Param<T> out, CParam<T> in, const tmat_t t,
     interp(out, loco, in, inoff, xidi, yidi, method, limages, clamp);
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/rotate.hpp b/src/backend/cuda/kernel/rotate.hpp
index 5c86b57edf..b31218047c 100644
--- a/src/backend/cuda/kernel/rotate.hpp
+++ b/src/backend/cuda/kernel/rotate.hpp
@@ -16,6 +16,7 @@
 #include <nvrtc_kernel_headers/rotate_cuh.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -33,7 +34,7 @@ template<typename T>
 void rotate(Param<T> out, CParam<T> in, const float theta,
             const af::interpType method, const int order) {
     auto rotate = common::getKernel(
-        "cuda::rotate", std::array{rotate_cuh_src},
+        "arrayfire::cuda::rotate", std::array{rotate_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(order)));
 
     const float c = cos(-theta), s = sin(-theta);
@@ -85,3 +86,4 @@ void rotate(Param<T> out, CParam<T> in, const float theta,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_by_key/scan_by_key_impl.cpp b/src/backend/cuda/kernel/scan_by_key/scan_by_key_impl.cpp
index 6b88c5e8e0..b1480e6628 100644
--- a/src/backend/cuda/kernel/scan_by_key/scan_by_key_impl.cpp
+++ b/src/backend/cuda/kernel/scan_by_key/scan_by_key_impl.cpp
@@ -14,6 +14,7 @@
 // The line below is read by CMake to determenine the instantiations
 // SBK_BINARY_OPS:af_add_t af_mul_t af_max_t af_min_t
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 // clang-format off
@@ -22,3 +23,4 @@ INSTANTIATE_SCAN_DIM_BY_KEY_OP( @SBK_BINARY_OP@ )
 // clang-format on
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_dim.cuh b/src/backend/cuda/kernel/scan_dim.cuh
index 3f019bb084..a7f4066c80 100644
--- a/src/backend/cuda/kernel/scan_dim.cuh
+++ b/src/backend/cuda/kernel/scan_dim.cuh
@@ -13,6 +13,7 @@
 #include <common/Transform.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename Ti, typename To, af_op_t op, int dim, bool isFinalPass,
@@ -168,3 +169,4 @@ __global__ void scan_dim_bcast(Param<To> out, CParam<To> tmp, uint blocks_x,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_dim.hpp b/src/backend/cuda/kernel/scan_dim.hpp
index 88c62e175e..a85c15a5ed 100644
--- a/src/backend/cuda/kernel/scan_dim.hpp
+++ b/src/backend/cuda/kernel/scan_dim.hpp
@@ -17,6 +17,7 @@
 #include <nvrtc_kernel_headers/scan_dim_cuh.hpp>
 #include "config.hpp"
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -25,7 +26,7 @@ static void scan_dim_launcher(Param<To> out, Param<To> tmp, CParam<Ti> in,
                               const uint threads_y, const dim_t blocks_all[4],
                               int dim, bool isFinalPass, bool inclusive_scan) {
     auto scan_dim = common::getKernel(
-        "cuda::scan_dim", std::array{scan_dim_cuh_src},
+        "arrayfire::cuda::scan_dim", std::array{scan_dim_cuh_src},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>(),
                      TemplateArg(op), TemplateArg(dim),
                      TemplateArg(isFinalPass), TemplateArg(threads_y),
@@ -36,10 +37,9 @@ static void scan_dim_launcher(Param<To> out, Param<To> tmp, CParam<Ti> in,
 
     dim3 blocks(blocks_all[0] * blocks_all[2], blocks_all[1] * blocks_all[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     uint lim = divup(out.dims[dim], (threads_y * blocks_all[dim]));
 
@@ -53,19 +53,18 @@ template<typename To, af_op_t op>
 static void bcast_dim_launcher(Param<To> out, CParam<To> tmp,
                                const uint threads_y, const dim_t blocks_all[4],
                                int dim, bool inclusive_scan) {
-    auto scan_dim_bcast =
-        common::getKernel("cuda::scan_dim_bcast", std::array{scan_dim_cuh_src},
-                          TemplateArgs(TemplateTypename<To>(), TemplateArg(op),
-                                       TemplateArg(dim)));
+    auto scan_dim_bcast = common::getKernel(
+        "arrayfire::cuda::scan_dim_bcast", std::array{scan_dim_cuh_src},
+        TemplateArgs(TemplateTypename<To>(), TemplateArg(op),
+                     TemplateArg(dim)));
 
     dim3 threads(THREADS_X, threads_y);
 
     dim3 blocks(blocks_all[0] * blocks_all[2], blocks_all[1] * blocks_all[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     uint lim = divup(out.dims[dim], (threads_y * blocks_all[dim]));
 
@@ -124,3 +123,4 @@ static void scan_dim(Param<To> out, CParam<Ti> in, int dim,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_dim_by_key.cuh b/src/backend/cuda/kernel/scan_dim_by_key.cuh
index 0c5875c2e1..06de7c1ae1 100644
--- a/src/backend/cuda/kernel/scan_dim_by_key.cuh
+++ b/src/backend/cuda/kernel/scan_dim_by_key.cuh
@@ -12,6 +12,7 @@
 #include <common/Transform.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename Tk>
@@ -368,3 +369,4 @@ __global__ void scanbykey_dim_bcast(Param<To> out, CParam<To> tmp,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_dim_by_key.hpp b/src/backend/cuda/kernel/scan_dim_by_key.hpp
index a36b95be39..05092499d6 100644
--- a/src/backend/cuda/kernel/scan_dim_by_key.hpp
+++ b/src/backend/cuda/kernel/scan_dim_by_key.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 template<typename Ti, typename Tk, typename To, af_op_t op>
@@ -18,3 +19,4 @@ void scan_dim_by_key(Param<To> out, CParam<Ti> in, CParam<Tk> key, int dim,
                      bool inclusive_scan);
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp b/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
index 0754e1fc22..0dda0b872f 100644
--- a/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
+++ b/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
@@ -21,6 +21,7 @@
 
 #include <algorithm>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -32,7 +33,8 @@ static void scan_dim_nonfinal_launcher(Param<To> out, Param<To> tmp,
                                        const dim_t blocks_all[4],
                                        bool inclusive_scan) {
     auto scanbykey_dim_nonfinal = common::getKernel(
-        "cuda::scanbykey_dim_nonfinal", std::array{scan_dim_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_dim_nonfinal",
+        std::array{scan_dim_by_key_cuh_src},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
                      TemplateTypename<To>(), TemplateArg(op)),
         std::array{DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)});
@@ -56,7 +58,8 @@ static void scan_dim_final_launcher(Param<To> out, CParam<Ti> in,
                                     const dim_t blocks_all[4],
                                     bool calculateFlags, bool inclusive_scan) {
     auto scanbykey_dim_final = common::getKernel(
-        "cuda::scanbykey_dim_final", std::array{scan_dim_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_dim_final",
+        std::array{scan_dim_by_key_cuh_src},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
                      TemplateTypename<To>(), TemplateArg(op)),
         std::array{DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)});
@@ -78,7 +81,8 @@ static void bcast_dim_launcher(Param<To> out, CParam<To> tmp, Param<int> tlid,
                                const int dim, const uint threads_y,
                                const dim_t blocks_all[4]) {
     auto scanbykey_dim_bcast = common::getKernel(
-        "cuda::scanbykey_dim_bcast", std::array{scan_dim_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_dim_bcast",
+        std::array{scan_dim_by_key_cuh_src},
         TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
     dim3 threads(THREADS_X, threads_y);
     dim3 blocks(blocks_all[0] * blocks_all[2], blocks_all[1] * blocks_all[3]);
@@ -167,3 +171,4 @@ void scan_dim_by_key(Param<To> out, CParam<Ti> in, CParam<Tk> key, int dim,
     INSTANTIATE_SCAN_DIM_BY_KEY_TYPES(ROp, intl) \
     INSTANTIATE_SCAN_DIM_BY_KEY_TYPES(ROp, uintl)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_first.cuh b/src/backend/cuda/kernel/scan_first.cuh
index 1bd3b52a53..31abbd57a5 100644
--- a/src/backend/cuda/kernel/scan_first.cuh
+++ b/src/backend/cuda/kernel/scan_first.cuh
@@ -13,6 +13,7 @@
 #include <common/Transform.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename Ti, typename To, af_op_t op, bool isFinalPass, uint DIMX,
@@ -134,3 +135,4 @@ __global__ void scan_first_bcast(Param<To> out, CParam<To> tmp, uint blocks_x,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_first.hpp b/src/backend/cuda/kernel/scan_first.hpp
index 0fe6ce1d5f..fec9d4be7a 100644
--- a/src/backend/cuda/kernel/scan_first.hpp
+++ b/src/backend/cuda/kernel/scan_first.hpp
@@ -17,6 +17,7 @@
 #include <nvrtc_kernel_headers/scan_first_cuh.hpp>
 #include "config.hpp"
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -26,7 +27,7 @@ static void scan_first_launcher(Param<To> out, Param<To> tmp, CParam<Ti> in,
                                 const uint threads_x, bool isFinalPass,
                                 bool inclusive_scan) {
     auto scan_first = common::getKernel(
-        "cuda::scan_first", std::array{scan_first_cuh_src},
+        "arrayfire::cuda::scan_first", std::array{scan_first_cuh_src},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>(),
                      TemplateArg(op), TemplateArg(isFinalPass),
                      TemplateArg(threads_x), TemplateArg(inclusive_scan)),
@@ -35,10 +36,9 @@ static void scan_first_launcher(Param<To> out, Param<To> tmp, CParam<Ti> in,
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     uint lim = divup(out.dims[0], (threads_x * blocks_x));
 
@@ -52,16 +52,15 @@ static void bcast_first_launcher(Param<To> out, CParam<To> tmp,
                                  const uint blocks_x, const uint blocks_y,
                                  const uint threads_x, bool inclusive_scan) {
     auto scan_first_bcast = common::getKernel(
-        "cuda::scan_first_bcast", std::array{scan_first_cuh_src},
+        "arrayfire::cuda::scan_first_bcast", std::array{scan_first_cuh_src},
         TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
 
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     uint lim = divup(out.dims[0], (threads_x * blocks_x));
 
@@ -114,3 +113,4 @@ static void scan_first(Param<To> out, CParam<Ti> in, bool inclusive_scan) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_first_by_key.cuh b/src/backend/cuda/kernel/scan_first_by_key.cuh
index ec894127a0..8f876e2470 100644
--- a/src/backend/cuda/kernel/scan_first_by_key.cuh
+++ b/src/backend/cuda/kernel/scan_first_by_key.cuh
@@ -12,6 +12,7 @@
 #include <common/Transform.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename Tk>
@@ -118,9 +119,9 @@ __global__ void scanbykey_first_nonfinal(Param<To> out, Param<To> tmp,
 #pragma unroll
         for (int off = 1; off < DIMX; off *= 2) {
             if (tidx >= off) {
-                val = sfptr[start + tidx]
-                          ? val
-                          : binop(val, sptr[(start - off) + tidx]);
+                val  = sfptr[start + tidx]
+                           ? val
+                           : binop(val, sptr[(start - off) + tidx]);
                 flag = sfptr[start + tidx] | sfptr[(start - off) + tidx];
             }
             start               = DIMX - start;
@@ -248,9 +249,9 @@ __global__ void scanbykey_first_final(Param<To> out, CParam<Ti> in,
 #pragma unroll
         for (int off = 1; off < DIMX; off *= 2) {
             if (tidx >= off) {
-                val = sfptr[start + tidx]
-                          ? val
-                          : binop(val, sptr[(start - off) + tidx]);
+                val  = sfptr[start + tidx]
+                           ? val
+                           : binop(val, sptr[(start - off) + tidx]);
                 flag = sfptr[start + tidx] | sfptr[(start - off) + tidx];
             }
             start               = DIMX - start;
@@ -313,3 +314,4 @@ __global__ void scanbykey_first_bcast(Param<To> out, Param<To> tmp,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_first_by_key.hpp b/src/backend/cuda/kernel/scan_first_by_key.hpp
index 41ae8d83c5..80491a1c65 100644
--- a/src/backend/cuda/kernel/scan_first_by_key.hpp
+++ b/src/backend/cuda/kernel/scan_first_by_key.hpp
@@ -11,6 +11,7 @@
 #include <Param.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 template<typename Ti, typename Tk, typename To, af_op_t op>
@@ -18,3 +19,4 @@ void scan_first_by_key(Param<To> out, CParam<Ti> in, CParam<Tk> key,
                        bool inclusive_scan);
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/scan_first_by_key_impl.hpp b/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
index 6f9fbd36dd..16abf56b3e 100644
--- a/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
+++ b/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
@@ -20,6 +20,7 @@
 
 #include <algorithm>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -30,7 +31,8 @@ static void scan_nonfinal_launcher(Param<To> out, Param<To> tmp,
                                    const uint blocks_x, const uint blocks_y,
                                    const uint threads_x, bool inclusive_scan) {
     auto scanbykey_first_nonfinal = common::getKernel(
-        "cuda::scanbykey_first_nonfinal", std::array{scan_first_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_first_nonfinal",
+        std::array{scan_first_by_key_cuh_src},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
                      TemplateTypename<To>(), TemplateArg(op)),
         std::array{DefineValue(THREADS_PER_BLOCK),
@@ -52,7 +54,8 @@ static void scan_final_launcher(Param<To> out, CParam<Ti> in, CParam<Tk> key,
                                 const uint threads_x, bool calculateFlags,
                                 bool inclusive_scan) {
     auto scanbykey_first_final = common::getKernel(
-        "cuda::scanbykey_first_final", std::array{scan_first_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_first_final",
+        std::array{scan_first_by_key_cuh_src},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
                      TemplateTypename<To>(), TemplateArg(op)),
         std::array{DefineValue(THREADS_PER_BLOCK),
@@ -73,7 +76,8 @@ static void bcast_first_launcher(Param<To> out, Param<To> tmp, Param<int> tlid,
                                  const dim_t blocks_x, const dim_t blocks_y,
                                  const uint threads_x) {
     auto scanbykey_first_bcast = common::getKernel(
-        "cuda::scanbykey_first_bcast", std::array{scan_first_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_first_bcast",
+        std::array{scan_first_by_key_cuh_src},
         TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
@@ -154,3 +158,4 @@ void scan_first_by_key(Param<To> out, CParam<Ti> in, CParam<Tk> key,
     INSTANTIATE_SCAN_FIRST_BY_KEY_TYPES(ROp, intl) \
     INSTANTIATE_SCAN_FIRST_BY_KEY_TYPES(ROp, uintl)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/select.cuh b/src/backend/cuda/kernel/select.cuh
index 36ab8e4991..c5988594cd 100644
--- a/src/backend/cuda/kernel/select.cuh
+++ b/src/backend/cuda/kernel/select.cuh
@@ -11,6 +11,7 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 int getOffset(dim_t *dims, dim_t *strides, dim_t *refdims, int ids[4]) {
@@ -99,3 +100,4 @@ __global__ void selectScalar(Param<T> out, CParam<char> cond, CParam<T> a, T b,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/select.hpp b/src/backend/cuda/kernel/select.hpp
index ceec068e96..1b6d78fa8f 100644
--- a/src/backend/cuda/kernel/select.hpp
+++ b/src/backend/cuda/kernel/select.hpp
@@ -16,6 +16,7 @@
 #include <math.hpp>
 #include <nvrtc_kernel_headers/select_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -30,7 +31,7 @@ void select(Param<T> out, CParam<char> cond, CParam<T> a, CParam<T> b,
     for (int i = 0; i < 4; i++) { is_same &= (a.dims[i] == b.dims[i]); }
 
     auto select = common::getKernel(
-        "cuda::select", std::array{select_cuh_src},
+        "arrayfire::cuda::select", std::array{select_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_same)));
 
     dim3 threads(DIMX, DIMY);
@@ -45,10 +46,9 @@ void select(Param<T> out, CParam<char> cond, CParam<T> a, CParam<T> b,
 
     dim3 blocks(blk_x * out.dims[2], blk_y * out.dims[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -60,7 +60,7 @@ template<typename T>
 void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const T b,
                    int ndims, bool flip) {
     auto selectScalar = common::getKernel(
-        "cuda::selectScalar", std::array{select_cuh_src},
+        "arrayfire::cuda::selectScalar", std::array{select_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(flip)));
 
     dim3 threads(DIMX, DIMY);
@@ -83,3 +83,4 @@ void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const T b,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/shared.hpp b/src/backend/cuda/kernel/shared.hpp
index 5ad92be9da..55d9f70a64 100644
--- a/src/backend/cuda/kernel/shared.hpp
+++ b/src/backend/cuda/kernel/shared.hpp
@@ -11,6 +11,7 @@
 
 #ifdef __CUDACC_RTC__
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 struct SharedMemory {
@@ -20,9 +21,11 @@ struct SharedMemory {
     }
 };
 }  // namespace cuda
+}  // namespace arrayfire
 
 #else
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -58,5 +61,6 @@ SPECIALIZE(uintl)
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
 
 #endif
diff --git a/src/backend/cuda/kernel/shfl_intrinsics.hpp b/src/backend/cuda/kernel/shfl_intrinsics.hpp
index ef12aafe29..687abf5144 100644
--- a/src/backend/cuda/kernel/shfl_intrinsics.hpp
+++ b/src/backend/cuda/kernel/shfl_intrinsics.hpp
@@ -7,6 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -51,25 +52,24 @@ __device__ T shfl_down_sync(unsigned mask, T var, int delta) {
 }
 // specialization for cfloat
 template<>
-inline __device__ cuda::cfloat shfl_down_sync(unsigned mask, cuda::cfloat var,
-                                              int delta) {
+inline __device__ cfloat shfl_down_sync(unsigned mask, cfloat var, int delta) {
 #if (CUDA_VERSION >= 9000)
-    cuda::cfloat res = {__shfl_down_sync(mask, var.x, delta),
-                        __shfl_down_sync(mask, var.y, delta)};
+    cfloat res = {__shfl_down_sync(mask, var.x, delta),
+                  __shfl_down_sync(mask, var.y, delta)};
 #else
-    cuda::cfloat res  = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)};
+    cfloat res  = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)};
 #endif
     return res;
 }
 // specialization for cdouble
 template<>
-inline __device__ cuda::cdouble shfl_down_sync(unsigned mask, cuda::cdouble var,
-                                               int delta) {
+inline __device__ cdouble shfl_down_sync(unsigned mask, cdouble var,
+                                         int delta) {
 #if (CUDA_VERSION >= 9000)
-    cuda::cdouble res = {__shfl_down_sync(mask, var.x, delta),
-                         __shfl_down_sync(mask, var.y, delta)};
+    cdouble res = {__shfl_down_sync(mask, var.x, delta),
+                   __shfl_down_sync(mask, var.y, delta)};
 #else
-    cuda::cdouble res = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)};
+    cdouble res = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)};
 #endif
     return res;
 }
@@ -85,28 +85,27 @@ __device__ T shfl_up_sync(unsigned mask, T var, int delta) {
 }
 // specialization for cfloat
 template<>
-inline __device__ cuda::cfloat shfl_up_sync(unsigned mask, cuda::cfloat var,
-                                            int delta) {
+inline __device__ cfloat shfl_up_sync(unsigned mask, cfloat var, int delta) {
 #if (CUDA_VERSION >= 9000)
-    cuda::cfloat res = {__shfl_up_sync(mask, var.x, delta),
-                        __shfl_up_sync(mask, var.y, delta)};
+    cfloat res = {__shfl_up_sync(mask, var.x, delta),
+                  __shfl_up_sync(mask, var.y, delta)};
 #else
-    cuda::cfloat res  = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)};
+    cfloat res  = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)};
 #endif
     return res;
 }
 // specialization for cdouble
 template<>
-inline __device__ cuda::cdouble shfl_up_sync(unsigned mask, cuda::cdouble var,
-                                             int delta) {
+inline __device__ cdouble shfl_up_sync(unsigned mask, cdouble var, int delta) {
 #if (CUDA_VERSION >= 9000)
-    cuda::cdouble res = {__shfl_up_sync(mask, var.x, delta),
-                         __shfl_up_sync(mask, var.y, delta)};
+    cdouble res = {__shfl_up_sync(mask, var.x, delta),
+                   __shfl_up_sync(mask, var.y, delta)};
 #else
-    cuda::cdouble res = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)};
+    cdouble res = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)};
 #endif
     return res;
 }
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sift.hpp b/src/backend/cuda/kernel/sift.hpp
index 509267402b..9c3e3bf7b8 100644
--- a/src/backend/cuda/kernel/sift.hpp
+++ b/src/backend/cuda/kernel/sift.hpp
@@ -35,6 +35,7 @@
 
 #include <cfloat>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -1066,10 +1067,9 @@ std::vector<Array<T>> buildGaussPyr(Param<T> init_img, const unsigned n_octaves,
             const unsigned imel   = tmp_pyr[idx].elements();
             const unsigned offset = imel * l;
 
-            CUDA_CHECK(cudaMemcpyAsync(gauss_pyr[o].get() + offset,
-                                       tmp_pyr[idx].get(), imel * sizeof(T),
-                                       cudaMemcpyDeviceToDevice,
-                                       cuda::getActiveStream()));
+            CUDA_CHECK(cudaMemcpyAsync(
+                gauss_pyr[o].get() + offset, tmp_pyr[idx].get(),
+                imel * sizeof(T), cudaMemcpyDeviceToDevice, getActiveStream()));
         }
     }
     return gauss_pyr;
@@ -1103,9 +1103,9 @@ std::vector<Array<T>> buildDoGPyr(std::vector<Array<T>>& gauss_pyr,
 
 template<typename T>
 void update_permutation(thrust::device_ptr<T>& keys,
-                        cuda::ThrustVector<int>& permutation) {
+                        arrayfire::cuda::ThrustVector<int>& permutation) {
     // temporary storage for keys
-    cuda::ThrustVector<T> temp(permutation.size());
+    arrayfire::cuda::ThrustVector<T> temp(permutation.size());
 
     // permute the keys with the current reordering
     THRUST_SELECT((thrust::gather), permutation.begin(), permutation.end(),
@@ -1118,9 +1118,9 @@ void update_permutation(thrust::device_ptr<T>& keys,
 
 template<typename T>
 void apply_permutation(thrust::device_ptr<T>& keys,
-                       cuda::ThrustVector<int>& permutation) {
+                       arrayfire::cuda::ThrustVector<int>& permutation) {
     // copy keys to temporary vector
-    cuda::ThrustVector<T> temp(keys, keys + permutation.size());
+    arrayfire::cuda::ThrustVector<T> temp(keys, keys + permutation.size());
 
     // permute the keys
     THRUST_SELECT((thrust::gather), permutation.begin(), permutation.end(),
@@ -1175,7 +1175,7 @@ void sift(unsigned* out_feat, unsigned* out_dlen, float** d_x, float** d_y,
         const unsigned max_feat = ceil(imel * feature_ratio);
 
         CUDA_CHECK(cudaMemsetAsync(d_count.get(), 0, sizeof(unsigned),
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
 
         uptr<float> d_extrema_x        = memAlloc<float>(max_feat);
         uptr<float> d_extrema_y        = memAlloc<float>(max_feat);
@@ -1200,14 +1200,14 @@ void sift(unsigned* out_feat, unsigned* out_dlen, float** d_x, float** d_y,
         unsigned extrema_feat = 0;
         CUDA_CHECK(cudaMemcpyAsync(&extrema_feat, d_count.get(),
                                    sizeof(unsigned), cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         extrema_feat = min(extrema_feat, max_feat);
 
         if (extrema_feat == 0) { continue; }
 
         CUDA_CHECK(cudaMemsetAsync(d_count.get(), 0, sizeof(unsigned),
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
 
         auto d_interp_x        = memAlloc<float>(extrema_feat);
         auto d_interp_y        = memAlloc<float>(extrema_feat);
@@ -1229,12 +1229,12 @@ void sift(unsigned* out_feat, unsigned* out_dlen, float** d_x, float** d_y,
         unsigned interp_feat = 0;
         CUDA_CHECK(cudaMemcpyAsync(&interp_feat, d_count.get(),
                                    sizeof(unsigned), cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         interp_feat = min(interp_feat, max_feat);
 
         CUDA_CHECK(cudaMemsetAsync(d_count.get(), 0, sizeof(unsigned),
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
 
         if (interp_feat == 0) { continue; }
 
@@ -1249,7 +1249,7 @@ void sift(unsigned* out_feat, unsigned* out_dlen, float** d_x, float** d_y,
         thrust::device_ptr<float> interp_size_ptr =
             thrust::device_pointer_cast(d_interp_size.get());
 
-        cuda::ThrustVector<int> permutation(interp_feat);
+        arrayfire::cuda::ThrustVector<int> permutation(interp_feat);
         thrust::sequence(permutation.begin(), permutation.end());
 
         update_permutation<float>(interp_size_ptr, permutation);
@@ -1282,11 +1282,10 @@ void sift(unsigned* out_feat, unsigned* out_dlen, float** d_x, float** d_y,
 
         unsigned nodup_feat = 0;
         CUDA_CHECK(cudaMemcpyAsync(&nodup_feat, d_count.get(), sizeof(unsigned),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
+                                   cudaMemcpyDeviceToHost, getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         CUDA_CHECK(cudaMemsetAsync(d_count.get(), 0, sizeof(unsigned),
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
 
         const unsigned max_oriented_feat = nodup_feat * 3;
 
@@ -1315,7 +1314,7 @@ void sift(unsigned* out_feat, unsigned* out_dlen, float** d_x, float** d_y,
         unsigned oriented_feat = 0;
         CUDA_CHECK(cudaMemcpyAsync(&oriented_feat, d_count.get(),
                                    sizeof(unsigned), cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         oriented_feat = min(oriented_feat, max_oriented_feat);
 
@@ -1377,25 +1376,25 @@ void sift(unsigned* out_feat, unsigned* out_dlen, float** d_x, float** d_y,
 
         CUDA_CHECK(cudaMemcpyAsync(
             *d_x + offset, d_x_pyr[i].get(), feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *d_y + offset, d_y_pyr[i].get(), feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(*d_score + offset, d_response_pyr[i].get(),
                                    feat_pyr[i] * sizeof(float),
                                    cudaMemcpyDeviceToDevice,
-                                   cuda::getActiveStream()));
+                                   getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *d_ori + offset, d_ori_pyr[i].get(), feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(
             *d_size + offset, d_size_pyr[i].get(), feat_pyr[i] * sizeof(float),
-            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+            cudaMemcpyDeviceToDevice, getActiveStream()));
 
         CUDA_CHECK(
             cudaMemcpyAsync(*d_desc + (offset * desc_len), d_desc_pyr[i].get(),
                             feat_pyr[i] * desc_len * sizeof(float),
-                            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+                            cudaMemcpyDeviceToDevice, getActiveStream()));
 
         offset += feat_pyr[i];
     }
@@ -1407,3 +1406,4 @@ void sift(unsigned* out_feat, unsigned* out_dlen, float** d_x, float** d_y,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sobel.cuh b/src/backend/cuda/kernel/sobel.cuh
index 1ed9b7b0af..03e333c414 100644
--- a/src/backend/cuda/kernel/sobel.cuh
+++ b/src/backend/cuda/kernel/sobel.cuh
@@ -10,18 +10,18 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
-__device__
-int reflect101(int index, int endIndex) {
+__device__ int reflect101(int index, int endIndex) {
     return abs(endIndex - abs(endIndex - index));
 }
 
 template<typename Ti>
 __device__ Ti load2ShrdMem(const Ti* in, int d0, int d1, int gx, int gy,
                            int inStride1, int inStride0) {
-    int idx = reflect101(gx, d0-1) * inStride0 +
-              reflect101(gy, d1-1) * inStride1;
+    int idx =
+        reflect101(gx, d0 - 1) * inStride0 + reflect101(gy, d1 - 1) * inStride1;
     return in[idx];
 }
 
@@ -77,14 +77,15 @@ __global__ void sobel3x3(Param<To> dx, Param<To> dy, CParam<Ti> in, int nBBS0,
         float NE = shrdMem[_i][j_];
         float SE = shrdMem[i_][j_];
 
-        float t1  = shrdMem[_i][j];
-        float t2  = shrdMem[i_][j];
+        float t1                       = shrdMem[_i][j];
+        float t2                       = shrdMem[i_][j];
         dxptr[gy * dx.strides[1] + gx] = (SW + SE - (NW + NE) + 2 * (t2 - t1));
 
-        t1 = shrdMem[i][_j];
-        t2 = shrdMem[i][j_];
+        t1                             = shrdMem[i][_j];
+        t2                             = shrdMem[i][j_];
         dyptr[gy * dy.strides[1] + gx] = (NE + SE - (NW + SW) + 2 * (t2 - t1));
     }
 }
 
-} // namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sobel.hpp b/src/backend/cuda/kernel/sobel.hpp
index 943d8d520e..130625c11b 100644
--- a/src/backend/cuda/kernel/sobel.hpp
+++ b/src/backend/cuda/kernel/sobel.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/sobel_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -27,7 +28,7 @@ void sobel(Param<To> dx, Param<To> dy, CParam<Ti> in,
     UNUSED(ker_size);
 
     auto sobel3x3 = common::getKernel(
-        "cuda::sobel3x3", std::array{sobel_cuh_src},
+        "arrayfire::cuda::sobel3x3", std::array{sobel_cuh_src},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>()),
         std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
 
@@ -49,3 +50,4 @@ void sobel(Param<To> dx, Param<To> dy, CParam<Ti> in,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sort.hpp b/src/backend/cuda/kernel/sort.hpp
index f99dcdf4ba..23ee41b820 100644
--- a/src/backend/cuda/kernel/sort.hpp
+++ b/src/backend/cuda/kernel/sort.hpp
@@ -18,6 +18,7 @@
 #include <thrust/sort.h>
 #include <thrust_utils.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 // Wrapper functions
@@ -80,3 +81,4 @@ void sort0(Param<T> val, bool isAscending) {
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sort_by_key.hpp b/src/backend/cuda/kernel/sort_by_key.hpp
index e2edb286e3..aea6bebb85 100644
--- a/src/backend/cuda/kernel/sort_by_key.hpp
+++ b/src/backend/cuda/kernel/sort_by_key.hpp
@@ -17,6 +17,7 @@
 #include <memory.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 // Wrapper functions
@@ -95,3 +96,4 @@ void sort0ByKey(Param<Tk> okey, Param<Tv> oval, bool isAscending) {
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sparse.cuh b/src/backend/cuda/kernel/sparse.cuh
index 81ad141f26..bdf0e20884 100644
--- a/src/backend/cuda/kernel/sparse.cuh
+++ b/src/backend/cuda/kernel/sparse.cuh
@@ -11,6 +11,7 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -33,3 +34,4 @@ __global__ void coo2Dense(Param<T> output, CParam<T> values, CParam<int> rowIdx,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sparse.hpp b/src/backend/cuda/kernel/sparse.hpp
index 66109b2934..efed1ed6d7 100644
--- a/src/backend/cuda/kernel/sparse.hpp
+++ b/src/backend/cuda/kernel/sparse.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/sparse_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -24,7 +25,7 @@ void coo2dense(Param<T> output, CParam<T> values, CParam<int> rowIdx,
     constexpr int reps = 4;
 
     auto coo2Dense = common::getKernel(
-        "cuda::coo2Dense", std::array{sparse_cuh_src},
+        "arrayfire::cuda::coo2Dense", std::array{sparse_cuh_src},
         TemplateArgs(TemplateTypename<T>()), std::array{DefineValue(reps)});
 
     dim3 threads(256, 1, 1);
@@ -39,3 +40,4 @@ void coo2dense(Param<T> output, CParam<T> values, CParam<int> rowIdx,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sparse_arith.cuh b/src/backend/cuda/kernel/sparse_arith.cuh
index a5d51bc8cc..5357805abe 100644
--- a/src/backend/cuda/kernel/sparse_arith.cuh
+++ b/src/backend/cuda/kernel/sparse_arith.cuh
@@ -13,6 +13,7 @@
 #include <math.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, af_op_t op>
@@ -152,3 +153,4 @@ __global__ void cooArithSSD(Param<T> values, Param<int> rowIdx,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/sparse_arith.hpp b/src/backend/cuda/kernel/sparse_arith.hpp
index fb66e19a79..13dd5ddb7e 100644
--- a/src/backend/cuda/kernel/sparse_arith.hpp
+++ b/src/backend/cuda/kernel/sparse_arith.hpp
@@ -16,6 +16,7 @@
 #include <nvrtc_kernel_headers/sparse_arith_cuh.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -26,10 +27,10 @@ constexpr unsigned THREADS = TX * TY;
 template<typename T, af_op_t op>
 void sparseArithOpCSR(Param<T> out, CParam<T> values, CParam<int> rowIdx,
                       CParam<int> colIdx, CParam<T> rhs, const bool reverse) {
-    auto csrArithDSD =
-        common::getKernel("cuda::csrArithDSD", std::array{sparse_arith_cuh_src},
-                          TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
-                          std::array{DefineValue(TX), DefineValue(TY)});
+    auto csrArithDSD = common::getKernel(
+        "arrayfire::cuda::csrArithDSD", std::array{sparse_arith_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
+        std::array{DefineValue(TX), DefineValue(TY)});
 
     // Each Y for threads does one row
     dim3 threads(TX, TY, 1);
@@ -46,10 +47,10 @@ void sparseArithOpCSR(Param<T> out, CParam<T> values, CParam<int> rowIdx,
 template<typename T, af_op_t op>
 void sparseArithOpCOO(Param<T> out, CParam<T> values, CParam<int> rowIdx,
                       CParam<int> colIdx, CParam<T> rhs, const bool reverse) {
-    auto cooArithDSD =
-        common::getKernel("cuda::cooArithDSD", std::array{sparse_arith_cuh_src},
-                          TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
-                          std::array{DefineValue(THREADS)});
+    auto cooArithDSD = common::getKernel(
+        "arrayfire::cuda::cooArithDSD", std::array{sparse_arith_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
+        std::array{DefineValue(THREADS)});
 
     // Linear indexing with one elements per thread
     dim3 threads(THREADS, 1, 1);
@@ -66,10 +67,10 @@ void sparseArithOpCOO(Param<T> out, CParam<T> values, CParam<int> rowIdx,
 template<typename T, af_op_t op>
 void sparseArithOpCSR(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
                       CParam<T> rhs, const bool reverse) {
-    auto csrArithSSD =
-        common::getKernel("cuda::csrArithSSD", std::array{sparse_arith_cuh_src},
-                          TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
-                          std::array{DefineValue(TX), DefineValue(TY)});
+    auto csrArithSSD = common::getKernel(
+        "arrayfire::cuda::csrArithSSD", std::array{sparse_arith_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
+        std::array{DefineValue(TX), DefineValue(TY)});
 
     // Each Y for threads does one row
     dim3 threads(TX, TY, 1);
@@ -86,10 +87,10 @@ void sparseArithOpCSR(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
 template<typename T, af_op_t op>
 void sparseArithOpCOO(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
                       CParam<T> rhs, const bool reverse) {
-    auto cooArithSSD =
-        common::getKernel("cuda::cooArithSSD", std::array{sparse_arith_cuh_src},
-                          TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
-                          std::array{DefineValue(THREADS)});
+    auto cooArithSSD = common::getKernel(
+        "arrayfire::cuda::cooArithSSD", std::array{sparse_arith_cuh_src},
+        TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
+        std::array{DefineValue(THREADS)});
 
     // Linear indexing with one elements per thread
     dim3 threads(THREADS, 1, 1);
@@ -105,3 +106,4 @@ void sparseArithOpCOO(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/susan.cuh b/src/backend/cuda/kernel/susan.cuh
index 0f23264454..e2a706e000 100644
--- a/src/backend/cuda/kernel/susan.cuh
+++ b/src/backend/cuda/kernel/susan.cuh
@@ -13,6 +13,7 @@
 #include <math.hpp>
 #include <shared.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 inline __device__ int max_val(const int x, const int y) { return max(x, y); }
@@ -121,3 +122,4 @@ __global__ void nonMax(float* x_out, float* y_out, float* resp_out,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/susan.hpp b/src/backend/cuda/kernel/susan.hpp
index e8246b5249..42082bd221 100644
--- a/src/backend/cuda/kernel/susan.hpp
+++ b/src/backend/cuda/kernel/susan.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/susan_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -26,7 +27,7 @@ void susan_responses(T* out, const T* in, const unsigned idim0,
                      const unsigned idim1, const int radius, const float t,
                      const float g, const unsigned edge) {
     auto susan = common::getKernel(
-        "cuda::susan", std::array{susan_cuh_src},
+        "arrayfire::cuda::susan", std::array{susan_cuh_src},
         TemplateArgs(TemplateTypename<T>()),
         std::array{DefineValue(BLOCK_X), DefineValue(BLOCK_Y)});
 
@@ -46,8 +47,9 @@ template<typename T>
 void nonMaximal(float* x_out, float* y_out, float* resp_out, unsigned* count,
                 const unsigned idim0, const unsigned idim1, const T* resp_in,
                 const unsigned edge, const unsigned max_corners) {
-    auto nonMax = common::getKernel("cuda::nonMax", std::array{susan_cuh_src},
-                                    TemplateArgs(TemplateTypename<T>()));
+    auto nonMax =
+        common::getKernel("arrayfire::cuda::nonMax", std::array{susan_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(BLOCK_X, BLOCK_Y);
     dim3 blocks(divup(idim0 - edge * 2, BLOCK_X),
@@ -55,7 +57,7 @@ void nonMaximal(float* x_out, float* y_out, float* resp_out, unsigned* count,
 
     auto d_corners_found = memAlloc<unsigned>(1);
     CUDA_CHECK(cudaMemsetAsync(d_corners_found.get(), 0, sizeof(unsigned),
-                               cuda::getActiveStream()));
+                               getActiveStream()));
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -64,10 +66,10 @@ void nonMaximal(float* x_out, float* y_out, float* resp_out, unsigned* count,
     POST_LAUNCH_CHECK();
 
     CUDA_CHECK(cudaMemcpyAsync(count, d_corners_found.get(), sizeof(unsigned),
-                               cudaMemcpyDeviceToHost,
-                               cuda::getActiveStream()));
+                               cudaMemcpyDeviceToHost, getActiveStream()));
     CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 }
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/thrust_sort_by_key.hpp b/src/backend/cuda/kernel/thrust_sort_by_key.hpp
index cb5cb376b1..9bf2a9b7a3 100644
--- a/src/backend/cuda/kernel/thrust_sort_by_key.hpp
+++ b/src/backend/cuda/kernel/thrust_sort_by_key.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 #include <Param.hpp>
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 // Wrapper functions
@@ -16,3 +17,4 @@ template<typename Tk, typename Tv>
 void thrustSortByKey(Tk *keyPtr, Tv *valPtr, int elements, bool isAscending);
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu b/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu
index 50996bb12e..19b291356c 100644
--- a/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu
+++ b/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu
@@ -14,6 +14,7 @@
 // SBK_TYPES:float double int uint intl uintl short ushort char uchar
 // SBK_INSTS:0 1
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 // clang-format off
@@ -21,3 +22,4 @@ namespace kernel {
 // clang-format on
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp b/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp
index 99d9ee7d9a..e4695ac48e 100644
--- a/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp
+++ b/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp
@@ -13,6 +13,7 @@
 #include <thrust_utils.hpp>
 #include <types.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 // Wrapper functions
@@ -50,3 +51,4 @@ void thrustSortByKey(Tk *keyPtr, Tv *valPtr, int elements, bool isAscending) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/tile.cuh b/src/backend/cuda/kernel/tile.cuh
index dd5047c46a..705ac70647 100644
--- a/src/backend/cuda/kernel/tile.cuh
+++ b/src/backend/cuda/kernel/tile.cuh
@@ -11,6 +11,7 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -52,3 +53,4 @@ __global__ void tile(Param<T> out, CParam<T> in, const int blocksPerMatX,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/tile.hpp b/src/backend/cuda/kernel/tile.hpp
index 5656fcf8e1..035cc39437 100644
--- a/src/backend/cuda/kernel/tile.hpp
+++ b/src/backend/cuda/kernel/tile.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/tile_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -25,8 +26,9 @@ void tile(Param<T> out, CParam<T> in) {
     constexpr unsigned TILEX = 512;
     constexpr unsigned TILEY = 32;
 
-    auto tile = common::getKernel("cuda::tile", std::array{tile_cuh_src},
-                                  TemplateArgs(TemplateTypename<T>()));
+    auto tile =
+        common::getKernel("arrayfire::cuda::tile", std::array{tile_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(TX, TY, 1);
 
@@ -34,10 +36,9 @@ void tile(Param<T> out, CParam<T> in) {
     int blocksPerMatY = divup(out.dims[1], TILEY);
     dim3 blocks(blocksPerMatX * out.dims[2], blocksPerMatY * out.dims[3], 1);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -47,3 +48,4 @@ void tile(Param<T> out, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/topk.hpp b/src/backend/cuda/kernel/topk.hpp
index 0d71d4949c..9418a9162d 100644
--- a/src/backend/cuda/kernel/topk.hpp
+++ b/src/backend/cuda/kernel/topk.hpp
@@ -22,6 +22,7 @@
 
 using cub::BlockRadixSort;
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 static const int TOPK_THRDS_PER_BLK = 256;
@@ -190,3 +191,4 @@ inline void topk(Param<T> ovals, Param<uint> oidxs, CParam<T> ivals,
 }
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/transform.cuh b/src/backend/cuda/kernel/transform.cuh
index 7bece00265..f2d2f2c909 100644
--- a/src/backend/cuda/kernel/transform.cuh
+++ b/src/backend/cuda/kernel/transform.cuh
@@ -13,11 +13,12 @@
 __constant__ float
     c_tmat[3072];  // Allows 512 Affine Transforms and 340 Persp. Transforms
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
-__device__
-void calc_transf_inverse(T *txo, const T *txi, const bool perspective) {
+__device__ void calc_transf_inverse(T *txo, const T *txi,
+                                    const bool perspective) {
     if (perspective) {
         txo[0] = txi[4] * txi[8] - txi[5] * txi[7];
         txo[1] = -(txi[1] * txi[8] - txi[2] * txi[7]);
@@ -56,13 +57,11 @@ void calc_transf_inverse(T *txo, const T *txi, const bool perspective) {
 }
 
 template<typename T, bool inverse, int order>
-__global__
-void transform(Param<T> out, CParam<T> in,
-               const int nImg2, const int nImg3,
-               const int nTfs2, const int nTfs3,
-               const int batchImg2,
-               const int blocksXPerImage, const int blocksYPerImage,
-               const bool perspective, af::interpType method) {
+__global__ void transform(Param<T> out, CParam<T> in, const int nImg2,
+                          const int nImg3, const int nTfs2, const int nTfs3,
+                          const int batchImg2, const int blocksXPerImage,
+                          const int blocksYPerImage, const bool perspective,
+                          af::interpType method) {
     // Image Ids
     const int imgId2 = blockIdx.x / blocksXPerImage;
     const int imgId3 = blockIdx.y / blocksYPerImage;
@@ -171,4 +170,5 @@ void transform(Param<T> out, CParam<T> in,
     interp(out, loco, in, inoff, xidi, yidi, method, limages, clamp);
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/transform.hpp b/src/backend/cuda/kernel/transform.hpp
index 489063cc8a..4ed94d7949 100644
--- a/src/backend/cuda/kernel/transform.hpp
+++ b/src/backend/cuda/kernel/transform.hpp
@@ -18,6 +18,7 @@
 
 #include <algorithm>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -31,7 +32,7 @@ template<typename T>
 void transform(Param<T> out, CParam<T> in, CParam<float> tf, const bool inverse,
                const bool perspective, const af::interpType method, int order) {
     auto transform = common::getKernel(
-        "cuda::transform", std::array{transform_cuh_src},
+        "arrayfire::cuda::transform", std::array{transform_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(inverse),
                      TemplateArg(order)));
 
@@ -74,3 +75,4 @@ void transform(Param<T> out, CParam<T> in, CParam<float> tf, const bool inverse,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/transpose.cuh b/src/backend/cuda/kernel/transpose.cuh
index 1307a043b3..444a61b819 100644
--- a/src/backend/cuda/kernel/transpose.cuh
+++ b/src/backend/cuda/kernel/transpose.cuh
@@ -10,6 +10,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool conjugate>
@@ -21,8 +22,7 @@ __device__ T doOp(T in) {
 }
 
 template<typename T, bool conjugate, bool is32Multiple>
-__global__ void transpose(Param<T> out, CParam<T> in,
-                          const int blocksPerMatX,
+__global__ void transpose(Param<T> out, CParam<T> in, const int blocksPerMatX,
                           const int blocksPerMatY) {
     __shared__ T shrdMem[TILE_DIM][TILE_DIM + 1];
 
@@ -75,4 +75,5 @@ __global__ void transpose(Param<T> out, CParam<T> in,
     }
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/transpose.hpp b/src/backend/cuda/kernel/transpose.hpp
index aca9efb9c6..7ec97b7127 100644
--- a/src/backend/cuda/kernel/transpose.hpp
+++ b/src/backend/cuda/kernel/transpose.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/transpose_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -26,7 +27,7 @@ template<typename T>
 void transpose(Param<T> out, CParam<T> in, const bool conjugate,
                const bool is32multiple) {
     auto transpose = common::getKernel(
-        "cuda::transpose", std::array{transpose_cuh_src},
+        "arrayfire::cuda::transpose", std::array{transpose_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(conjugate),
                      TemplateArg(is32multiple)),
         std::array{DefineValue(TILE_DIM), DefineValue(THREADS_Y)});
@@ -36,10 +37,9 @@ void transpose(Param<T> out, CParam<T> in, const bool conjugate,
     int blk_x = divup(in.dims[0], TILE_DIM);
     int blk_y = divup(in.dims[1], TILE_DIM);
     dim3 blocks(blk_x * in.dims[2], blk_y * in.dims[3]);
-    const int maxBlocksY =
-        cuda::getDeviceProp(getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -50,3 +50,4 @@ void transpose(Param<T> out, CParam<T> in, const bool conjugate,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/transpose_inplace.cuh b/src/backend/cuda/kernel/transpose_inplace.cuh
index 733db729c0..8d0b3cdb04 100644
--- a/src/backend/cuda/kernel/transpose_inplace.cuh
+++ b/src/backend/cuda/kernel/transpose_inplace.cuh
@@ -10,6 +10,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool conjugate>
@@ -117,4 +118,5 @@ __global__ void transposeIP(Param<T> in, const int blocksPerMatX,
     }
 }
 
-} //namespace cuda
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/transpose_inplace.hpp b/src/backend/cuda/kernel/transpose_inplace.hpp
index d603a08653..b5374b6025 100644
--- a/src/backend/cuda/kernel/transpose_inplace.hpp
+++ b/src/backend/cuda/kernel/transpose_inplace.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/transpose_inplace_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -26,7 +27,7 @@ template<typename T>
 void transpose_inplace(Param<T> in, const bool conjugate,
                        const bool is32multiple) {
     auto transposeIP = common::getKernel(
-        "cuda::transposeIP", std::array{transpose_inplace_cuh_src},
+        "arrayfire::cuda::transposeIP", std::array{transpose_inplace_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(conjugate),
                      TemplateArg(is32multiple)),
         std::array{DefineValue(TILE_DIM), DefineValue(THREADS_Y)});
@@ -49,3 +50,4 @@ void transpose_inplace(Param<T> in, const bool conjugate,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/triangle.cuh b/src/backend/cuda/kernel/triangle.cuh
index 44d3342f2b..841a7c636f 100644
--- a/src/backend/cuda/kernel/triangle.cuh
+++ b/src/backend/cuda/kernel/triangle.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool is_upper, bool is_unit_diag>
@@ -59,3 +60,4 @@ __global__ void triangle(Param<T> r, CParam<T> in, const int blocksPerMatX,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/triangle.hpp b/src/backend/cuda/kernel/triangle.hpp
index e6efac7be6..3c1841a324 100644
--- a/src/backend/cuda/kernel/triangle.hpp
+++ b/src/backend/cuda/kernel/triangle.hpp
@@ -15,6 +15,7 @@
 #include <debug_cuda.hpp>
 #include <nvrtc_kernel_headers/triangle_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -26,7 +27,7 @@ void triangle(Param<T> r, CParam<T> in, bool is_upper, bool is_unit_diag) {
     constexpr unsigned TILEY = 32;
 
     auto triangle = common::getKernel(
-        "cuda::triangle", std::array{triangle_cuh_src},
+        "arrayfire::cuda::triangle", std::array{triangle_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_upper),
                      TemplateArg(is_unit_diag)));
 
@@ -36,10 +37,9 @@ void triangle(Param<T> r, CParam<T> in, bool is_upper, bool is_unit_diag) {
     int blocksPerMatY = divup(r.dims[1], TILEY);
     dim3 blocks(blocksPerMatX * r.dims[2], blocksPerMatY * r.dims[3], 1);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -49,3 +49,4 @@ void triangle(Param<T> r, CParam<T> in, bool is_upper, bool is_unit_diag) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/unwrap.cuh b/src/backend/cuda/kernel/unwrap.cuh
index b8668356b0..415727a281 100644
--- a/src/backend/cuda/kernel/unwrap.cuh
+++ b/src/backend/cuda/kernel/unwrap.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool is_column>
@@ -79,3 +80,4 @@ __global__ void unwrap(Param<T> out, CParam<T> in, const int wx, const int wy,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/unwrap.hpp b/src/backend/cuda/kernel/unwrap.hpp
index 15f74df963..6105b8b0a1 100644
--- a/src/backend/cuda/kernel/unwrap.hpp
+++ b/src/backend/cuda/kernel/unwrap.hpp
@@ -16,6 +16,7 @@
 #include <kernel/config.hpp>
 #include <nvrtc_kernel_headers/unwrap_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -24,7 +25,7 @@ void unwrap(Param<T> out, CParam<T> in, const int wx, const int wy,
             const int sx, const int sy, const int px, const int py,
             const int dx, const int dy, const int nx, const bool is_column) {
     auto unwrap = common::getKernel(
-        "cuda::unwrap", std::array{unwrap_cuh_src},
+        "arrayfire::cuda::unwrap", std::array{unwrap_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     dim3 threads, blocks;
@@ -44,10 +45,9 @@ void unwrap(Param<T> out, CParam<T> in, const int wx, const int wy,
         reps = divup((wx * wy), threads.y);
     }
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -57,3 +57,4 @@ void unwrap(Param<T> out, CParam<T> in, const int wx, const int wy,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/where.cuh b/src/backend/cuda/kernel/where.cuh
index ac1f81cfa9..a9e31d2739 100644
--- a/src/backend/cuda/kernel/where.cuh
+++ b/src/backend/cuda/kernel/where.cuh
@@ -11,12 +11,12 @@
 #include <backend.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
-__global__
-void where(uint *optr, CParam<uint> otmp, CParam<uint> rtmp, CParam<T> in,
-           uint blocks_x, uint blocks_y, uint lim) {
+__global__ void where(uint *optr, CParam<uint> otmp, CParam<uint> rtmp,
+                      CParam<T> in, uint blocks_x, uint blocks_y, uint lim) {
     const uint tidx = threadIdx.x;
     const uint tidy = threadIdx.y;
 
@@ -56,4 +56,5 @@ void where(uint *optr, CParam<uint> otmp, CParam<uint> rtmp, CParam<T> in,
     }
 }
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/where.hpp b/src/backend/cuda/kernel/where.hpp
index bf992648d3..0dddc456b9 100644
--- a/src/backend/cuda/kernel/where.hpp
+++ b/src/backend/cuda/kernel/where.hpp
@@ -18,13 +18,15 @@
 #include "config.hpp"
 #include "scan_first.hpp"
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
 template<typename T>
 static void where(Param<uint> &out, CParam<T> in) {
-    auto where = common::getKernel("cuda::where", std::array{where_cuh_src},
-                                   TemplateArgs(TemplateTypename<T>()));
+    auto where =
+        common::getKernel("arrayfire::cuda::where", std::array{where_cuh_src},
+                          TemplateArgs(TemplateTypename<T>()));
 
     uint threads_x = nextpow2(std::max(32u, (uint)in.dims[0]));
     threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
@@ -72,7 +74,7 @@ static void where(Param<uint> &out, CParam<T> in) {
     uint total;
     CUDA_CHECK(cudaMemcpyAsync(&total, rtmp.ptr + rtmp_elements - 1,
                                sizeof(uint), cudaMemcpyDeviceToHost,
-                               cuda::getActiveStream()));
+                               getActiveStream()));
     CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
 
     auto out_alloc = memAlloc<uint>(total);
@@ -90,10 +92,9 @@ static void where(Param<uint> &out, CParam<T> in) {
 
     uint lim = divup(otmp.dims[0], (threads_x * blocks_x));
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
     where(qArgs, out.ptr, otmp, rtmp, in, blocks_x, blocks_y, lim);
@@ -104,3 +105,4 @@ static void where(Param<uint> &out, CParam<T> in) {
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/wrap.cuh b/src/backend/cuda/kernel/wrap.cuh
index f8f1db20ca..9200d78f13 100644
--- a/src/backend/cuda/kernel/wrap.cuh
+++ b/src/backend/cuda/kernel/wrap.cuh
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, bool is_column>
@@ -144,3 +145,4 @@ __global__ void wrap_dilated(Param<T> out, CParam<T> in, const int wx,
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/kernel/wrap.hpp b/src/backend/cuda/kernel/wrap.hpp
index 7185ea38bb..37b9e97cf9 100644
--- a/src/backend/cuda/kernel/wrap.hpp
+++ b/src/backend/cuda/kernel/wrap.hpp
@@ -16,6 +16,7 @@
 #include <kernel/config.hpp>
 #include <nvrtc_kernel_headers/wrap_cuh.hpp>
 
+namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
@@ -23,7 +24,7 @@ template<typename T>
 void wrap(Param<T> out, CParam<T> in, const int wx, const int wy, const int sx,
           const int sy, const int px, const int py, const bool is_column) {
     auto wrap = common::getKernel(
-        "cuda::wrap", std::array{wrap_cuh_src},
+        "arrayfire::cuda::wrap", std::array{wrap_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     int nx = (out.dims[0] + 2 * px - wx) / sx + 1;
@@ -35,10 +36,9 @@ void wrap(Param<T> out, CParam<T> in, const int wx, const int wy, const int sx,
 
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -52,7 +52,7 @@ void wrap_dilated(Param<T> out, CParam<T> in, const dim_t wx, const dim_t wy,
                   const dim_t py, const dim_t dx, const dim_t dy,
                   const bool is_column) {
     auto wrap = common::getKernel(
-        "cuda::wrap_dilated", std::array{wrap_cuh_src},
+        "arrayfire::cuda::wrap_dilated", std::array{wrap_cuh_src},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     int nx = 1 + (out.dims[0] + 2 * px - (((wx - 1) * dx) + 1)) / sx;
@@ -64,10 +64,9 @@ void wrap_dilated(Param<T> out, CParam<T> in, const dim_t wx, const dim_t wy,
 
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
-    const int maxBlocksY =
-        cuda::getDeviceProp(cuda::getActiveDeviceId()).maxGridSize[1];
-    blocks.z = divup(blocks.y, maxBlocksY);
-    blocks.y = divup(blocks.y, blocks.z);
+    const int maxBlocksY = getDeviceProp(getActiveDeviceId()).maxGridSize[1];
+    blocks.z             = divup(blocks.y, maxBlocksY);
+    blocks.y             = divup(blocks.y, blocks.z);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -78,3 +77,4 @@ void wrap_dilated(Param<T> out, CParam<T> in, const dim_t wx, const dim_t wy,
 
 }  // namespace kernel
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/logic.hpp b/src/backend/cuda/logic.hpp
index e32a15548f..88c11b3d09 100644
--- a/src/backend/cuda/logic.hpp
+++ b/src/backend/cuda/logic.hpp
@@ -11,6 +11,7 @@
 #include <common/jit/BinaryNode.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T, af_op_t op>
 Array<char> logicOp(const Array<T> &lhs, const Array<T> &rhs,
@@ -24,3 +25,4 @@ Array<T> bitOp(const Array<T> &lhs, const Array<T> &rhs,
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/lookup.cpp b/src/backend/cuda/lookup.cpp
index f5e6bebc69..133db5ba26 100644
--- a/src/backend/cuda/lookup.cpp
+++ b/src/backend/cuda/lookup.cpp
@@ -14,8 +14,9 @@
 #include <err_cuda.hpp>
 #include <kernel/lookup.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
@@ -72,3 +73,4 @@ INSTANTIATE(short);
 INSTANTIATE(ushort);
 INSTANTIATE(half);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/lookup.hpp b/src/backend/cuda/lookup.hpp
index 0a3c25414a..0dc298805b 100644
--- a/src/backend/cuda/lookup.hpp
+++ b/src/backend/cuda/lookup.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                    const unsigned dim);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/lu.cpp b/src/backend/cuda/lu.cpp
index cf3dcc11ea..addae1e7ba 100644
--- a/src/backend/cuda/lu.cpp
+++ b/src/backend/cuda/lu.cpp
@@ -18,6 +18,7 @@
 
 #include <algorithm>
 
+namespace arrayfire {
 namespace cuda {
 
 // cusolverStatus_t CUDENSEAPI cusolverDn<>getrf_bufferSize(
@@ -147,3 +148,4 @@ INSTANTIATE_LU(cfloat)
 INSTANTIATE_LU(double)
 INSTANTIATE_LU(cdouble)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/lu.hpp b/src/backend/cuda/lu.hpp
index 335d6b3376..7ed639bef4 100644
--- a/src/backend/cuda/lu.hpp
+++ b/src/backend/cuda/lu.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
@@ -19,3 +20,4 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot = true);
 
 bool isLAPACKAvailable();
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/match_template.cpp b/src/backend/cuda/match_template.cpp
index 19043b7cb7..d82137bb5c 100644
--- a/src/backend/cuda/match_template.cpp
+++ b/src/backend/cuda/match_template.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename inType, typename outType>
@@ -42,3 +43,4 @@ INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/match_template.hpp b/src/backend/cuda/match_template.hpp
index a7f24fc833..fe98cea5e9 100644
--- a/src/backend/cuda/match_template.hpp
+++ b/src/backend/cuda/match_template.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename inType, typename outType>
 Array<outType> match_template(const Array<inType> &sImg,
                               const Array<inType> &tImg,
                               const af::matchType mType);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index 23aa1a449b..4c48e6990f 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -30,6 +30,7 @@
 #include <cuda_fp16.h>
 #include <math_constants.h>
 
+namespace arrayfire {
 namespace cuda {
 
 #ifdef AF_WITH_FAST_MATH
@@ -393,14 +394,20 @@ template<typename T>
 constexpr const __DH__ T clamp(const T value, const T lo, const T hi) {
     return clamp(value, lo, hi, [](auto lhs, auto rhs) { return lhs < rhs; });
 }
-
 }  // namespace cuda
+}  // namespace arrayfire
 
-__SDH__ bool operator==(cuda::cfloat a, cuda::cfloat b) {
+__SDH__ bool operator==(arrayfire::cuda::cfloat a, arrayfire::cuda::cfloat b) {
     return (a.x == b.x) && (a.y == b.y);
 }
-__SDH__ bool operator!=(cuda::cfloat a, cuda::cfloat b) { return !(a == b); }
-__SDH__ bool operator==(cuda::cdouble a, cuda::cdouble b) {
+__SDH__ bool operator!=(arrayfire::cuda::cfloat a, arrayfire::cuda::cfloat b) {
+    return !(a == b);
+}
+__SDH__ bool operator==(arrayfire::cuda::cdouble a,
+                        arrayfire::cuda::cdouble b) {
     return (a.x == b.x) && (a.y == b.y);
 }
-__SDH__ bool operator!=(cuda::cdouble a, cuda::cdouble b) { return !(a == b); }
+__SDH__ bool operator!=(arrayfire::cuda::cdouble a,
+                        arrayfire::cuda::cdouble b) {
+    return !(a == b);
+}
diff --git a/src/backend/cuda/max.cu b/src/backend/cuda/max.cu
index 337262dc15..03f712b303 100644
--- a/src/backend/cuda/max.cu
+++ b/src/backend/cuda/max.cu
@@ -7,11 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include "reduce_impl.hpp"
 #include <common/half.hpp>
+#include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 // max
 INSTANTIATE(af_max_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_max_t, short, short)
 INSTANTIATE(af_max_t, ushort, ushort)
 INSTANTIATE(af_max_t, half, half)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/mean.cu b/src/backend/cuda/mean.cu
index cf692ea48c..9b1eea74e9 100644
--- a/src/backend/cuda/mean.cu
+++ b/src/backend/cuda/mean.cu
@@ -11,15 +11,16 @@
 #include <af/dim4.hpp>
 
 #undef _GLIBCXX_USE_INT128
+#include <common/half.hpp>
 #include <err_cuda.hpp>
 #include <kernel/mean.hpp>
 #include <mean.hpp>
 #include <complex>
-#include <common/half.hpp>
 
-using common::half;
 using af::dim4;
+using arrayfire::common::half;
 using std::swap;
+namespace arrayfire {
 namespace cuda {
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in) {
@@ -80,3 +81,4 @@ INSTANTIATE_WGT(cdouble, double);
 INSTANTIATE_WGT(half, float);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/mean.hpp b/src/backend/cuda/mean.hpp
index 7871bb2aab..af1810550c 100644
--- a/src/backend/cuda/mean.hpp
+++ b/src/backend/cuda/mean.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in);
@@ -24,3 +25,4 @@ template<typename T, typename Tw>
 Array<T> mean(const Array<T>& in, const Array<Tw>& wts, const int dim);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/meanshift.cpp b/src/backend/cuda/meanshift.cpp
index c2f552df2b..d72d1aa041 100644
--- a/src/backend/cuda/meanshift.cpp
+++ b/src/backend/cuda/meanshift.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
@@ -43,3 +44,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/meanshift.hpp b/src/backend/cuda/meanshift.hpp
index d27ff71279..267a978cb1 100644
--- a/src/backend/cuda/meanshift.hpp
+++ b/src/backend/cuda/meanshift.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
                    const float &chromaticSigma, const unsigned &numIterations,
                    const bool &isColor);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/medfilt.cpp b/src/backend/cuda/medfilt.cpp
index 6561419ddd..c80c95c21f 100644
--- a/src/backend/cuda/medfilt.cpp
+++ b/src/backend/cuda/medfilt.cpp
@@ -16,6 +16,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -62,3 +63,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/medfilt.hpp b/src/backend/cuda/medfilt.hpp
index 9fa6868859..e9bc1d2f2d 100644
--- a/src/backend/cuda/medfilt.hpp
+++ b/src/backend/cuda/medfilt.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -20,3 +21,4 @@ Array<T> medfilt2(const Array<T> &in, const int w_len, const int w_wid,
                   const af::borderType edge_pad);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/memory.cpp b/src/backend/cuda/memory.cpp
index 969574a1c4..6c86a6244a 100644
--- a/src/backend/cuda/memory.cpp
+++ b/src/backend/cuda/memory.cpp
@@ -28,11 +28,12 @@
 #include <mutex>
 
 using af::dim4;
-using common::bytesToString;
-using common::half;
+using arrayfire::common::bytesToString;
+using arrayfire::common::half;
 
 using std::move;
 
+namespace arrayfire {
 namespace cuda {
 float getMemoryPressure() { return memoryManager().getMemoryPressure(); }
 float getMemoryPressureThreshold() {
@@ -136,9 +137,9 @@ template void memFree(void *ptr);
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
 void Allocator::shutdown() {
-    for (int n = 0; n < cuda::getDeviceCount(); n++) {
+    for (int n = 0; n < getDeviceCount(); n++) {
         try {
-            cuda::setDevice(n);
+            setDevice(n);
             shutdownMemoryManager();
         } catch (const AfError &err) {
             continue;  // Do not throw any errors while shutting down
@@ -148,9 +149,7 @@ void Allocator::shutdown() {
 
 int Allocator::getActiveDeviceId() { return cuda::getActiveDeviceId(); }
 
-size_t Allocator::getMaxMemorySize(int id) {
-    return cuda::getDeviceMemorySize(id);
-}
+size_t Allocator::getMaxMemorySize(int id) { return getDeviceMemorySize(id); }
 
 void *Allocator::nativeAlloc(const size_t bytes) {
     void *ptr = NULL;
@@ -175,7 +174,7 @@ int AllocatorPinned::getActiveDeviceId() {
 
 size_t AllocatorPinned::getMaxMemorySize(int id) {
     UNUSED(id);
-    return cuda::getHostMemorySize();
+    return getHostMemorySize();
 }
 
 void *AllocatorPinned::nativeAlloc(const size_t bytes) {
@@ -191,3 +190,4 @@ void AllocatorPinned::nativeFree(void *ptr) {
     if (err != cudaErrorCudartUnloading) { CUDA_CHECK(err); }
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/memory.hpp b/src/backend/cuda/memory.hpp
index d033ba0443..935c788769 100644
--- a/src/backend/cuda/memory.hpp
+++ b/src/backend/cuda/memory.hpp
@@ -14,6 +14,7 @@
 #include <functional>
 #include <memory>
 
+namespace arrayfire {
 namespace cuda {
 float getMemoryPressure();
 float getMemoryPressureThreshold();
@@ -58,7 +59,7 @@ bool jitTreeExceedsMemoryPressure(size_t bytes);
 void setMemStepSize(size_t step_bytes);
 size_t getMemStepSize(void);
 
-class Allocator final : public common::memory::AllocatorInterface {
+class Allocator final : public arrayfire::common::AllocatorInterface {
    public:
     Allocator();
     ~Allocator() = default;
@@ -73,7 +74,7 @@ class Allocator final : public common::memory::AllocatorInterface {
 // So we pass 1 as numDevices to the constructor so that it creates 1 vector
 // of memory_info
 // When allocating and freeing, it doesn't really matter which device is active
-class AllocatorPinned final : public common::memory::AllocatorInterface {
+class AllocatorPinned final : public arrayfire::common::AllocatorInterface {
    public:
     AllocatorPinned();
     ~AllocatorPinned() = default;
@@ -85,3 +86,4 @@ class AllocatorPinned final : public common::memory::AllocatorInterface {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/min.cu b/src/backend/cuda/min.cu
index 30ad8bc186..72a3f1beef 100644
--- a/src/backend/cuda/min.cu
+++ b/src/backend/cuda/min.cu
@@ -7,11 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include "reduce_impl.hpp"
 #include <common/half.hpp>
+#include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 // min
 INSTANTIATE(af_min_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_min_t, short, short)
 INSTANTIATE(af_min_t, ushort, ushort)
 INSTANTIATE(af_min_t, half, half)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/minmax_op.hpp b/src/backend/cuda/minmax_op.hpp
index 83040d7248..4fcc995c0b 100644
--- a/src/backend/cuda/minmax_op.hpp
+++ b/src/backend/cuda/minmax_op.hpp
@@ -11,6 +11,7 @@
 
 #include <common/Binary.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -83,3 +84,4 @@ struct MinMaxOp<af_max_t, T> {
 };
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/moments.cpp b/src/backend/cuda/moments.cpp
index a8c1a53ab7..34c8cf753f 100644
--- a/src/backend/cuda/moments.cpp
+++ b/src/backend/cuda/moments.cpp
@@ -14,6 +14,7 @@
 #include <err_cuda.hpp>
 #include <kernel/moments.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 static inline unsigned bitCount(unsigned v) {
@@ -56,3 +57,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/moments.hpp b/src/backend/cuda/moments.hpp
index d8361d8896..54791ac590 100644
--- a/src/backend/cuda/moments.hpp
+++ b/src/backend/cuda/moments.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<float> moments(const Array<T> &in, const af_moment_type moment);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/morph.cpp b/src/backend/cuda/morph.cpp
index ba4cf98683..a49fd5a40e 100644
--- a/src/backend/cuda/morph.cpp
+++ b/src/backend/cuda/morph.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -57,3 +58,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/morph.hpp b/src/backend/cuda/morph.hpp
index b1276dfbf2..7b072ef669 100644
--- a/src/backend/cuda/morph.hpp
+++ b/src/backend/cuda/morph.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
@@ -16,3 +17,4 @@ Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
 template<typename T>
 Array<T> morph3d(const Array<T> &in, const Array<T> &mask, bool isDilation);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/nearest_neighbour.cu b/src/backend/cuda/nearest_neighbour.cu
index 53e22a29fc..ca6a11a1c6 100644
--- a/src/backend/cuda/nearest_neighbour.cu
+++ b/src/backend/cuda/nearest_neighbour.cu
@@ -17,6 +17,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename To>
@@ -73,3 +74,4 @@ INSTANTIATE(ushort, uint)
 INSTANTIATE(uintl, uint)  // For Hamming
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/nearest_neighbour.hpp b/src/backend/cuda/nearest_neighbour.hpp
index 8de98e6924..a1e8bd21bf 100644
--- a/src/backend/cuda/nearest_neighbour.hpp
+++ b/src/backend/cuda/nearest_neighbour.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename To>
@@ -20,4 +21,5 @@ void nearest_neighbour(Array<uint>& idx, Array<To>& dist, const Array<T>& query,
                        const uint n_dist,
                        const af_match_type dist_type = AF_SSD);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/orb.cu b/src/backend/cuda/orb.cu
index 86e463ed42..83da734ce2 100644
--- a/src/backend/cuda/orb.cu
+++ b/src/backend/cuda/orb.cu
@@ -21,6 +21,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename convAccT>
@@ -99,3 +100,4 @@ INSTANTIATE(float, float)
 INSTANTIATE(double, double)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/orb.hpp b/src/backend/cuda/orb.hpp
index e7a03ad9e1..c40a1f9026 100644
--- a/src/backend/cuda/orb.hpp
+++ b/src/backend/cuda/orb.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename convAccT>
@@ -21,4 +22,5 @@ unsigned orb(Array<float> &x, Array<float> &y, Array<float> &score,
              const unsigned max_feat, const float scl_fctr,
              const unsigned levels, const bool blur_img);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/pad_array_borders.cpp b/src/backend/cuda/pad_array_borders.cpp
index 2250f7f363..bf41b5f2e7 100644
--- a/src/backend/cuda/pad_array_borders.cpp
+++ b/src/backend/cuda/pad_array_borders.cpp
@@ -14,6 +14,7 @@
 #include <err_cuda.hpp>
 #include <kernel/pad_array_borders.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> padArrayBorders(Array<T> const& in, dim4 const& lowerBoundPadding,
@@ -53,3 +54,4 @@ INSTANTIATE_PAD_ARRAY_BORDERS(ushort)
 INSTANTIATE_PAD_ARRAY_BORDERS(short)
 INSTANTIATE_PAD_ARRAY_BORDERS(common::half)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index d3b7c2efd9..5ad8c27a7f 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -60,13 +60,14 @@ using std::to_string;
 using std::unique_ptr;
 using std::vector;
 
-using common::getEnvVar;
-using common::int_version_to_string;
-using common::unique_handle;
-using common::memory::MemoryManagerBase;
-using cuda::Allocator;
-using cuda::AllocatorPinned;
-
+using arrayfire::common::getEnvVar;
+using arrayfire::common::int_version_to_string;
+using arrayfire::common::MemoryManagerBase;
+using arrayfire::common::unique_handle;
+using arrayfire::cuda::Allocator;
+using arrayfire::cuda::AllocatorPinned;
+
+namespace arrayfire {
 namespace cuda {
 
 static string get_system() {
@@ -92,8 +93,7 @@ unique_handle<cublasHandle_t> *cublasManager(const int deviceId) {
         // TODO(pradeep) When multiple streams per device
         // is added to CUDA backend, move the cublasSetStream
         // call outside of call_once scope.
-        CUBLAS_CHECK(
-            cublasSetStream(handles[deviceId], cuda::getStream(deviceId)));
+        CUBLAS_CHECK(cublasSetStream(handles[deviceId], getStream(deviceId)));
 #ifdef AF_WITH_FAST_MATH
         CUBLAS_CHECK(
             cublasSetMathMode(handles[deviceId], CUBLAS_TF32_TENSOR_OP_MATH));
@@ -128,7 +128,7 @@ unique_handle<cudnnHandle_t> *nnManager(const int deviceId) {
         AF_ERROR(error_msg, AF_ERR_RUNTIME);
     }
     CUDNN_CHECK(getCudnnPlugin().cudnnSetStream(cudnnHandles[deviceId],
-                                                cuda::getStream(deviceId)));
+                                                getStream(deviceId)));
 
     return handle;
 }
@@ -152,14 +152,14 @@ unique_handle<cusolverDnHandle_t> *cusolverManager(const int deviceId) {
         // is added to CUDA backend, move the cublasSetStream
         // call outside of call_once scope.
         CUSOLVER_CHECK(
-            cusolverDnSetStream(handles[deviceId], cuda::getStream(deviceId)));
+            cusolverDnSetStream(handles[deviceId], getStream(deviceId)));
     });
     // TODO(pradeep) prior to this change, stream was being synced in get solver
     // handle because of some cusolver bug. Re-enable that if this change
     // doesn't work and sovler tests fail.
     // https://gist.github.com/shehzan10/414c3d04a40e7c4a03ed3c2e1b9072e7
     // cuSolver Streams patch:
-    // CUDA_CHECK(cudaStreamSynchronize(cuda::getStream(deviceId)));
+    // CUDA_CHECK(cudaStreamSynchronize(getStream(deviceId)));
 
     return &handles[deviceId];
 }
@@ -175,7 +175,7 @@ unique_handle<cusparseHandle_t> *cusparseManager(const int deviceId) {
         // is added to CUDA backend, move the cublasSetStream
         // call outside of call_once scope.
         CUSPARSE_CHECK(
-            _.cusparseSetStream(handles[deviceId], cuda::getStream(deviceId)));
+            _.cusparseSetStream(handles[deviceId], getStream(deviceId)));
     });
     return &handles[deviceId];
 }
@@ -486,7 +486,7 @@ void resetMemoryManagerPinned() {
     return DeviceManager::getInstance().resetMemoryManagerPinned();
 }
 
-graphics::ForgeManager &forgeManager() {
+arrayfire::common::ForgeManager &forgeManager() {
     return *(DeviceManager::getInstance().fgMngr);
 }
 
@@ -504,11 +504,9 @@ GraphicsResourceManager &interopManager() {
     return *(inst.gfxManagers[id].get());
 }
 
-PlanCache &fftManager() {
-    return *(cufftManager(cuda::getActiveDeviceId()).get());
-}
+PlanCache &fftManager() { return *(cufftManager(getActiveDeviceId()).get()); }
 
-BlasHandle blasHandle() { return *cublasManager(cuda::getActiveDeviceId()); }
+BlasHandle blasHandle() { return *cublasManager(getActiveDeviceId()); }
 
 #ifdef WITH_CUDNN
 cudnnHandle_t nnHandle() {
@@ -519,7 +517,7 @@ cudnnHandle_t nnHandle() {
     static cudnnModule keep_me_to_avoid_exceptions_exceptions =
         getCudnnPlugin();
     static unique_handle<cudnnHandle_t> *handle =
-        nnManager(cuda::getActiveDeviceId());
+        nnManager(getActiveDeviceId());
     if (*handle) {
         return *handle;
     } else {
@@ -528,13 +526,9 @@ cudnnHandle_t nnHandle() {
 }
 #endif
 
-SolveHandle solverDnHandle() {
-    return *cusolverManager(cuda::getActiveDeviceId());
-}
+SolveHandle solverDnHandle() { return *cusolverManager(getActiveDeviceId()); }
 
-SparseHandle sparseHandle() {
-    return *cusparseManager(cuda::getActiveDeviceId());
-}
+SparseHandle sparseHandle() { return *cusparseManager(getActiveDeviceId()); }
 
 void sync(int device) {
     int currDevice = getActiveDeviceId();
@@ -554,10 +548,11 @@ bool &evalFlag() {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
 
 af_err afcu_get_stream(cudaStream_t *stream, int id) {
     try {
-        *stream = cuda::getStream(id);
+        *stream = arrayfire::cuda::getStream(id);
     }
     CATCHALL;
     return AF_SUCCESS;
@@ -565,7 +560,7 @@ af_err afcu_get_stream(cudaStream_t *stream, int id) {
 
 af_err afcu_get_native_id(int *nativeid, int id) {
     try {
-        *nativeid = cuda::getDeviceNativeId(id);
+        *nativeid = arrayfire::cuda::getDeviceNativeId(id);
     }
     CATCHALL;
     return AF_SUCCESS;
@@ -573,7 +568,8 @@ af_err afcu_get_native_id(int *nativeid, int id) {
 
 af_err afcu_set_native_id(int nativeid) {
     try {
-        cuda::setDevice(cuda::getDeviceIdFromNativeId(nativeid));
+        arrayfire::cuda::setDevice(
+            arrayfire::cuda::getDeviceIdFromNativeId(nativeid));
     }
     CATCHALL;
     return AF_SUCCESS;
@@ -581,7 +577,7 @@ af_err afcu_set_native_id(int nativeid) {
 
 af_err afcu_cublasSetMathMode(cublasMath_t mode) {
     try {
-        CUBLAS_CHECK(cublasSetMathMode(cuda::blasHandle(), mode));
+        CUBLAS_CHECK(cublasSetMathMode(arrayfire::cuda::blasHandle(), mode));
     }
     CATCHALL;
     return AF_SUCCESS;
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index bbdf5a8d6d..946c6addf1 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -38,18 +38,16 @@ namespace spdlog {
 class logger;
 }
 
-namespace graphics {
-class ForgeManager;
-}
-
+namespace arrayfire {
 namespace common {
-namespace memory {
+class ForgeManager;
 class MemoryManagerBase;
-}
 }  // namespace common
+}  // namespace arrayfire
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 
+namespace arrayfire {
 namespace cuda {
 
 class GraphicsResourceManager;
@@ -132,7 +130,7 @@ void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
 
 void resetMemoryManagerPinned();
 
-graphics::ForgeManager& forgeManager();
+arrayfire::common::ForgeManager& forgeManager();
 
 GraphicsResourceManager& interopManager();
 
@@ -149,3 +147,4 @@ SolveHandle solverDnHandle();
 SparseHandle sparseHandle();
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp
index c454b0dff1..e012377305 100644
--- a/src/backend/cuda/plot.cpp
+++ b/src/backend/cuda/plot.cpp
@@ -15,12 +15,16 @@
 #include <plot.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_plot(const Array<T> &P, fg_plot plot) {
-    auto stream = cuda::getActiveStream();
+    auto stream = getActiveStream();
     if (DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
@@ -38,7 +42,7 @@ void copy_plot(const Array<T> &P, fg_plot plot) {
 
         POST_LAUNCH_CHECK();
     } else {
-        ForgeModule &_ = graphics::forgePlugin();
+        ForgeModule &_ = common::forgePlugin();
         unsigned bytes = 0, buffer = 0;
         FG_CHECK(_.fg_get_plot_vertex_buffer(&buffer, plot));
         FG_CHECK(_.fg_get_plot_vertex_buffer_size(&bytes, plot));
@@ -69,3 +73,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/plot.hpp b/src/backend/cuda/plot.hpp
index 7b0a7473f3..ff0739105d 100644
--- a/src/backend/cuda/plot.hpp
+++ b/src/backend/cuda/plot.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_plot(const Array<T> &P, fg_plot plot);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/print.hpp b/src/backend/cuda/print.hpp
index 97fe7a22ff..2343992350 100644
--- a/src/backend/cuda/print.hpp
+++ b/src/backend/cuda/print.hpp
@@ -12,6 +12,7 @@
 #include <types.hpp>
 #include <ostream>
 
+namespace arrayfire {
 namespace cuda {
 static std::ostream& operator<<(std::ostream& out, const cfloat& var) {
     out << "(" << var.x << "," << var.y << ")";
@@ -23,3 +24,4 @@ static std::ostream& operator<<(std::ostream& out, const cdouble& var) {
     return out;
 }
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/product.cu b/src/backend/cuda/product.cu
index 42a38dae3a..c4fff43b93 100644
--- a/src/backend/cuda/product.cu
+++ b/src/backend/cuda/product.cu
@@ -7,11 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include "reduce_impl.hpp"
 #include <common/half.hpp>
+#include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 // mul
 INSTANTIATE(af_mul_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_mul_t, short, int)
 INSTANTIATE(af_mul_t, ushort, uint)
 INSTANTIATE(af_mul_t, half, float)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/qr.cpp b/src/backend/cuda/qr.cpp
index 3663f43570..c28a41523f 100644
--- a/src/backend/cuda/qr.cpp
+++ b/src/backend/cuda/qr.cpp
@@ -18,6 +18,7 @@
 #include <memory.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 // cusolverStatus_t cusolverDn<>geqrf_bufferSize(
@@ -183,3 +184,4 @@ INSTANTIATE_QR(cfloat)
 INSTANTIATE_QR(double)
 INSTANTIATE_QR(cdouble)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/qr.hpp b/src/backend/cuda/qr.hpp
index 450a3555a6..46121cc211 100644
--- a/src/backend/cuda/qr.hpp
+++ b/src/backend/cuda/qr.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in);
@@ -16,3 +17,4 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in);
 template<typename T>
 Array<T> qr_inplace(Array<T> &in);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/random_engine.cu b/src/backend/cuda/random_engine.cu
index d03eb51e91..a63ead0bf8 100644
--- a/src/backend/cuda/random_engine.cu
+++ b/src/backend/cuda/random_engine.cu
@@ -13,8 +13,9 @@
 #include <af/dim4.hpp>
 #include <cassert>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl) {
@@ -158,3 +159,4 @@ COMPLEX_NORMAL_DISTRIBUTION(cdouble, double)
 COMPLEX_NORMAL_DISTRIBUTION(cfloat, float)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/random_engine.hpp b/src/backend/cuda/random_engine.hpp
index ca7bd1a233..8062f6feb7 100644
--- a/src/backend/cuda/random_engine.hpp
+++ b/src/backend/cuda/random_engine.hpp
@@ -13,6 +13,7 @@
 #include <backend.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl);
@@ -39,3 +40,4 @@ Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
                             Array<uint> recursion_table,
                             Array<uint> temper_table, Array<uint> state);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/range.cpp b/src/backend/cuda/range.cpp
index 54cc76268e..55a2553649 100644
--- a/src/backend/cuda/range.cpp
+++ b/src/backend/cuda/range.cpp
@@ -16,8 +16,9 @@
 
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> range(const dim4& dim, const int seq_dim) {
@@ -52,3 +53,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/range.hpp b/src/backend/cuda/range.hpp
index 904fe139a9..7ad50970aa 100644
--- a/src/backend/cuda/range.hpp
+++ b/src/backend/cuda/range.hpp
@@ -10,7 +10,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> range(const dim4& dim, const int seq_dim = -1);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/reduce.hpp b/src/backend/cuda/reduce.hpp
index d606153650..70f7cf848d 100644
--- a/src/backend/cuda/reduce.hpp
+++ b/src/backend/cuda/reduce.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<af_op_t op, typename Ti, typename To>
 Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan = false,
@@ -24,3 +25,4 @@ template<af_op_t op, typename Ti, typename To>
 Array<To> reduce_all(const Array<Ti> &in, bool change_nan = false,
                      double nanval = 0);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/reduce_impl.hpp b/src/backend/cuda/reduce_impl.hpp
index 0c4e2e3e87..eb8a5b9a48 100644
--- a/src/backend/cuda/reduce_impl.hpp
+++ b/src/backend/cuda/reduce_impl.hpp
@@ -27,6 +27,7 @@
 using af::dim4;
 using std::swap;
 
+namespace arrayfire {
 namespace cuda {
 template<af_op_t op, typename Ti, typename To>
 Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan,
@@ -360,6 +361,7 @@ Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
 
 #define INSTANTIATE(Op, Ti, To)                                                \
     template Array<To> reduce<Op, Ti, To>(const Array<Ti> &in, const int dim,  \
diff --git a/src/backend/cuda/regions.cu b/src/backend/cuda/regions.cu
index a79717a5bf..7de5c54c05 100644
--- a/src/backend/cuda/regions.cu
+++ b/src/backend/cuda/regions.cu
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -73,3 +74,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/regions.hpp b/src/backend/cuda/regions.hpp
index f94b2f7f79..34959c4f62 100644
--- a/src/backend/cuda/regions.hpp
+++ b/src/backend/cuda/regions.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 Array<T> regions(const Array<char> &in, af_connectivity connectivity);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/reorder.cpp b/src/backend/cuda/reorder.cpp
index fcc0e6a830..c81fd02f6a 100644
--- a/src/backend/cuda/reorder.cpp
+++ b/src/backend/cuda/reorder.cpp
@@ -16,8 +16,9 @@
 
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -51,3 +52,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/reorder.hpp b/src/backend/cuda/reorder.hpp
index 525b50001f..bda5fc449c 100644
--- a/src/backend/cuda/reorder.hpp
+++ b/src/backend/cuda/reorder.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> reorder(const Array<T> &in, const af::dim4 &rdims);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/reshape.cpp b/src/backend/cuda/reshape.cpp
index 8d48000457..9d6e57549f 100644
--- a/src/backend/cuda/reshape.cpp
+++ b/src/backend/cuda/reshape.cpp
@@ -13,8 +13,9 @@
 #include <common/half.hpp>
 #include <kernel/memcopy.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename inType, typename outType>
@@ -77,3 +78,4 @@ INSTANTIATE_COMPLEX(cfloat)
 INSTANTIATE_COMPLEX(cdouble)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/resize.cpp b/src/backend/cuda/resize.cpp
index 25678976e3..97dc8a7da8 100644
--- a/src/backend/cuda/resize.cpp
+++ b/src/backend/cuda/resize.cpp
@@ -13,6 +13,7 @@
 #include <err_cuda.hpp>
 #include <kernel/resize.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
@@ -45,3 +46,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/resize.hpp b/src/backend/cuda/resize.hpp
index 602a071b24..ee2f1a0117 100644
--- a/src/backend/cuda/resize.hpp
+++ b/src/backend/cuda/resize.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
                 const af_interp_type method);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/rotate.cpp b/src/backend/cuda/rotate.cpp
index 7c26164a8c..2f46894aef 100644
--- a/src/backend/cuda/rotate.cpp
+++ b/src/backend/cuda/rotate.cpp
@@ -12,6 +12,7 @@
 #include <kernel/rotate.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -40,3 +41,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/rotate.hpp b/src/backend/cuda/rotate.hpp
index 0686fd40bd..a9e271de04 100644
--- a/src/backend/cuda/rotate.hpp
+++ b/src/backend/cuda/rotate.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
                 const af_interp_type method);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/scalar.hpp b/src/backend/cuda/scalar.hpp
index c08c201a73..250062b535 100644
--- a/src/backend/cuda/scalar.hpp
+++ b/src/backend/cuda/scalar.hpp
@@ -13,6 +13,7 @@
 #include <optypes.hpp>
 #include <memory>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -33,3 +34,4 @@ Array<T> createScalarNode(const dim4 &size, const T val) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/scan.cpp b/src/backend/cuda/scan.cpp
index c6f2da12d2..10002cbbad 100644
--- a/src/backend/cuda/scan.cpp
+++ b/src/backend/cuda/scan.cpp
@@ -17,6 +17,7 @@
 #include <scan.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace cuda {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusive_scan) {
@@ -56,3 +57,4 @@ INSTANTIATE_SCAN_ALL(af_mul_t)
 INSTANTIATE_SCAN_ALL(af_min_t)
 INSTANTIATE_SCAN_ALL(af_max_t)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/scan.hpp b/src/backend/cuda/scan.hpp
index 4ee9e84d5c..b26202fba7 100644
--- a/src/backend/cuda/scan.hpp
+++ b/src/backend/cuda/scan.hpp
@@ -10,7 +10,9 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusive_scan = true);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/scan_by_key.cpp b/src/backend/cuda/scan_by_key.cpp
index 30ae778a3d..b7d476cc56 100644
--- a/src/backend/cuda/scan_by_key.cpp
+++ b/src/backend/cuda/scan_by_key.cpp
@@ -16,6 +16,7 @@
 #include <scan_by_key.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace cuda {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
@@ -57,3 +58,4 @@ INSTANTIATE_SCAN_OP(af_mul_t)
 INSTANTIATE_SCAN_OP(af_min_t)
 INSTANTIATE_SCAN_OP(af_max_t)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/scan_by_key.hpp b/src/backend/cuda/scan_by_key.hpp
index 366453b3ad..5b95c75978 100644
--- a/src/backend/cuda/scan_by_key.hpp
+++ b/src/backend/cuda/scan_by_key.hpp
@@ -10,8 +10,10 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
                bool inclusive_scan);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/select.cpp b/src/backend/cuda/select.cpp
index 739e150c05..b13df55bfe 100644
--- a/src/backend/cuda/select.cpp
+++ b/src/backend/cuda/select.cpp
@@ -18,12 +18,13 @@
 
 #include <memory>
 
-using common::half;
-using common::NaryNode;
-using common::Node_ptr;
+using arrayfire::common::half;
+using arrayfire::common::NaryNode;
+using arrayfire::common::Node_ptr;
 using std::make_shared;
 using std::max;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -132,3 +133,4 @@ INSTANTIATE(ushort);
 INSTANTIATE(half);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/select.hpp b/src/backend/cuda/select.hpp
index 6552ca3ccd..530aab097f 100644
--- a/src/backend/cuda/select.hpp
+++ b/src/backend/cuda/select.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
@@ -27,3 +28,4 @@ template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
                           const T &b_val, const af::dim4 &odims);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/set.cu b/src/backend/cuda/set.cu
index a768c31e15..fbbbc28c0a 100644
--- a/src/backend/cuda/set.cu
+++ b/src/backend/cuda/set.cu
@@ -10,9 +10,9 @@
 #include <Array.hpp>
 #include <copy.hpp>
 #include <debug_cuda.hpp>
-#include <thrust_utils.hpp>
 #include <set.hpp>
 #include <sort.hpp>
+#include <thrust_utils.hpp>
 #include <af/dim4.hpp>
 
 #include <thrust/device_ptr.h>
@@ -22,6 +22,7 @@
 
 #include <algorithm>
 
+namespace arrayfire {
 namespace cuda {
 using af::dim4;
 
@@ -127,3 +128,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/set.hpp b/src/backend/cuda/set.hpp
index 7b72447bcf..872599ad40 100644
--- a/src/backend/cuda/set.hpp
+++ b/src/backend/cuda/set.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> setUnique(const Array<T> &in, const bool is_sorted);
@@ -21,3 +22,4 @@ template<typename T>
 Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
                       const bool is_unique);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/shift.cpp b/src/backend/cuda/shift.cpp
index f83bba9802..82aab5e1fe 100644
--- a/src/backend/cuda/shift.cpp
+++ b/src/backend/cuda/shift.cpp
@@ -17,16 +17,17 @@
 
 using af::dim4;
 
-using common::Node_ptr;
-using common::ShiftNodeBase;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::ShiftNodeBase;
 
-using cuda::jit::BufferNode;
+using arrayfire::cuda::jit::BufferNode;
 
 using std::array;
 using std::make_shared;
 using std::static_pointer_cast;
 using std::string;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 using ShiftNode = ShiftNodeBase<BufferNode<T>>;
@@ -74,3 +75,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/shift.hpp b/src/backend/cuda/shift.hpp
index e651c2b0d3..68c4ccd9bf 100644
--- a/src/backend/cuda/shift.hpp
+++ b/src/backend/cuda/shift.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> shift(const Array<T> &in, const int sdims[4]);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sift.cu b/src/backend/cuda/sift.cu
index 78314981cd..dbfb46a63b 100644
--- a/src/backend/cuda/sift.cu
+++ b/src/backend/cuda/sift.cu
@@ -14,6 +14,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename convAccT>
@@ -71,3 +72,4 @@ INSTANTIATE(float, float)
 INSTANTIATE(double, double)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sift.hpp b/src/backend/cuda/sift.hpp
index 1ec8638b41..a177c345ae 100644
--- a/src/backend/cuda/sift.hpp
+++ b/src/backend/cuda/sift.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, typename convAccT>
@@ -23,4 +24,5 @@ unsigned sift(Array<float>& x, Array<float>& y, Array<float>& score,
               const float img_scale, const float feature_ratio,
               const bool compute_GLOH);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sobel.cpp b/src/backend/cuda/sobel.cpp
index c58bb17974..5200f69a45 100644
--- a/src/backend/cuda/sobel.cpp
+++ b/src/backend/cuda/sobel.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename Ti, typename To>
@@ -42,3 +43,4 @@ INSTANTIATE(short, int)
 INSTANTIATE(ushort, int)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sobel.hpp b/src/backend/cuda/sobel.hpp
index 4cba95b4cf..f566459138 100644
--- a/src/backend/cuda/sobel.hpp
+++ b/src/backend/cuda/sobel.hpp
@@ -10,10 +10,12 @@
 #include <Array.hpp>
 #include <utility>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename Ti, typename To>
 std::pair<Array<To>, Array<To>> sobelDerivatives(const Array<Ti> &img,
                                                  const unsigned &ker_size);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/solve.cu b/src/backend/cuda/solve.cu
index f9e80efdf0..f762785818 100644
--- a/src/backend/cuda/solve.cu
+++ b/src/backend/cuda/solve.cu
@@ -23,6 +23,7 @@
 #include <qr.hpp>
 #include <transpose.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 // cublasStatus_t cublas<>getrsBatched( cublasHandle_t handle,
@@ -271,8 +272,10 @@ Array<T> generalSolveBatched(const Array<T> &a, const Array<T> &b) {
         }
     }
 
-    unique_mem_ptr aBatched_device_mem(pinnedAlloc<char>(bytes), pinnedFree<char>);
-    unique_mem_ptr bBatched_device_mem(pinnedAlloc<char>(bytes), pinnedFree<char>);
+    unique_mem_ptr aBatched_device_mem(pinnedAlloc<char>(bytes),
+                                       pinnedFree<char>);
+    unique_mem_ptr bBatched_device_mem(pinnedAlloc<char>(bytes),
+                                       pinnedFree<char>);
 
     T **aBatched_device_ptrs = (T **)aBatched_device_mem.get();
     T **bBatched_device_ptrs = (T **)bBatched_device_mem.get();
@@ -477,3 +480,4 @@ INSTANTIATE_SOLVE(double)
 INSTANTIATE_SOLVE(cdouble)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/solve.hpp b/src/backend/cuda/solve.hpp
index 72c80000d0..20205aa771 100644
--- a/src/backend/cuda/solve.hpp
+++ b/src/backend/cuda/solve.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> solve(const Array<T> &a, const Array<T> &b,
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> solveLU(const Array<T> &a, const Array<int> &pivot, const Array<T> &b,
                  const af_mat_prop options = AF_MAT_NONE);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sort.cu b/src/backend/cuda/sort.cu
index 8596c3b894..9970ddd8b2 100644
--- a/src/backend/cuda/sort.cu
+++ b/src/backend/cuda/sort.cu
@@ -16,6 +16,7 @@
 #include <sort.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending) {
@@ -59,3 +60,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sort.hpp b/src/backend/cuda/sort.hpp
index 74473bb981..f6b8832f01 100644
--- a/src/backend/cuda/sort.hpp
+++ b/src/backend/cuda/sort.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sort_by_key.cu b/src/backend/cuda/sort_by_key.cu
index 4cc64e2aed..bd19d16240 100644
--- a/src/backend/cuda/sort_by_key.cu
+++ b/src/backend/cuda/sort_by_key.cu
@@ -16,6 +16,7 @@
 #include <sort_by_key.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace cuda {
 template<typename Tk, typename Tv>
 void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
@@ -82,3 +83,4 @@ INSTANTIATE1(intl)
 INSTANTIATE1(uintl)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sort_by_key.hpp b/src/backend/cuda/sort_by_key.hpp
index 5eb7c1e716..e44badc6a8 100644
--- a/src/backend/cuda/sort_by_key.hpp
+++ b/src/backend/cuda/sort_by_key.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename Tk, typename Tv>
 void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
                  const Array<Tv> &ival, const unsigned dim, bool isAscending);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sort_index.cu b/src/backend/cuda/sort_index.cu
index 9d1a88822e..039e77a147 100644
--- a/src/backend/cuda/sort_index.cu
+++ b/src/backend/cuda/sort_index.cu
@@ -17,6 +17,7 @@
 #include <sort_index.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void sort_index(Array<T> &okey, Array<uint> &oval, const Array<T> &in,
@@ -69,3 +70,4 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sort_index.hpp b/src/backend/cuda/sort_index.hpp
index 970e7c9b48..1355f9ea8a 100644
--- a/src/backend/cuda/sort_index.hpp
+++ b/src/backend/cuda/sort_index.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void sort_index(Array<T> &val, Array<unsigned> &idx, const Array<T> &in,
                 const unsigned dim, bool isAscending);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sparse.cu b/src/backend/cuda/sparse.cu
index 27b805e9ea..6dec35090c 100644
--- a/src/backend/cuda/sparse.cu
+++ b/src/backend/cuda/sparse.cu
@@ -25,6 +25,7 @@
 #include <stdexcept>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 
 using namespace common;
@@ -307,7 +308,7 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
         CUDA_CHECK(
             cudaMemcpyAsync(converted.getColIdx().get(), in.getColIdx().get(),
                             in.getColIdx().elements() * sizeof(int),
-                            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+                            cudaMemcpyDeviceToDevice, getActiveStream()));
 
         // cusparse function to expand compressed row into coordinate
         CUSPARSE_CHECK(_.cusparseXcsr2coo(
@@ -374,11 +375,11 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
         CUDA_CHECK(
             cudaMemcpyAsync(converted.getValues().get(), cooT.getValues().get(),
                             cooT.getValues().elements() * sizeof(T),
-                            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+                            cudaMemcpyDeviceToDevice, getActiveStream()));
         CUDA_CHECK(
             cudaMemcpyAsync(converted.getColIdx().get(), cooT.getColIdx().get(),
                             cooT.getColIdx().elements() * sizeof(int),
-                            cudaMemcpyDeviceToDevice, cuda::getActiveStream()));
+                            cudaMemcpyDeviceToDevice, getActiveStream()));
 
         // cusparse function to compress row from coordinate
         CUSPARSE_CHECK(_.cusparseXcoo2csr(
@@ -446,3 +447,4 @@ INSTANTIATE_SPARSE(cdouble)
 #undef INSTANTIATE_SPARSE
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sparse.hpp b/src/backend/cuda/sparse.hpp
index 5b571d4eb9..ae4f42ccf6 100644
--- a/src/backend/cuda/sparse.hpp
+++ b/src/backend/cuda/sparse.hpp
@@ -12,6 +12,7 @@
 #include <Array.hpp>
 #include <common/SparseArray.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T, af_storage stype>
@@ -25,3 +26,4 @@ common::SparseArray<T> sparseConvertStorageToStorage(
     const common::SparseArray<T> &in);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sparse_arith.cu b/src/backend/cuda/sparse_arith.cu
index a41c356397..63bda7f733 100644
--- a/src/backend/cuda/sparse_arith.cu
+++ b/src/backend/cuda/sparse_arith.cu
@@ -26,6 +26,7 @@
 #include <stdexcept>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 
 using namespace common;
@@ -235,11 +236,9 @@ SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
         nnzC = *nnzcDevHostPtr;
     } else {
         CUDA_CHECK(cudaMemcpyAsync(&nnzC, csrRowPtrC + M, sizeof(int),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
+                                   cudaMemcpyDeviceToHost, getActiveStream()));
         CUDA_CHECK(cudaMemcpyAsync(&baseC, csrRowPtrC, sizeof(int),
-                                   cudaMemcpyDeviceToHost,
-                                   cuda::getActiveStream()));
+                                   cudaMemcpyDeviceToHost, getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         nnzC -= baseC;
     }
@@ -295,3 +294,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sparse_arith.hpp b/src/backend/cuda/sparse_arith.hpp
index bd1839d058..a3628df405 100644
--- a/src/backend/cuda/sparse_arith.hpp
+++ b/src/backend/cuda/sparse_arith.hpp
@@ -12,6 +12,7 @@
 #include <optypes.hpp>
 #include <sparse.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 // These two functions cannot be overloaded by return type.
@@ -28,3 +29,4 @@ template<typename T, af_op_t op>
 common::SparseArray<T> arithOp(const common::SparseArray<T> &lhs,
                                const common::SparseArray<T> &rhs);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sparse_blas.cu b/src/backend/cuda/sparse_blas.cu
index 33a2957a62..965186a915 100644
--- a/src/backend/cuda/sparse_blas.cu
+++ b/src/backend/cuda/sparse_blas.cu
@@ -22,6 +22,7 @@
 #include <stdexcept>
 #include <string>
 
+namespace arrayfire {
 namespace cuda {
 
 cusparseOperation_t toCusparseTranspose(af_mat_prop opt) {
@@ -222,3 +223,4 @@ INSTANTIATE_SPARSE(cfloat)
 INSTANTIATE_SPARSE(cdouble)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sparse_blas.hpp b/src/backend/cuda/sparse_blas.hpp
index 3ff5e38520..d4b41defd0 100644
--- a/src/backend/cuda/sparse_blas.hpp
+++ b/src/backend/cuda/sparse_blas.hpp
@@ -10,10 +10,12 @@
 #include <Array.hpp>
 #include <common/SparseArray.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhs,
                 af_mat_prop optLhs, af_mat_prop optRhs);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/sum.cu b/src/backend/cuda/sum.cu
index 3dcd357700..44cfec9449 100644
--- a/src/backend/cuda/sum.cu
+++ b/src/backend/cuda/sum.cu
@@ -7,11 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include "reduce_impl.hpp"
 #include <common/half.hpp>
+#include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 // sum
 INSTANTIATE(af_add_t, float, float)
@@ -38,3 +39,4 @@ INSTANTIATE(af_add_t, half, half)
 INSTANTIATE(af_add_t, half, float)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp
index ca38716f39..bef751239b 100644
--- a/src/backend/cuda/surface.cpp
+++ b/src/backend/cuda/surface.cpp
@@ -15,12 +15,16 @@
 #include <surface.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_surface(const Array<T> &P, fg_surface surface) {
-    auto stream = cuda::getActiveStream();
+    auto stream = getActiveStream();
     if (DeviceManager::checkGraphicsInteropCapability()) {
         const T *d_P = P.get();
 
@@ -38,7 +42,7 @@ void copy_surface(const Array<T> &P, fg_surface surface) {
 
         POST_LAUNCH_CHECK();
     } else {
-        ForgeModule &_ = graphics::forgePlugin();
+        ForgeModule &_ = forgePlugin();
         unsigned bytes = 0, buffer = 0;
         FG_CHECK(_.fg_get_surface_vertex_buffer(&buffer, surface));
         FG_CHECK(_.fg_get_surface_vertex_buffer_size(&bytes, surface));
@@ -70,3 +74,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/surface.hpp b/src/backend/cuda/surface.hpp
index a9fef84fb6..896344c73b 100644
--- a/src/backend/cuda/surface.hpp
+++ b/src/backend/cuda/surface.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_surface(const Array<T> &P, fg_surface surface);
 
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/susan.cpp b/src/backend/cuda/susan.cpp
index 1f2a367e88..4d0fcc078c 100644
--- a/src/backend/cuda/susan.cpp
+++ b/src/backend/cuda/susan.cpp
@@ -18,6 +18,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -78,3 +79,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/susan.hpp b/src/backend/cuda/susan.hpp
index bc27d5bc7f..2266320485 100644
--- a/src/backend/cuda/susan.hpp
+++ b/src/backend/cuda/susan.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -19,4 +20,5 @@ unsigned susan(Array<float> &x_out, Array<float> &y_out, Array<float> &resp_out,
                const Array<T> &in, const unsigned radius, const float diff_thr,
                const float geom_thr, const float feature_ratio,
                const unsigned edge);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/svd.cpp b/src/backend/cuda/svd.cpp
index 7c51fefc51..6ec71739ba 100644
--- a/src/backend/cuda/svd.cpp
+++ b/src/backend/cuda/svd.cpp
@@ -19,6 +19,7 @@
 
 #include <cusolverDn.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 cusolverStatus_t gesvd_buf_func(cusolverDnHandle_t /*handle*/, int /*m*/,
@@ -114,3 +115,4 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/svd.hpp b/src/backend/cuda/svd.hpp
index 39192f95bb..21cd52b684 100644
--- a/src/backend/cuda/svd.hpp
+++ b/src/backend/cuda/svd.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T, typename Tr>
 void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
@@ -16,3 +17,4 @@ void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
 template<typename T, typename Tr>
 void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/threadsMgt.hpp b/src/backend/cuda/threadsMgt.hpp
index 06fccdb0a3..147dff5586 100644
--- a/src/backend/cuda/threadsMgt.hpp
+++ b/src/backend/cuda/threadsMgt.hpp
@@ -11,13 +11,14 @@
 #include <common/dispatch.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace cuda {
 // OVERALL USAGE (With looping):
 // ...                                                      // OWN CODE
 // threadsMgt<T> th(...);                                   // backend.hpp
 // const dim3 threads{th.genThreads()};                     // backend.hpp
 // const dim3 blocks{th.genBlocks(threads,..)};             // backend.hpp
-// cuda::Kernel KER{GETKERNEL(..., th.loop0, th.loop1, th.loop2,
+// arrayfire::cuda::Kernel KER{GETKERNEL(..., th.loop0, th.loop1, th.loop2,
 //                               th.loop3)};                // OWN CODE
 // KER(threads,blocks,...);                                 // OWN CODE
 // ...                                                      // OWN CODE
@@ -27,8 +28,8 @@ namespace cuda {
 // threadsMgt<T> th(...);                                   // backend.hpp
 // const dim3 threads{th.genThreads()};                     // backend.hpp
 // const dim3 blocks{th.genBlocksFull(threads,...)};        // backend.hpp
-// cuda::Kernel KER{GETKERNEL(...)};                        // OWN CODE
-// KER(threads,blocks,...);                                 // OWN CODE
+// arrayfire::cuda::Kernel KER{GETKERNEL(...)};             // OWN
+// CODE KER(threads,blocks,...);                            // OWN CODE
 // ...                                                      // OWN CODE
 template<typename T>
 class threadsMgt {
@@ -324,4 +325,5 @@ inline dim3 threadsMgt<T>::genBlocks(const dim3& threads,
 
     return blocks;
 };
-}  // namespace cuda
\ No newline at end of file
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/thrust_utils.hpp b/src/backend/cuda/thrust_utils.hpp
index ed468b74a5..8aafbc1752 100644
--- a/src/backend/cuda/thrust_utils.hpp
+++ b/src/backend/cuda/thrust_utils.hpp
@@ -13,29 +13,32 @@
 #include <thrust/version.h>
 #include <ThrustAllocator.cuh>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
-using ThrustVector = thrust::device_vector<T, cuda::ThrustAllocator<T>>;
-}
+using ThrustVector = thrust::device_vector<T, ThrustAllocator<T>>;
+}  // namespace cuda
+}  // namespace arrayfire
 
 #if THRUST_MAJOR_VERSION >= 1 && THRUST_MINOR_VERSION >= 8
 
-#define THRUST_SELECT(fn, ...) fn(cuda::ThrustArrayFirePolicy(), __VA_ARGS__)
+#define THRUST_SELECT(fn, ...) \
+    fn(arrayfire::cuda::ThrustArrayFirePolicy(), __VA_ARGS__)
 #define THRUST_SELECT_OUT(res, fn, ...) \
-    res = fn(cuda::ThrustArrayFirePolicy(), __VA_ARGS__)
+    res = fn(arrayfire::cuda::ThrustArrayFirePolicy(), __VA_ARGS__)
 
 #else
 
-#define THRUST_SELECT(fn, ...)                                      \
-    do {                                                            \
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream())); \
-        fn(__VA_ARGS__);                                            \
+#define THRUST_SELECT(fn, ...)                                                 \
+    do {                                                                       \
+        CUDA_CHECK(cudaStreamSynchronize(arrayfire::cuda::getActiveStream())); \
+        fn(__VA_ARGS__);                                                       \
     } while (0)
 
-#define THRUST_SELECT_OUT(res, fn, ...)                             \
-    do {                                                            \
-        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream())); \
-        res = fn(__VA_ARGS__);                                      \
+#define THRUST_SELECT_OUT(res, fn, ...)                                        \
+    do {                                                                       \
+        CUDA_CHECK(cudaStreamSynchronize(arrayfire::cuda::getActiveStream())); \
+        res = fn(__VA_ARGS__);                                                 \
     } while (0)
 
 #endif
diff --git a/src/backend/cuda/tile.cpp b/src/backend/cuda/tile.cpp
index 4b2839232e..f93982eb43 100644
--- a/src/backend/cuda/tile.cpp
+++ b/src/backend/cuda/tile.cpp
@@ -16,8 +16,9 @@
 
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> tile(const Array<T> &in, const af::dim4 &tileDims) {
@@ -54,3 +55,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/tile.hpp b/src/backend/cuda/tile.hpp
index d58795a629..888e77aa13 100644
--- a/src/backend/cuda/tile.hpp
+++ b/src/backend/cuda/tile.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> tile(const Array<T> &in, const af::dim4 &tileDims);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/topk.cu b/src/backend/cuda/topk.cu
index 5901c5e5b1..12dde72684 100644
--- a/src/backend/cuda/topk.cu
+++ b/src/backend/cuda/topk.cu
@@ -13,8 +13,9 @@
 #include <topk.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void topk(Array<T>& ovals, Array<uint>& oidxs, const Array<T>& ivals,
@@ -40,3 +41,4 @@ INSTANTIATE(long long)
 INSTANTIATE(unsigned long long)
 INSTANTIATE(half)
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/topk.hpp b/src/backend/cuda/topk.hpp
index 3b87427eb3..f3c27f433c 100644
--- a/src/backend/cuda/topk.hpp
+++ b/src/backend/cuda/topk.hpp
@@ -8,8 +8,10 @@
  ********************************************************/
 
 #include <Array.hpp>
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void topk(Array<T>& keys, Array<unsigned>& vals, const Array<T>& in,
           const int k, const int dim, const af::topkFunction order);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/transform.cpp b/src/backend/cuda/transform.cpp
index a143d74963..baba9b1a04 100644
--- a/src/backend/cuda/transform.cpp
+++ b/src/backend/cuda/transform.cpp
@@ -12,6 +12,7 @@
 #include <kernel/transform.hpp>
 #include <utility.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -42,3 +43,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/transform.hpp b/src/backend/cuda/transform.hpp
index ee3596d3ef..8e9e4b6990 100644
--- a/src/backend/cuda/transform.hpp
+++ b/src/backend/cuda/transform.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const af_interp_type method, const bool inverse,
                const bool perspective);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/transpose.cpp b/src/backend/cuda/transpose.cpp
index 25f882b667..faa4659b68 100644
--- a/src/backend/cuda/transpose.cpp
+++ b/src/backend/cuda/transpose.cpp
@@ -14,8 +14,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -52,3 +53,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/transpose.hpp b/src/backend/cuda/transpose.hpp
index 5a26aa8b14..e612754323 100644
--- a/src/backend/cuda/transpose.hpp
+++ b/src/backend/cuda/transpose.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 void transpose_inplace(Array<T> &in, const bool conjugate);
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/transpose_inplace.cpp b/src/backend/cuda/transpose_inplace.cpp
index d0c9163f89..ff89730d47 100644
--- a/src/backend/cuda/transpose_inplace.cpp
+++ b/src/backend/cuda/transpose_inplace.cpp
@@ -14,8 +14,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -44,3 +45,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/triangle.cpp b/src/backend/cuda/triangle.cpp
index 8e5f7eec76..4ec0a04e6f 100644
--- a/src/backend/cuda/triangle.cpp
+++ b/src/backend/cuda/triangle.cpp
@@ -15,8 +15,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -53,3 +54,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/triangle.hpp b/src/backend/cuda/triangle.hpp
index 801dfdd900..98c3480126 100644
--- a/src/backend/cuda/triangle.hpp
+++ b/src/backend/cuda/triangle.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void triangle(Array<T> &out, const Array<T> &in, const bool is_upper,
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> triangle(const Array<T> &in, const bool is_upper,
                   const bool is_unit_diag);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/types.hpp b/src/backend/cuda/types.hpp
index 91bcdbbda7..34815cba66 100644
--- a/src/backend/cuda/types.hpp
+++ b/src/backend/cuda/types.hpp
@@ -13,9 +13,11 @@
 #include <cuComplex.h>
 #include <cuda_fp16.h>
 
+namespace arrayfire {
 namespace common {
 class half;
-}
+}  // namespace common
+}  // namespace arrayfire
 
 #ifdef __CUDACC_RTC__
 
@@ -27,6 +29,7 @@ using dim_t = long long;
 
 #endif  //__CUDACC_RTC__
 
+namespace arrayfire {
 namespace cuda {
 
 using cdouble = cuDoubleComplex;
@@ -99,7 +102,7 @@ inline const char *shortname<ushort>(bool caps) {
     return caps ? "Q" : "q";
 }
 template<>
-inline const char *shortname<common::half>(bool caps) {
+inline const char *shortname<arrayfire::common::half>(bool caps) {
     return caps ? "H" : "h";
 }
 
@@ -133,9 +136,7 @@ inline const char *getFullName<common::half>() {
 }  // namespace
 #endif  //__CUDACC_RTC__
 
-//#ifndef __CUDACC_RTC__
 }  // namespace cuda
-//#endif  //__CUDACC_RTC__
 
 namespace common {
 
@@ -143,8 +144,8 @@ template<typename T>
 struct kernel_type;
 
 template<>
-struct kernel_type<common::half> {
-    using data = common::half;
+struct kernel_type<arrayfire::common::half> {
+    using data = arrayfire::common::half;
 
 #ifdef __CUDA_ARCH__
 
@@ -170,3 +171,4 @@ struct kernel_type<common::half> {
 #endif  // __CUDA_ARCH__
 };
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/cuda/unary.hpp b/src/backend/cuda/unary.hpp
index a94c84dfa2..5fd9e48f52 100644
--- a/src/backend/cuda/unary.hpp
+++ b/src/backend/cuda/unary.hpp
@@ -14,6 +14,7 @@
 #include <math.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<af_op_t op>
@@ -78,8 +79,8 @@ UNARY_DECL(noop, "__noop")
 
 template<typename T, af_op_t op>
 Array<T> unaryOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
-    using common::Node;
-    using common::Node_ptr;
+    using arrayfire::common::Node;
+    using arrayfire::common::Node_ptr;
     using std::array;
 
     auto createUnary = [](array<Node_ptr, 1> &operands) {
@@ -95,7 +96,7 @@ Array<T> unaryOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
 
 template<typename T, af_op_t op>
 Array<char> checkOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
-    using common::Node_ptr;
+    using arrayfire::common::Node_ptr;
 
     auto createUnary = [](std::array<Node_ptr, 1> &operands) {
         return Node_ptr(new common::UnaryNode(
@@ -109,3 +110,4 @@ Array<char> checkOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/unwrap.cpp b/src/backend/cuda/unwrap.cpp
index 0f9b4dd0c1..6eae7d428b 100644
--- a/src/backend/cuda/unwrap.cpp
+++ b/src/backend/cuda/unwrap.cpp
@@ -16,8 +16,9 @@
 
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -62,3 +63,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/unwrap.hpp b/src/backend/cuda/unwrap.hpp
index 1a348d93e2..dbb1f8ee24 100644
--- a/src/backend/cuda/unwrap.hpp
+++ b/src/backend/cuda/unwrap.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<T> unwrap(const Array<T> &in, const dim_t wx, const dim_t wy,
                 const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
                 const dim_t dx, const dim_t dy, const bool is_column);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/utility.cpp b/src/backend/cuda/utility.cpp
index a315f4d28d..724f546326 100644
--- a/src/backend/cuda/utility.cpp
+++ b/src/backend/cuda/utility.cpp
@@ -11,6 +11,7 @@
 
 #include <err_cuda.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 int interpOrder(const af_interp_type p) noexcept {
@@ -31,3 +32,4 @@ int interpOrder(const af_interp_type p) noexcept {
 }
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/utility.hpp b/src/backend/cuda/utility.hpp
index bf602eacc9..d3ff338bf6 100644
--- a/src/backend/cuda/utility.hpp
+++ b/src/backend/cuda/utility.hpp
@@ -12,6 +12,7 @@
 #include <backend.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace cuda {
 
 [[gnu::unused]] static __DH__ dim_t trimIndex(const int &idx,
@@ -30,3 +31,4 @@ namespace cuda {
 int interpOrder(const af_interp_type p) noexcept;
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp
index eba52ad532..2868979772 100644
--- a/src/backend/cuda/vector_field.cpp
+++ b/src/backend/cuda/vector_field.cpp
@@ -15,13 +15,17 @@
 #include <vector_field.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        fg_vector_field vfield) {
-    auto stream = cuda::getActiveStream();
+    auto stream = getActiveStream();
     if (DeviceManager::checkGraphicsInteropCapability()) {
         auto res = interopManager().getVectorFieldResources(vfield);
         cudaGraphicsResource_t resources[2] = {*res[0].get(), *res[1].get()};
@@ -54,7 +58,7 @@ void copy_vector_field(const Array<T> &points, const Array<T> &directions,
 
         POST_LAUNCH_CHECK();
     } else {
-        ForgeModule &_ = graphics::forgePlugin();
+        ForgeModule &_ = forgePlugin();
         CheckGL("Begin CUDA fallback-resource copy");
         unsigned size1 = 0, size2 = 0;
         unsigned buff1 = 0, buff2 = 0;
@@ -104,3 +108,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/vector_field.hpp b/src/backend/cuda/vector_field.hpp
index abb375bcbc..086e1bbf27 100644
--- a/src/backend/cuda/vector_field.hpp
+++ b/src/backend/cuda/vector_field.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        fg_vector_field vfield);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/where.cpp b/src/backend/cuda/where.cpp
index fd39c88eb6..efd488d26e 100644
--- a/src/backend/cuda/where.cpp
+++ b/src/backend/cuda/where.cpp
@@ -16,6 +16,7 @@
 #include <where.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<uint> where(const Array<T> &in) {
@@ -40,3 +41,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/where.hpp b/src/backend/cuda/where.hpp
index 6a2069f344..a2e9ccdab6 100644
--- a/src/backend/cuda/where.hpp
+++ b/src/backend/cuda/where.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 Array<uint> where(const Array<T>& in);
-}
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/wrap.cpp b/src/backend/cuda/wrap.cpp
index 76834e6a10..d8963cacd9 100644
--- a/src/backend/cuda/wrap.cpp
+++ b/src/backend/cuda/wrap.cpp
@@ -18,8 +18,9 @@
 
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace cuda {
 
 template<typename T>
@@ -74,3 +75,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/wrap.hpp b/src/backend/cuda/wrap.hpp
index d324975379..312b24a23e 100644
--- a/src/backend/cuda/wrap.hpp
+++ b/src/backend/cuda/wrap.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace cuda {
 template<typename T>
 void wrap(Array<T> &out, const Array<T> &in, const dim_t wx, const dim_t wy,
@@ -21,3 +22,4 @@ Array<T> wrap_dilated(const Array<T> &in, const dim_t ox, const dim_t oy,
                       const dim_t sy, const dim_t px, const dim_t py,
                       const dim_t dx, const dim_t dy, const bool is_column);
 }  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index f3dd8d97ed..225e9686ac 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -39,11 +39,11 @@ using af::dtype_traits;
 
 using cl::Buffer;
 
-using common::half;
-using common::Node;
-using common::Node_ptr;
-using common::NodeIterator;
-using opencl::jit::BufferNode;
+using arrayfire::common::half;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::NodeIterator;
+using arrayfire::opencl::jit::BufferNode;
 
 using nonstd::span;
 using std::accumulate;
@@ -52,6 +52,7 @@ using std::make_shared;
 using std::shared_ptr;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 shared_ptr<BufferNode> bufferNodePtr() {
@@ -549,3 +550,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/Array.hpp b/src/backend/opencl/Array.hpp
index d3362cfa9a..2d2ca97c94 100644
--- a/src/backend/opencl/Array.hpp
+++ b/src/backend/opencl/Array.hpp
@@ -34,6 +34,7 @@ template<typename T>
 class SparseArray;
 }
 
+namespace arrayfire {
 namespace opencl {
 typedef std::shared_ptr<cl::Buffer> Buffer_ptr;
 using af::dim4;
@@ -315,3 +316,4 @@ class Array {
 };
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 069609b95e..cf31204415 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -28,7 +28,7 @@ file_to_string(
     EXTENSION "hpp"
     OUTPUT_DIR ${kernel_headers_dir}
     TARGETS cl_kernel_targets
-    NAMESPACE "opencl"
+    NAMESPACE "arrayfire opencl"
     )
 
 set(opencl_compile_definitions
diff --git a/src/backend/opencl/Event.cpp b/src/backend/opencl/Event.cpp
index 21523891d9..bc93b60a62 100644
--- a/src/backend/opencl/Event.cpp
+++ b/src/backend/opencl/Event.cpp
@@ -20,6 +20,7 @@
 using std::make_unique;
 using std::unique_ptr;
 
+namespace arrayfire {
 namespace opencl {
 /// \brief Creates a new event and marks it in the queue
 Event makeEvent(cl::CommandQueue& queue) {
@@ -70,3 +71,4 @@ af_event createAndMarkEvent() {
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/Event.hpp b/src/backend/opencl/Event.hpp
index 51505d5489..c8420a9dff 100644
--- a/src/backend/opencl/Event.hpp
+++ b/src/backend/opencl/Event.hpp
@@ -12,6 +12,7 @@
 #include <common/EventBase.hpp>
 #include <af/event.h>
 
+namespace arrayfire {
 namespace opencl {
 class OpenCLEventPolicy {
    public:
@@ -57,3 +58,4 @@ void block(af_event eventHandle);
 af_event createAndMarkEvent();
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/GraphicsResourceManager.cpp b/src/backend/opencl/GraphicsResourceManager.cpp
index e2cd64150f..fe1f703a5f 100644
--- a/src/backend/opencl/GraphicsResourceManager.cpp
+++ b/src/backend/opencl/GraphicsResourceManager.cpp
@@ -10,6 +10,7 @@
 #include <GraphicsResourceManager.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace opencl {
 GraphicsResourceManager::ShrdResVector
 GraphicsResourceManager::registerResources(
@@ -25,3 +26,4 @@ GraphicsResourceManager::registerResources(
     return output;
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/GraphicsResourceManager.hpp b/src/backend/opencl/GraphicsResourceManager.hpp
index 618e46e2f4..130a564df1 100644
--- a/src/backend/opencl/GraphicsResourceManager.hpp
+++ b/src/backend/opencl/GraphicsResourceManager.hpp
@@ -18,6 +18,7 @@ namespace cl {
 class Buffer;
 }
 
+namespace arrayfire {
 namespace opencl {
 class GraphicsResourceManager
     : public common::InteropManager<GraphicsResourceManager, cl::Buffer> {
@@ -33,3 +34,4 @@ class GraphicsResourceManager
     void operator=(GraphicsResourceManager const&);
 };
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/Kernel.cpp b/src/backend/opencl/Kernel.cpp
index a096979f9a..b5d818b6d2 100644
--- a/src/backend/opencl/Kernel.cpp
+++ b/src/backend/opencl/Kernel.cpp
@@ -14,6 +14,7 @@
 #include <common/defines.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 Kernel::DevPtrType Kernel::getDevPtr(const char* name) {
@@ -39,3 +40,4 @@ int Kernel::getFlag(Kernel::DevPtrType src) {
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/Kernel.hpp b/src/backend/opencl/Kernel.hpp
index 92eb28be1e..e3a05e7da8 100644
--- a/src/backend/opencl/Kernel.hpp
+++ b/src/backend/opencl/Kernel.hpp
@@ -16,6 +16,7 @@
 #include <cl2hpp.hpp>
 #include <string>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel_logger {
 inline auto getLogger() -> spdlog::logger* {
@@ -63,3 +64,4 @@ class Kernel
 };
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/Module.hpp b/src/backend/opencl/Module.hpp
index c918797699..b8a8d6a3b5 100644
--- a/src/backend/opencl/Module.hpp
+++ b/src/backend/opencl/Module.hpp
@@ -13,6 +13,7 @@
 
 #include <cl2hpp.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 /// OpenCL backend wrapper for cl::Program object
@@ -35,3 +36,4 @@ class Module : public common::ModuleInterface<cl::Program> {
 };
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/Param.cpp b/src/backend/opencl/Param.cpp
index 25358310ae..3b791c96ea 100644
--- a/src/backend/opencl/Param.cpp
+++ b/src/backend/opencl/Param.cpp
@@ -12,6 +12,7 @@
 #include <platform.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace opencl {
 Param::Param() : data(nullptr), info{{0, 0, 0, 0}, {0, 0, 0, 0}, 0} {}
 Param::Param(cl::Buffer *data_, KParam info_) : data(data_), info(info_) {}
@@ -28,3 +29,4 @@ Param makeParam(cl::Buffer &mem, int off, const int dims[4],
     return out;
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/Param.hpp b/src/backend/opencl/Param.hpp
index 6cf63f356b..aaf19dea62 100644
--- a/src/backend/opencl/Param.hpp
+++ b/src/backend/opencl/Param.hpp
@@ -12,6 +12,7 @@
 #include <cl2hpp.hpp>
 #include <kernel/KParam.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 struct Param {
@@ -32,3 +33,4 @@ struct Param {
 Param makeParam(cl::Buffer& mem, int off, const int dims[4],
                 const int strides[4]);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/all.cpp b/src/backend/opencl/all.cpp
index 5825b3af4a..2d2a1d4717 100644
--- a/src/backend/opencl/all.cpp
+++ b/src/backend/opencl/all.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 // alltrue
 INSTANTIATE(af_and_t, float, char)
@@ -28,3 +29,4 @@ INSTANTIATE(af_and_t, short, char)
 INSTANTIATE(af_and_t, ushort, char)
 INSTANTIATE(af_and_t, half, char)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/anisotropic_diffusion.cpp b/src/backend/opencl/anisotropic_diffusion.cpp
index e71a78cfc8..19e065c14f 100644
--- a/src/backend/opencl/anisotropic_diffusion.cpp
+++ b/src/backend/opencl/anisotropic_diffusion.cpp
@@ -13,6 +13,7 @@
 #include <kernel/anisotropic_diffusion.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
@@ -33,3 +34,4 @@ void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
 INSTANTIATE(double)
 INSTANTIATE(float)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/anisotropic_diffusion.hpp b/src/backend/opencl/anisotropic_diffusion.hpp
index 816cae3359..a1a76a29dc 100644
--- a/src/backend/opencl/anisotropic_diffusion.hpp
+++ b/src/backend/opencl/anisotropic_diffusion.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
                           const af::fluxFunction fftype,
                           const af::diffusionEq eq);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/any.cpp b/src/backend/opencl/any.cpp
index c9668f3451..ce36f8ed90 100644
--- a/src/backend/opencl/any.cpp
+++ b/src/backend/opencl/any.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 // anytrue
 INSTANTIATE(af_or_t, float, char)
@@ -28,3 +29,4 @@ INSTANTIATE(af_or_t, short, char)
 INSTANTIATE(af_or_t, ushort, char)
 INSTANTIATE(af_or_t, half, char)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/approx.cpp b/src/backend/opencl/approx.cpp
index dc4f851e4f..cc8c6994a9 100644
--- a/src/backend/opencl/approx.cpp
+++ b/src/backend/opencl/approx.cpp
@@ -11,6 +11,7 @@
 
 #include <kernel/approx.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
@@ -83,3 +84,4 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/approx.hpp b/src/backend/opencl/approx.hpp
index addb8fe73c..5a2b7e3212 100644
--- a/src/backend/opencl/approx.hpp
+++ b/src/backend/opencl/approx.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
@@ -22,3 +23,4 @@ void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
              const Tp &yi_step, const af_interp_type method,
              const float offGrid);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/arith.hpp b/src/backend/opencl/arith.hpp
index 48bab53038..932a86d814 100644
--- a/src/backend/opencl/arith.hpp
+++ b/src/backend/opencl/arith.hpp
@@ -14,6 +14,7 @@
 #include <optypes.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, af_op_t op>
@@ -28,3 +29,4 @@ Array<T> arithOp(const Array<T> &lhs, const Array<T> &rhs,
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/assign.cpp b/src/backend/opencl/assign.cpp
index b11a2398a9..9e0f8074a3 100644
--- a/src/backend/opencl/assign.cpp
+++ b/src/backend/opencl/assign.cpp
@@ -18,8 +18,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -87,3 +88,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/assign.hpp b/src/backend/opencl/assign.hpp
index 4dd07541d5..6283ad8ceb 100644
--- a/src/backend/opencl/assign.hpp
+++ b/src/backend/opencl/assign.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/index.h>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/backend.hpp b/src/backend/opencl/backend.hpp
index 527d379168..30392a7b9a 100644
--- a/src/backend/opencl/backend.hpp
+++ b/src/backend/opencl/backend.hpp
@@ -21,4 +21,4 @@
 
 #include "types.hpp"
 
-namespace detail = opencl;
+namespace detail = arrayfire::opencl;
diff --git a/src/backend/opencl/bilateral.cpp b/src/backend/opencl/bilateral.cpp
index d75f62d2fc..21ec82e2b6 100644
--- a/src/backend/opencl/bilateral.cpp
+++ b/src/backend/opencl/bilateral.cpp
@@ -14,6 +14,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename inType, typename outType>
@@ -38,3 +39,4 @@ INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/bilateral.hpp b/src/backend/opencl/bilateral.hpp
index ab9775f3b2..05fd52c429 100644
--- a/src/backend/opencl/bilateral.hpp
+++ b/src/backend/opencl/bilateral.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename inType, typename outType>
 Array<outType> bilateral(const Array<inType> &in, const float &spatialSigma,
                          const float &chromaticSigma);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/binary.hpp b/src/backend/opencl/binary.hpp
index 700a1b3c49..02291d566a 100644
--- a/src/backend/opencl/binary.hpp
+++ b/src/backend/opencl/binary.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename To, typename Ti, af_op_t op>
@@ -125,3 +126,4 @@ struct BinOp<To, Ti, af_hypot_t> {
 };
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/blas.cpp b/src/backend/opencl/blas.cpp
index 263d07bd9f..45b4149599 100644
--- a/src/backend/opencl/blas.cpp
+++ b/src/backend/opencl/blas.cpp
@@ -26,8 +26,9 @@
 #include <cpu/cpu_blas.hpp>
 #include <magma/magma_blas.h>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 void initBlas() { gpu_blas_init(); }
@@ -164,3 +165,4 @@ INSTANTIATE_DOT(cdouble)
 INSTANTIATE_DOT(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/blas.hpp b/src/backend/opencl/blas.hpp
index 22c2e1ec02..4416960f46 100644
--- a/src/backend/opencl/blas.hpp
+++ b/src/backend/opencl/blas.hpp
@@ -14,6 +14,7 @@
 // functions. They can be implemented in different back-ends,
 // such as CLBlast or clBLAS.
 
+namespace arrayfire {
 namespace opencl {
 
 void initBlas();
@@ -40,3 +41,4 @@ template<typename T>
 Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
              af_mat_prop optRhs);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/canny.cpp b/src/backend/opencl/canny.cpp
index ab2ec78c2f..cf4965fd5c 100644
--- a/src/backend/opencl/canny.cpp
+++ b/src/backend/opencl/canny.cpp
@@ -14,6 +14,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 Array<float> nonMaximumSuppression(const Array<float>& mag,
                                    const Array<float>& gx,
@@ -34,3 +35,4 @@ Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
     return out;
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/canny.hpp b/src/backend/opencl/canny.hpp
index 173937b521..e7ad6dda0d 100644
--- a/src/backend/opencl/canny.hpp
+++ b/src/backend/opencl/canny.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 Array<float> nonMaximumSuppression(const Array<float>& mag,
                                    const Array<float>& gx,
@@ -17,3 +18,4 @@ Array<float> nonMaximumSuppression(const Array<float>& mag,
 Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
                                      const Array<char>& weak);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cast.hpp b/src/backend/opencl/cast.hpp
index 3f3a0c1001..999d6188d9 100644
--- a/src/backend/opencl/cast.hpp
+++ b/src/backend/opencl/cast.hpp
@@ -18,6 +18,7 @@
 #include <af/dim4.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename To, typename Ti>
@@ -71,3 +72,4 @@ struct CastOp<cdouble, cdouble> {
 #undef CAST_CFN
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cholesky.cpp b/src/backend/opencl/cholesky.cpp
index eac4490baf..4d140ba099 100644
--- a/src/backend/opencl/cholesky.cpp
+++ b/src/backend/opencl/cholesky.cpp
@@ -17,6 +17,7 @@
 #include <magma/magma.h>
 #include <triangle.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -58,9 +59,11 @@ INSTANTIATE_CH(double)
 INSTANTIATE_CH(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -84,5 +87,6 @@ INSTANTIATE_CH(double)
 INSTANTIATE_CH(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cholesky.hpp b/src/backend/opencl/cholesky.hpp
index aa4e56bf29..be1805bc96 100644
--- a/src/backend/opencl/cholesky.hpp
+++ b/src/backend/opencl/cholesky.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
@@ -16,3 +17,4 @@ Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
 template<typename T>
 int cholesky_inplace(Array<T> &in, const bool is_upper);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/clfft.cpp b/src/backend/opencl/clfft.cpp
index 21ef1f37d7..68a17cbd50 100644
--- a/src/backend/opencl/clfft.cpp
+++ b/src/backend/opencl/clfft.cpp
@@ -18,6 +18,7 @@
 using std::make_unique;
 using std::string;
 
+namespace arrayfire {
 namespace opencl {
 const char *_clfftGetResultString(clfftStatus st) {
     switch (st) {
@@ -178,3 +179,4 @@ SharedPlan findPlan(clfftLayout iLayout, clfftLayout oLayout, clfftDim rank,
     return retVal;
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/clfft.hpp b/src/backend/opencl/clfft.hpp
index f0f1bc28f6..c7b9d9949f 100644
--- a/src/backend/opencl/clfft.hpp
+++ b/src/backend/opencl/clfft.hpp
@@ -15,6 +15,7 @@
 
 #include <cstdio>
 
+namespace arrayfire {
 namespace opencl {
 typedef clfftPlanHandle PlanType;
 typedef std::shared_ptr<PlanType> SharedPlan;
@@ -34,6 +35,7 @@ class PlanCache : public common::FFTPlanCache<PlanCache, PlanType> {
                                size_t batch);
 };
 }  // namespace opencl
+}  // namespace arrayfire
 
 #define CLFFT_CHECK(fn)                                          \
     do {                                                         \
diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp
index f931bb554a..32ea5809f5 100644
--- a/src/backend/opencl/compile_module.cpp
+++ b/src/backend/opencl/compile_module.cpp
@@ -30,16 +30,16 @@
 #include <string>
 #include <vector>
 
+using arrayfire::common::getEnvVar;
+using arrayfire::common::loggerFactory;
+using arrayfire::opencl::getActiveDeviceId;
+using arrayfire::opencl::getDevice;
+using arrayfire::opencl::Kernel;
+using arrayfire::opencl::Module;
 using cl::Error;
 using cl::Program;
-using common::getEnvVar;
-using common::loggerFactory;
 using fmt::format;
 using nonstd::span;
-using opencl::getActiveDeviceId;
-using opencl::getDevice;
-using opencl::Kernel;
-using opencl::Module;
 using spdlog::logger;
 
 using std::begin;
@@ -86,6 +86,7 @@ string getProgramBuildLog(const Program &prog) {
         AF_ERROR(build_error, AF_ERR_INTERNAL);                      \
     } while (0)
 
+namespace arrayfire {
 namespace opencl {
 
 const static string DEFAULT_MACROS_STR(
@@ -141,9 +142,10 @@ Program buildProgram(span<const string> kernelSources,
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 string getKernelCacheFilename(const int device, const string &key) {
-    auto &dev = opencl::getDevice(device);
+    auto &dev = arrayfire::opencl::getDevice(device);
 
     unsigned vendorId = dev.getInfo<CL_DEVICE_VENDOR_ID>();
     auto devName      = dev.getInfo<CL_DEVICE_NAME>();
@@ -157,6 +159,7 @@ string getKernelCacheFilename(const int device, const string &key) {
            to_string(AF_API_VERSION_CURRENT) + ".bin";
 }
 
+namespace arrayfire {
 namespace common {
 
 Module compileModule(const string &moduleKey, span<const string> sources,
@@ -166,11 +169,11 @@ Module compileModule(const string &moduleKey, span<const string> sources,
     UNUSED(isJIT);
 
     auto compileBegin = high_resolution_clock::now();
-    auto program      = opencl::buildProgram(sources, options);
+    auto program      = arrayfire::opencl::buildProgram(sources, options);
     auto compileEnd   = high_resolution_clock::now();
 
 #ifdef AF_CACHE_KERNELS_TO_DISK
-    const int device             = opencl::getActiveDeviceId();
+    const int device             = arrayfire::opencl::getActiveDeviceId();
     const string &cacheDirectory = getCacheDirectory();
     if (!cacheDirectory.empty()) {
         const string cacheFile = cacheDirectory + AF_PATH_SEPARATOR +
@@ -202,15 +205,17 @@ Module compileModule(const string &moduleKey, span<const string> sources,
             // before the current thread.
             if (!renameFile(tempFile, cacheFile)) { removeFile(tempFile); }
         } catch (const cl::Error &e) {
-            AF_TRACE("{{{:<20} : Failed to fetch opencl binary for {}, {}}}",
-                     moduleKey,
-                     opencl::getDevice(device).getInfo<CL_DEVICE_NAME>(),
-                     e.what());
+            AF_TRACE(
+                "{{{:<20} : Failed to fetch opencl binary for {}, {}}}",
+                moduleKey,
+                arrayfire::opencl::getDevice(device).getInfo<CL_DEVICE_NAME>(),
+                e.what());
         } catch (const std::ios_base::failure &e) {
-            AF_TRACE("{{{:<20} : Failed writing binary to {} for {}, {}}}",
-                     moduleKey, cacheFile,
-                     opencl::getDevice(device).getInfo<CL_DEVICE_NAME>(),
-                     e.what());
+            AF_TRACE(
+                "{{{:<20} : Failed writing binary to {} for {}, {}}}",
+                moduleKey, cacheFile,
+                arrayfire::opencl::getDevice(device).getInfo<CL_DEVICE_NAME>(),
+                e.what());
         }
     }
 #endif
@@ -228,7 +233,7 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
     const string &cacheDirectory = getCacheDirectory();
     if (cacheDirectory.empty()) return Module{};
 
-    auto &dev              = opencl::getDevice(device);
+    auto &dev              = arrayfire::opencl::getDevice(device);
     const string cacheFile = cacheDirectory + AF_PATH_SEPARATOR +
                              getKernelCacheFilename(device, moduleKey);
     Program program;
@@ -255,7 +260,7 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
         if (recomputedHash != clbinHash) {
             AF_ERROR("Binary on disk seems to be corrupted", AF_ERR_LOAD_SYM);
         }
-        program = Program(opencl::getContext(), {dev}, {clbin});
+        program = Program(arrayfire::opencl::getContext(), {dev}, {clbin});
         program.build();
 
         AF_TRACE("{{{:<20} : loaded from {} for {} }}", moduleKey, cacheFile,
@@ -293,3 +298,4 @@ Kernel getKernel(const Module &mod, const string &nameExpr,
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/opencl/complex.hpp b/src/backend/opencl/complex.hpp
index 124d3b49ca..a4306c7be3 100644
--- a/src/backend/opencl/complex.hpp
+++ b/src/backend/opencl/complex.hpp
@@ -15,6 +15,7 @@
 #include <traits.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename To, typename Ti>
 Array<To> cplx(const Array<Ti> &lhs, const Array<Ti> &rhs,
@@ -88,3 +89,4 @@ Array<T> conj(const Array<T> &in) {
     return createNodeArray<T>(in.dims(), common::Node_ptr(node));
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/convolve.cpp b/src/backend/opencl/convolve.cpp
index a4924303f3..edc28e4e35 100644
--- a/src/backend/opencl/convolve.cpp
+++ b/src/backend/opencl/convolve.cpp
@@ -24,11 +24,12 @@
 #include <vector>
 
 using af::dim4;
-using common::flip;
-using common::half;
-using common::modDims;
+using arrayfire::common::flip;
+using arrayfire::common::half;
+using arrayfire::common::modDims;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename accT>
@@ -249,3 +250,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/convolve.hpp b/src/backend/opencl/convolve.hpp
index 6e52ed6e56..0cf040c417 100644
--- a/src/backend/opencl/convolve.hpp
+++ b/src/backend/opencl/convolve.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename accT>
@@ -37,3 +38,4 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
                              const Array<T> &convolved_output, af::dim4 stride,
                              af::dim4 padding, af::dim4 dilation);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/convolve_separable.cpp b/src/backend/opencl/convolve_separable.cpp
index fc337e718f..03da468ac4 100644
--- a/src/backend/opencl/convolve_separable.cpp
+++ b/src/backend/opencl/convolve_separable.cpp
@@ -16,6 +16,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename accT>
@@ -72,3 +73,4 @@ INSTANTIATE(intl, float)
 INSTANTIATE(uintl, float)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/copy.cpp b/src/backend/opencl/copy.cpp
index cfb5e5b61d..970deae518 100644
--- a/src/backend/opencl/copy.cpp
+++ b/src/backend/opencl/copy.cpp
@@ -15,9 +15,10 @@
 #include <err_opencl.hpp>
 #include <math.hpp>
 
-using common::half;
-using common::is_complex;
+using arrayfire::common::half;
+using arrayfire::common::is_complex;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -209,3 +210,4 @@ INSTANTIATE_GETSCALAR(ushort)
 INSTANTIATE_GETSCALAR(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/copy.hpp b/src/backend/opencl/copy.hpp
index 9f6b19bcae..1b8576a5d9 100644
--- a/src/backend/opencl/copy.hpp
+++ b/src/backend/opencl/copy.hpp
@@ -11,6 +11,7 @@
 #include <Array.hpp>
 #include <kernel/pad_array_borders.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void copyData(T *data, const Array<T> &A);
@@ -65,3 +66,4 @@ void multiply_inplace(Array<T> &in, double val);
 template<typename T>
 T getScalar(const Array<T> &in);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/count.cpp b/src/backend/opencl/count.cpp
index fd1f6b3381..80f12e68cd 100644
--- a/src/backend/opencl/count.cpp
+++ b/src/backend/opencl/count.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 // count
 INSTANTIATE(af_notzero_t, float, uint)
@@ -28,3 +29,4 @@ INSTANTIATE(af_notzero_t, short, uint)
 INSTANTIATE(af_notzero_t, ushort, uint)
 INSTANTIATE(af_notzero_t, half, uint)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_blas.cpp b/src/backend/opencl/cpu/cpu_blas.cpp
index 8f80b044f3..8fbef46443 100644
--- a/src/backend/opencl/cpu/cpu_blas.cpp
+++ b/src/backend/opencl/cpu/cpu_blas.cpp
@@ -16,7 +16,7 @@
 #include <math.hpp>
 #include <traits.hpp>
 
-using common::is_complex;
+using arrayfire::common::is_complex;
 
 using std::add_const;
 using std::add_pointer;
@@ -25,6 +25,7 @@ using std::enable_if;
 using std::is_floating_point;
 using std::remove_const;
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -246,4 +247,5 @@ INSTANTIATE_GEMM(cdouble)
 
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cpu/cpu_blas.hpp b/src/backend/opencl/cpu/cpu_blas.hpp
index b39d8ae205..ae44d0ea91 100644
--- a/src/backend/opencl/cpu/cpu_blas.hpp
+++ b/src/backend/opencl/cpu/cpu_blas.hpp
@@ -9,11 +9,13 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
 template<typename T>
 void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
           const Array<T> &lhs, const Array<T> &rhs, const T *beta);
-}
+}  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_cholesky.cpp b/src/backend/opencl/cpu/cpu_cholesky.cpp
index fc066bd710..8878c8adf2 100644
--- a/src/backend/opencl/cpu/cpu_cholesky.cpp
+++ b/src/backend/opencl/cpu/cpu_cholesky.cpp
@@ -13,6 +13,7 @@
 #include <cpu/cpu_helper.hpp>
 #include <cpu/cpu_triangle.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -81,4 +82,5 @@ INSTANTIATE_CH(cdouble)
 
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cpu/cpu_cholesky.hpp b/src/backend/opencl/cpu/cpu_cholesky.hpp
index 3fdecfcd4a..489221304c 100644
--- a/src/backend/opencl/cpu/cpu_cholesky.hpp
+++ b/src/backend/opencl/cpu/cpu_cholesky.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 int cholesky_inplace(Array<T> &in, const bool is_upper);
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_helper.hpp b/src/backend/opencl/cpu/cpu_helper.hpp
index b614e53be1..0f979d1f90 100644
--- a/src/backend/opencl/cpu/cpu_helper.hpp
+++ b/src/backend/opencl/cpu/cpu_helper.hpp
@@ -20,8 +20,8 @@
 //********************************************************/
 #if defined(WITH_LINEAR_ALGEBRA)
 
-#define lapack_complex_float opencl::cfloat
-#define lapack_complex_double opencl::cdouble
+#define lapack_complex_float arrayfire::opencl::cfloat
+#define lapack_complex_double arrayfire::opencl::cdouble
 #define LAPACK_PREFIX LAPACKE_
 #define ORDER_TYPE int
 #define AF_LAPACK_COL_MAJOR LAPACK_COL_MAJOR
diff --git a/src/backend/opencl/cpu/cpu_inverse.cpp b/src/backend/opencl/cpu/cpu_inverse.cpp
index 7adcacc17c..b31e70b857 100644
--- a/src/backend/opencl/cpu/cpu_inverse.cpp
+++ b/src/backend/opencl/cpu/cpu_inverse.cpp
@@ -13,6 +13,7 @@
 #include <cpu/cpu_inverse.hpp>
 #include <cpu/cpu_lu.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -68,4 +69,5 @@ INSTANTIATE(cdouble)
 
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cpu/cpu_inverse.hpp b/src/backend/opencl/cpu/cpu_inverse.hpp
index b5be9e1ee0..04ed32b7d4 100644
--- a/src/backend/opencl/cpu/cpu_inverse.hpp
+++ b/src/backend/opencl/cpu/cpu_inverse.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 template<typename T>
 Array<T> inverse(const Array<T> &in);
-}
+}  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_lu.cpp b/src/backend/opencl/cpu/cpu_lu.cpp
index 7793a3590e..a754535025 100644
--- a/src/backend/opencl/cpu/cpu_lu.cpp
+++ b/src/backend/opencl/cpu/cpu_lu.cpp
@@ -16,6 +16,7 @@
 
 #include <numeric>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -156,4 +157,5 @@ INSTANTIATE_LU(cdouble)
 
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cpu/cpu_lu.hpp b/src/backend/opencl/cpu/cpu_lu.hpp
index f3cf4aaa1d..936add16e3 100644
--- a/src/backend/opencl/cpu/cpu_lu.hpp
+++ b/src/backend/opencl/cpu/cpu_lu.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 template<typename T>
@@ -19,3 +20,4 @@ template<typename T>
 Array<int> lu_inplace(Array<T> &in, const bool convert_pivot = true);
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_qr.cpp b/src/backend/opencl/cpu/cpu_qr.cpp
index fd5526792d..1e1b926d0f 100644
--- a/src/backend/opencl/cpu/cpu_qr.cpp
+++ b/src/backend/opencl/cpu/cpu_qr.cpp
@@ -13,6 +13,7 @@
 #include <cpu/cpu_qr.hpp>
 #include <cpu/cpu_triangle.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -115,4 +116,5 @@ INSTANTIATE_QR(cdouble)
 
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cpu/cpu_qr.hpp b/src/backend/opencl/cpu/cpu_qr.hpp
index 5d755dbd0b..d9c9345115 100644
--- a/src/backend/opencl/cpu/cpu_qr.hpp
+++ b/src/backend/opencl/cpu/cpu_qr.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> qr_inplace(Array<T> &in);
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_solve.cpp b/src/backend/opencl/cpu/cpu_solve.cpp
index 8b2cd79f64..4e0349d2dc 100644
--- a/src/backend/opencl/cpu/cpu_solve.cpp
+++ b/src/backend/opencl/cpu/cpu_solve.cpp
@@ -18,6 +18,7 @@
 #include <algorithm>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -313,4 +314,5 @@ INSTANTIATE_SOLVE(cdouble)
 
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cpu/cpu_solve.hpp b/src/backend/opencl/cpu/cpu_solve.hpp
index 9ef13caa8f..1223a96531 100644
--- a/src/backend/opencl/cpu/cpu_solve.hpp
+++ b/src/backend/opencl/cpu/cpu_solve.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 template<typename T>
@@ -20,3 +21,4 @@ Array<T> solveLU(const Array<T> &a, const Array<int> &pivot, const Array<T> &b,
                  const af_mat_prop options = AF_MAT_NONE);
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_sparse_blas.cpp b/src/backend/opencl/cpu/cpu_sparse_blas.cpp
index 0699c44717..66fba7cdbe 100644
--- a/src/backend/opencl/cpu/cpu_sparse_blas.cpp
+++ b/src/backend/opencl/cpu/cpu_sparse_blas.cpp
@@ -20,7 +20,7 @@
 #include <stdexcept>
 #include <string>
 
-using common::is_complex;
+using arrayfire::common::is_complex;
 
 using std::add_const;
 using std::add_pointer;
@@ -30,6 +30,7 @@ using std::is_floating_point;
 using std::is_same;
 using std::remove_const;
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -487,4 +488,5 @@ INSTANTIATE_SPARSE(cdouble)
 
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cpu/cpu_sparse_blas.hpp b/src/backend/opencl/cpu/cpu_sparse_blas.hpp
index 90e53e30d6..dee21c7c01 100644
--- a/src/backend/opencl/cpu/cpu_sparse_blas.hpp
+++ b/src/backend/opencl/cpu/cpu_sparse_blas.hpp
@@ -18,10 +18,11 @@
 using sp_cfloat  = MKL_Complex8;
 using sp_cdouble = MKL_Complex16;
 #else
-using sp_cfloat  = opencl::cfloat;
-using sp_cdouble = opencl::cdouble;
+using sp_cfloat  = arrayfire::opencl::cfloat;
+using sp_cdouble = arrayfire::opencl::cdouble;
 #endif
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -29,5 +30,6 @@ template<typename T>
 Array<T> matmul(const common::SparseArray<T> lhs, const Array<T> rhs,
                 af_mat_prop optLhs, af_mat_prop optRhs);
 
-}
+}  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_svd.cpp b/src/backend/opencl/cpu/cpu_svd.cpp
index 2b0e23db1e..6d865e8520 100644
--- a/src/backend/opencl/cpu/cpu_svd.cpp
+++ b/src/backend/opencl/cpu/cpu_svd.cpp
@@ -12,6 +12,7 @@
 #include <cpu/cpu_helper.hpp>
 #include <cpu/cpu_svd.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -93,4 +94,5 @@ INSTANTIATE_SVD(cfloat, float)
 INSTANTIATE_SVD(cdouble, double)
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/cpu/cpu_svd.hpp b/src/backend/opencl/cpu/cpu_svd.hpp
index 783c1664fe..2cb163de43 100644
--- a/src/backend/opencl/cpu/cpu_svd.hpp
+++ b/src/backend/opencl/cpu/cpu_svd.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 template<typename T, typename Tr>
@@ -18,3 +19,4 @@ template<typename T, typename Tr>
 void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in);
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/cpu/cpu_triangle.hpp b/src/backend/opencl/cpu/cpu_triangle.hpp
index 51bc242428..6bf2a4ceda 100644
--- a/src/backend/opencl/cpu/cpu_triangle.hpp
+++ b/src/backend/opencl/cpu/cpu_triangle.hpp
@@ -13,6 +13,7 @@
 
 #include <math.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace cpu {
 
@@ -50,6 +51,7 @@ void triangle(T *o, const T *i, const dim4 odm, const dim4 ost,
 
 }  // namespace cpu
 }  // namespace opencl
+}  // namespace arrayfire
 
 #endif
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index a9cfbc02e2..c1fa920a97 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -40,11 +40,11 @@
 #include <string>
 #include <vector>
 
+using arrayfire::common::getEnvVar;
 using cl::CommandQueue;
 using cl::Context;
 using cl::Device;
 using cl::Platform;
-using common::getEnvVar;
 using std::begin;
 using std::end;
 using std::find;
@@ -54,6 +54,7 @@ using std::stringstream;
 using std::unique_ptr;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 
 #if defined(OS_MAC)
@@ -197,7 +198,7 @@ DeviceManager::DeviceManager()
         }
 #endif
     }
-    fgMngr = std::make_unique<graphics::ForgeManager>();
+    fgMngr = std::make_unique<arrayfire::common::ForgeManager>();
 
     // This is all we need because the sort takes care of the order of devices
 #ifdef OS_MAC
@@ -543,3 +544,4 @@ void DeviceManager::markDeviceForInterop(const int device,
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/device_manager.hpp b/src/backend/opencl/device_manager.hpp
index b68297b511..8789675fe2 100644
--- a/src/backend/opencl/device_manager.hpp
+++ b/src/backend/opencl/device_manager.hpp
@@ -40,18 +40,16 @@ namespace spdlog {
 class logger;
 }
 
-namespace graphics {
-class ForgeManager;
-}
-
+namespace arrayfire {
 namespace common {
-namespace memory {
+class ForgeManager;
 class MemoryManagerBase;
-}
 }  // namespace common
+}  // namespace arrayfire
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 
+namespace arrayfire {
 namespace opencl {
 
 // opencl namespace forward declarations
@@ -60,27 +58,31 @@ struct kc_entry_t;  // kernel cache entry
 class PlanCache;    // clfft
 
 class DeviceManager {
-    friend MemoryManagerBase& memoryManager();
+    friend arrayfire::common::MemoryManagerBase& memoryManager();
 
-    friend void setMemoryManager(std::unique_ptr<MemoryManagerBase> mgr);
+    friend void setMemoryManager(
+        std::unique_ptr<arrayfire::common::MemoryManagerBase> mgr);
 
-    void setMemoryManager(std::unique_ptr<MemoryManagerBase> mgr);
+    void setMemoryManager(
+        std::unique_ptr<arrayfire::common::MemoryManagerBase> mgr);
 
     friend void resetMemoryManager();
 
     void resetMemoryManager();
 
-    friend MemoryManagerBase& pinnedMemoryManager();
+    friend arrayfire::common::MemoryManagerBase& pinnedMemoryManager();
 
-    friend void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
+    friend void setMemoryManagerPinned(
+        std::unique_ptr<arrayfire::common::MemoryManagerBase> mgr);
 
-    void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
+    void setMemoryManagerPinned(
+        std::unique_ptr<arrayfire::common::MemoryManagerBase> mgr);
 
     friend void resetMemoryManagerPinned();
 
     void resetMemoryManagerPinned();
 
-    friend graphics::ForgeManager& forgeManager();
+    friend arrayfire::common::ForgeManager& forgeManager();
 
     friend GraphicsResourceManager& interopManager();
 
@@ -163,7 +165,7 @@ class DeviceManager {
     std::vector<int> mPlatforms;
     unsigned mUserDeviceOffset;
 
-    std::unique_ptr<graphics::ForgeManager> fgMngr;
+    std::unique_ptr<arrayfire::common::ForgeManager> fgMngr;
     std::unique_ptr<MemoryManagerBase> memManager;
     std::unique_ptr<MemoryManagerBase> pinnedMemManager;
     std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
@@ -175,3 +177,4 @@ class DeviceManager {
 };
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/diagonal.cpp b/src/backend/opencl/diagonal.cpp
index 96624f90b7..094906a77a 100644
--- a/src/backend/opencl/diagonal.cpp
+++ b/src/backend/opencl/diagonal.cpp
@@ -15,8 +15,9 @@
 #include <math.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> diagCreate(const Array<T> &in, const int num) {
@@ -59,3 +60,4 @@ INSTANTIATE_DIAGONAL(ushort)
 INSTANTIATE_DIAGONAL(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/diagonal.hpp b/src/backend/opencl/diagonal.hpp
index 2d08df817e..5ba6daed79 100644
--- a/src/backend/opencl/diagonal.hpp
+++ b/src/backend/opencl/diagonal.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> diagCreate(const Array<T> &in, const int num);
@@ -16,3 +17,4 @@ Array<T> diagCreate(const Array<T> &in, const int num);
 template<typename T>
 Array<T> diagExtract(const Array<T> &in, const int num);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/diff.cpp b/src/backend/opencl/diff.cpp
index 8c99eee837..020365d24c 100644
--- a/src/backend/opencl/diff.cpp
+++ b/src/backend/opencl/diff.cpp
@@ -13,6 +13,7 @@
 #include <af/dim4.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -56,3 +57,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(char)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/diff.hpp b/src/backend/opencl/diff.hpp
index d670ebcf33..ff60455fe8 100644
--- a/src/backend/opencl/diff.hpp
+++ b/src/backend/opencl/diff.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> diff1(const Array<T> &in, const int dim);
@@ -16,3 +17,4 @@ Array<T> diff1(const Array<T> &in, const int dim);
 template<typename T>
 Array<T> diff2(const Array<T> &in, const int dim);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/exampleFunction.cpp b/src/backend/opencl/exampleFunction.cpp
index fd0f7c3e18..10af977382 100644
--- a/src/backend/opencl/exampleFunction.cpp
+++ b/src/backend/opencl/exampleFunction.cpp
@@ -23,6 +23,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -62,3 +63,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/exampleFunction.hpp b/src/backend/opencl/exampleFunction.hpp
index 2ee89e8f42..35f844dc4e 100644
--- a/src/backend/opencl/exampleFunction.hpp
+++ b/src/backend/opencl/exampleFunction.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> exampleFunction(const Array<T> &a, const Array<T> &b,
                          const af_someenum_t method);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/fast.cpp b/src/backend/opencl/fast.cpp
index faf9914b96..bfe6c84177 100644
--- a/src/backend/opencl/fast.cpp
+++ b/src/backend/opencl/fast.cpp
@@ -16,6 +16,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -57,3 +58,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/fast.hpp b/src/backend/opencl/fast.hpp
index 2eda909eb1..4a1d7cc3cd 100644
--- a/src/backend/opencl/fast.hpp
+++ b/src/backend/opencl/fast.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -20,4 +21,5 @@ unsigned fast(Array<float> &x_out, Array<float> &y_out, Array<float> &score_out,
               const bool non_max, const float feature_ratio,
               const unsigned edge);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/fft.cpp b/src/backend/opencl/fft.cpp
index 071ef4b9e4..36ebd70a63 100644
--- a/src/backend/opencl/fft.cpp
+++ b/src/backend/opencl/fft.cpp
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 
 void setFFTPlanCacheSize(size_t numPlans) {
@@ -167,3 +168,4 @@ INSTANTIATE(cdouble)
 INSTANTIATE_REAL(float, cfloat)
 INSTANTIATE_REAL(double, cdouble)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/fft.hpp b/src/backend/opencl/fft.hpp
index 28adbdfbfa..f071b9a8c5 100644
--- a/src/backend/opencl/fft.hpp
+++ b/src/backend/opencl/fft.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 void setFFTPlanCacheSize(size_t numPlans);
@@ -23,3 +24,4 @@ template<typename Tr, typename Tc>
 Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank);
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/fftconvolve.cpp b/src/backend/opencl/fftconvolve.cpp
index 10b3015b6b..a4f8b1f1f1 100644
--- a/src/backend/opencl/fftconvolve.cpp
+++ b/src/backend/opencl/fftconvolve.cpp
@@ -25,6 +25,7 @@ using std::is_integral;
 using std::is_same;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -143,3 +144,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/fftconvolve.hpp b/src/backend/opencl/fftconvolve.hpp
index fde659d2b0..a00f978adc 100644
--- a/src/backend/opencl/fftconvolve.hpp
+++ b/src/backend/opencl/fftconvolve.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
                      const bool expand, AF_BATCH_KIND kind, const int rank);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/flood_fill.cpp b/src/backend/opencl/flood_fill.cpp
index 500a9219db..b57de824bd 100644
--- a/src/backend/opencl/flood_fill.cpp
+++ b/src/backend/opencl/flood_fill.cpp
@@ -12,6 +12,7 @@
 #include <err_opencl.hpp>
 #include <kernel/flood_fill.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -36,3 +37,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/flood_fill.hpp b/src/backend/opencl/flood_fill.hpp
index 0cdea7fd62..b4210c2d57 100644
--- a/src/backend/opencl/flood_fill.hpp
+++ b/src/backend/opencl/flood_fill.hpp
@@ -12,6 +12,7 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
@@ -19,3 +20,4 @@ Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
                    const T lowValue, const T highValue,
                    const af::connectivity nlookup = AF_CONNECTIVITY_8);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/gradient.cpp b/src/backend/opencl/gradient.cpp
index 0ecf94f06b..711e579295 100644
--- a/src/backend/opencl/gradient.cpp
+++ b/src/backend/opencl/gradient.cpp
@@ -13,6 +13,7 @@
 #include <math.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in) {
@@ -28,3 +29,4 @@ INSTANTIATE(double)
 INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/gradient.hpp b/src/backend/opencl/gradient.hpp
index c5108ae93f..88d663f436 100644
--- a/src/backend/opencl/gradient.hpp
+++ b/src/backend/opencl/gradient.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/harris.cpp b/src/backend/opencl/harris.cpp
index eedb054add..ce2f21fced 100644
--- a/src/backend/opencl/harris.cpp
+++ b/src/backend/opencl/harris.cpp
@@ -16,6 +16,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename convAccT>
@@ -53,3 +54,4 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/harris.hpp b/src/backend/opencl/harris.hpp
index b68dfbf098..73ac64bbfd 100644
--- a/src/backend/opencl/harris.hpp
+++ b/src/backend/opencl/harris.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename convAccT>
@@ -21,4 +22,5 @@ unsigned harris(Array<float> &x_out, Array<float> &y_out,
                 const float sigma, const unsigned filter_len,
                 const float k_thr);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/hist_graphics.cpp b/src/backend/opencl/hist_graphics.cpp
index a1875686bc..6c2a06e0b1 100644
--- a/src/backend/opencl/hist_graphics.cpp
+++ b/src/backend/opencl/hist_graphics.cpp
@@ -13,11 +13,15 @@
 #include <err_opencl.hpp>
 #include <hist_graphics.hpp>
 
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_histogram(const Array<T> &data, fg_histogram hist) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = forgePlugin();
     if (isGLSharingSupported()) {
         CheckGL("Begin OpenCL resource copy");
         const cl::Buffer *d_P = data.get();
@@ -73,3 +77,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/hist_graphics.hpp b/src/backend/opencl/hist_graphics.hpp
index fa49bfe43f..40dd57e5e9 100644
--- a/src/backend/opencl/hist_graphics.hpp
+++ b/src/backend/opencl/hist_graphics.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_histogram(const Array<T> &data, fg_histogram hist);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/histogram.cpp b/src/backend/opencl/histogram.cpp
index 7963d07d3c..7c3d432228 100644
--- a/src/backend/opencl/histogram.cpp
+++ b/src/backend/opencl/histogram.cpp
@@ -15,8 +15,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -48,3 +49,4 @@ INSTANTIATE(uintl)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/histogram.hpp b/src/backend/opencl/histogram.hpp
index 583a8150cd..5b0c21e970 100644
--- a/src/backend/opencl/histogram.hpp
+++ b/src/backend/opencl/histogram.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<uint> histogram(const Array<T> &in, const unsigned &nbins,
                       const double &minval, const double &maxval,
                       const bool isLinear);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/homography.cpp b/src/backend/opencl/homography.cpp
index 9153336471..1bd958de55 100644
--- a/src/backend/opencl/homography.cpp
+++ b/src/backend/opencl/homography.cpp
@@ -19,6 +19,7 @@
 using af::dim4;
 using std::numeric_limits;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -74,3 +75,4 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/homography.hpp b/src/backend/opencl/homography.hpp
index 3453abc11f..2fa7c76690 100644
--- a/src/backend/opencl/homography.hpp
+++ b/src/backend/opencl/homography.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -18,4 +19,5 @@ int homography(Array<T> &H, const Array<float> &x_src,
                const af_homography_type htype, const float inlier_thr,
                const unsigned iterations);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/hsv_rgb.cpp b/src/backend/opencl/hsv_rgb.cpp
index 5ca8521236..06ab6b9856 100644
--- a/src/backend/opencl/hsv_rgb.cpp
+++ b/src/backend/opencl/hsv_rgb.cpp
@@ -11,6 +11,7 @@
 
 #include <kernel/hsv_rgb.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -35,3 +36,4 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/hsv_rgb.hpp b/src/backend/opencl/hsv_rgb.hpp
index fbbaf66569..4c87fa9479 100644
--- a/src/backend/opencl/hsv_rgb.hpp
+++ b/src/backend/opencl/hsv_rgb.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> rgb2hsv(const Array<T>& in);
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/identity.cpp b/src/backend/opencl/identity.cpp
index 27a092448c..9d9ae55718 100644
--- a/src/backend/opencl/identity.cpp
+++ b/src/backend/opencl/identity.cpp
@@ -14,8 +14,9 @@
 #include <debug_opencl.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> identity(const dim4& dims) {
@@ -42,3 +43,4 @@ INSTANTIATE_IDENTITY(ushort)
 INSTANTIATE_IDENTITY(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/identity.hpp b/src/backend/opencl/identity.hpp
index cb5512d1b5..0a401099b8 100644
--- a/src/backend/opencl/identity.hpp
+++ b/src/backend/opencl/identity.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> identity(const dim4& dim);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/iir.cpp b/src/backend/opencl/iir.cpp
index 63d34be2bd..9b53708212 100644
--- a/src/backend/opencl/iir.cpp
+++ b/src/backend/opencl/iir.cpp
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x) {
@@ -57,3 +58,4 @@ INSTANTIATE(double)
 INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/iir.hpp b/src/backend/opencl/iir.hpp
index c278a86b05..0b939ab3fe 100644
--- a/src/backend/opencl/iir.hpp
+++ b/src/backend/opencl/iir.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/image.cpp b/src/backend/opencl/image.cpp
index 15b6a614a6..cffc2b8194 100644
--- a/src/backend/opencl/image.cpp
+++ b/src/backend/opencl/image.cpp
@@ -16,11 +16,15 @@
 #include <stdexcept>
 #include <vector>
 
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
+
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_image(const Array<T> &in, fg_image image) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = forgePlugin();
     if (isGLSharingSupported()) {
         CheckGL("Begin opencl resource copy");
 
@@ -80,3 +84,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/image.hpp b/src/backend/opencl/image.hpp
index 7f4d37efa5..f9ee5db1eb 100644
--- a/src/backend/opencl/image.hpp
+++ b/src/backend/opencl/image.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_image(const Array<T> &in, fg_image image);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/index.cpp b/src/backend/opencl/index.cpp
index a5d00b8373..0911229936 100644
--- a/src/backend/opencl/index.cpp
+++ b/src/backend/opencl/index.cpp
@@ -16,8 +16,9 @@
 #include <memory.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -87,3 +88,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/index.hpp b/src/backend/opencl/index.hpp
index b0d933a4f3..2164305a62 100644
--- a/src/backend/opencl/index.hpp
+++ b/src/backend/opencl/index.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/index.h>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 Array<T> index(const Array<T>& in, const af_index_t idxrs[]);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/inverse.cpp b/src/backend/opencl/inverse.cpp
index c5b62a861f..860c449c3c 100644
--- a/src/backend/opencl/inverse.cpp
+++ b/src/backend/opencl/inverse.cpp
@@ -15,6 +15,7 @@
 #include <cpu/cpu_inverse.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -34,9 +35,11 @@ INSTANTIATE(double)
 INSTANTIATE(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -52,5 +55,6 @@ INSTANTIATE(double)
 INSTANTIATE(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #endif
diff --git a/src/backend/opencl/inverse.hpp b/src/backend/opencl/inverse.hpp
index 9316532a1a..1695798720 100644
--- a/src/backend/opencl/inverse.hpp
+++ b/src/backend/opencl/inverse.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> inverse(const Array<T> &in);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/iota.cpp b/src/backend/opencl/iota.cpp
index ebd0b5824d..de69ca6595 100644
--- a/src/backend/opencl/iota.cpp
+++ b/src/backend/opencl/iota.cpp
@@ -16,8 +16,9 @@
 
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> iota(const dim4 &dims, const dim4 &tile_dims) {
@@ -43,3 +44,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/iota.hpp b/src/backend/opencl/iota.hpp
index 5552e63332..26869554b8 100644
--- a/src/backend/opencl/iota.hpp
+++ b/src/backend/opencl/iota.hpp
@@ -10,7 +10,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> iota(const dim4 &dim, const dim4 &tile_dims = dim4(1));
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/ireduce.cpp b/src/backend/opencl/ireduce.cpp
index 86ff0fd1db..ca4c916f63 100644
--- a/src/backend/opencl/ireduce.cpp
+++ b/src/backend/opencl/ireduce.cpp
@@ -17,8 +17,9 @@
 #include <complex>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<af_op_t op, typename T>
@@ -77,3 +78,4 @@ INSTANTIATE(af_max_t, short)
 INSTANTIATE(af_max_t, ushort)
 INSTANTIATE(af_max_t, half)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/ireduce.hpp b/src/backend/opencl/ireduce.hpp
index 05bea7bd19..1b60a7a745 100644
--- a/src/backend/opencl/ireduce.hpp
+++ b/src/backend/opencl/ireduce.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<af_op_t op, typename T>
 void ireduce(Array<T> &out, Array<uint> &loc, const Array<T> &in,
@@ -22,3 +23,4 @@ void rreduce(Array<T> &out, Array<uint> &loc, const Array<T> &in, const int dim,
 template<af_op_t op, typename T>
 T ireduce_all(unsigned *loc, const Array<T> &in);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index dddf1ecd0d..30a942d2dd 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -33,15 +33,15 @@
 #include <string>
 #include <vector>
 
-using common::findModule;
-using common::getFuncName;
-using common::ModdimNode;
-using common::Node;
-using common::Node_ids;
-using common::Node_map_t;
-using common::Node_ptr;
-using common::NodeIterator;
-using common::saveKernel;
+using arrayfire::common::findModule;
+using arrayfire::common::getFuncName;
+using arrayfire::common::ModdimNode;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ids;
+using arrayfire::common::Node_map_t;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::NodeIterator;
+using arrayfire::common::saveKernel;
 
 using cl::Kernel;
 using cl::NDRange;
@@ -56,6 +56,7 @@ using std::stringstream;
 using std::to_string;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 using jit::BufferNode;
 
@@ -490,3 +491,4 @@ void evalNodes(Param& out, Node* node) {
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/jit/BufferNode.hpp b/src/backend/opencl/jit/BufferNode.hpp
index 0746c0538e..e188fb429f 100644
--- a/src/backend/opencl/jit/BufferNode.hpp
+++ b/src/backend/opencl/jit/BufferNode.hpp
@@ -13,10 +13,11 @@
 
 #include <memory>
 
+namespace arrayfire {
 namespace opencl {
 namespace jit {
 using BufferNode = common::BufferNodeBase<std::shared_ptr<cl::Buffer>, KParam>;
-}
+}  // namespace jit
 }  // namespace opencl
 
 namespace common {
@@ -32,3 +33,4 @@ bool BufferNodeBase<DataType, ParamType>::operator==(
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/opencl/jit/kernel_generators.hpp b/src/backend/opencl/jit/kernel_generators.hpp
index 5c111fdedb..d4700260c4 100644
--- a/src/backend/opencl/jit/kernel_generators.hpp
+++ b/src/backend/opencl/jit/kernel_generators.hpp
@@ -11,6 +11,7 @@
 #include <sstream>
 #include <string>
 
+namespace arrayfire {
 namespace opencl {
 
 namespace {
@@ -106,3 +107,4 @@ inline void generateShiftNodeRead(std::stringstream& kerStream, int id,
 }
 }  // namespace
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/join.cpp b/src/backend/opencl/join.cpp
index 7eda4fc307..22875d0e61 100644
--- a/src/backend/opencl/join.cpp
+++ b/src/backend/opencl/join.cpp
@@ -19,11 +19,12 @@
 #include <vector>
 
 using af::dim4;
-using common::half;
-using common::Node;
-using common::Node_ptr;
+using arrayfire::common::half;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ptr;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
@@ -252,3 +253,4 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/join.hpp b/src/backend/opencl/join.hpp
index ea101d03f2..9caf52d863 100644
--- a/src/backend/opencl/join.hpp
+++ b/src/backend/opencl/join.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
@@ -16,3 +17,4 @@ Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
 template<typename T>
 void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/anisotropic_diffusion.hpp b/src/backend/opencl/kernel/anisotropic_diffusion.hpp
index 84af9db4a7..bf13bb4cd5 100644
--- a/src/backend/opencl/kernel/anisotropic_diffusion.hpp
+++ b/src/backend/opencl/kernel/anisotropic_diffusion.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -68,3 +69,4 @@ void anisotropicDiffusion(Param inout, const float dt, const float mct,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/approx.hpp b/src/backend/opencl/kernel/approx.hpp
index 1d702ed090..797ac19d4b 100644
--- a/src/backend/opencl/kernel/approx.hpp
+++ b/src/backend/opencl/kernel/approx.hpp
@@ -24,6 +24,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -133,3 +134,4 @@ void approx2(Param zo, const Param zi, const Param xo, const int xdim,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/assign.hpp b/src/backend/opencl/kernel/assign.hpp
index 0b9ae34472..447e4e8c60 100644
--- a/src/backend/opencl/kernel/assign.hpp
+++ b/src/backend/opencl/kernel/assign.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -59,3 +60,4 @@ void assign(Param out, const Param in, const AssignKernelParam_t& p,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/bilateral.hpp b/src/backend/opencl/kernel/bilateral.hpp
index a191d53815..832611dcdb 100644
--- a/src/backend/opencl/kernel/bilateral.hpp
+++ b/src/backend/opencl/kernel/bilateral.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -77,3 +78,4 @@ void bilateral(Param out, const Param in, const float s_sigma,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/canny.hpp b/src/backend/opencl/kernel/canny.hpp
index 7444ac00aa..3659e1fb4b 100644
--- a/src/backend/opencl/kernel/canny.hpp
+++ b/src/backend/opencl/kernel/canny.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 constexpr int THREADS_X = 16;
@@ -174,3 +175,4 @@ void edgeTrackingHysteresis(Param output, const Param strong,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/config.cpp b/src/backend/opencl/kernel/config.cpp
index 97d91c510a..363a876d95 100644
--- a/src/backend/opencl/kernel/config.cpp
+++ b/src/backend/opencl/kernel/config.cpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include "config.hpp"
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -22,3 +23,4 @@ std::ostream& operator<<(std::ostream& out, const cdouble& var) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/config.hpp b/src/backend/opencl/kernel/config.hpp
index 38a47399a4..9e3d07868a 100644
--- a/src/backend/opencl/kernel/config.hpp
+++ b/src/backend/opencl/kernel/config.hpp
@@ -11,6 +11,7 @@
 #include <types.hpp>
 #include <ostream>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -24,3 +25,4 @@ static const uint THREADS_Y         = THREADS_PER_GROUP / THREADS_X;
 static const uint REPEAT            = 32;
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve.hpp b/src/backend/opencl/kernel/convolve.hpp
index 6c9e2e5d6d..39d2c77564 100644
--- a/src/backend/opencl/kernel/convolve.hpp
+++ b/src/backend/opencl/kernel/convolve.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <kernel/convolve/conv_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 namespace kernel {
@@ -57,3 +58,4 @@ void convolve_nd(Param out, const Param signal, const Param filter,
 }  // namespace kernel
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv1.cpp b/src/backend/opencl/kernel/convolve/conv1.cpp
index d870faaf80..10ae600888 100644
--- a/src/backend/opencl/kernel/convolve/conv1.cpp
+++ b/src/backend/opencl/kernel/convolve/conv1.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -66,3 +67,4 @@ INSTANTIATE(intl, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_b8.cpp b/src/backend/opencl/kernel/convolve/conv2_b8.cpp
index c9e61d1fee..18c41628a6 100644
--- a/src/backend/opencl/kernel/convolve/conv2_b8.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_b8.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(char, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_c32.cpp b/src/backend/opencl/kernel/convolve/conv2_c32.cpp
index 53b05d2cea..5be66c8040 100644
--- a/src/backend/opencl/kernel/convolve/conv2_c32.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_c32.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(cfloat, cfloat)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_c64.cpp b/src/backend/opencl/kernel/convolve/conv2_c64.cpp
index e8a5af8a4f..87e787ceed 100644
--- a/src/backend/opencl/kernel/convolve/conv2_c64.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_c64.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(cdouble, cdouble)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_f32.cpp b/src/backend/opencl/kernel/convolve/conv2_f32.cpp
index 2f92484942..89dc63dd6d 100644
--- a/src/backend/opencl/kernel/convolve/conv2_f32.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_f32.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(float, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_f64.cpp b/src/backend/opencl/kernel/convolve/conv2_f64.cpp
index 84dd2ac4bb..97a8044cdd 100644
--- a/src/backend/opencl/kernel/convolve/conv2_f64.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_f64.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(double, double)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_impl.hpp b/src/backend/opencl/kernel/convolve/conv2_impl.hpp
index 61f9d1d56d..59f0523de8 100644
--- a/src/backend/opencl/kernel/convolve/conv2_impl.hpp
+++ b/src/backend/opencl/kernel/convolve/conv2_impl.hpp
@@ -12,6 +12,7 @@
 #include <common/kernel_cache.hpp>
 #include <kernel/convolve/conv_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -95,3 +96,4 @@ void conv2(conv_kparam_t& p, Param& out, const Param& sig, const Param& filt,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_s16.cpp b/src/backend/opencl/kernel/convolve/conv2_s16.cpp
index 2a8b7866d3..d5c1e5cc3d 100644
--- a/src/backend/opencl/kernel/convolve/conv2_s16.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_s16.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(short, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_s32.cpp b/src/backend/opencl/kernel/convolve/conv2_s32.cpp
index 4fa785d738..dc621d45f5 100644
--- a/src/backend/opencl/kernel/convolve/conv2_s32.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_s32.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(int, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_s64.cpp b/src/backend/opencl/kernel/convolve/conv2_s64.cpp
index 93dca03a3b..cdfde44ab1 100644
--- a/src/backend/opencl/kernel/convolve/conv2_s64.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_s64.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(intl, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_u16.cpp b/src/backend/opencl/kernel/convolve/conv2_u16.cpp
index ad06327135..05b525ea5c 100644
--- a/src/backend/opencl/kernel/convolve/conv2_u16.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_u16.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(ushort, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_u32.cpp b/src/backend/opencl/kernel/convolve/conv2_u32.cpp
index 6ad074843e..c4b6667c32 100644
--- a/src/backend/opencl/kernel/convolve/conv2_u32.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_u32.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(uint, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_u64.cpp b/src/backend/opencl/kernel/convolve/conv2_u64.cpp
index d682084197..b7f410bc9c 100644
--- a/src/backend/opencl/kernel/convolve/conv2_u64.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_u64.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(uintl, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv2_u8.cpp b/src/backend/opencl/kernel/convolve/conv2_u8.cpp
index 23879b269d..bfe74b4c6b 100644
--- a/src/backend/opencl/kernel/convolve/conv2_u8.cpp
+++ b/src/backend/opencl/kernel/convolve/conv2_u8.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv2_impl.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -16,3 +17,4 @@ INSTANTIATE(uchar, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv3.cpp b/src/backend/opencl/kernel/convolve/conv3.cpp
index 411ff85372..9a1baf9c6b 100644
--- a/src/backend/opencl/kernel/convolve/conv3.cpp
+++ b/src/backend/opencl/kernel/convolve/conv3.cpp
@@ -9,6 +9,7 @@
 
 #include <kernel/convolve/conv_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -53,3 +54,4 @@ INSTANTIATE(intl, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv_common.hpp b/src/backend/opencl/kernel/convolve/conv_common.hpp
index 987e623dcf..93c4781976 100644
--- a/src/backend/opencl/kernel/convolve/conv_common.hpp
+++ b/src/backend/opencl/kernel/convolve/conv_common.hpp
@@ -24,6 +24,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -136,3 +137,4 @@ void conv3(conv_kparam_t& p, Param& out, const Param& sig, const Param& filt,
            const bool expand);
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve_separable.cpp b/src/backend/opencl/kernel/convolve_separable.cpp
index 7017170e41..6f7611428b 100644
--- a/src/backend/opencl/kernel/convolve_separable.cpp
+++ b/src/backend/opencl/kernel/convolve_separable.cpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -103,3 +104,4 @@ INSTANTIATE(intl, float)
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve_separable.hpp b/src/backend/opencl/kernel/convolve_separable.hpp
index 0d7feddd44..2651856c92 100644
--- a/src/backend/opencl/kernel/convolve_separable.hpp
+++ b/src/backend/opencl/kernel/convolve_separable.hpp
@@ -11,6 +11,7 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -25,3 +26,4 @@ void convSep(Param out, const Param sig, const Param filt, const int cDim,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/cscmm.hpp b/src/backend/opencl/kernel/cscmm.hpp
index 9857133f9d..4fb0cc3479 100644
--- a/src/backend/opencl/kernel/cscmm.hpp
+++ b/src/backend/opencl/kernel/cscmm.hpp
@@ -24,6 +24,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -52,7 +53,7 @@ void cscmm_nn(Param out, const Param &values, const Param &colIdx,
         DefineKeyValue(THREADS, threads),
         DefineKeyValue(ROWS_PER_GROUP, rows_per_group),
         DefineKeyValue(COLS_PER_GROUP, cols_per_group),
-        DefineKeyValue(IS_CPLX, (af::iscplx<T>() ? 1 : 0)),
+        DefineKeyValue(IS_CPLX, (iscplx<T>() ? 1 : 0)),
         getTypeBuildDefinition<T>()};
 
     auto cscmmNN =
@@ -74,3 +75,4 @@ void cscmm_nn(Param out, const Param &values, const Param &colIdx,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/cscmv.hpp b/src/backend/opencl/kernel/cscmv.hpp
index a3b66714c3..675176e393 100644
--- a/src/backend/opencl/kernel/cscmv.hpp
+++ b/src/backend/opencl/kernel/cscmv.hpp
@@ -23,6 +23,7 @@
 
 #include <string>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -50,7 +51,7 @@ void cscmv(Param out, const Param &values, const Param &colIdx,
         DefineKeyValue(IS_CONJ, is_conj),
         DefineKeyValue(THREADS, local[0]),
         DefineKeyValue(ROWS_PER_GROUP, rows_per_group),
-        DefineKeyValue(IS_CPLX, (af::iscplx<T>() ? 1 : 0)),
+        DefineKeyValue(IS_CPLX, (iscplx<T>() ? 1 : 0)),
         getTypeBuildDefinition<T>()};
 
     auto cscmvBlock = common::getKernel("cscmv_block", std::array{cscmv_cl_src},
@@ -68,3 +69,4 @@ void cscmv(Param out, const Param &values, const Param &colIdx,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/csrmm.hpp b/src/backend/opencl/kernel/csrmm.hpp
index 42b5cc093a..a786f7cafb 100644
--- a/src/backend/opencl/kernel/csrmm.hpp
+++ b/src/backend/opencl/kernel/csrmm.hpp
@@ -24,6 +24,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -50,7 +51,7 @@ void csrmm_nt(Param out, const Param &values, const Param &rowIdx,
         DefineKeyValue(USE_BETA, use_beta),
         DefineKeyValue(USE_GREEDY, use_greedy),
         DefineValue(THREADS_PER_GROUP),
-        DefineKeyValue(IS_CPLX, (af::iscplx<T>() ? 1 : 0)),
+        DefineKeyValue(IS_CPLX, (iscplx<T>() ? 1 : 0)),
         getTypeBuildDefinition<T>()};
 
     // FIXME: Switch to perf (thread vs block) baesd kernel
@@ -76,3 +77,4 @@ void csrmm_nt(Param out, const Param &values, const Param &rowIdx,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/csrmv.hpp b/src/backend/opencl/kernel/csrmv.hpp
index 2d7abaa190..3c948f0177 100644
--- a/src/backend/opencl/kernel/csrmv.hpp
+++ b/src/backend/opencl/kernel/csrmv.hpp
@@ -24,6 +24,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -53,7 +54,7 @@ void csrmv(Param out, const Param &values, const Param &rowIdx,
         DefineKeyValue(USE_BETA, use_beta),
         DefineKeyValue(USE_GREEDY, use_greedy),
         DefineKeyValue(THREADS, local[0]),
-        DefineKeyValue(IS_CPLX, (af::iscplx<T>() ? 1 : 0)),
+        DefineKeyValue(IS_CPLX, (iscplx<T>() ? 1 : 0)),
         getTypeBuildDefinition<T>()};
 
     auto csrmv =
@@ -87,3 +88,4 @@ void csrmv(Param out, const Param &values, const Param &rowIdx,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/diagonal.hpp b/src/backend/opencl/kernel/diagonal.hpp
index e4320aa6dc..9f2ded02c7 100644
--- a/src/backend/opencl/kernel/diagonal.hpp
+++ b/src/backend/opencl/kernel/diagonal.hpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -32,7 +33,7 @@ static void diagCreate(Param out, Param in, int num) {
     };
     std::array<std::string, 3> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
+        DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
         getTypeBuildDefinition<T>()};
 
     auto diagCreate = common::getKernel(
@@ -56,7 +57,7 @@ static void diagExtract(Param out, Param in, int num) {
     };
     std::array<std::string, 3> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
+        DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
         getTypeBuildDefinition<T>()};
 
     auto diagExtract = common::getKernel(
@@ -75,3 +76,4 @@ static void diagExtract(Param out, Param in, int num) {
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/diff.hpp b/src/backend/opencl/kernel/diff.hpp
index c249e55d94..817bd92bac 100644
--- a/src/backend/opencl/kernel/diff.hpp
+++ b/src/backend/opencl/kernel/diff.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -58,3 +59,4 @@ void diff(Param out, const Param in, const unsigned indims, const unsigned dim,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/exampleFunction.hpp b/src/backend/opencl/kernel/exampleFunction.hpp
index 4b5e506c13..8de171e908 100644
--- a/src/backend/opencl/kernel/exampleFunction.hpp
+++ b/src/backend/opencl/kernel/exampleFunction.hpp
@@ -33,6 +33,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -82,3 +83,4 @@ void exampleFunc(Param c, const Param a, const Param b, const af_someenum_t p) {
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/fast.hpp b/src/backend/opencl/kernel/fast.hpp
index 9b4fc4341f..5e75bd1995 100644
--- a/src/backend/opencl/kernel/fast.hpp
+++ b/src/backend/opencl/kernel/fast.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -145,3 +146,4 @@ void fast(const unsigned arc_length, unsigned *out_feat, Param &x_out,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/fftconvolve.hpp b/src/backend/opencl/kernel/fftconvolve.hpp
index 222bde02e8..c43e750a89 100644
--- a/src/backend/opencl/kernel/fftconvolve.hpp
+++ b/src/backend/opencl/kernel/fftconvolve.hpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -227,3 +228,4 @@ void reorderOutputHelper(Param out, Param packed, Param sig, Param filter,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/flood_fill.hpp b/src/backend/opencl/kernel/flood_fill.hpp
index 45b8dc7bf7..d0af9aa7c9 100644
--- a/src/backend/opencl/kernel/flood_fill.hpp
+++ b/src/backend/opencl/kernel/flood_fill.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -114,3 +115,4 @@ void floodFill(Param out, const Param image, const Param seedsx,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/gradient.hpp b/src/backend/opencl/kernel/gradient.hpp
index ad7ce75c84..cab0a98abf 100644
--- a/src/backend/opencl/kernel/gradient.hpp
+++ b/src/backend/opencl/kernel/gradient.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -36,8 +37,8 @@ void gradient(Param grad0, Param grad1, const Param in) {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineValue(TX),
         DefineValue(TY),
-        DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
-        DefineKeyValue(CPLX, static_cast<int>(af::iscplx<T>())),
+        DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
+        DefineKeyValue(CPLX, static_cast<int>(iscplx<T>())),
         getTypeBuildDefinition<T>()};
 
     auto gradOp = common::getKernel("gradient", std::array{gradient_cl_src},
@@ -57,3 +58,4 @@ void gradient(Param grad0, Param grad1, const Param in) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/harris.hpp b/src/backend/opencl/kernel/harris.hpp
index eb57c8ad71..942fb44d1b 100644
--- a/src/backend/opencl/kernel/harris.hpp
+++ b/src/backend/opencl/kernel/harris.hpp
@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -274,3 +275,4 @@ void harris(unsigned *corners_out, Param &x_out, Param &y_out, Param &resp_out,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/histogram.hpp b/src/backend/opencl/kernel/histogram.hpp
index 03a2c2c892..a05bad05f6 100644
--- a/src/backend/opencl/kernel/histogram.hpp
+++ b/src/backend/opencl/kernel/histogram.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -58,3 +59,4 @@ void histogram(Param out, const Param in, int nbins, float minval, float maxval,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/homography.hpp b/src/backend/opencl/kernel/homography.hpp
index 2c192ef6b7..328f39d753 100644
--- a/src/backend/opencl/kernel/homography.hpp
+++ b/src/backend/opencl/kernel/homography.hpp
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 constexpr int HG_THREADS_X = 16;
@@ -213,3 +214,4 @@ int computeH(Param bestH, Param H, Param err, Param x_src, Param y_src,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/hsv_rgb.hpp b/src/backend/opencl/kernel/hsv_rgb.hpp
index 5e30938b17..1f46cc5085 100644
--- a/src/backend/opencl/kernel/hsv_rgb.hpp
+++ b/src/backend/opencl/kernel/hsv_rgb.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -54,3 +55,4 @@ void hsv2rgb_convert(Param out, const Param in, bool isHSV2RGB) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/identity.hpp b/src/backend/opencl/kernel/identity.hpp
index 6369beb3ce..19afcdaea7 100644
--- a/src/backend/opencl/kernel/identity.hpp
+++ b/src/backend/opencl/kernel/identity.hpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -32,8 +33,8 @@ static void identity(Param out) {
     };
     std::array<std::string, 4> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineKeyValue(ONE, af::scalar_to_option(scalar<T>(1))),
-        DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
+        DefineKeyValue(ONE, scalar_to_option(scalar<T>(1))),
+        DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
         getTypeBuildDefinition<T>()};
 
     auto identityOp = common::getKernel(
@@ -52,3 +53,4 @@ static void identity(Param out) {
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/iir.hpp b/src/backend/opencl/kernel/iir.hpp
index 2bbb407fe9..7786197da4 100644
--- a/src/backend/opencl/kernel/iir.hpp
+++ b/src/backend/opencl/kernel/iir.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -36,7 +37,7 @@ void iir(Param y, Param c, Param a) {
     std::array<std::string, 5> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(MAX_A_SIZE),
         DefineKeyValue(BATCH_A, batch_a),
-        DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
+        DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
         getTypeBuildDefinition<T>()};
 
     auto iir =
@@ -63,3 +64,4 @@ void iir(Param y, Param c, Param a) {
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/index.hpp b/src/backend/opencl/kernel/index.hpp
index 881f000697..6a496d1ade 100644
--- a/src/backend/opencl/kernel/index.hpp
+++ b/src/backend/opencl/kernel/index.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -63,3 +64,4 @@ void index(Param out, const Param in, const IndexKernelParam_t& p,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/interp.hpp b/src/backend/opencl/kernel/interp.hpp
index 0c3a744c42..d827bedc5a 100644
--- a/src/backend/opencl/kernel/interp.hpp
+++ b/src/backend/opencl/kernel/interp.hpp
@@ -15,6 +15,7 @@
 #include <array>
 #include <string>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -40,3 +41,4 @@ static void addInterpEnumOptions(std::vector<std::string>& options) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/iota.hpp b/src/backend/opencl/kernel/iota.hpp
index cbf490fbf0..3308ee23e1 100644
--- a/src/backend/opencl/kernel/iota.hpp
+++ b/src/backend/opencl/kernel/iota.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -52,3 +53,4 @@ void iota(Param out, const af::dim4& sdims) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/ireduce.hpp b/src/backend/opencl/kernel/ireduce.hpp
index 5bdd55c180..775ee044d7 100644
--- a/src/backend/opencl/kernel/ireduce.hpp
+++ b/src/backend/opencl/kernel/ireduce.hpp
@@ -25,6 +25,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -44,7 +45,7 @@ void ireduceDimLauncher(Param out, cl::Buffer *oidx, Param in, cl::Buffer *iidx,
         DefineValue(THREADS_X),
         DefineKeyValue(init, toNumStr(common::Binary<T, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<T>()),
+        DefineKeyValue(CPLX, iscplx<T>()),
         DefineKeyValue(IS_FIRST, is_first),
         getTypeBuildDefinition<T>()};
 
@@ -120,7 +121,7 @@ void ireduceFirstLauncher(Param out, cl::Buffer *oidx, Param in,
         DefineValue(THREADS_PER_GROUP),
         DefineKeyValue(init, toNumStr(common::Binary<T, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<T>()),
+        DefineKeyValue(CPLX, iscplx<T>()),
         DefineKeyValue(IS_FIRST, is_first),
         getTypeBuildDefinition<T>()};
 
@@ -333,3 +334,4 @@ T ireduceAll(uint *loc, Param in) {
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/laset.hpp b/src/backend/opencl/kernel/laset.hpp
index fb52f3571f..504cf9244f 100644
--- a/src/backend/opencl/kernel/laset.hpp
+++ b/src/backend/opencl/kernel/laset.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -53,7 +54,7 @@ void laset(int m, int n, T offdiag, T diag, cl_mem dA, size_t dA_offset,
     std::array<std::string, 5> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(BLK_X),
         DefineValue(BLK_Y),
-        DefineKeyValue(IS_CPLX, static_cast<int>(af::iscplx<T>())),
+        DefineKeyValue(IS_CPLX, static_cast<int>(iscplx<T>())),
         getTypeBuildDefinition<T>()};
 
     auto lasetOp = common::getKernel(laset_name<uplo>(),
@@ -74,3 +75,4 @@ void laset(int m, int n, T offdiag, T diag, cl_mem dA, size_t dA_offset,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/laset_band.hpp b/src/backend/opencl/kernel/laset_band.hpp
index 9ceffec9e0..daa1f73b0c 100644
--- a/src/backend/opencl/kernel/laset_band.hpp
+++ b/src/backend/opencl/kernel/laset_band.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -42,7 +43,7 @@ void laset_band(int m, int  n, int k,
     std::array<std::string, 4> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineValue(NB),
-        DefineKeyValue(IS_CPLX, static_cast<int>(af::iscplx<T>())),
+        DefineKeyValue(IS_CPLX, static_cast<int>(iscplx<T>())),
         getTypeBuildDefinition<T>()
     };
 
@@ -68,3 +69,4 @@ void laset_band(int m, int  n, int k,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/laswp.hpp b/src/backend/opencl/kernel/laswp.hpp
index 0fd58eb961..5db0b388ff 100644
--- a/src/backend/opencl/kernel/laswp.hpp
+++ b/src/backend/opencl/kernel/laswp.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -69,3 +70,4 @@ void laswp(int n, cl_mem in, size_t offset, int ldda, int k1, int k2,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/lookup.hpp b/src/backend/opencl/kernel/lookup.hpp
index ed82d58b6a..1e99e82780 100644
--- a/src/backend/opencl/kernel/lookup.hpp
+++ b/src/backend/opencl/kernel/lookup.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -57,3 +58,4 @@ void lookup(Param out, const Param in, const Param indices,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/lu_split.hpp b/src/backend/opencl/kernel/lu_split.hpp
index e27eb78955..65fc511415 100644
--- a/src/backend/opencl/kernel/lu_split.hpp
+++ b/src/backend/opencl/kernel/lu_split.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -36,8 +37,8 @@ void luSplitLauncher(Param lower, Param upper, const Param in, bool same_dims) {
     };
     std::array<std::string, 5> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(same_dims),
-        DefineKeyValue(ZERO, af::scalar_to_option(scalar<T>(0))),
-        DefineKeyValue(ONE, af::scalar_to_option(scalar<T>(1))),
+        DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
+        DefineKeyValue(ONE, scalar_to_option(scalar<T>(1))),
         getTypeBuildDefinition<T>()};
 
     auto luSplit = common::getKernel("luSplit", std::array{lu_split_cl_src},
@@ -64,3 +65,4 @@ void luSplit(Param lower, Param upper, const Param in) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/match_template.hpp b/src/backend/opencl/kernel/match_template.hpp
index 5b7c471c33..21041eb73b 100644
--- a/src/backend/opencl/kernel/match_template.hpp
+++ b/src/backend/opencl/kernel/match_template.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -67,3 +68,4 @@ void matchTemplate(Param out, const Param srch, const Param tmplt,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/mean.hpp b/src/backend/opencl/kernel/mean.hpp
index 3149da3280..13f74453a8 100644
--- a/src/backend/opencl/kernel/mean.hpp
+++ b/src/backend/opencl/kernel/mean.hpp
@@ -27,6 +27,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -467,3 +468,4 @@ To meanAll(Param in) {
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/meanshift.hpp b/src/backend/opencl/kernel/meanshift.hpp
index fb92f18866..24fa61374d 100644
--- a/src/backend/opencl/kernel/meanshift.hpp
+++ b/src/backend/opencl/kernel/meanshift.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -66,3 +67,4 @@ void meanshift(Param out, const Param in, const float spatialSigma,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/medfilt.hpp b/src/backend/opencl/kernel/medfilt.hpp
index e8af452eda..d38943e50d 100644
--- a/src/backend/opencl/kernel/medfilt.hpp
+++ b/src/backend/opencl/kernel/medfilt.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -103,3 +104,4 @@ void medfilt2(Param out, const Param in, const af_border_type pad,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/memcopy.hpp b/src/backend/opencl/kernel/memcopy.hpp
index c63d1e42b3..d9fe825107 100644
--- a/src/backend/opencl/kernel/memcopy.hpp
+++ b/src/backend/opencl/kernel/memcopy.hpp
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 typedef struct {
@@ -249,3 +250,4 @@ void copy(const Param out, const Param in, dim_t ondims,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/moments.hpp b/src/backend/opencl/kernel/moments.hpp
index 6da71b9833..3f269686c3 100644
--- a/src/backend/opencl/kernel/moments.hpp
+++ b/src/backend/opencl/kernel/moments.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -53,3 +54,4 @@ void moments(Param out, const Param in, af_moment_type moment) {
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/morph.hpp b/src/backend/opencl/kernel/morph.hpp
index 43b5d6d443..730a424eed 100644
--- a/src/backend/opencl/kernel/morph.hpp
+++ b/src/backend/opencl/kernel/morph.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -145,3 +146,4 @@ void morph3d(Param out, const Param in, const Param mask, bool isDilation) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/nearest_neighbour.hpp b/src/backend/opencl/kernel/nearest_neighbour.hpp
index 841a844038..b4f7e5fa36 100644
--- a/src/backend/opencl/kernel/nearest_neighbour.hpp
+++ b/src/backend/opencl/kernel/nearest_neighbour.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -94,3 +95,4 @@ void allDistances(Param dist, Param query, Param train, const dim_t dist_dim,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/orb.hpp b/src/backend/opencl/kernel/orb.hpp
index f2e72c7317..b3e4014d05 100644
--- a/src/backend/opencl/kernel/orb.hpp
+++ b/src/backend/opencl/kernel/orb.hpp
@@ -44,6 +44,7 @@
 /* Other */
 #endif
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -498,6 +499,7 @@ void orb(unsigned* out_feat, Param& x_out, Param& y_out, Param& score_out,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
 
 #if defined(__clang__)
 /* Clang/LLVM */
diff --git a/src/backend/opencl/kernel/pad_array_borders.hpp b/src/backend/opencl/kernel/pad_array_borders.hpp
index 4d18b06099..8e75e5fbd5 100644
--- a/src/backend/opencl/kernel/pad_array_borders.hpp
+++ b/src/backend/opencl/kernel/pad_array_borders.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 static const int PADB_THREADS_X = 16;
@@ -65,3 +66,4 @@ void padBorders(Param out, const Param in, dim4 const& lBPadding,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/random_engine.hpp b/src/backend/opencl/kernel/random_engine.hpp
index c15f9e292f..96c230f133 100644
--- a/src/backend/opencl/kernel/random_engine.hpp
+++ b/src/backend/opencl/kernel/random_engine.hpp
@@ -30,6 +30,7 @@ static const int TABLE_SIZE = 16;
 static const int MAX_BLOCKS = 32;
 static const int STATE_SIZE = (256 * 3);
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 static const uint THREADS = 256;
@@ -170,3 +171,4 @@ void initMersenneState(cl::Buffer state, cl::Buffer table, const uintl &seed) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/range.hpp b/src/backend/opencl/kernel/range.hpp
index d4a5acbd33..ddb946d307 100644
--- a/src/backend/opencl/kernel/range.hpp
+++ b/src/backend/opencl/kernel/range.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -51,3 +52,4 @@ void range(Param out, const int dim) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/reduce.hpp b/src/backend/opencl/kernel/reduce.hpp
index f52d044bcb..21db6e2edc 100644
--- a/src/backend/opencl/kernel/reduce.hpp
+++ b/src/backend/opencl/kernel/reduce.hpp
@@ -30,6 +30,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -51,7 +52,7 @@ void reduceDimLauncher(Param out, Param in, const int dim, const uint threads_y,
         DefineValue(THREADS_X),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
         getTypeBuildDefinition<Ti, To>()};
 
     auto reduceDim = common::getKernel(
@@ -129,7 +130,7 @@ void reduceAllLauncher(Param out, Param in, const uint groups_x,
         DefineValue(THREADS_PER_GROUP),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
         getTypeBuildDefinition<Ti, To>()};
 
     auto reduceAll = common::getKernel(
@@ -177,7 +178,7 @@ void reduceFirstLauncher(Param out, Param in, const uint groups_x,
         DefineValue(THREADS_PER_GROUP),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
         getTypeBuildDefinition<Ti, To>()};
 
     auto reduceFirst = common::getKernel(
@@ -271,3 +272,4 @@ void reduceAll(Param out, Param in, int change_nan, double nanval) {
 }  // namespace kernel
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/reduce_by_key.hpp b/src/backend/opencl/kernel/reduce_by_key.hpp
index 79779ca320..eeb0e119df 100644
--- a/src/backend/opencl/kernel/reduce_by_key.hpp
+++ b/src/backend/opencl/kernel/reduce_by_key.hpp
@@ -36,6 +36,7 @@
 
 namespace compute = boost::compute;
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -59,7 +60,7 @@ void reduceBlocksByKeyDim(cl::Buffer *reduced_block_sizes, Param keys_out,
         DefineKeyValue(DIM, dim),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
@@ -102,7 +103,7 @@ void reduceBlocksByKey(cl::Buffer *reduced_block_sizes, Param keys_out,
         DefineKeyValue(DIMX, threads_x),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
@@ -143,7 +144,7 @@ void finalBoundaryReduce(cl::Buffer *reduced_block_sizes, Param keys_out,
         DefineKeyValue(DIMX, threads_x),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<To>()),
+        DefineKeyValue(CPLX, iscplx<To>()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
@@ -182,7 +183,7 @@ void finalBoundaryReduceDim(cl::Buffer *reduced_block_sizes, Param keys_out,
         DefineKeyValue(DIM, dim),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<To>()),
+        DefineKeyValue(CPLX, iscplx<To>()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
@@ -218,7 +219,7 @@ void compact(cl::Buffer *reduced_block_sizes, Param keys_out, Param vals_out,
         DefineKeyValue(To, dtype_traits<To>::getName()),
         DefineKeyValue(T, "To"),
         DefineKeyValue(DIMX, threads_x),
-        DefineKeyValue(CPLX, af::iscplx<To>()),
+        DefineKeyValue(CPLX, iscplx<To>()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
@@ -253,7 +254,7 @@ void compactDim(cl::Buffer *reduced_block_sizes, Param keys_out, Param vals_out,
         DefineKeyValue(T, "To"),
         DefineKeyValue(DIMX, threads_x),
         DefineKeyValue(DIM, dim),
-        DefineKeyValue(CPLX, af::iscplx<To>()),
+        DefineKeyValue(CPLX, iscplx<To>()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
@@ -572,3 +573,4 @@ void reduceByKey(Array<Tk> &keys_out, Array<To> &vals_out,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/regions.hpp b/src/backend/opencl/kernel/regions.hpp
index 710ccdf64b..63716ba8ea 100644
--- a/src/backend/opencl/kernel/regions.hpp
+++ b/src/backend/opencl/kernel/regions.hpp
@@ -37,6 +37,7 @@ AF_DEPRECATED_WARNINGS_ON
 
 namespace compute = boost::compute;
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -195,3 +196,4 @@ void regions(Param out, Param in, const bool full_conn,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/reorder.hpp b/src/backend/opencl/kernel/reorder.hpp
index e2dc87f481..9322647cd2 100644
--- a/src/backend/opencl/kernel/reorder.hpp
+++ b/src/backend/opencl/kernel/reorder.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -53,3 +54,4 @@ void reorder(Param out, const Param in, const dim_t* rdims) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/resize.hpp b/src/backend/opencl/kernel/resize.hpp
index ae0184a4a1..bc813393c5 100644
--- a/src/backend/opencl/kernel/resize.hpp
+++ b/src/backend/opencl/kernel/resize.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -87,3 +88,4 @@ void resize(Param out, const Param in, const af_interp_type method) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/rotate.hpp b/src/backend/opencl/kernel/rotate.hpp
index 999a7f25a5..dec52c8962 100644
--- a/src/backend/opencl/kernel/rotate.hpp
+++ b/src/backend/opencl/kernel/rotate.hpp
@@ -24,6 +24,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -131,3 +132,4 @@ void rotate(Param out, const Param in, const float theta, af_interp_type method,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/scan_by_key/scan_by_key_impl.cpp b/src/backend/opencl/kernel/scan_by_key/scan_by_key_impl.cpp
index db44fb59c7..46cac6723d 100644
--- a/src/backend/opencl/kernel/scan_by_key/scan_by_key_impl.cpp
+++ b/src/backend/opencl/kernel/scan_by_key/scan_by_key_impl.cpp
@@ -15,9 +15,11 @@
 // The line below is read by CMake to determenine the instantiations
 // SBK_BINARY_OPS:af_add_t af_mul_t af_max_t af_min_t
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 INSTANTIATE_SCAN_FIRST_BY_KEY_OP(TYPE)
 INSTANTIATE_SCAN_DIM_BY_KEY_OP(TYPE)
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/scan_dim.hpp b/src/backend/opencl/kernel/scan_dim.hpp
index 00c4cfc8ef..2edc7f68c0 100644
--- a/src/backend/opencl/kernel/scan_dim.hpp
+++ b/src/backend/opencl/kernel/scan_dim.hpp
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename Ti, typename To, af_op_t op>
@@ -51,7 +52,7 @@ static opencl::Kernel getScanDimKernel(const std::string key, int dim,
         DefineValue(THREADS_X),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
         DefineKeyValue(IS_FINAL_PASS, (isFinalPass ? 1 : 0)),
         DefineKeyValue(INCLUSIVE_SCAN, inclusiveScan),
     };
@@ -156,3 +157,4 @@ static void scanDim(Param out, const Param in, const int dim,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/scan_dim_by_key.hpp b/src/backend/opencl/kernel/scan_dim_by_key.hpp
index d975fbe03e..f698c4176d 100644
--- a/src/backend/opencl/kernel/scan_dim_by_key.hpp
+++ b/src/backend/opencl/kernel/scan_dim_by_key.hpp
@@ -11,6 +11,7 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename Ti, typename Tk, typename To, af_op_t op>
@@ -18,3 +19,4 @@ void scanDimByKey(Param out, const Param in, const Param key, int dim,
                   const bool inclusive_scan);
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp b/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
index 8376c3a876..3d9745923c 100644
--- a/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
@@ -25,6 +25,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename Ti, typename Tk, typename To, af_op_t op>
@@ -51,7 +52,7 @@ static opencl::Kernel getScanDimKernel(const std::string key, int dim,
         DefineValue(THREADS_X),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
         DefineKeyValue(calculateFlags, (calculateFlags ? 1 : 0)),
         DefineKeyValue(INCLUSIVE_SCAN, inclusiveScan),
     };
@@ -210,3 +211,4 @@ void scanDimByKey(Param out, const Param in, const Param key, int dim,
     INSTANTIATE_SCAN_DIM_BY_KEY_TYPES(ROp, intl) \
     INSTANTIATE_SCAN_DIM_BY_KEY_TYPES(ROp, uintl)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/scan_first.hpp b/src/backend/opencl/kernel/scan_first.hpp
index a8031ecc5e..4354d27b49 100644
--- a/src/backend/opencl/kernel/scan_first.hpp
+++ b/src/backend/opencl/kernel/scan_first.hpp
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -52,7 +53,7 @@ static opencl::Kernel getScanFirstKernel(const std::string key,
         DefineKeyFromStr(binOpName<op>()),
         DefineValue(SHARED_MEM_SIZE),
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
         DefineKeyValue(IS_FINAL_PASS, (isFinalPass ? 1 : 0)),
         DefineKeyValue(INCLUSIVE_SCAN, inclusiveScan),
     };
@@ -152,3 +153,4 @@ static void scanFirst(Param &out, const Param &in,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/scan_first_by_key.hpp b/src/backend/opencl/kernel/scan_first_by_key.hpp
index 609e918f56..1e520bcebb 100644
--- a/src/backend/opencl/kernel/scan_first_by_key.hpp
+++ b/src/backend/opencl/kernel/scan_first_by_key.hpp
@@ -11,6 +11,7 @@
 
 #include <Param.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename Ti, typename Tk, typename To, af_op_t op>
@@ -18,3 +19,4 @@ void scanFirstByKey(Param &out, const Param &in, const Param &key,
                     const bool inclusive_scan);
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/scan_first_by_key_impl.hpp b/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
index f8835e18a8..d0351add52 100644
--- a/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -55,7 +56,7 @@ static opencl::Kernel getScanFirstKernel(const std::string key,
         DefineKeyValue(init, toNumStr(common::Binary<To, op>::init())),
         DefineValue(SHARED_MEM_SIZE),
         DefineKeyFromStr(binOpName<op>()),
-        DefineKeyValue(CPLX, af::iscplx<Ti>()),
+        DefineKeyValue(CPLX, iscplx<Ti>()),
         DefineKeyValue(calculateFlags, (calculateFlags ? 1 : 0)),
         DefineKeyValue(INCLUSIVE_SCAN, inclusiveScan),
     };
@@ -206,3 +207,4 @@ void scanFirstByKey(Param &out, const Param &in, const Param &key,
     INSTANTIATE_SCAN_FIRST_BY_KEY_TYPES(ROp, intl) \
     INSTANTIATE_SCAN_FIRST_BY_KEY_TYPES(ROp, uintl)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/select.hpp b/src/backend/opencl/kernel/select.hpp
index 69602817a9..fc37e6cb86 100644
--- a/src/backend/opencl/kernel/select.hpp
+++ b/src/backend/opencl/kernel/select.hpp
@@ -17,9 +17,10 @@
 #include <math.hpp>
 #include <traits.hpp>
 
+#include <array>
 #include <string>
-#include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 constexpr uint DIMX  = 32;
@@ -103,3 +104,4 @@ void select_scalar(Param out, Param cond, Param a, const T b, const int ndims,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sift.hpp b/src/backend/opencl/kernel/sift.hpp
index 90b063b2d0..d5b248f007 100644
--- a/src/backend/opencl/kernel/sift.hpp
+++ b/src/backend/opencl/kernel/sift.hpp
@@ -38,6 +38,7 @@ AF_DEPRECATED_WARNINGS_ON
 
 namespace compute = boost::compute;
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -729,3 +730,4 @@ void sift(unsigned* out_feat, unsigned* out_dlen, Param& x_out, Param& y_out,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sobel.hpp b/src/backend/opencl/kernel/sobel.hpp
index 8e0c406f4a..9e92213adf 100644
--- a/src/backend/opencl/kernel/sobel.hpp
+++ b/src/backend/opencl/kernel/sobel.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename Ti, typename To, unsigned ker_size>
@@ -58,3 +59,4 @@ void sobel(Param dx, Param dy, const Param in) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sort.hpp b/src/backend/opencl/kernel/sort.hpp
index a55eb2b966..dd8bbe1390 100644
--- a/src/backend/opencl/kernel/sort.hpp
+++ b/src/backend/opencl/kernel/sort.hpp
@@ -26,6 +26,7 @@ AF_DEPRECATED_WARNINGS_ON
 
 namespace compute = boost::compute;
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -128,3 +129,4 @@ void sort0(Param val, bool isAscending) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sort_by_key.hpp b/src/backend/opencl/kernel/sort_by_key.hpp
index 7a25662667..4333a7830c 100644
--- a/src/backend/opencl/kernel/sort_by_key.hpp
+++ b/src/backend/opencl/kernel/sort_by_key.hpp
@@ -13,6 +13,7 @@
 #include <debug_opencl.hpp>
 #include <traits.hpp>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename Tk, typename Tv>
@@ -25,3 +26,4 @@ template<typename Tk, typename Tv>
 void sort0ByKey(Param pKey, Param pVal, bool isAscending);
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp
index ab20be6a33..dd74cccc7e 100644
--- a/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp
+++ b/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp
@@ -11,8 +11,10 @@
 
 // SBK_TYPES:float double int uint intl uintl short ushort char uchar half
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 INSTANTIATE1(TYPE)
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sort_by_key_impl.hpp b/src/backend/opencl/kernel/sort_by_key_impl.hpp
index 2d6f84493b..a070a60c67 100644
--- a/src/backend/opencl/kernel/sort_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/sort_by_key_impl.hpp
@@ -36,7 +36,7 @@ AF_DEPRECATED_WARNINGS_ON
 
 namespace compute = boost::compute;
 
-using common::half;
+using arrayfire::common::half;
 
 template<typename Tk, typename Tv, bool isAscending>
 inline boost::compute::function<bool(const std::pair<Tk, Tv>,
@@ -79,6 +79,7 @@ INSTANTIATE_FLIP(cl_ulong, ULONG_MAX)
 
 #undef INSTANTIATE_FLIP
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 static const int copyPairIter = 4;
@@ -254,3 +255,4 @@ void sort0ByKey(Param pKey, Param pVal, bool isAscending) {
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sort_helper.hpp b/src/backend/opencl/kernel/sort_helper.hpp
index 1c9db6cab7..971b4077e9 100644
--- a/src/backend/opencl/kernel/sort_helper.hpp
+++ b/src/backend/opencl/kernel/sort_helper.hpp
@@ -14,6 +14,7 @@
 #include <debug_opencl.hpp>
 #include <type_traits>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -44,3 +45,4 @@ using type_t = typename std::conditional<std::is_same<T, uintl>::value,
                                          cl_ulong, ltype_t<T>>::type;
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sparse.hpp b/src/backend/opencl/kernel/sparse.hpp
index 6cfed4b554..f7ef69e248 100644
--- a/src/backend/opencl/kernel/sparse.hpp
+++ b/src/backend/opencl/kernel/sparse.hpp
@@ -27,6 +27,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -227,3 +228,4 @@ void coo2csr(Param ovalues, Param orowIdx, Param ocolIdx, const Param ivalues,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/sparse_arith.hpp b/src/backend/opencl/kernel/sparse_arith.hpp
index f10b3327a0..048a6d4876 100644
--- a/src/backend/opencl/kernel/sparse_arith.hpp
+++ b/src/backend/opencl/kernel/sparse_arith.hpp
@@ -25,6 +25,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -171,7 +172,7 @@ void ssArithCSR(Param oVals, Param oColIdx, const Param oRowIdx, const uint M,
 
     auto arithOp = fetchKernel<T, op>(
         "ssarith_csr", sp_sp_arith_csr_cl_src,
-        {DefineKeyValue(IDENTITY_VALUE, af::scalar_to_option(iden_val))});
+        {DefineKeyValue(IDENTITY_VALUE, scalar_to_option(iden_val))});
 
     cl::NDRange local(256, 1);
     cl::NDRange global(divup(M, local[0]) * local[0], 1, 1);
@@ -184,3 +185,4 @@ void ssArithCSR(Param oVals, Param oColIdx, const Param oRowIdx, const uint M,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/susan.hpp b/src/backend/opencl/kernel/susan.hpp
index d3cdfb8af2..d407755f31 100644
--- a/src/backend/opencl/kernel/susan.hpp
+++ b/src/backend/opencl/kernel/susan.hpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 constexpr unsigned SUSAN_THREADS_X = 16;
@@ -95,3 +96,4 @@ unsigned nonMaximal(cl::Buffer* x_out, cl::Buffer* y_out, cl::Buffer* resp_out,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/swapdblk.hpp b/src/backend/opencl/kernel/swapdblk.hpp
index ff875e25da..820db15094 100644
--- a/src/backend/opencl/kernel/swapdblk.hpp
+++ b/src/backend/opencl/kernel/swapdblk.hpp
@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -81,3 +82,4 @@ void swapdblk(int n, int nb, cl_mem dA, size_t dA_offset, int ldda, int inca,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/tile.hpp b/src/backend/opencl/kernel/tile.hpp
index cc65a1fc54..fa097ba58f 100644
--- a/src/backend/opencl/kernel/tile.hpp
+++ b/src/backend/opencl/kernel/tile.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -57,3 +58,4 @@ void tile(Param out, const Param in) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/transform.hpp b/src/backend/opencl/kernel/transform.hpp
index a64468ea26..a3f81fd75b 100644
--- a/src/backend/opencl/kernel/transform.hpp
+++ b/src/backend/opencl/kernel/transform.hpp
@@ -24,6 +24,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -109,3 +110,4 @@ void transform(Param out, const Param in, const Param tf, bool isInverse,
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/transpose.hpp b/src/backend/opencl/kernel/transpose.hpp
index 87e6b65fee..3397596179 100644
--- a/src/backend/opencl/kernel/transpose.hpp
+++ b/src/backend/opencl/kernel/transpose.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -43,7 +44,7 @@ void transpose(Param out, const Param in, cl::CommandQueue queue,
         DefineValue(TILE_DIM),
         DefineValue(THREADS_Y),
         DefineValue(IS32MULTIPLE),
-        DefineKeyValue(DOCONJUGATE, (conjugate && af::iscplx<T>())),
+        DefineKeyValue(DOCONJUGATE, (conjugate && iscplx<T>())),
         DefineKeyValue(T, dtype_traits<T>::getName()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
@@ -66,3 +67,4 @@ void transpose(Param out, const Param in, cl::CommandQueue queue,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/transpose_inplace.hpp b/src/backend/opencl/kernel/transpose_inplace.hpp
index 06020a6e3c..b55f2e4d43 100644
--- a/src/backend/opencl/kernel/transpose_inplace.hpp
+++ b/src/backend/opencl/kernel/transpose_inplace.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -43,7 +44,7 @@ void transpose_inplace(Param in, cl::CommandQueue& queue, const bool conjugate,
         DefineValue(TILE_DIM),
         DefineValue(THREADS_Y),
         DefineValue(IS32MULTIPLE),
-        DefineKeyValue(DOCONJUGATE, (conjugate && af::iscplx<T>())),
+        DefineKeyValue(DOCONJUGATE, (conjugate && iscplx<T>())),
         DefineKeyValue(T, dtype_traits<T>::getName()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
@@ -69,3 +70,4 @@ void transpose_inplace(Param in, cl::CommandQueue& queue, const bool conjugate,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/triangle.hpp b/src/backend/opencl/kernel/triangle.hpp
index 8380894b07..c0be0de33f 100644
--- a/src/backend/opencl/kernel/triangle.hpp
+++ b/src/backend/opencl/kernel/triangle.hpp
@@ -21,12 +21,13 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
 template<typename T>
 void triangle(Param out, const Param in, bool is_upper, bool is_unit_diag) {
-    using af::scalar_to_option;
+    using arrayfire::opencl::scalar_to_option;
     using cl::EnqueueArgs;
     using cl::NDRange;
     using std::string;
@@ -68,3 +69,4 @@ void triangle(Param out, const Param in, bool is_upper, bool is_unit_diag) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/unwrap.hpp b/src/backend/opencl/kernel/unwrap.hpp
index 68d6846893..08e535f713 100644
--- a/src/backend/opencl/kernel/unwrap.hpp
+++ b/src/backend/opencl/kernel/unwrap.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -79,3 +80,4 @@ void unwrap(Param out, const Param in, const dim_t wx, const dim_t wy,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/where.hpp b/src/backend/opencl/kernel/where.hpp
index 9c17143398..88e89fd26b 100644
--- a/src/backend/opencl/kernel/where.hpp
+++ b/src/backend/opencl/kernel/where.hpp
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 template<typename T>
@@ -41,7 +42,7 @@ static void get_out_idx(cl::Buffer *out_data, Param &otmp, Param &rtmp,
     vector<string> compileOpts = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(ZERO, toNumStr(scalar<T>(0))),
-        DefineKeyValue(CPLX, af::iscplx<T>()),
+        DefineKeyValue(CPLX, iscplx<T>()),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
@@ -132,3 +133,4 @@ static void where(Param &out, Param &in) {
 }
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/wrap.hpp b/src/backend/opencl/kernel/wrap.hpp
index 72797bd5f5..b527cd8bce 100644
--- a/src/backend/opencl/kernel/wrap.hpp
+++ b/src/backend/opencl/kernel/wrap.hpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace opencl {
 namespace kernel {
 
@@ -118,3 +119,4 @@ void wrap_dilated(Param out, const Param in, const dim_t wx, const dim_t wy,
 
 }  // namespace kernel
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/logic.hpp b/src/backend/opencl/logic.hpp
index b7132ac01c..78efdcadd3 100644
--- a/src/backend/opencl/logic.hpp
+++ b/src/backend/opencl/logic.hpp
@@ -15,6 +15,7 @@
 #include <af/defines.h>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T, af_op_t op>
 Array<char> logicOp(const Array<T> &lhs, const Array<T> &rhs,
@@ -28,3 +29,4 @@ Array<T> bitOp(const Array<T> &lhs, const Array<T> &rhs,
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/lookup.cpp b/src/backend/opencl/lookup.cpp
index 724538604e..2fee6f6ae0 100644
--- a/src/backend/opencl/lookup.cpp
+++ b/src/backend/opencl/lookup.cpp
@@ -15,8 +15,9 @@
 #include <err_opencl.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
@@ -71,3 +72,4 @@ INSTANTIATE(ushort);
 INSTANTIATE(short);
 INSTANTIATE(half);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/lookup.hpp b/src/backend/opencl/lookup.hpp
index 5164648cfa..abf10d5902 100644
--- a/src/backend/opencl/lookup.hpp
+++ b/src/backend/opencl/lookup.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                    const unsigned dim);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/lu.cpp b/src/backend/opencl/lu.cpp
index 8fe05b3bf6..ff6f54d0d9 100644
--- a/src/backend/opencl/lu.cpp
+++ b/src/backend/opencl/lu.cpp
@@ -18,6 +18,7 @@
 #include <magma/magma.h>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 Array<int> convertPivot(int *ipiv, int in_sz, int out_sz) {
@@ -91,9 +92,11 @@ INSTANTIATE_LU(double)
 INSTANTIATE_LU(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -121,5 +124,6 @@ INSTANTIATE_LU(double)
 INSTANTIATE_LU(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/lu.hpp b/src/backend/opencl/lu.hpp
index 6ba417baa7..2186aef62e 100644
--- a/src/backend/opencl/lu.hpp
+++ b/src/backend/opencl/lu.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
@@ -19,3 +20,4 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot = true);
 
 bool isLAPACKAvailable();
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/magma/geqrf2.cpp b/src/backend/opencl/magma/geqrf2.cpp
index bcb71ad51f..daba1f4328 100644
--- a/src/backend/opencl/magma/geqrf2.cpp
+++ b/src/backend/opencl/magma/geqrf2.cpp
@@ -230,7 +230,7 @@ magma_int_t magma_geqrf2_gpu(magma_int_t m, magma_int_t n, cl_mem dA,
     }
     */
 
-    cl_mem buffer = clCreateBuffer(opencl::getContext()(),
+    cl_mem buffer = clCreateBuffer(arrayfire::opencl::getContext()(),
                                    CL_MEM_READ_WRITE | CL_MEM_ALLOC_HOST_PTR,
                                    sizeof(Ty) * lwork, NULL, NULL);
     work          = (Ty *)clEnqueueMapBuffer(queue[0], buffer, CL_TRUE,
diff --git a/src/backend/opencl/magma/getrs.cpp b/src/backend/opencl/magma/getrs.cpp
index 1f4578db6b..a689408a26 100644
--- a/src/backend/opencl/magma/getrs.cpp
+++ b/src/backend/opencl/magma/getrs.cpp
@@ -165,7 +165,7 @@ magma_int_t magma_getrs_gpu(magma_trans_t trans, magma_int_t n,
             : (trans == MagmaTrans ? OPENCL_BLAS_TRANS
                                    : OPENCL_BLAS_CONJ_TRANS);
 
-    bool cond  = opencl::getActivePlatform() == AFCL_PLATFORM_NVIDIA;
+    bool cond  = arrayfire::opencl::getActivePlatform() == AFCL_PLATFORM_NVIDIA;
     cl_mem dAT = 0;
     if (nrhs > 1 && cond) {
         magma_malloc<Ty>(&dAT, n * n);
diff --git a/src/backend/opencl/magma/labrd.cpp b/src/backend/opencl/magma/labrd.cpp
index 010a3675a7..c2f5fd0698 100644
--- a/src/backend/opencl/magma/labrd.cpp
+++ b/src/backend/opencl/magma/labrd.cpp
@@ -203,7 +203,7 @@ magma_int_t magma_labrd_gpu(magma_int_t m, magma_int_t n, magma_int_t nb, Ty *a,
 
     using Tr = typename af::dtype_traits<Ty>::base_type;
 
-    constexpr bool is_cplx = common::is_complex<Ty>::value;
+    constexpr bool is_cplx = arrayfire::common::is_complex<Ty>::value;
 
     Tr *d = (Tr *)_d;
     Tr *e = (Tr *)_e;
diff --git a/src/backend/opencl/magma/laset.cpp b/src/backend/opencl/magma/laset.cpp
index a08b7af2fa..520bdea59e 100644
--- a/src/backend/opencl/magma/laset.cpp
+++ b/src/backend/opencl/magma/laset.cpp
@@ -60,6 +60,7 @@ template<typename T>
 void magmablas_laset(magma_uplo_t uplo, magma_int_t m, magma_int_t n, T offdiag,
                      T diag, cl_mem dA, size_t dA_offset, magma_int_t ldda,
                      magma_queue_t queue) {
+    using arrayfire::opencl::kernel::laset;
     magma_int_t info = 0;
     if (uplo != MagmaLower && uplo != MagmaUpper && uplo != MagmaFull) {
         info = -1;
@@ -79,14 +80,11 @@ void magmablas_laset(magma_uplo_t uplo, magma_int_t m, magma_int_t n, T offdiag,
 
     switch (uplo) {
         case MagmaFull:
-            return opencl::kernel::laset<T, 0>(m, n, offdiag, diag, dA,
-                                               dA_offset, ldda, queue);
+            return laset<T, 0>(m, n, offdiag, diag, dA, dA_offset, ldda, queue);
         case MagmaLower:
-            return opencl::kernel::laset<T, 1>(m, n, offdiag, diag, dA,
-                                               dA_offset, ldda, queue);
+            return laset<T, 1>(m, n, offdiag, diag, dA, dA_offset, ldda, queue);
         case MagmaUpper:
-            return opencl::kernel::laset<T, 2>(m, n, offdiag, diag, dA,
-                                               dA_offset, ldda, queue);
+            return laset<T, 2>(m, n, offdiag, diag, dA, dA_offset, ldda, queue);
         default: return;
     }
 }
diff --git a/src/backend/opencl/magma/laswp.cpp b/src/backend/opencl/magma/laswp.cpp
index 53f4cccbea..14d24e61c7 100644
--- a/src/backend/opencl/magma/laswp.cpp
+++ b/src/backend/opencl/magma/laswp.cpp
@@ -78,7 +78,8 @@ void magmablas_laswp(magma_int_t n, cl_mem dAT, size_t dAT_offset,
     }
 
     cl::CommandQueue q(queue, true);
-    opencl::kernel::laswp<T>(n, dAT, dAT_offset, ldda, k1, k2, ipiv, inci, q);
+    arrayfire::opencl::kernel::laswp<T>(n, dAT, dAT_offset, ldda, k1, k2, ipiv,
+                                        inci, q);
 }
 
 #define INSTANTIATE(T)                                                  \
diff --git a/src/backend/opencl/magma/magma_blas.h b/src/backend/opencl/magma/magma_blas.h
index d34d04c29a..62f3290121 100644
--- a/src/backend/opencl/magma/magma_blas.h
+++ b/src/backend/opencl/magma/magma_blas.h
@@ -17,8 +17,8 @@
 #include <types.hpp>
 #include "magma_common.h"
 
-using opencl::cdouble;
-using opencl::cfloat;
+using arrayfire::opencl::cdouble;
+using arrayfire::opencl::cfloat;
 
 template<typename T>
 struct gpu_blas_gemm_func;
diff --git a/src/backend/opencl/magma/magma_blas_clblast.h b/src/backend/opencl/magma/magma_blas_clblast.h
index 905b5fc723..bb2bfbeee5 100644
--- a/src/backend/opencl/magma/magma_blas_clblast.h
+++ b/src/backend/opencl/magma/magma_blas_clblast.h
@@ -60,7 +60,7 @@ struct CLBlastType<cdouble> {
     using Type = std::complex<double>;
 };
 template<>
-struct CLBlastType<common::half> {
+struct CLBlastType<arrayfire::common::half> {
     using Type = cl_half;
 };
 
@@ -78,7 +78,7 @@ double inline toCLBlastConstant(const double val) {
     return val;
 }
 template<>
-cl_half inline toCLBlastConstant(const common::half val) {
+cl_half inline toCLBlastConstant(const arrayfire::common::half val) {
     cl_half out;
     memcpy(&out, &val, sizeof(cl_half));
     return out;
@@ -98,7 +98,7 @@ struct CLBlastBasicType {
     using Type = T;
 };
 template<>
-struct CLBlastBasicType<common::half> {
+struct CLBlastBasicType<arrayfire::common::half> {
     using Type = cl_half;
 };
 template<>
diff --git a/src/backend/opencl/magma/magma_data.h b/src/backend/opencl/magma/magma_data.h
index 4d6834b42e..69bd5e36a8 100644
--- a/src/backend/opencl/magma/magma_data.h
+++ b/src/backend/opencl/magma/magma_data.h
@@ -71,8 +71,8 @@ static magma_int_t magma_malloc(magma_ptr* ptrPtr, int num) {
     // size
     if (size == 0) size = sizeof(T);
     cl_int err;
-    *ptrPtr = clCreateBuffer(opencl::getContext()(), CL_MEM_READ_WRITE, size,
-                             NULL, &err);
+    *ptrPtr = clCreateBuffer(arrayfire::opencl::getContext()(),
+                             CL_MEM_READ_WRITE, size, NULL, &err);
     if (err != CL_SUCCESS) { return MAGMA_ERR_DEVICE_ALLOC; }
     return MAGMA_SUCCESS;
 }
diff --git a/src/backend/opencl/magma/swapdblk.cpp b/src/backend/opencl/magma/swapdblk.cpp
index d6751b2c0f..6a669a54ce 100644
--- a/src/backend/opencl/magma/swapdblk.cpp
+++ b/src/backend/opencl/magma/swapdblk.cpp
@@ -16,8 +16,8 @@ void magmablas_swapdblk(magma_int_t n, magma_int_t nb, cl_mem dA,
                         magma_int_t inca, cl_mem dB, magma_int_t dB_offset,
                         magma_int_t lddb, magma_int_t incb,
                         magma_queue_t queue) {
-    opencl::kernel::swapdblk<T>(n, nb, dA, dA_offset, ldda, inca, dB, dB_offset,
-                                lddb, incb, queue);
+    arrayfire::opencl::kernel::swapdblk<T>(n, nb, dA, dA_offset, ldda, inca, dB,
+                                           dB_offset, lddb, incb, queue);
 }
 
 #define INSTANTIATE(T)                                                        \
diff --git a/src/backend/opencl/magma/transpose.cpp b/src/backend/opencl/magma/transpose.cpp
index e9ff2243ca..a33d440f95 100644
--- a/src/backend/opencl/magma/transpose.cpp
+++ b/src/backend/opencl/magma/transpose.cpp
@@ -54,10 +54,10 @@
 #include "kernel/transpose.hpp"
 #include "magma_data.h"
 
+using arrayfire::opencl::makeParam;
+using arrayfire::opencl::kernel::transpose;
 using cl::Buffer;
 using cl::CommandQueue;
-using opencl::makeParam;
-using opencl::kernel::transpose;
 
 template<typename T>
 void magmablas_transpose(magma_int_t m, magma_int_t n, cl_mem dA,
diff --git a/src/backend/opencl/magma/transpose_inplace.cpp b/src/backend/opencl/magma/transpose_inplace.cpp
index 21770f98be..7705edb7b3 100644
--- a/src/backend/opencl/magma/transpose_inplace.cpp
+++ b/src/backend/opencl/magma/transpose_inplace.cpp
@@ -54,10 +54,10 @@
 #include "kernel/transpose_inplace.hpp"
 #include "magma_data.h"
 
+using arrayfire::opencl::makeParam;
+using arrayfire::opencl::kernel::transpose_inplace;
 using cl::Buffer;
 using cl::CommandQueue;
-using opencl::makeParam;
-using opencl::kernel::transpose_inplace;
 
 template<typename T>
 void magmablas_transpose_inplace(magma_int_t n, cl_mem dA, size_t dA_offset,
diff --git a/src/backend/opencl/match_template.cpp b/src/backend/opencl/match_template.cpp
index 8b2d0dd025..f97bc6d353 100644
--- a/src/backend/opencl/match_template.cpp
+++ b/src/backend/opencl/match_template.cpp
@@ -11,6 +11,7 @@
 
 #include <kernel/match_template.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename inType, typename outType>
@@ -41,3 +42,4 @@ INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/match_template.hpp b/src/backend/opencl/match_template.hpp
index bf2a76f55d..7b493d2ca0 100644
--- a/src/backend/opencl/match_template.hpp
+++ b/src/backend/opencl/match_template.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace opencl {
 template<typename inType, typename outType>
 Array<outType> match_template(const Array<inType> &sImg,
                               const Array<inType> &tImg,
                               const af::matchType mType);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/math.cpp b/src/backend/opencl/math.cpp
index 31c09c3b96..bbe78dfc94 100644
--- a/src/backend/opencl/math.cpp
+++ b/src/backend/opencl/math.cpp
@@ -10,6 +10,7 @@
 #include "math.hpp"
 #include <common/half.hpp>
 
+namespace arrayfire {
 namespace opencl {
 cfloat operator+(cfloat lhs, cfloat rhs) {
     cfloat res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
@@ -53,3 +54,4 @@ cdouble division(cdouble lhs, double rhs) {
     return retVal;
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/math.hpp b/src/backend/opencl/math.hpp
index e7cf8d1928..e4745d9e92 100644
--- a/src/backend/opencl/math.hpp
+++ b/src/backend/opencl/math.hpp
@@ -28,6 +28,7 @@
 /* Other */
 #endif
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -142,19 +143,22 @@ cfloat operator*(cfloat lhs, cfloat rhs);
 cdouble operator*(cdouble lhs, cdouble rhs);
 common::half operator+(common::half lhs, common::half rhs) noexcept;
 }  // namespace opencl
+}  // namespace arrayfire
 
-static inline bool operator==(opencl::cfloat lhs, opencl::cfloat rhs) noexcept {
+static inline bool operator==(arrayfire::opencl::cfloat lhs,
+                              arrayfire::opencl::cfloat rhs) noexcept {
     return (lhs.s[0] == rhs.s[0]) && (lhs.s[1] == rhs.s[1]);
 }
-static inline bool operator!=(opencl::cfloat lhs, opencl::cfloat rhs) noexcept {
+static inline bool operator!=(arrayfire::opencl::cfloat lhs,
+                              arrayfire::opencl::cfloat rhs) noexcept {
     return !(lhs == rhs);
 }
-static inline bool operator==(opencl::cdouble lhs,
-                              opencl::cdouble rhs) noexcept {
+static inline bool operator==(arrayfire::opencl::cdouble lhs,
+                              arrayfire::opencl::cdouble rhs) noexcept {
     return (lhs.s[0] == rhs.s[0]) && (lhs.s[1] == rhs.s[1]);
 }
-static inline bool operator!=(opencl::cdouble lhs,
-                              opencl::cdouble rhs) noexcept {
+static inline bool operator!=(arrayfire::opencl::cdouble lhs,
+                              arrayfire::opencl::cdouble rhs) noexcept {
     return !(lhs == rhs);
 }
 
diff --git a/src/backend/opencl/max.cpp b/src/backend/opencl/max.cpp
index d4a7640acf..b2a2cdfdf0 100644
--- a/src/backend/opencl/max.cpp
+++ b/src/backend/opencl/max.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 // max
 INSTANTIATE(af_max_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_max_t, short, short)
 INSTANTIATE(af_max_t, ushort, ushort)
 INSTANTIATE(af_max_t, half, half)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/mean.cpp b/src/backend/opencl/mean.cpp
index adce4be841..7bd586e587 100644
--- a/src/backend/opencl/mean.cpp
+++ b/src/backend/opencl/mean.cpp
@@ -14,9 +14,10 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using std::swap;
 
+namespace arrayfire {
 namespace opencl {
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in) {
@@ -77,3 +78,4 @@ INSTANTIATE_WGT(cdouble, double);
 INSTANTIATE_WGT(half, float);
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/mean.hpp b/src/backend/opencl/mean.hpp
index 7f98f439d8..61f44aa86a 100644
--- a/src/backend/opencl/mean.hpp
+++ b/src/backend/opencl/mean.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in);
@@ -24,3 +25,4 @@ template<typename T, typename Tw>
 Array<T> mean(const Array<T>& in, const Array<Tw>& wts, const int dim);
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/meanshift.cpp b/src/backend/opencl/meanshift.cpp
index bceed64bb1..3c6f140c98 100644
--- a/src/backend/opencl/meanshift.cpp
+++ b/src/backend/opencl/meanshift.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
@@ -43,3 +44,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/meanshift.hpp b/src/backend/opencl/meanshift.hpp
index eafd6dbd93..54e8dd588f 100644
--- a/src/backend/opencl/meanshift.hpp
+++ b/src/backend/opencl/meanshift.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
                    const float &chromaticSigma, const unsigned &numIterations,
                    const bool &isColor);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/medfilt.cpp b/src/backend/opencl/medfilt.cpp
index 0e63834253..66a4c6969e 100644
--- a/src/backend/opencl/medfilt.cpp
+++ b/src/backend/opencl/medfilt.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -59,3 +60,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/medfilt.hpp b/src/backend/opencl/medfilt.hpp
index 0a010c3154..439282b1f1 100644
--- a/src/backend/opencl/medfilt.hpp
+++ b/src/backend/opencl/medfilt.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -20,3 +21,4 @@ Array<T> medfilt2(const Array<T> &in, const int w_len, const int w_wid,
                   const af::borderType edge_pad);
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index 8dab1f428b..6c37d873a2 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -20,13 +20,14 @@
 
 #include <utility>
 
-using common::bytesToString;
+using arrayfire::common::bytesToString;
 
 using af::dim4;
 using std::function;
 using std::move;
 using std::unique_ptr;
 
+namespace arrayfire {
 namespace opencl {
 float getMemoryPressure() { return memoryManager().getMemoryPressure(); }
 float getMemoryPressureThreshold() {
@@ -272,3 +273,4 @@ void AllocatorPinned::nativeFree(void *ptr) {
     }
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/memory.hpp b/src/backend/opencl/memory.hpp
index ba7e340d32..4f618d7956 100644
--- a/src/backend/opencl/memory.hpp
+++ b/src/backend/opencl/memory.hpp
@@ -20,6 +20,7 @@ namespace cl {
 class Buffer;  // Forward declaration of cl::Buffer from CL/cl2.hpp
 }
 
+namespace arrayfire {
 namespace opencl {
 cl::Buffer *bufferAlloc(const size_t &bytes);
 void bufferFree(cl::Buffer *buf);
@@ -60,7 +61,7 @@ bool jitTreeExceedsMemoryPressure(size_t bytes);
 void setMemStepSize(size_t step_bytes);
 size_t getMemStepSize(void);
 
-class Allocator final : public common::memory::AllocatorInterface {
+class Allocator final : public common::AllocatorInterface {
    public:
     Allocator();
     ~Allocator() = default;
@@ -71,7 +72,7 @@ class Allocator final : public common::memory::AllocatorInterface {
     void nativeFree(void *ptr) override;
 };
 
-class AllocatorPinned final : public common::memory::AllocatorInterface {
+class AllocatorPinned final : public common::AllocatorInterface {
    public:
     AllocatorPinned();
     ~AllocatorPinned() = default;
@@ -86,3 +87,4 @@ class AllocatorPinned final : public common::memory::AllocatorInterface {
 };
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/min.cpp b/src/backend/opencl/min.cpp
index 69aa38efae..9cc6a09272 100644
--- a/src/backend/opencl/min.cpp
+++ b/src/backend/opencl/min.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 // min
 INSTANTIATE(af_min_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_min_t, short, short)
 INSTANTIATE(af_min_t, ushort, ushort)
 INSTANTIATE(af_min_t, half, half)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/moments.cpp b/src/backend/opencl/moments.cpp
index ef378762e2..0b03d203c9 100644
--- a/src/backend/opencl/moments.cpp
+++ b/src/backend/opencl/moments.cpp
@@ -12,6 +12,7 @@
 #include <err_opencl.hpp>
 #include <kernel/moments.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 static inline unsigned bitCount(unsigned v) {
@@ -52,3 +53,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/moments.hpp b/src/backend/opencl/moments.hpp
index 90666f710a..c0e3cb4058 100644
--- a/src/backend/opencl/moments.hpp
+++ b/src/backend/opencl/moments.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<float> moments(const Array<T> &in, const af_moment_type moment);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/morph.cpp b/src/backend/opencl/morph.cpp
index 10ac7397c5..e77b7a063c 100644
--- a/src/backend/opencl/morph.cpp
+++ b/src/backend/opencl/morph.cpp
@@ -16,6 +16,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -61,3 +62,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/morph.hpp b/src/backend/opencl/morph.hpp
index 9435abef85..aee753c8d7 100644
--- a/src/backend/opencl/morph.hpp
+++ b/src/backend/opencl/morph.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
@@ -16,3 +17,4 @@ Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
 template<typename T>
 Array<T> morph3d(const Array<T> &in, const Array<T> &mask, bool isDilation);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/nearest_neighbour.cpp b/src/backend/opencl/nearest_neighbour.cpp
index fc3727b860..535be4083f 100644
--- a/src/backend/opencl/nearest_neighbour.cpp
+++ b/src/backend/opencl/nearest_neighbour.cpp
@@ -18,6 +18,7 @@
 using af::dim4;
 using cl::Device;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename To, af_match_type dist_type>
@@ -84,3 +85,4 @@ INSTANTIATE(uchar, uint)
 INSTANTIATE(uintl, uint)  // For Hamming
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/nearest_neighbour.hpp b/src/backend/opencl/nearest_neighbour.hpp
index 2f64436874..65a7a3d1c5 100644
--- a/src/backend/opencl/nearest_neighbour.hpp
+++ b/src/backend/opencl/nearest_neighbour.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename To>
@@ -20,4 +21,5 @@ void nearest_neighbour(Array<uint>& idx, Array<To>& dist, const Array<T>& query,
                        const uint n_dist,
                        const af_match_type dist_type = AF_SSD);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/orb.cpp b/src/backend/opencl/orb.cpp
index 44971f9d02..5e1d2b42d0 100644
--- a/src/backend/opencl/orb.cpp
+++ b/src/backend/opencl/orb.cpp
@@ -17,6 +17,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename convAccT>
@@ -63,3 +64,4 @@ INSTANTIATE(float, float)
 INSTANTIATE(double, double)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/orb.hpp b/src/backend/opencl/orb.hpp
index 6b5906ae18..012113886e 100644
--- a/src/backend/opencl/orb.hpp
+++ b/src/backend/opencl/orb.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename convAccT>
@@ -21,4 +22,5 @@ unsigned orb(Array<float> &x, Array<float> &y, Array<float> &score,
              const unsigned max_feat, const float scl_fctr,
              const unsigned levels, const bool blur_img);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 04859ad40a..c040c04b09 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -66,12 +66,13 @@ using std::to_string;
 using std::unique_ptr;
 using std::vector;
 
-using common::getEnvVar;
-using common::ltrim;
-using common::memory::MemoryManagerBase;
-using opencl::Allocator;
-using opencl::AllocatorPinned;
+using arrayfire::common::getEnvVar;
+using arrayfire::common::ltrim;
+using arrayfire::common::MemoryManagerBase;
+using arrayfire::opencl::Allocator;
+using arrayfire::opencl::AllocatorPinned;
 
+namespace arrayfire {
 namespace opencl {
 
 static string get_system() {
@@ -645,7 +646,7 @@ void resetMemoryManagerPinned() {
     return DeviceManager::getInstance().resetMemoryManagerPinned();
 }
 
-graphics::ForgeManager& forgeManager() {
+arrayfire::common::ForgeManager& forgeManager() {
     return *(DeviceManager::getInstance().fgMngr);
 }
 
@@ -670,8 +671,9 @@ PlanCache& fftManager() {
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
 
-using namespace opencl;
+using namespace arrayfire::opencl;
 
 af_err afcl_get_device_type(afcl_device_type* res) {
     try {
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index fa937b0e0f..07eca8f856 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -29,18 +29,18 @@ namespace spdlog {
 class logger;
 }
 
-namespace graphics {
+namespace arrayfire {
+namespace common {
+
 class ForgeManager;
-}
 
-namespace common {
-namespace memory {
 class MemoryManagerBase;
-}
 }  // namespace common
+}  // namespace arrayfire
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 
+namespace arrayfire {
 namespace opencl {
 
 // Forward declarations
@@ -165,7 +165,7 @@ void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
 
 void resetMemoryManagerPinned();
 
-graphics::ForgeManager& forgeManager();
+arrayfire::common::ForgeManager& forgeManager();
 
 GraphicsResourceManager& interopManager();
 
@@ -176,3 +176,4 @@ afcl::platform getPlatformEnum(cl::Device dev);
 void setActiveContext(int device);
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/plot.cpp b/src/backend/opencl/plot.cpp
index bf4a1e7370..cc7f93262e 100644
--- a/src/backend/opencl/plot.cpp
+++ b/src/backend/opencl/plot.cpp
@@ -14,12 +14,15 @@
 #include <plot.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_plot(const Array<T> &P, fg_plot plot) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = forgePlugin();
     if (isGLSharingSupported()) {
         CheckGL("Begin OpenCL resource copy");
         const cl::Buffer *d_P = P.get();
@@ -75,3 +78,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/plot.hpp b/src/backend/opencl/plot.hpp
index 1d8c2e9f10..4a6849e01a 100644
--- a/src/backend/opencl/plot.hpp
+++ b/src/backend/opencl/plot.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_plot(const Array<T> &P, fg_plot plot);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/print.hpp b/src/backend/opencl/print.hpp
index d78e1a36a2..40919135a7 100644
--- a/src/backend/opencl/print.hpp
+++ b/src/backend/opencl/print.hpp
@@ -11,6 +11,7 @@
 #include <backend.hpp>
 #include <ostream>
 
+namespace arrayfire {
 namespace opencl {
 static std::ostream& operator<<(std::ostream& out, const cfloat& var) {
     out << "(" << var.s[0] << "," << var.s[1] << ")";
@@ -22,3 +23,4 @@ static std::ostream& operator<<(std::ostream& out, const cdouble& var) {
     return out;
 }
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/product.cpp b/src/backend/opencl/product.cpp
index 3ea554e2f6..f13a9b9ae3 100644
--- a/src/backend/opencl/product.cpp
+++ b/src/backend/opencl/product.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 // sum
 INSTANTIATE(af_mul_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_mul_t, short, int)
 INSTANTIATE(af_mul_t, ushort, uint)
 INSTANTIATE(af_mul_t, half, float)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/qr.cpp b/src/backend/opencl/qr.cpp
index 3588147aed..bb8d5c1205 100644
--- a/src/backend/opencl/qr.cpp
+++ b/src/backend/opencl/qr.cpp
@@ -23,6 +23,7 @@
 #include <magma/magma_helper.h>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -112,9 +113,11 @@ INSTANTIATE_QR(double)
 INSTANTIATE_QR(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -138,5 +141,6 @@ INSTANTIATE_QR(double)
 INSTANTIATE_QR(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/qr.hpp b/src/backend/opencl/qr.hpp
index b202aec88a..6c7b564ebc 100644
--- a/src/backend/opencl/qr.hpp
+++ b/src/backend/opencl/qr.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &orig);
@@ -16,3 +17,4 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &orig);
 template<typename T>
 Array<T> qr_inplace(Array<T> &in);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/random_engine.cpp b/src/backend/opencl/random_engine.cpp
index c112df4196..f2110c8be0 100644
--- a/src/backend/opencl/random_engine.cpp
+++ b/src/backend/opencl/random_engine.cpp
@@ -12,8 +12,9 @@
 #include <kernel/random_engine.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl) {
@@ -153,3 +154,4 @@ COMPLEX_NORMAL_DISTRIBUTION(cdouble, double)
 COMPLEX_NORMAL_DISTRIBUTION(cfloat, float)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/random_engine.hpp b/src/backend/opencl/random_engine.hpp
index 279db75fc1..93c190942e 100644
--- a/src/backend/opencl/random_engine.hpp
+++ b/src/backend/opencl/random_engine.hpp
@@ -13,6 +13,7 @@
 #include <backend.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace opencl {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl);
@@ -39,3 +40,4 @@ Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
                             Array<uint> recursion_table,
                             Array<uint> temper_table, Array<uint> state);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/range.cpp b/src/backend/opencl/range.cpp
index b98d9ba584..92340d34eb 100644
--- a/src/backend/opencl/range.cpp
+++ b/src/backend/opencl/range.cpp
@@ -15,8 +15,9 @@
 #include <math.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> range(const dim4& dim, const int seq_dim) {
@@ -51,3 +52,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/range.hpp b/src/backend/opencl/range.hpp
index 610d31933f..e34f302536 100644
--- a/src/backend/opencl/range.hpp
+++ b/src/backend/opencl/range.hpp
@@ -10,7 +10,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> range(const dim4& dim, const int seq_dim = -1);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/reduce.hpp b/src/backend/opencl/reduce.hpp
index 4c9581c761..8660f9f1d8 100644
--- a/src/backend/opencl/reduce.hpp
+++ b/src/backend/opencl/reduce.hpp
@@ -11,6 +11,7 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<af_op_t op, typename Ti, typename To>
 Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan = false,
@@ -25,3 +26,4 @@ template<af_op_t op, typename Ti, typename To>
 Array<To> reduce_all(const Array<Ti> &in, bool change_nan = false,
                      double nanval = 0);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/reduce_impl.hpp b/src/backend/opencl/reduce_impl.hpp
index 4211dc9050..7b68187e4e 100644
--- a/src/backend/opencl/reduce_impl.hpp
+++ b/src/backend/opencl/reduce_impl.hpp
@@ -17,6 +17,7 @@
 
 using af::dim4;
 using std::swap;
+namespace arrayfire {
 namespace opencl {
 template<af_op_t op, typename Ti, typename To>
 Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan,
@@ -44,6 +45,7 @@ Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #define INSTANTIATE(Op, Ti, To)                                                \
     template Array<To> reduce<Op, Ti, To>(const Array<Ti> &in, const int dim,  \
diff --git a/src/backend/opencl/regions.cpp b/src/backend/opencl/regions.cpp
index 66d67ee448..06df18dd4c 100644
--- a/src/backend/opencl/regions.cpp
+++ b/src/backend/opencl/regions.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -37,3 +38,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/regions.hpp b/src/backend/opencl/regions.hpp
index 89eab2714c..1c4d26f6c0 100644
--- a/src/backend/opencl/regions.hpp
+++ b/src/backend/opencl/regions.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 Array<T> regions(const Array<char> &in, af_connectivity connectivity);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/reorder.cpp b/src/backend/opencl/reorder.cpp
index 720d415883..da485911e6 100644
--- a/src/backend/opencl/reorder.cpp
+++ b/src/backend/opencl/reorder.cpp
@@ -14,8 +14,9 @@
 #include <reorder.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> reorder(const Array<T> &in, const af::dim4 &rdims) {
@@ -47,3 +48,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/reorder.hpp b/src/backend/opencl/reorder.hpp
index bd49a074f9..6aa860c769 100644
--- a/src/backend/opencl/reorder.hpp
+++ b/src/backend/opencl/reorder.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> reorder(const Array<T> &in, const af::dim4 &rdims);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/reshape.cpp b/src/backend/opencl/reshape.cpp
index 0ec77e27bc..78c83cc086 100644
--- a/src/backend/opencl/reshape.cpp
+++ b/src/backend/opencl/reshape.cpp
@@ -13,8 +13,9 @@
 #include <common/half.hpp>
 #include <kernel/memcopy.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename inType, typename outType>
@@ -77,3 +78,4 @@ INSTANTIATE_COMPLEX(cfloat)
 INSTANTIATE_COMPLEX(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/resize.cpp b/src/backend/opencl/resize.cpp
index 67257cc214..ee7776b82f 100644
--- a/src/backend/opencl/resize.cpp
+++ b/src/backend/opencl/resize.cpp
@@ -13,6 +13,7 @@
 #include <af/dim4.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
@@ -42,3 +43,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/resize.hpp b/src/backend/opencl/resize.hpp
index 0741be36b5..bec5bc8ce3 100644
--- a/src/backend/opencl/resize.hpp
+++ b/src/backend/opencl/resize.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
                 const af_interp_type method);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/rotate.cpp b/src/backend/opencl/rotate.cpp
index a7f969e55e..46caa65c88 100644
--- a/src/backend/opencl/rotate.cpp
+++ b/src/backend/opencl/rotate.cpp
@@ -11,6 +11,7 @@
 
 #include <kernel/rotate.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
@@ -53,3 +54,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/rotate.hpp b/src/backend/opencl/rotate.hpp
index 94916e7441..dddc164718 100644
--- a/src/backend/opencl/rotate.hpp
+++ b/src/backend/opencl/rotate.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
                 const af_interp_type method);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/scalar.hpp b/src/backend/opencl/scalar.hpp
index 420b38144d..1e497af867 100644
--- a/src/backend/opencl/scalar.hpp
+++ b/src/backend/opencl/scalar.hpp
@@ -12,6 +12,7 @@
 #include <math.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -21,3 +22,4 @@ Array<T> createScalarNode(const dim4 &size, const T val) {
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/scan.cpp b/src/backend/opencl/scan.cpp
index c069beb537..0fc36366ef 100644
--- a/src/backend/opencl/scan.cpp
+++ b/src/backend/opencl/scan.cpp
@@ -12,6 +12,7 @@
 #include <kernel/scan_dim.hpp>
 #include <kernel/scan_first.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusiveScan) {
@@ -52,3 +53,4 @@ INSTANTIATE_SCAN_ALL(af_mul_t)
 INSTANTIATE_SCAN_ALL(af_min_t)
 INSTANTIATE_SCAN_ALL(af_max_t)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/scan.hpp b/src/backend/opencl/scan.hpp
index d72f86dc64..77fef74c02 100644
--- a/src/backend/opencl/scan.hpp
+++ b/src/backend/opencl/scan.hpp
@@ -10,7 +10,9 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusive_scan = true);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/scan_by_key.cpp b/src/backend/opencl/scan_by_key.cpp
index 606a1b00f9..8af8d2a31b 100644
--- a/src/backend/opencl/scan_by_key.cpp
+++ b/src/backend/opencl/scan_by_key.cpp
@@ -16,6 +16,7 @@
 #include <kernel/scan_dim_by_key.hpp>
 #include <kernel/scan_first_by_key.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
@@ -60,3 +61,4 @@ INSTANTIATE_SCAN_BY_KEY_OP(af_mul_t)
 INSTANTIATE_SCAN_BY_KEY_OP(af_min_t)
 INSTANTIATE_SCAN_BY_KEY_OP(af_max_t)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/scan_by_key.hpp b/src/backend/opencl/scan_by_key.hpp
index 58fb5cacdd..f2ad2b2fc7 100644
--- a/src/backend/opencl/scan_by_key.hpp
+++ b/src/backend/opencl/scan_by_key.hpp
@@ -10,8 +10,10 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
                bool inclusive_scan = true);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/select.cpp b/src/backend/opencl/select.cpp
index d652df25c6..bbafbe989c 100644
--- a/src/backend/opencl/select.cpp
+++ b/src/backend/opencl/select.cpp
@@ -20,12 +20,13 @@
 
 using af::dim4;
 
-using common::half;
-using common::NaryNode;
+using arrayfire::common::half;
+using arrayfire::common::NaryNode;
 
 using std::make_shared;
 using std::max;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
@@ -133,3 +134,4 @@ INSTANTIATE(half);
 
 #undef INSTANTIATE
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/select.hpp b/src/backend/opencl/select.hpp
index 4dbd0635da..a026f9c04d 100644
--- a/src/backend/opencl/select.hpp
+++ b/src/backend/opencl/select.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
@@ -27,3 +28,4 @@ template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
                           const T &b_val, const af::dim4 &odims);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/set.cpp b/src/backend/opencl/set.cpp
index 30aa475a01..195cf23047 100644
--- a/src/backend/opencl/set.cpp
+++ b/src/backend/opencl/set.cpp
@@ -24,6 +24,7 @@ AF_DEPRECATED_WARNINGS_ON
 
 namespace compute = boost::compute;
 
+namespace arrayfire {
 namespace opencl {
 using af::dim4;
 
@@ -152,3 +153,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/set.hpp b/src/backend/opencl/set.hpp
index e67acc1ffd..2a3ea83594 100644
--- a/src/backend/opencl/set.hpp
+++ b/src/backend/opencl/set.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> setUnique(const Array<T> &in, const bool is_sorted);
@@ -21,3 +22,4 @@ template<typename T>
 Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
                       const bool is_unique);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/shift.cpp b/src/backend/opencl/shift.cpp
index 0266c5e6d5..512c113ed1 100644
--- a/src/backend/opencl/shift.cpp
+++ b/src/backend/opencl/shift.cpp
@@ -14,14 +14,15 @@
 #include <traits.hpp>
 
 using af::dim4;
-using common::Node_ptr;
-using common::ShiftNodeBase;
-using opencl::jit::BufferNode;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::ShiftNodeBase;
+using arrayfire::opencl::jit::BufferNode;
 using std::array;
 using std::make_shared;
 using std::static_pointer_cast;
 using std::string;
 
+namespace arrayfire {
 namespace opencl {
 using ShiftNode = ShiftNodeBase<BufferNode>;
 
@@ -68,3 +69,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/shift.hpp b/src/backend/opencl/shift.hpp
index 5ee21f063c..1797d6d1a7 100644
--- a/src/backend/opencl/shift.hpp
+++ b/src/backend/opencl/shift.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> shift(const Array<T> &in, const int sdims[4]);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sift.cpp b/src/backend/opencl/sift.cpp
index aa4dea46e5..d4b32c3820 100644
--- a/src/backend/opencl/sift.cpp
+++ b/src/backend/opencl/sift.cpp
@@ -15,6 +15,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename convAccT>
@@ -69,3 +70,4 @@ INSTANTIATE(float, float)
 INSTANTIATE(double, double)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sift.hpp b/src/backend/opencl/sift.hpp
index 3544405315..078841bf69 100644
--- a/src/backend/opencl/sift.hpp
+++ b/src/backend/opencl/sift.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename convAccT>
@@ -23,4 +24,5 @@ unsigned sift(Array<float>& x, Array<float>& y, Array<float>& score,
               const float img_scale, const float feature_ratio,
               const bool compute_GLOH);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sobel.cpp b/src/backend/opencl/sobel.cpp
index 9716140019..e718021b42 100644
--- a/src/backend/opencl/sobel.cpp
+++ b/src/backend/opencl/sobel.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename Ti, typename To>
@@ -44,3 +45,4 @@ INSTANTIATE(short, int)
 INSTANTIATE(ushort, int)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sobel.hpp b/src/backend/opencl/sobel.hpp
index 63b25bd316..74ccb2ebcf 100644
--- a/src/backend/opencl/sobel.hpp
+++ b/src/backend/opencl/sobel.hpp
@@ -10,10 +10,12 @@
 #include <Array.hpp>
 #include <utility>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename Ti, typename To>
 std::pair<Array<To>, Array<To>> sobelDerivatives(const Array<Ti> &img,
                                                  const unsigned &ker_size);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/solve.cpp b/src/backend/opencl/solve.cpp
index ad73e21d27..60d8f3a59b 100644
--- a/src/backend/opencl/solve.cpp
+++ b/src/backend/opencl/solve.cpp
@@ -32,6 +32,7 @@ using cl::Buffer;
 using std::min;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -325,9 +326,11 @@ INSTANTIATE_SOLVE(cfloat)
 INSTANTIATE_SOLVE(double)
 INSTANTIATE_SOLVE(cdouble)
 }  // namespace opencl
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -355,5 +358,6 @@ INSTANTIATE_SOLVE(double)
 INSTANTIATE_SOLVE(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/solve.hpp b/src/backend/opencl/solve.hpp
index c2b22810e4..390871856c 100644
--- a/src/backend/opencl/solve.hpp
+++ b/src/backend/opencl/solve.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> solve(const Array<T> &a, const Array<T> &b,
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> solveLU(const Array<T> &a, const Array<int> &pivot, const Array<T> &b,
                  const af_mat_prop options = AF_MAT_NONE);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sort.cpp b/src/backend/opencl/sort.cpp
index e73f4db312..8b977316f1 100644
--- a/src/backend/opencl/sort.cpp
+++ b/src/backend/opencl/sort.cpp
@@ -16,6 +16,7 @@
 #include <sort.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending) {
@@ -62,3 +63,4 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sort.hpp b/src/backend/opencl/sort.hpp
index 91e57b560c..092995aeec 100644
--- a/src/backend/opencl/sort.hpp
+++ b/src/backend/opencl/sort.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sort_by_key.cpp b/src/backend/opencl/sort_by_key.cpp
index f98a70e057..2e4b2dd616 100644
--- a/src/backend/opencl/sort_by_key.cpp
+++ b/src/backend/opencl/sort_by_key.cpp
@@ -16,6 +16,7 @@
 #include <sort_by_key.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace opencl {
 template<typename Tk, typename Tv>
 void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
@@ -83,3 +84,4 @@ INSTANTIATE1(uchar)
 INSTANTIATE1(intl)
 INSTANTIATE1(uintl)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sort_by_key.hpp b/src/backend/opencl/sort_by_key.hpp
index a1e616c3e5..78223de9be 100644
--- a/src/backend/opencl/sort_by_key.hpp
+++ b/src/backend/opencl/sort_by_key.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename Tk, typename Tv>
 void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
                  const Array<Tv> &ival, const unsigned dim, bool isAscending);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sort_index.cpp b/src/backend/opencl/sort_index.cpp
index 869dd7bdc0..9c92f8406c 100644
--- a/src/backend/opencl/sort_index.cpp
+++ b/src/backend/opencl/sort_index.cpp
@@ -18,8 +18,9 @@
 #include <sort_index.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void sort_index(Array<T> &okey, Array<uint> &oval, const Array<T> &in,
@@ -77,3 +78,4 @@ INSTANTIATE(uintl)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sort_index.hpp b/src/backend/opencl/sort_index.hpp
index 573a61d247..0979a1aa37 100644
--- a/src/backend/opencl/sort_index.hpp
+++ b/src/backend/opencl/sort_index.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void sort_index(Array<T> &okey, Array<unsigned> &oval, const Array<T> &in,
                 const unsigned dim, bool isAscending);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sparse.cpp b/src/backend/opencl/sparse.cpp
index 580822d5d1..de220563f7 100644
--- a/src/backend/opencl/sparse.cpp
+++ b/src/backend/opencl/sparse.cpp
@@ -26,6 +26,7 @@
 #include <stdexcept>
 #include <string>
 
+namespace arrayfire {
 namespace opencl {
 
 using namespace common;
@@ -217,3 +218,4 @@ INSTANTIATE_SPARSE(cdouble)
 #undef INSTANTIATE_SPARSE
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sparse.hpp b/src/backend/opencl/sparse.hpp
index e8496a533e..32a118df0e 100644
--- a/src/backend/opencl/sparse.hpp
+++ b/src/backend/opencl/sparse.hpp
@@ -12,6 +12,7 @@
 #include <Array.hpp>
 #include <common/SparseArray.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, af_storage stype>
@@ -25,3 +26,4 @@ common::SparseArray<T> sparseConvertStorageToStorage(
     const common::SparseArray<T> &in);
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sparse_arith.cpp b/src/backend/opencl/sparse_arith.cpp
index 5de05b873a..cfc868b0a6 100644
--- a/src/backend/opencl/sparse_arith.cpp
+++ b/src/backend/opencl/sparse_arith.cpp
@@ -24,6 +24,7 @@
 #include <scan.hpp>
 #include <where.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 using namespace common;
@@ -174,3 +175,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sparse_arith.hpp b/src/backend/opencl/sparse_arith.hpp
index c0ac32c180..3d45738c76 100644
--- a/src/backend/opencl/sparse_arith.hpp
+++ b/src/backend/opencl/sparse_arith.hpp
@@ -12,6 +12,7 @@
 #include <optypes.hpp>
 #include <sparse.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 // These two functions cannot be overloaded by return type.
@@ -28,3 +29,4 @@ template<typename T, af_op_t op>
 common::SparseArray<T> arithOp(const common::SparseArray<T> &lhs,
                                const common::SparseArray<T> &rhs);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sparse_blas.cpp b/src/backend/opencl/sparse_blas.cpp
index 4b214e821e..42b6547127 100644
--- a/src/backend/opencl/sparse_blas.cpp
+++ b/src/backend/opencl/sparse_blas.cpp
@@ -30,6 +30,7 @@
 #include <cpu/cpu_sparse_blas.hpp>
 #endif  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace opencl {
 
 using namespace common;
@@ -96,3 +97,4 @@ INSTANTIATE_SPARSE(cfloat)
 INSTANTIATE_SPARSE(cdouble)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sparse_blas.hpp b/src/backend/opencl/sparse_blas.hpp
index 788fe3fd3c..f51eeac9b4 100644
--- a/src/backend/opencl/sparse_blas.hpp
+++ b/src/backend/opencl/sparse_blas.hpp
@@ -11,10 +11,12 @@
 #include <common/SparseArray.hpp>
 #include <sparse.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhs,
                 af_mat_prop optLhs, af_mat_prop optRhs);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/sum.cpp b/src/backend/opencl/sum.cpp
index fc02b072c9..890280ba92 100644
--- a/src/backend/opencl/sum.cpp
+++ b/src/backend/opencl/sum.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 // sum
 INSTANTIATE(af_add_t, float, float)
@@ -37,3 +38,4 @@ INSTANTIATE(af_add_t, ushort, float)
 INSTANTIATE(af_add_t, half, half)
 INSTANTIATE(af_add_t, half, float)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/surface.cpp b/src/backend/opencl/surface.cpp
index d1ab53196d..a0de95fb19 100644
--- a/src/backend/opencl/surface.cpp
+++ b/src/backend/opencl/surface.cpp
@@ -14,14 +14,17 @@
 #include <surface.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 using cl::Memory;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_surface(const Array<T> &P, fg_surface surface) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = forgePlugin();
     if (isGLSharingSupported()) {
         CheckGL("Begin OpenCL resource copy");
         const cl::Buffer *d_P = P.get();
@@ -78,3 +81,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/surface.hpp b/src/backend/opencl/surface.hpp
index 6eedbfec66..62a1095a84 100644
--- a/src/backend/opencl/surface.hpp
+++ b/src/backend/opencl/surface.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_surface(const Array<T> &P, fg_surface surface);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/susan.cpp b/src/backend/opencl/susan.cpp
index 35f22a953b..6bd78e2540 100644
--- a/src/backend/opencl/susan.cpp
+++ b/src/backend/opencl/susan.cpp
@@ -17,6 +17,7 @@
 using af::features;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -70,3 +71,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/susan.hpp b/src/backend/opencl/susan.hpp
index a82fa4418b..ca6c779c8a 100644
--- a/src/backend/opencl/susan.hpp
+++ b/src/backend/opencl/susan.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -21,4 +22,5 @@ unsigned susan(Array<float> &x_out, Array<float> &y_out,
                const float geom_thr, const float feature_ratio,
                const unsigned edge);
 
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/svd.cpp b/src/backend/opencl/svd.cpp
index 5c7aed92c4..7bda5306ca 100644
--- a/src/backend/opencl/svd.cpp
+++ b/src/backend/opencl/svd.cpp
@@ -24,6 +24,7 @@
 #include <magma/magma_helper.h>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename Tr>
@@ -231,9 +232,11 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T, typename Tr>
@@ -258,5 +261,6 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace opencl
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/opencl/svd.hpp b/src/backend/opencl/svd.hpp
index 6dd4eb6dc6..ddf3f4a1bb 100644
--- a/src/backend/opencl/svd.hpp
+++ b/src/backend/opencl/svd.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T, typename Tr>
 void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
@@ -16,3 +17,4 @@ void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
 template<typename T, typename Tr>
 void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/threadsMgt.hpp b/src/backend/opencl/threadsMgt.hpp
index 4fb3838e5b..1fdc136613 100644
--- a/src/backend/opencl/threadsMgt.hpp
+++ b/src/backend/opencl/threadsMgt.hpp
@@ -13,6 +13,7 @@
 #include <platform.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace opencl {
 // OVERALL USAGE (With looping):
 // ...                                                      // OWN CODE
@@ -325,4 +326,5 @@ inline cl::NDRange threadsMgt<T>::genGlobal(const cl::NDRange& local) const {
         return genGlobalFull(local);
     }
 };
-}  // namespace opencl
\ No newline at end of file
+}  // namespace opencl
+}  // namespace arrayfire
\ No newline at end of file
diff --git a/src/backend/opencl/tile.cpp b/src/backend/opencl/tile.cpp
index c3e2604970..14e2d5beac 100644
--- a/src/backend/opencl/tile.cpp
+++ b/src/backend/opencl/tile.cpp
@@ -13,8 +13,9 @@
 #include <common/half.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> tile(const Array<T> &in, const af::dim4 &tileDims) {
@@ -47,3 +48,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/tile.hpp b/src/backend/opencl/tile.hpp
index 8326b034e2..172cbadbed 100644
--- a/src/backend/opencl/tile.hpp
+++ b/src/backend/opencl/tile.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> tile(const Array<T> &in, const af::dim4 &tileDims);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/topk.cpp b/src/backend/opencl/topk.cpp
index 5fcf157946..9ff966ed65 100644
--- a/src/backend/opencl/topk.cpp
+++ b/src/backend/opencl/topk.cpp
@@ -20,9 +20,9 @@
 #include <numeric>
 #include <vector>
 
+using arrayfire::common::half;
 using cl::Buffer;
 using cl::Event;
-using common::half;
 
 using std::iota;
 using std::min;
@@ -30,6 +30,7 @@ using std::partial_sort_copy;
 using std::transform;
 using std::vector;
 
+namespace arrayfire {
 namespace opencl {
 vector<af_index_t> indexForTopK(const int k) {
     af_index_t idx;
@@ -177,3 +178,4 @@ INSTANTIATE(long long)
 INSTANTIATE(unsigned long long)
 INSTANTIATE(half)
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/topk.hpp b/src/backend/opencl/topk.hpp
index 5767d8a0d2..d4c67878e7 100644
--- a/src/backend/opencl/topk.hpp
+++ b/src/backend/opencl/topk.hpp
@@ -7,8 +7,14 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <Array.hpp>
+
+#include <af/defines.h>
+
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void topk(Array<T>& keys, Array<unsigned>& vals, const Array<T>& in,
           const int k, const int dim, const af::topkFunction order);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/traits.hpp b/src/backend/opencl/traits.hpp
index 6610c7aee1..00af1d17b0 100644
--- a/src/backend/opencl/traits.hpp
+++ b/src/backend/opencl/traits.hpp
@@ -19,36 +19,40 @@
 namespace af {
 
 template<>
-struct dtype_traits<opencl::cfloat> {
+struct dtype_traits<arrayfire::opencl::cfloat> {
     enum { af_type = c32 };
     typedef float base_type;
     static const char *getName() { return "float2"; }
 };
 
 template<>
-struct dtype_traits<opencl::cdouble> {
+struct dtype_traits<arrayfire::opencl::cdouble> {
     enum { af_type = c64 };
     typedef double base_type;
     static const char *getName() { return "double2"; }
 };
+}  // namespace af
+
+namespace arrayfire {
+namespace opencl {
 
 template<typename T>
 static bool iscplx() {
     return false;
 }
 template<>
-inline bool iscplx<opencl::cfloat>() {
+inline bool iscplx<cfloat>() {
     return true;
 }
 template<>
-inline bool iscplx<opencl::cdouble>() {
+inline bool iscplx<cdouble>() {
     return true;
 }
 
 template<typename T>
 inline std::string scalar_to_option(const T &val) {
-    using namespace common;
-    using namespace std;
+    using namespace arrayfire::common;
+    using std::to_string;
     return to_string(+val);
 }
 
@@ -65,6 +69,7 @@ inline std::string scalar_to_option<cl_double2>(const cl_double2 &val) {
     ss << val.s[0] << "," << val.s[1];
     return ss.str();
 }
-}  // namespace af
 
 using af::dtype_traits;
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/transform.cpp b/src/backend/opencl/transform.cpp
index 253ff6ccb4..14ee03c962 100644
--- a/src/backend/opencl/transform.cpp
+++ b/src/backend/opencl/transform.cpp
@@ -11,6 +11,7 @@
 
 #include <kernel/transform.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -54,3 +55,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/transform.hpp b/src/backend/opencl/transform.hpp
index 809294fc6f..50c1455be0 100644
--- a/src/backend/opencl/transform.hpp
+++ b/src/backend/opencl/transform.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const af_interp_type method, const bool inverse,
                const bool perspective);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/transpose.cpp b/src/backend/opencl/transpose.cpp
index 819e73fb29..a25fa9be28 100644
--- a/src/backend/opencl/transpose.cpp
+++ b/src/backend/opencl/transpose.cpp
@@ -14,8 +14,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -50,3 +51,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/transpose.hpp b/src/backend/opencl/transpose.hpp
index f9d363f11b..7bb1f66bbf 100644
--- a/src/backend/opencl/transpose.hpp
+++ b/src/backend/opencl/transpose.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 void transpose_inplace(Array<T> &in, const bool conjugate);
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/transpose_inplace.cpp b/src/backend/opencl/transpose_inplace.cpp
index 4ee4a740cd..dc23873814 100644
--- a/src/backend/opencl/transpose_inplace.cpp
+++ b/src/backend/opencl/transpose_inplace.cpp
@@ -14,8 +14,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -46,3 +47,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/triangle.cpp b/src/backend/opencl/triangle.cpp
index 9713c906c8..cb781eeef4 100644
--- a/src/backend/opencl/triangle.cpp
+++ b/src/backend/opencl/triangle.cpp
@@ -14,8 +14,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -52,3 +53,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/triangle.hpp b/src/backend/opencl/triangle.hpp
index d616337c7e..51061d51b8 100644
--- a/src/backend/opencl/triangle.hpp
+++ b/src/backend/opencl/triangle.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 void triangle(Array<T> &out, const Array<T> &in, const bool is_upper,
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> triangle(const Array<T> &in, const bool is_upper,
                   const bool is_unit_diag);
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/types.cpp b/src/backend/opencl/types.cpp
index aba15fe693..35c2b5745a 100644
--- a/src/backend/opencl/types.cpp
+++ b/src/backend/opencl/types.cpp
@@ -17,12 +17,13 @@
 #include <sstream>
 #include <string>
 
-using common::half;
-using common::toString;
+using arrayfire::common::half;
+using arrayfire::common::toString;
 
 using std::isinf;
 using std::stringstream;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -101,3 +102,4 @@ INSTANTIATE(half);
 #undef INSTANTIATE
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/types.hpp b/src/backend/opencl/types.hpp
index e88086b262..2bc96996aa 100644
--- a/src/backend/opencl/types.hpp
+++ b/src/backend/opencl/types.hpp
@@ -18,6 +18,7 @@
 #include <array>
 #include <string>
 
+namespace arrayfire {
 namespace common {
 /// This is a CPU based half which need to be converted into floats before they
 /// are used
@@ -31,7 +32,9 @@ struct kernel_type<common::half> {
     using compute = float;
 };
 }  // namespace common
+}  // namespace arrayfire
 
+namespace arrayfire {
 namespace opencl {
 using cdouble = cl_double2;
 using cfloat  = cl_float2;
@@ -127,7 +130,7 @@ inline const char *getFullName<cdouble>() {
 
 template<typename... ARGS>
 AF_CONSTEXPR const char *getTypeBuildDefinition() {
-    using common::half;
+    using arrayfire::common::half;
     using std::any_of;
     using std::array;
     using std::begin;
@@ -157,3 +160,4 @@ AF_CONSTEXPR const char *getTypeBuildDefinition() {
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/unary.hpp b/src/backend/opencl/unary.hpp
index 65da1b690b..9ff2fea8c6 100644
--- a/src/backend/opencl/unary.hpp
+++ b/src/backend/opencl/unary.hpp
@@ -13,6 +13,7 @@
 #include <math.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<af_op_t op>
@@ -77,8 +78,8 @@ UNARY_DECL(bitnot, "__bitnot")
 
 template<typename T, af_op_t op>
 Array<T> unaryOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
-    using common::Node;
-    using common::Node_ptr;
+    using arrayfire::common::Node;
+    using arrayfire::common::Node_ptr;
     using std::array;
 
     auto createUnary = [](array<Node_ptr, 1> &operands) {
@@ -94,7 +95,7 @@ Array<T> unaryOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
 
 template<typename T, af_op_t op>
 Array<char> checkOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
-    using common::Node_ptr;
+    using arrayfire::common::Node_ptr;
 
     auto createUnary = [](std::array<Node_ptr, 1> &operands) {
         return Node_ptr(new common::UnaryNode(
@@ -108,3 +109,4 @@ Array<char> checkOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
 }
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/unwrap.cpp b/src/backend/opencl/unwrap.cpp
index 26c720e3c1..c6c7a12d4f 100644
--- a/src/backend/opencl/unwrap.cpp
+++ b/src/backend/opencl/unwrap.cpp
@@ -14,8 +14,9 @@
 #include <unwrap.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -60,3 +61,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/unwrap.hpp b/src/backend/opencl/unwrap.hpp
index 35b6b617f5..f65e324c67 100644
--- a/src/backend/opencl/unwrap.hpp
+++ b/src/backend/opencl/unwrap.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<T> unwrap(const Array<T> &in, const dim_t wx, const dim_t wy,
                 const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
                 const dim_t dx, const dim_t dy, const bool is_column);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/vector_field.cpp b/src/backend/opencl/vector_field.cpp
index 508ff0ded9..e470f73c9a 100644
--- a/src/backend/opencl/vector_field.cpp
+++ b/src/backend/opencl/vector_field.cpp
@@ -14,13 +14,16 @@
 #include <vector_field.hpp>
 
 using af::dim4;
+using arrayfire::common::ForgeModule;
+using arrayfire::common::forgePlugin;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        fg_vector_field vfield) {
-    ForgeModule &_ = graphics::forgePlugin();
+    ForgeModule &_ = common::forgePlugin();
     if (isGLSharingSupported()) {
         CheckGL("Begin OpenCL resource copy");
         const cl::Buffer *d_points     = points.get();
@@ -101,3 +104,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/vector_field.hpp b/src/backend/opencl/vector_field.hpp
index 2c3447aa4a..33d4d61dff 100644
--- a/src/backend/opencl/vector_field.hpp
+++ b/src/backend/opencl/vector_field.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        fg_vector_field vfield);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/where.cpp b/src/backend/opencl/where.cpp
index 4ad6a870d9..c3ac797454 100644
--- a/src/backend/opencl/where.cpp
+++ b/src/backend/opencl/where.cpp
@@ -14,6 +14,7 @@
 #include <af/dim4.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<uint> where(const Array<T> &in) {
@@ -39,3 +40,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/where.hpp b/src/backend/opencl/where.hpp
index c67a235e66..a5ee5feca4 100644
--- a/src/backend/opencl/where.hpp
+++ b/src/backend/opencl/where.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 template<typename T>
 Array<uint> where(const Array<T>& in);
-}
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/wrap.cpp b/src/backend/opencl/wrap.cpp
index 76847e1988..42d684857a 100644
--- a/src/backend/opencl/wrap.cpp
+++ b/src/backend/opencl/wrap.cpp
@@ -16,8 +16,9 @@
 #include <wrap.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -72,3 +73,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/wrap.hpp b/src/backend/opencl/wrap.hpp
index 7a7815caa1..cceb47ee43 100644
--- a/src/backend/opencl/wrap.hpp
+++ b/src/backend/opencl/wrap.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace opencl {
 
 template<typename T>
@@ -22,3 +23,4 @@ Array<T> wrap_dilated(const Array<T> &in, const dim_t ox, const dim_t oy,
                       const dim_t sy, const dim_t px, const dim_t py,
                       const dim_t dx, const dim_t dy, const bool is_column);
 }  // namespace opencl
+}  // namespace arrayfire

From 5e66211164521f61ce9793abef134d7209d4d6e0 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 31 Dec 2022 14:25:06 -0500
Subject: [PATCH 153/473] Move oneapi namespace into the arrayfire namespace

---
 src/backend/oneapi/Array.cpp                  | 14 +++---
 src/backend/oneapi/Array.hpp                  |  2 +
 src/backend/oneapi/Event.cpp                  |  2 +
 src/backend/oneapi/Event.hpp                  |  2 +
 .../oneapi/GraphicsResourceManager.cpp        |  2 +
 .../oneapi/GraphicsResourceManager.hpp        |  2 +
 src/backend/oneapi/Kernel.hpp                 |  2 +
 src/backend/oneapi/Module.hpp                 |  2 +
 src/backend/oneapi/Param.cpp                  |  2 +
 src/backend/oneapi/Param.hpp                  |  2 +
 src/backend/oneapi/all.cpp                    |  4 +-
 src/backend/oneapi/anisotropic_diffusion.cpp  |  2 +
 src/backend/oneapi/anisotropic_diffusion.hpp  |  4 +-
 src/backend/oneapi/any.cpp                    |  4 +-
 src/backend/oneapi/approx.cpp                 |  2 +
 src/backend/oneapi/approx.hpp                 |  2 +
 src/backend/oneapi/approx1.cpp                |  2 +
 src/backend/oneapi/approx2.cpp                |  2 +
 src/backend/oneapi/arith.hpp                  |  2 +
 src/backend/oneapi/assign.cpp                 |  4 +-
 src/backend/oneapi/assign.hpp                 |  4 +-
 src/backend/oneapi/backend.hpp                |  2 +-
 src/backend/oneapi/bilateral.cpp              |  2 +
 src/backend/oneapi/bilateral.hpp              |  4 +-
 src/backend/oneapi/binary.hpp                 |  2 +
 src/backend/oneapi/blas.cpp                   |  4 +-
 src/backend/oneapi/blas.hpp                   |  2 +
 src/backend/oneapi/canny.cpp                  |  2 +
 src/backend/oneapi/canny.hpp                  |  2 +
 src/backend/oneapi/cast.hpp                   |  2 +
 src/backend/oneapi/cholesky.cpp               |  4 ++
 src/backend/oneapi/cholesky.hpp               |  2 +
 src/backend/oneapi/compile_module.cpp         | 12 +++--
 src/backend/oneapi/complex.hpp                |  2 +
 src/backend/oneapi/convolve.cpp               |  8 ++--
 src/backend/oneapi/convolve.hpp               |  2 +
 src/backend/oneapi/convolve_separable.cpp     |  2 +
 src/backend/oneapi/copy.cpp                   |  6 ++-
 src/backend/oneapi/copy.hpp                   |  2 +
 src/backend/oneapi/count.cpp                  |  4 +-
 src/backend/oneapi/device_manager.cpp         |  7 ++-
 src/backend/oneapi/device_manager.hpp         | 17 ++++---
 src/backend/oneapi/diagonal.cpp               |  4 +-
 src/backend/oneapi/diagonal.hpp               |  2 +
 src/backend/oneapi/diff.cpp                   |  2 +
 src/backend/oneapi/diff.hpp                   |  2 +
 src/backend/oneapi/exampleFunction.cpp        |  2 +
 src/backend/oneapi/exampleFunction.hpp        |  4 +-
 src/backend/oneapi/fast.cpp                   |  2 +
 src/backend/oneapi/fast.hpp                   |  4 +-
 src/backend/oneapi/fft.cpp                    |  2 +
 src/backend/oneapi/fft.hpp                    |  2 +
 src/backend/oneapi/fftconvolve.cpp            |  2 +
 src/backend/oneapi/fftconvolve.hpp            |  4 +-
 src/backend/oneapi/flood_fill.cpp             |  2 +
 src/backend/oneapi/flood_fill.hpp             |  4 +-
 src/backend/oneapi/gradient.cpp               |  2 +
 src/backend/oneapi/gradient.hpp               |  4 +-
 src/backend/oneapi/harris.cpp                 |  2 +
 src/backend/oneapi/harris.hpp                 |  4 +-
 src/backend/oneapi/hist_graphics.cpp          |  2 +
 src/backend/oneapi/hist_graphics.hpp          |  4 +-
 src/backend/oneapi/histogram.cpp              |  4 +-
 src/backend/oneapi/histogram.hpp              |  4 +-
 src/backend/oneapi/homography.cpp             |  2 +
 src/backend/oneapi/homography.hpp             |  4 +-
 src/backend/oneapi/hsv_rgb.cpp                |  2 +
 src/backend/oneapi/hsv_rgb.hpp                |  2 +
 src/backend/oneapi/identity.cpp               |  4 +-
 src/backend/oneapi/identity.hpp               |  4 +-
 src/backend/oneapi/iir.cpp                    |  2 +
 src/backend/oneapi/iir.hpp                    |  4 +-
 src/backend/oneapi/image.cpp                  |  2 +
 src/backend/oneapi/image.hpp                  |  5 +-
 src/backend/oneapi/index.cpp                  |  4 +-
 src/backend/oneapi/index.hpp                  |  4 +-
 src/backend/oneapi/inverse.cpp                |  4 ++
 src/backend/oneapi/inverse.hpp                |  4 +-
 src/backend/oneapi/iota.cpp                   |  4 +-
 src/backend/oneapi/iota.hpp                   |  4 +-
 src/backend/oneapi/ireduce.cpp                |  4 +-
 src/backend/oneapi/ireduce.hpp                |  2 +
 src/backend/oneapi/jit.cpp                    | 10 ++--
 src/backend/oneapi/jit/BufferNode.hpp         |  2 +
 src/backend/oneapi/jit/kernel_generators.hpp  |  2 +
 src/backend/oneapi/join.cpp                   |  4 +-
 src/backend/oneapi/join.hpp                   |  2 +
 src/backend/oneapi/kernel/approx1.hpp         |  2 +
 src/backend/oneapi/kernel/approx2.hpp         |  2 +
 src/backend/oneapi/kernel/assign.hpp          |  2 +
 src/backend/oneapi/kernel/bilateral.hpp       |  2 +
 src/backend/oneapi/kernel/convolve.hpp        |  2 +
 src/backend/oneapi/kernel/default_config.hpp  |  2 +
 src/backend/oneapi/kernel/diagonal.hpp        |  2 +
 src/backend/oneapi/kernel/diff.hpp            |  2 +
 src/backend/oneapi/kernel/histogram.hpp       |  2 +
 src/backend/oneapi/kernel/interp.hpp          |  2 +
 src/backend/oneapi/kernel/iota.hpp            |  2 +
 src/backend/oneapi/kernel/mean.hpp            |  2 +
 src/backend/oneapi/kernel/memcopy.hpp         | 21 +++++----
 src/backend/oneapi/kernel/random_engine.hpp   |  2 +
 .../oneapi/kernel/random_engine_mersenne.hpp  |  2 +
 .../oneapi/kernel/random_engine_philox.hpp    |  2 +
 .../oneapi/kernel/random_engine_threefry.hpp  |  2 +
 .../oneapi/kernel/random_engine_write.hpp     | 46 ++++++++++---------
 src/backend/oneapi/kernel/range.hpp           |  6 ++-
 src/backend/oneapi/kernel/reduce.hpp          |  2 +
 src/backend/oneapi/kernel/reduce_all.hpp      |  2 +
 src/backend/oneapi/kernel/reduce_config.hpp   |  2 +
 src/backend/oneapi/kernel/reduce_dim.hpp      |  2 +
 src/backend/oneapi/kernel/reduce_first.hpp    |  2 +
 src/backend/oneapi/kernel/reorder.hpp         |  2 +
 src/backend/oneapi/kernel/scan_dim.hpp        |  2 +
 src/backend/oneapi/kernel/scan_first.hpp      |  2 +
 src/backend/oneapi/kernel/transpose.hpp       |  2 +
 .../oneapi/kernel/transpose_inplace.hpp       |  2 +
 src/backend/oneapi/kernel/triangle.hpp        |  2 +
 src/backend/oneapi/kernel/unwrap.hpp          |  2 +
 src/backend/oneapi/kernel/where.hpp           |  2 +
 src/backend/oneapi/kernel/wrap.hpp            |  2 +
 src/backend/oneapi/kernel/wrap_dilated.hpp    |  2 +
 src/backend/oneapi/logic.hpp                  |  2 +
 src/backend/oneapi/lookup.cpp                 |  4 +-
 src/backend/oneapi/lookup.hpp                 |  4 +-
 src/backend/oneapi/lu.cpp                     |  4 ++
 src/backend/oneapi/lu.hpp                     |  2 +
 src/backend/oneapi/match_template.cpp         |  2 +
 src/backend/oneapi/match_template.hpp         |  4 +-
 src/backend/oneapi/math.cpp                   |  2 +
 src/backend/oneapi/math.hpp                   | 16 ++++---
 src/backend/oneapi/max.cpp                    |  4 +-
 src/backend/oneapi/mean.cpp                   |  4 +-
 src/backend/oneapi/mean.hpp                   |  2 +
 src/backend/oneapi/meanshift.cpp              |  2 +
 src/backend/oneapi/meanshift.hpp              |  4 +-
 src/backend/oneapi/medfilt.cpp                |  2 +
 src/backend/oneapi/medfilt.hpp                |  2 +
 src/backend/oneapi/memory.cpp                 |  6 ++-
 src/backend/oneapi/memory.hpp                 |  6 ++-
 src/backend/oneapi/min.cpp                    |  4 +-
 src/backend/oneapi/moments.cpp                |  2 +
 src/backend/oneapi/moments.hpp                |  4 +-
 src/backend/oneapi/morph.cpp                  |  2 +
 src/backend/oneapi/morph.hpp                  |  2 +
 src/backend/oneapi/nearest_neighbour.cpp      |  2 +
 src/backend/oneapi/nearest_neighbour.hpp      |  5 +-
 src/backend/oneapi/orb.cpp                    |  2 +
 src/backend/oneapi/orb.hpp                    |  4 +-
 src/backend/oneapi/platform.cpp               | 14 +++---
 src/backend/oneapi/platform.hpp               | 15 +++---
 src/backend/oneapi/plot.cpp                   |  4 +-
 src/backend/oneapi/plot.hpp                   |  4 +-
 src/backend/oneapi/print.hpp                  |  2 +
 src/backend/oneapi/product.cpp                |  4 +-
 src/backend/oneapi/qr.cpp                     |  4 ++
 src/backend/oneapi/qr.hpp                     |  2 +
 src/backend/oneapi/random_engine.cpp          |  4 +-
 src/backend/oneapi/random_engine.hpp          |  2 +
 src/backend/oneapi/range.cpp                  |  4 +-
 src/backend/oneapi/range.hpp                  |  4 +-
 src/backend/oneapi/reduce.hpp                 |  2 +
 src/backend/oneapi/reduce_impl.hpp            |  2 +
 src/backend/oneapi/regions.cpp                |  2 +
 src/backend/oneapi/regions.hpp                |  4 +-
 src/backend/oneapi/reorder.cpp                |  4 +-
 src/backend/oneapi/reorder.hpp                |  4 +-
 src/backend/oneapi/reshape.cpp                |  4 +-
 src/backend/oneapi/resize.cpp                 |  2 +
 src/backend/oneapi/resize.hpp                 |  4 +-
 src/backend/oneapi/rotate.cpp                 |  2 +
 src/backend/oneapi/rotate.hpp                 |  4 +-
 src/backend/oneapi/scalar.hpp                 |  2 +
 src/backend/oneapi/scan.cpp                   |  2 +
 src/backend/oneapi/scan.hpp                   |  4 +-
 src/backend/oneapi/scan_by_key.cpp            |  2 +
 src/backend/oneapi/scan_by_key.hpp            |  4 +-
 src/backend/oneapi/select.cpp                 |  6 ++-
 src/backend/oneapi/select.hpp                 |  2 +
 src/backend/oneapi/set.cpp                    |  2 +
 src/backend/oneapi/set.hpp                    |  2 +
 src/backend/oneapi/shift.cpp                  |  6 ++-
 src/backend/oneapi/shift.hpp                  |  4 +-
 src/backend/oneapi/sift.cpp                   |  2 +
 src/backend/oneapi/sift.hpp                   |  4 +-
 src/backend/oneapi/sobel.cpp                  |  2 +
 src/backend/oneapi/sobel.hpp                  |  4 +-
 src/backend/oneapi/solve.cpp                  |  4 ++
 src/backend/oneapi/solve.hpp                  |  2 +
 src/backend/oneapi/sort.cpp                   |  2 +
 src/backend/oneapi/sort.hpp                   |  4 +-
 src/backend/oneapi/sort_by_key.cpp            |  2 +
 src/backend/oneapi/sort_by_key.hpp            |  4 +-
 src/backend/oneapi/sort_index.cpp             |  4 +-
 src/backend/oneapi/sort_index.hpp             |  4 +-
 src/backend/oneapi/sparse.cpp                 |  2 +
 src/backend/oneapi/sparse.hpp                 |  2 +
 src/backend/oneapi/sparse_arith.cpp           |  2 +
 src/backend/oneapi/sparse_arith.hpp           |  2 +
 src/backend/oneapi/sparse_blas.cpp            |  2 +
 src/backend/oneapi/sparse_blas.hpp            |  4 +-
 src/backend/oneapi/sum.cpp                    |  4 +-
 src/backend/oneapi/surface.cpp                |  4 +-
 src/backend/oneapi/surface.hpp                |  4 +-
 src/backend/oneapi/susan.cpp                  |  2 +
 src/backend/oneapi/susan.hpp                  |  4 +-
 src/backend/oneapi/svd.cpp                    |  4 ++
 src/backend/oneapi/svd.hpp                    |  2 +
 src/backend/oneapi/tile.cpp                   |  4 +-
 src/backend/oneapi/tile.hpp                   |  5 +-
 src/backend/oneapi/topk.cpp                   |  4 +-
 src/backend/oneapi/topk.hpp                   |  5 +-
 src/backend/oneapi/traits.hpp                 |  6 +--
 src/backend/oneapi/transform.cpp              |  2 +
 src/backend/oneapi/transform.hpp              |  4 +-
 src/backend/oneapi/transpose.cpp              |  4 +-
 src/backend/oneapi/transpose.hpp              |  2 +
 src/backend/oneapi/transpose_inplace.cpp      |  4 +-
 src/backend/oneapi/triangle.cpp               |  4 +-
 src/backend/oneapi/triangle.hpp               |  2 +
 src/backend/oneapi/types.hpp                  |  6 ++-
 src/backend/oneapi/unary.hpp                  |  8 ++--
 src/backend/oneapi/unwrap.cpp                 |  4 +-
 src/backend/oneapi/unwrap.hpp                 |  4 +-
 src/backend/oneapi/vector_field.cpp           |  2 +
 src/backend/oneapi/vector_field.hpp           |  4 +-
 src/backend/oneapi/where.cpp                  |  2 +
 src/backend/oneapi/where.hpp                  |  4 +-
 src/backend/oneapi/wrap.cpp                   |  4 +-
 src/backend/oneapi/wrap.hpp                   |  2 +
 229 files changed, 648 insertions(+), 180 deletions(-)
 mode change 100755 => 100644 src/backend/oneapi/convolve.cpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/bilateral.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/convolve.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/histogram.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/interp.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/reorder.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/unwrap.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/wrap.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/wrap_dilated.hpp

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 24330ee3ae..16ab7e5b5a 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -36,11 +36,11 @@
 using af::dim4;
 using af::dtype_traits;
 
-using common::half;
-using common::Node;
-using common::Node_ptr;
-using common::NodeIterator;
-using oneapi::jit::BufferNode;
+using arrayfire::common::half;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::NodeIterator;
+using arrayfire::oneapi::jit::BufferNode;
 
 using nonstd::span;
 using std::accumulate;
@@ -51,6 +51,7 @@ using std::vector;
 
 using sycl::buffer;
 
+namespace arrayfire {
 namespace oneapi {
 namespace {
 template<typename T>
@@ -77,7 +78,7 @@ void verifyTypeSupport<cdouble>() {
 }
 
 template<>
-void verifyTypeSupport<common::half>() {
+void verifyTypeSupport<arrayfire::common::half>() {
     if (!isHalfSupported(getActiveDeviceId())) {
         AF_ERROR("Half precision not supported", AF_ERR_NO_HALF);
     }
@@ -575,3 +576,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index ae7234fb02..c3e0d38b98 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -35,6 +35,7 @@ template<typename T>
 class SparseArray;
 }
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -326,3 +327,4 @@ class Array {
 };
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/Event.cpp b/src/backend/oneapi/Event.cpp
index a86d74f8ab..056c6cf950 100644
--- a/src/backend/oneapi/Event.cpp
+++ b/src/backend/oneapi/Event.cpp
@@ -20,6 +20,7 @@
 using std::make_unique;
 using std::unique_ptr;
 
+namespace arrayfire {
 namespace oneapi {
 /// \brief Creates a new event and marks it in the queue
 Event makeEvent(sycl::queue& queue) {
@@ -76,3 +77,4 @@ af_event createAndMarkEvent() {
 }
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/Event.hpp b/src/backend/oneapi/Event.hpp
index ff600ebbcb..1bdedf34ad 100644
--- a/src/backend/oneapi/Event.hpp
+++ b/src/backend/oneapi/Event.hpp
@@ -12,6 +12,7 @@
 #include <common/EventBase.hpp>
 #include <af/event.h>
 
+namespace arrayfire {
 namespace oneapi {
 class OneAPIEventPolicy {
    public:
@@ -62,3 +63,4 @@ void block(af_event eventHandle);
 af_event createAndMarkEvent();
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/GraphicsResourceManager.cpp b/src/backend/oneapi/GraphicsResourceManager.cpp
index 8cf078e8be..cb03ce0a4f 100644
--- a/src/backend/oneapi/GraphicsResourceManager.cpp
+++ b/src/backend/oneapi/GraphicsResourceManager.cpp
@@ -10,6 +10,7 @@
 #include <GraphicsResourceManager.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 GraphicsResourceManager::ShrdResVector
 GraphicsResourceManager::registerResources(
@@ -18,3 +19,4 @@ GraphicsResourceManager::registerResources(
     return output;
 }
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/GraphicsResourceManager.hpp b/src/backend/oneapi/GraphicsResourceManager.hpp
index 6374f1ef7e..1f19c6f8c0 100644
--- a/src/backend/oneapi/GraphicsResourceManager.hpp
+++ b/src/backend/oneapi/GraphicsResourceManager.hpp
@@ -15,6 +15,7 @@
 #include <memory>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 class GraphicsResourceManager
     : public common::InteropManager<GraphicsResourceManager, std::byte> {
@@ -30,3 +31,4 @@ class GraphicsResourceManager
     void operator=(GraphicsResourceManager const&);
 };
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/Kernel.hpp b/src/backend/oneapi/Kernel.hpp
index 704237de24..e36e202387 100644
--- a/src/backend/oneapi/Kernel.hpp
+++ b/src/backend/oneapi/Kernel.hpp
@@ -16,6 +16,7 @@
 #include <backend.hpp>
 #include <string>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel_logger {
 inline auto getLogger() -> spdlog::logger* {
@@ -91,3 +92,4 @@ class Kernel {
 };
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/Module.hpp b/src/backend/oneapi/Module.hpp
index 5637fa5d06..c4de202761 100644
--- a/src/backend/oneapi/Module.hpp
+++ b/src/backend/oneapi/Module.hpp
@@ -12,6 +12,7 @@
 #include <CL/sycl.hpp>
 #include <common/ModuleInterface.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 /// oneapi backend wrapper for cl::Program object
@@ -39,3 +40,4 @@ class Module
 };
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/Param.cpp b/src/backend/oneapi/Param.cpp
index 87a539ce67..6528f707f4 100644
--- a/src/backend/oneapi/Param.cpp
+++ b/src/backend/oneapi/Param.cpp
@@ -12,6 +12,7 @@
 #include <platform.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -28,3 +29,4 @@ Param<T> makeParam(sycl::buffer<T> &mem, int off, const int dims[4],
 }
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index 4a0d6ff9cc..01088f86b7 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -12,6 +12,7 @@
 #include <CL/sycl.hpp>
 #include <kernel/KParam.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -36,3 +37,4 @@ template<typename T>
 Param<T> makeParam(sycl::buffer<T>& mem, int off, const int dims[4],
                    const int strides[4]);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/all.cpp b/src/backend/oneapi/all.cpp
index e74df9806c..ad09e4aff1 100644
--- a/src/backend/oneapi/all.cpp
+++ b/src/backend/oneapi/all.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 // alltrue
 INSTANTIATE(af_and_t, float, char)
@@ -28,3 +29,4 @@ INSTANTIATE(af_and_t, short, char)
 INSTANTIATE(af_and_t, ushort, char)
 INSTANTIATE(af_and_t, half, char)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/anisotropic_diffusion.cpp b/src/backend/oneapi/anisotropic_diffusion.cpp
index a68b8aaa8f..912ee6d986 100644
--- a/src/backend/oneapi/anisotropic_diffusion.cpp
+++ b/src/backend/oneapi/anisotropic_diffusion.cpp
@@ -13,6 +13,7 @@
 #include <err_oneapi.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
@@ -29,3 +30,4 @@ void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
 INSTANTIATE(double)
 INSTANTIATE(float)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/anisotropic_diffusion.hpp b/src/backend/oneapi/anisotropic_diffusion.hpp
index e71d8928ef..71ed5a9bc4 100644
--- a/src/backend/oneapi/anisotropic_diffusion.hpp
+++ b/src/backend/oneapi/anisotropic_diffusion.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void anisotropicDiffusion(Array<T>& inout, const float dt, const float mct,
                           const af::fluxFunction fftype,
                           const af::diffusionEq eq);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/any.cpp b/src/backend/oneapi/any.cpp
index 3a3e62431f..bdf600e9a9 100644
--- a/src/backend/oneapi/any.cpp
+++ b/src/backend/oneapi/any.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 // anytrue
 INSTANTIATE(af_or_t, float, char)
@@ -28,3 +29,4 @@ INSTANTIATE(af_or_t, short, char)
 INSTANTIATE(af_or_t, ushort, char)
 INSTANTIATE(af_or_t, half, char)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/approx.cpp b/src/backend/oneapi/approx.cpp
index 4ad0c27d9b..825c9072fb 100644
--- a/src/backend/oneapi/approx.cpp
+++ b/src/backend/oneapi/approx.cpp
@@ -12,6 +12,7 @@
 #include <kernel/approx1.hpp>
 #include <kernel/approx2.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
@@ -84,3 +85,4 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/approx.hpp b/src/backend/oneapi/approx.hpp
index 68d06967eb..b895dac8aa 100644
--- a/src/backend/oneapi/approx.hpp
+++ b/src/backend/oneapi/approx.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
@@ -22,3 +23,4 @@ void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
              const Tp &yi_step, const af_interp_type method,
              const float offGrid);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/approx1.cpp b/src/backend/oneapi/approx1.cpp
index 8906f57016..0271d0a4ed 100644
--- a/src/backend/oneapi/approx1.cpp
+++ b/src/backend/oneapi/approx1.cpp
@@ -10,6 +10,7 @@
 #include <err_oneapi.hpp>
 #include <kernel/approx1.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename Ty, typename Tp>
 void approx1(Array<Ty> &yo, const Array<Ty> &yi, const Array<Tp> &xo,
@@ -47,3 +48,4 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/approx2.cpp b/src/backend/oneapi/approx2.cpp
index 3330aaa42f..e491a5be5e 100644
--- a/src/backend/oneapi/approx2.cpp
+++ b/src/backend/oneapi/approx2.cpp
@@ -10,6 +10,7 @@
 #include <err_oneapi.hpp>
 #include <kernel/approx2.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename Ty, typename Tp>
 void approx2(Array<Ty> &zo, const Array<Ty> &zi, const Array<Tp> &xo,
@@ -54,3 +55,4 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/arith.hpp b/src/backend/oneapi/arith.hpp
index 1311d4d607..8f31a5383e 100644
--- a/src/backend/oneapi/arith.hpp
+++ b/src/backend/oneapi/arith.hpp
@@ -15,6 +15,7 @@
 #include <optypes.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, af_op_t op>
@@ -31,3 +32,4 @@ Array<T> arithOp(const Array<T> &lhs, const Array<T> &rhs,
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/assign.cpp b/src/backend/oneapi/assign.cpp
index 5517793411..0f2b96e5d5 100644
--- a/src/backend/oneapi/assign.cpp
+++ b/src/backend/oneapi/assign.cpp
@@ -18,8 +18,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -86,3 +87,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/assign.hpp b/src/backend/oneapi/assign.hpp
index 7cb69fb9f4..cb26fd515b 100644
--- a/src/backend/oneapi/assign.hpp
+++ b/src/backend/oneapi/assign.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/index.h>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/backend.hpp b/src/backend/oneapi/backend.hpp
index 5c805903c5..3366912b3b 100644
--- a/src/backend/oneapi/backend.hpp
+++ b/src/backend/oneapi/backend.hpp
@@ -19,4 +19,4 @@
 #define __DH__
 #endif
 
-namespace detail = oneapi;
+namespace detail = arrayfire::oneapi;
diff --git a/src/backend/oneapi/bilateral.cpp b/src/backend/oneapi/bilateral.cpp
index 75b97d5509..d7d5dd33b9 100644
--- a/src/backend/oneapi/bilateral.cpp
+++ b/src/backend/oneapi/bilateral.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename inType, typename outType>
@@ -39,3 +40,4 @@ INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/bilateral.hpp b/src/backend/oneapi/bilateral.hpp
index 14a221f48f..f88145cd7b 100644
--- a/src/backend/oneapi/bilateral.hpp
+++ b/src/backend/oneapi/bilateral.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename inType, typename outType>
 Array<outType> bilateral(const Array<inType> &in, const float &spatialSigma,
                          const float &chromaticSigma);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/binary.hpp b/src/backend/oneapi/binary.hpp
index b0d02195b6..a9bc4900e8 100644
--- a/src/backend/oneapi/binary.hpp
+++ b/src/backend/oneapi/binary.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename To, typename Ti, af_op_t op>
@@ -125,3 +126,4 @@ struct BinOp<To, Ti, af_hypot_t> {
 };
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
index c8e8d69c98..4a3b5e180d 100644
--- a/src/backend/oneapi/blas.cpp
+++ b/src/backend/oneapi/blas.cpp
@@ -22,8 +22,9 @@
 #include <complex>
 #include <vector>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 void initBlas() { /*gpu_blas_init();*/
@@ -84,3 +85,4 @@ INSTANTIATE_DOT(cdouble)
 INSTANTIATE_DOT(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/blas.hpp b/src/backend/oneapi/blas.hpp
index 7371d4884f..605b3f6d6c 100644
--- a/src/backend/oneapi/blas.hpp
+++ b/src/backend/oneapi/blas.hpp
@@ -13,6 +13,7 @@
 // This file contains the common interface for OneAPI BLAS
 // functions
 
+namespace arrayfire {
 namespace oneapi {
 
 void initBlas();
@@ -39,3 +40,4 @@ template<typename T>
 Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
              af_mat_prop optRhs);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/canny.cpp b/src/backend/oneapi/canny.cpp
index ac85af2e1b..4e9e7fceb2 100644
--- a/src/backend/oneapi/canny.cpp
+++ b/src/backend/oneapi/canny.cpp
@@ -13,6 +13,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 Array<float> nonMaximumSuppression(const Array<float>& mag,
                                    const Array<float>& gx,
@@ -26,3 +27,4 @@ Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
 }
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/canny.hpp b/src/backend/oneapi/canny.hpp
index 25f7f5458b..c9bbe36edd 100644
--- a/src/backend/oneapi/canny.hpp
+++ b/src/backend/oneapi/canny.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 Array<float> nonMaximumSuppression(const Array<float>& mag,
                                    const Array<float>& gx,
@@ -17,3 +18,4 @@ Array<float> nonMaximumSuppression(const Array<float>& mag,
 Array<char> edgeTrackingByHysteresis(const Array<char>& strong,
                                      const Array<char>& weak);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/cast.hpp b/src/backend/oneapi/cast.hpp
index aef3711589..c9b015c4f2 100644
--- a/src/backend/oneapi/cast.hpp
+++ b/src/backend/oneapi/cast.hpp
@@ -18,6 +18,7 @@
 #include <af/dim4.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename To, typename Ti>
@@ -71,3 +72,4 @@ struct CastOp<cdouble, cdouble> {
 #undef CAST_CFN
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/cholesky.cpp b/src/backend/oneapi/cholesky.cpp
index bd6b286654..905a3208c5 100644
--- a/src/backend/oneapi/cholesky.cpp
+++ b/src/backend/oneapi/cholesky.cpp
@@ -15,6 +15,7 @@
 #if defined(WITH_LINEAR_ALGEBRA)
 //#include <triangle.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -40,9 +41,11 @@ INSTANTIATE_CH(double)
 INSTANTIATE_CH(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -66,5 +69,6 @@ INSTANTIATE_CH(double)
 INSTANTIATE_CH(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/cholesky.hpp b/src/backend/oneapi/cholesky.hpp
index d934beb566..ab2bef5cc8 100644
--- a/src/backend/oneapi/cholesky.hpp
+++ b/src/backend/oneapi/cholesky.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
@@ -16,3 +17,4 @@ Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper);
 template<typename T>
 int cholesky_inplace(Array<T> &in, const bool is_upper);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/compile_module.cpp b/src/backend/oneapi/compile_module.cpp
index cc85d37005..39783a3c53 100644
--- a/src/backend/oneapi/compile_module.cpp
+++ b/src/backend/oneapi/compile_module.cpp
@@ -28,12 +28,12 @@
 #include <string>
 #include <vector>
 
-using common::loggerFactory;
+using arrayfire::common::loggerFactory;
+using arrayfire::oneapi::Kernel;
+using arrayfire::oneapi::Module;
 using fmt::format;
-// using oneapi::getActiveDeviceId;
-// using oneapi::getDevice;
-using oneapi::Kernel;
-using oneapi::Module;
+// using arrayfire::oneapi::getActiveDeviceId;
+// using arrayfire::oneapi::getDevice;
 using spdlog::logger;
 using sycl::bundle_state;
 using sycl::kernel_bundle;
@@ -69,6 +69,7 @@ string getProgramBuildLog(const kernel_bundle<bundle_state::executable> &prog) {
 //        AF_ERROR(build_error, AF_ERR_INTERNAL);                      \
 //    } while (0)
 
+namespace arrayfire {
 namespace oneapi {
 
 // const static string DEFAULT_MACROS_STR(
@@ -98,6 +99,7 @@ kernel_bundle<bundle_state::executable> buildProgram(const vector<string>
 */
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 string getKernelCacheFilename(const int device, const string &key) {
     ONEAPI_NOT_SUPPORTED("");
diff --git a/src/backend/oneapi/complex.hpp b/src/backend/oneapi/complex.hpp
index c087959b42..c480fa6474 100644
--- a/src/backend/oneapi/complex.hpp
+++ b/src/backend/oneapi/complex.hpp
@@ -15,6 +15,7 @@
 #include <traits.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename To, typename Ti>
 Array<To> cplx(const Array<Ti> &lhs, const Array<Ti> &rhs,
@@ -88,3 +89,4 @@ Array<T> conj(const Array<T> &in) {
     return createNodeArray<T>(in.dims(), common::Node_ptr(node));
 }
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/convolve.cpp b/src/backend/oneapi/convolve.cpp
old mode 100755
new mode 100644
index a7a2fc9aee..ac940f501d
--- a/src/backend/oneapi/convolve.cpp
+++ b/src/backend/oneapi/convolve.cpp
@@ -24,11 +24,12 @@
 #include <vector>
 
 using af::dim4;
-using common::flip;
-using common::half;
-using common::modDims;
+using arrayfire::common::flip;
+using arrayfire::common::half;
+using arrayfire::common::modDims;
 using std::vector;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename accT>
@@ -173,3 +174,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/convolve.hpp b/src/backend/oneapi/convolve.hpp
index 7fbf2e86a1..6551416170 100644
--- a/src/backend/oneapi/convolve.hpp
+++ b/src/backend/oneapi/convolve.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename accT>
@@ -37,3 +38,4 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
                              const Array<T> &convolved_output, af::dim4 stride,
                              af::dim4 padding, af::dim4 dilation);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/convolve_separable.cpp b/src/backend/oneapi/convolve_separable.cpp
index d9b1e1f64a..969aff66e2 100644
--- a/src/backend/oneapi/convolve_separable.cpp
+++ b/src/backend/oneapi/convolve_separable.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename accT>
@@ -43,3 +44,4 @@ INSTANTIATE(intl, float)
 INSTANTIATE(uintl, float)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index f49689a423..23106f7dd1 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -15,9 +15,10 @@
 #include <kernel/memcopy.hpp>
 #include <math.hpp>
 
-using common::half;
-using common::is_complex;
+using arrayfire::common::half;
+using arrayfire::common::is_complex;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -246,3 +247,4 @@ INSTANTIATE_GETSCALAR(ushort)
 INSTANTIATE_GETSCALAR(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/copy.hpp b/src/backend/oneapi/copy.hpp
index 30d6196aa2..048c89260a 100644
--- a/src/backend/oneapi/copy.hpp
+++ b/src/backend/oneapi/copy.hpp
@@ -11,6 +11,7 @@
 #include <Array.hpp>
 //#include <kernel/pad_array_borders.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void copyData(T *data, const Array<T> &A);
@@ -65,3 +66,4 @@ void multiply_inplace(Array<T> &in, double val);
 template<typename T>
 T getScalar(const Array<T> &in);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/count.cpp b/src/backend/oneapi/count.cpp
index d50f35b694..f8ef354169 100644
--- a/src/backend/oneapi/count.cpp
+++ b/src/backend/oneapi/count.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 // count
 INSTANTIATE(af_notzero_t, float, uint)
@@ -28,3 +29,4 @@ INSTANTIATE(af_notzero_t, short, uint)
 INSTANTIATE(af_notzero_t, ushort, uint)
 INSTANTIATE(af_notzero_t, half, uint)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index 48201b7ebc..54878e3fea 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -30,7 +30,8 @@
 #include <string>
 #include <vector>
 
-using common::getEnvVar;
+using arrayfire::common::ForgeManager;
+using arrayfire::common::getEnvVar;
 using std::begin;
 using std::end;
 using std::find;
@@ -43,6 +44,7 @@ using std::vector;
 using sycl::device;
 using sycl::platform;
 
+namespace arrayfire {
 namespace oneapi {
 
 static inline bool compare_default(const unique_ptr<sycl::device>& ldev,
@@ -68,7 +70,7 @@ DeviceManager::DeviceManager()
             AF_ERR_RUNTIME);
     }
 
-    fgMngr = std::make_unique<graphics::ForgeManager>();
+    fgMngr = std::make_unique<ForgeManager>();
 
     AF_TRACE("Found {} sycl platforms", platforms.size());
     // Iterate through platforms, get all available devices and store them
@@ -223,3 +225,4 @@ void DeviceManager::markDeviceForInterop(const int device,
 }
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
index d84994226c..df14603147 100644
--- a/src/backend/oneapi/device_manager.hpp
+++ b/src/backend/oneapi/device_manager.hpp
@@ -23,18 +23,16 @@ namespace spdlog {
 class logger;
 }
 
-namespace graphics {
-class ForgeManager;
-}
-
+namespace arrayfire {
 namespace common {
-namespace memory {
+class ForgeManager;
 class MemoryManagerBase;
-}
 }  // namespace common
+}  // namespace arrayfire
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 
+namespace arrayfire {
 namespace oneapi {
 
 // opencl namespace forward declarations
@@ -62,7 +60,7 @@ class DeviceManager {
 
     void resetMemoryManagerPinned();
 
-    friend graphics::ForgeManager& forgeManager();
+    friend arrayfire::common::ForgeManager& forgeManager();
 
     friend GraphicsResourceManager& interopManager();
 
@@ -141,7 +139,7 @@ class DeviceManager {
     std::vector<int> mPlatforms;
     unsigned mUserDeviceOffset;
 
-    std::unique_ptr<graphics::ForgeManager> fgMngr;
+    std::unique_ptr<arrayfire::common::ForgeManager> fgMngr;
     std::unique_ptr<MemoryManagerBase> memManager;
     std::unique_ptr<MemoryManagerBase> pinnedMemManager;
     std::unique_ptr<GraphicsResourceManager> gfxManagers[MAX_DEVICES];
@@ -152,3 +150,4 @@ class DeviceManager {
 };
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/diagonal.cpp b/src/backend/oneapi/diagonal.cpp
index b9d443c662..a18d024585 100644
--- a/src/backend/oneapi/diagonal.cpp
+++ b/src/backend/oneapi/diagonal.cpp
@@ -15,8 +15,9 @@
 #include <math.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> diagCreate(const Array<T> &in, const int num) {
@@ -59,3 +60,4 @@ INSTANTIATE_DIAGONAL(ushort)
 INSTANTIATE_DIAGONAL(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/diagonal.hpp b/src/backend/oneapi/diagonal.hpp
index 28b4f46df6..1329cdd9d2 100644
--- a/src/backend/oneapi/diagonal.hpp
+++ b/src/backend/oneapi/diagonal.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> diagCreate(const Array<T> &in, const int num);
@@ -16,3 +17,4 @@ Array<T> diagCreate(const Array<T> &in, const int num);
 template<typename T>
 Array<T> diagExtract(const Array<T> &in, const int num);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/diff.cpp b/src/backend/oneapi/diff.cpp
index ad9da16697..a3c37f6a4a 100644
--- a/src/backend/oneapi/diff.cpp
+++ b/src/backend/oneapi/diff.cpp
@@ -13,6 +13,7 @@
 #include <af/dim4.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -56,3 +57,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(char)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/diff.hpp b/src/backend/oneapi/diff.hpp
index d7f5aaf477..9679f90c59 100644
--- a/src/backend/oneapi/diff.hpp
+++ b/src/backend/oneapi/diff.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> diff1(const Array<T> &in, const int dim);
@@ -16,3 +17,4 @@ Array<T> diff1(const Array<T> &in, const int dim);
 template<typename T>
 Array<T> diff2(const Array<T> &in, const int dim);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/exampleFunction.cpp b/src/backend/oneapi/exampleFunction.cpp
index bc5c52b031..9e6d81e9d5 100644
--- a/src/backend/oneapi/exampleFunction.cpp
+++ b/src/backend/oneapi/exampleFunction.cpp
@@ -24,6 +24,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -64,3 +65,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/exampleFunction.hpp b/src/backend/oneapi/exampleFunction.hpp
index 7f51018f83..5e5978a057 100644
--- a/src/backend/oneapi/exampleFunction.hpp
+++ b/src/backend/oneapi/exampleFunction.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> exampleFunction(const Array<T> &a, const Array<T> &b,
                          const af_someenum_t method);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/fast.cpp b/src/backend/oneapi/fast.cpp
index 25f8c47e6a..cb9ae28d4c 100644
--- a/src/backend/oneapi/fast.cpp
+++ b/src/backend/oneapi/fast.cpp
@@ -15,6 +15,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -42,3 +43,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/fast.hpp b/src/backend/oneapi/fast.hpp
index 19667cf49e..4f9c7cf7f4 100644
--- a/src/backend/oneapi/fast.hpp
+++ b/src/backend/oneapi/fast.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -20,4 +21,5 @@ unsigned fast(Array<float> &x_out, Array<float> &y_out, Array<float> &score_out,
               const bool non_max, const float feature_ratio,
               const unsigned edge);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
index 1591e4b4cf..9ccdcfcb86 100644
--- a/src/backend/oneapi/fft.cpp
+++ b/src/backend/oneapi/fft.cpp
@@ -17,6 +17,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 void setFFTPlanCacheSize(size_t numPlans) {}
@@ -103,3 +104,4 @@ INSTANTIATE(cdouble)
 INSTANTIATE_REAL(float, cfloat)
 INSTANTIATE_REAL(double, cdouble)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/fft.hpp b/src/backend/oneapi/fft.hpp
index 57de589db2..0138970ba9 100644
--- a/src/backend/oneapi/fft.hpp
+++ b/src/backend/oneapi/fft.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 void setFFTPlanCacheSize(size_t numPlans);
@@ -23,3 +24,4 @@ template<typename Tr, typename Tc>
 Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank);
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/fftconvolve.cpp b/src/backend/oneapi/fftconvolve.cpp
index dad10f492e..c4aea5689c 100644
--- a/src/backend/oneapi/fftconvolve.cpp
+++ b/src/backend/oneapi/fftconvolve.cpp
@@ -26,6 +26,7 @@ using std::is_integral;
 using std::is_same;
 using std::vector;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -80,3 +81,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/fftconvolve.hpp b/src/backend/oneapi/fftconvolve.hpp
index 7eac7750aa..88ad3c9b9d 100644
--- a/src/backend/oneapi/fftconvolve.hpp
+++ b/src/backend/oneapi/fftconvolve.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
                      const bool expand, AF_BATCH_KIND kind, const int rank);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/flood_fill.cpp b/src/backend/oneapi/flood_fill.cpp
index a336a441ec..2d9d22d696 100644
--- a/src/backend/oneapi/flood_fill.cpp
+++ b/src/backend/oneapi/flood_fill.cpp
@@ -11,6 +11,7 @@
 
 #include <err_oneapi.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -34,3 +35,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/flood_fill.hpp b/src/backend/oneapi/flood_fill.hpp
index 6590f33e59..00ddce1b70 100644
--- a/src/backend/oneapi/flood_fill.hpp
+++ b/src/backend/oneapi/flood_fill.hpp
@@ -12,10 +12,12 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
                    const Array<uint>& seedsY, const T newValue,
                    const T lowValue, const T highValue,
                    const af::connectivity nlookup = AF_CONNECTIVITY_8);
-}  // namespace oneapi
+}
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/gradient.cpp b/src/backend/oneapi/gradient.cpp
index 40b557a4ae..dc45b67cc6 100644
--- a/src/backend/oneapi/gradient.cpp
+++ b/src/backend/oneapi/gradient.cpp
@@ -14,6 +14,7 @@
 #include <math.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in) {
@@ -29,3 +30,4 @@ INSTANTIATE(double)
 INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/gradient.hpp b/src/backend/oneapi/gradient.hpp
index e5ebff012c..b90fb6ecc7 100644
--- a/src/backend/oneapi/gradient.hpp
+++ b/src/backend/oneapi/gradient.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/harris.cpp b/src/backend/oneapi/harris.cpp
index ef6b844fd4..d266a18bad 100644
--- a/src/backend/oneapi/harris.cpp
+++ b/src/backend/oneapi/harris.cpp
@@ -15,6 +15,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename convAccT>
@@ -38,3 +39,4 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/harris.hpp b/src/backend/oneapi/harris.hpp
index 8eeef1dcc3..eba87bd404 100644
--- a/src/backend/oneapi/harris.hpp
+++ b/src/backend/oneapi/harris.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename convAccT>
@@ -21,4 +22,5 @@ unsigned harris(Array<float> &x_out, Array<float> &y_out,
                 const float sigma, const unsigned filter_len,
                 const float k_thr);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/hist_graphics.cpp b/src/backend/oneapi/hist_graphics.cpp
index 12d9bb2b33..3b280592b1 100644
--- a/src/backend/oneapi/hist_graphics.cpp
+++ b/src/backend/oneapi/hist_graphics.cpp
@@ -12,6 +12,7 @@
 #include <err_oneapi.hpp>
 #include <hist_graphics.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -30,3 +31,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/hist_graphics.hpp b/src/backend/oneapi/hist_graphics.hpp
index 4be3935750..578a9bde70 100644
--- a/src/backend/oneapi/hist_graphics.hpp
+++ b/src/backend/oneapi/hist_graphics.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 void copy_histogram(const Array<T> &data, fg_histogram hist);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/histogram.cpp b/src/backend/oneapi/histogram.cpp
index 62ccd879af..4036a5229b 100644
--- a/src/backend/oneapi/histogram.cpp
+++ b/src/backend/oneapi/histogram.cpp
@@ -15,8 +15,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -50,3 +51,4 @@ INSTANTIATE(uintl)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/histogram.hpp b/src/backend/oneapi/histogram.hpp
index f899faffbe..67be10a0d3 100644
--- a/src/backend/oneapi/histogram.hpp
+++ b/src/backend/oneapi/histogram.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<uint> histogram(const Array<T> &in, const unsigned &nbins,
                       const double &minval, const double &maxval,
                       const bool isLinear);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/homography.cpp b/src/backend/oneapi/homography.cpp
index 5060cd50ae..2bf05ef672 100644
--- a/src/backend/oneapi/homography.cpp
+++ b/src/backend/oneapi/homography.cpp
@@ -19,6 +19,7 @@
 using af::dim4;
 using std::numeric_limits;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -42,3 +43,4 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/homography.hpp b/src/backend/oneapi/homography.hpp
index 6c4e54be66..456b692330 100644
--- a/src/backend/oneapi/homography.hpp
+++ b/src/backend/oneapi/homography.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -18,4 +19,5 @@ int homography(Array<T> &H, const Array<float> &x_src,
                const af_homography_type htype, const float inlier_thr,
                const unsigned iterations);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/hsv_rgb.cpp b/src/backend/oneapi/hsv_rgb.cpp
index 6902f0f6c2..fb9d86b5ec 100644
--- a/src/backend/oneapi/hsv_rgb.cpp
+++ b/src/backend/oneapi/hsv_rgb.cpp
@@ -11,6 +11,7 @@
 
 #include <err_oneapi.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -35,3 +36,4 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/hsv_rgb.hpp b/src/backend/oneapi/hsv_rgb.hpp
index e46da55a80..73abd86410 100644
--- a/src/backend/oneapi/hsv_rgb.hpp
+++ b/src/backend/oneapi/hsv_rgb.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> rgb2hsv(const Array<T>& in);
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/identity.cpp b/src/backend/oneapi/identity.cpp
index ccb633aef2..c7db8e7d44 100644
--- a/src/backend/oneapi/identity.cpp
+++ b/src/backend/oneapi/identity.cpp
@@ -13,8 +13,9 @@
 #include <err_oneapi.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> identity(const dim4& dims) {
@@ -41,3 +42,4 @@ INSTANTIATE_IDENTITY(ushort)
 INSTANTIATE_IDENTITY(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/identity.hpp b/src/backend/oneapi/identity.hpp
index b9fed4aa03..4b1057d04a 100644
--- a/src/backend/oneapi/identity.hpp
+++ b/src/backend/oneapi/identity.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> identity(const dim4& dim);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/iir.cpp b/src/backend/oneapi/iir.cpp
index 9051e34b5f..e0223ca6f1 100644
--- a/src/backend/oneapi/iir.cpp
+++ b/src/backend/oneapi/iir.cpp
@@ -18,6 +18,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x) {
@@ -35,3 +36,4 @@ INSTANTIATE(double)
 INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/iir.hpp b/src/backend/oneapi/iir.hpp
index 6f7d052119..3c50f539ee 100644
--- a/src/backend/oneapi/iir.hpp
+++ b/src/backend/oneapi/iir.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/image.cpp b/src/backend/oneapi/image.cpp
index 8406294a44..723c29fb8b 100644
--- a/src/backend/oneapi/image.cpp
+++ b/src/backend/oneapi/image.cpp
@@ -15,6 +15,7 @@
 #include <stdexcept>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -34,3 +35,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/image.hpp b/src/backend/oneapi/image.hpp
index 5647efea36..6e644a3e48 100644
--- a/src/backend/oneapi/image.hpp
+++ b/src/backend/oneapi/image.hpp
@@ -10,9 +10,10 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 void copy_image(const Array<T> &in, fg_image image);
-
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/index.cpp b/src/backend/oneapi/index.cpp
index 481da0f9ec..03a6b74c56 100644
--- a/src/backend/oneapi/index.cpp
+++ b/src/backend/oneapi/index.cpp
@@ -15,8 +15,9 @@
 #include <memory.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -44,3 +45,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/index.hpp b/src/backend/oneapi/index.hpp
index d8fdb674b5..cebd4c3ea5 100644
--- a/src/backend/oneapi/index.hpp
+++ b/src/backend/oneapi/index.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/index.h>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 Array<T> index(const Array<T>& in, const af_index_t idxrs[]);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/inverse.cpp b/src/backend/oneapi/inverse.cpp
index 079250d4f7..97d91f4db4 100644
--- a/src/backend/oneapi/inverse.cpp
+++ b/src/backend/oneapi/inverse.cpp
@@ -14,6 +14,7 @@
 #if defined(WITH_LINEAR_ALGEBRA)
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -31,9 +32,11 @@ INSTANTIATE(double)
 INSTANTIATE(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -51,5 +54,6 @@ INSTANTIATE(double)
 INSTANTIATE(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #endif
diff --git a/src/backend/oneapi/inverse.hpp b/src/backend/oneapi/inverse.hpp
index 2011950ed1..5b37d94978 100644
--- a/src/backend/oneapi/inverse.hpp
+++ b/src/backend/oneapi/inverse.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> inverse(const Array<T> &in);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/iota.cpp b/src/backend/oneapi/iota.cpp
index 18077e5199..84bf693f1b 100644
--- a/src/backend/oneapi/iota.cpp
+++ b/src/backend/oneapi/iota.cpp
@@ -16,8 +16,9 @@
 
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> iota(const dim4 &dims, const dim4 &tile_dims) {
@@ -50,3 +51,4 @@ INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/iota.hpp b/src/backend/oneapi/iota.hpp
index fe9b1cdf8c..ffce49d1bd 100644
--- a/src/backend/oneapi/iota.hpp
+++ b/src/backend/oneapi/iota.hpp
@@ -10,7 +10,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> iota(const dim4 &dim, const dim4 &tile_dims = dim4(1));
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/ireduce.cpp b/src/backend/oneapi/ireduce.cpp
index cf97ad3a4a..6cca678b20 100644
--- a/src/backend/oneapi/ireduce.cpp
+++ b/src/backend/oneapi/ireduce.cpp
@@ -16,8 +16,9 @@
 #include <complex>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<af_op_t op, typename T>
@@ -76,3 +77,4 @@ INSTANTIATE(af_max_t, short)
 INSTANTIATE(af_max_t, ushort)
 INSTANTIATE(af_max_t, half)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/ireduce.hpp b/src/backend/oneapi/ireduce.hpp
index 3ae1b6c476..99a1e45aac 100644
--- a/src/backend/oneapi/ireduce.hpp
+++ b/src/backend/oneapi/ireduce.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<af_op_t op, typename T>
 void ireduce(Array<T> &out, Array<uint> &loc, const Array<T> &in,
@@ -22,3 +23,4 @@ void rreduce(Array<T> &out, Array<uint> &loc, const Array<T> &in, const int dim,
 template<af_op_t op, typename T>
 T ireduce_all(unsigned *loc, const Array<T> &in);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index c957c86c1d..3233f97430 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -29,16 +29,17 @@
 #include <string>
 #include <vector>
 
-using common::getFuncName;
-using common::Node;
-using common::Node_ids;
-using common::Node_map_t;
+using arrayfire::common::getFuncName;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ids;
+using arrayfire::common::Node_map_t;
 
 using std::string;
 using std::stringstream;
 using std::to_string;
 using std::vector;
 
+namespace arrayfire {
 namespace oneapi {
 
 string getKernelString(const string &funcName, const vector<Node *> &full_nodes,
@@ -69,3 +70,4 @@ void evalNodes(Param &out, Node *node) {
 */
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/jit/BufferNode.hpp b/src/backend/oneapi/jit/BufferNode.hpp
index 9925ec7211..5f8ead77e0 100644
--- a/src/backend/oneapi/jit/BufferNode.hpp
+++ b/src/backend/oneapi/jit/BufferNode.hpp
@@ -12,6 +12,7 @@
 
 #include <memory>
 
+namespace arrayfire {
 namespace oneapi {
 namespace jit {
 template<typename T>
@@ -33,3 +34,4 @@ bool BufferNodeBase<DataType, ParamType>::operator==(
 }
 
 }  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
index 202403f4cb..b3753955b9 100644
--- a/src/backend/oneapi/jit/kernel_generators.hpp
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -13,6 +13,7 @@
 
 #include <err_oneapi.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 namespace {
@@ -112,3 +113,4 @@ inline void generateShiftNodeRead(std::stringstream& kerStream, int id,
 }
 }  // namespace
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/join.cpp b/src/backend/oneapi/join.cpp
index a645ea56f5..9e8aa2f743 100644
--- a/src/backend/oneapi/join.cpp
+++ b/src/backend/oneapi/join.cpp
@@ -17,10 +17,11 @@
 #include <vector>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using std::transform;
 using std::vector;
 
+namespace arrayfire {
 namespace oneapi {
 dim4 calcOffset(const dim4 &dims, int dim) {
     dim4 offset;
@@ -89,3 +90,4 @@ INSTANTIATE(half)
 
 #undef INSTANTIATE
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/join.hpp b/src/backend/oneapi/join.hpp
index 25763f063e..818047cae2 100644
--- a/src/backend/oneapi/join.hpp
+++ b/src/backend/oneapi/join.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
@@ -16,3 +17,4 @@ Array<T> join(const int dim, const Array<T> &first, const Array<T> &second);
 template<typename T>
 void join(Array<T> &out, const int dim, const std::vector<Array<T>> &inputs);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/approx1.hpp b/src/backend/oneapi/kernel/approx1.hpp
index 95b4ceb65c..f520719749 100644
--- a/src/backend/oneapi/kernel/approx1.hpp
+++ b/src/backend/oneapi/kernel/approx1.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -163,3 +164,4 @@ void approx1(Param<Ty> yo, const Param<Ty> yi, const Param<Tp> xo,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/approx2.hpp b/src/backend/oneapi/kernel/approx2.hpp
index 94b2f7060c..5b7e509f9b 100644
--- a/src/backend/oneapi/kernel/approx2.hpp
+++ b/src/backend/oneapi/kernel/approx2.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -192,3 +193,4 @@ void approx2(Param<Ty> zo, const Param<Ty> zi, const Param<Tp> xo,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index d4cc7e2b6c..162c1d5254 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -17,6 +17,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -144,3 +145,4 @@ void assign(Param<T> out, const Param<T> in, const AssignKernelParam_t& p,
 }
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/bilateral.hpp b/src/backend/oneapi/kernel/bilateral.hpp
old mode 100755
new mode 100644
index aba8b93d87..3814084c1b
--- a/src/backend/oneapi/kernel/bilateral.hpp
+++ b/src/backend/oneapi/kernel/bilateral.hpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -215,3 +216,4 @@ void bilateral(Param<outType> out, const Param<inType> in, const float s_sigma,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/convolve.hpp b/src/backend/oneapi/kernel/convolve.hpp
old mode 100755
new mode 100644
index 39abe603ad..9f868ce729
--- a/src/backend/oneapi/kernel/convolve.hpp
+++ b/src/backend/oneapi/kernel/convolve.hpp
@@ -17,6 +17,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -143,3 +144,4 @@ void convolve_nd(Param<T> out, const Param<T> signal, const Param<aT> filter,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/default_config.hpp b/src/backend/oneapi/kernel/default_config.hpp
index c279fd98bb..c2ed8ae3dc 100644
--- a/src/backend/oneapi/kernel/default_config.hpp
+++ b/src/backend/oneapi/kernel/default_config.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -19,3 +20,4 @@ static const uint REPEAT            = 32;
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/diagonal.hpp b/src/backend/oneapi/kernel/diagonal.hpp
index 4668fee5bd..a21c1abd11 100644
--- a/src/backend/oneapi/kernel/diagonal.hpp
+++ b/src/backend/oneapi/kernel/diagonal.hpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -161,3 +162,4 @@ static void diagExtract(Param<T> out, Param<T> in, int num) {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/diff.hpp b/src/backend/oneapi/kernel/diff.hpp
index d624cd5283..bd3d925d3b 100644
--- a/src/backend/oneapi/kernel/diff.hpp
+++ b/src/backend/oneapi/kernel/diff.hpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -122,3 +123,4 @@ void diff(Param<T> out, const Param<T> in, const unsigned indims,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/histogram.hpp b/src/backend/oneapi/kernel/histogram.hpp
old mode 100755
new mode 100644
index bc9f74f88c..99ee437ae3
--- a/src/backend/oneapi/kernel/histogram.hpp
+++ b/src/backend/oneapi/kernel/histogram.hpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -165,3 +166,4 @@ void histogram(Param<uint> out, const Param<T> in, int nbins, float minval,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
old mode 100755
new mode 100644
index 6f43fb52f2..af430ca031
--- a/src/backend/oneapi/kernel/interp.hpp
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -13,6 +13,7 @@
 #include <af/constants.h>
 #include <algorithm>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -341,3 +342,4 @@ struct Interp2<Ty, Tp, 3> {
 };
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index ee0b16d23a..956bbc401a 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -118,3 +119,4 @@ void iota(Param<T> out, const af::dim4& sdims) {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 17d2eb2164..d0361a18dc 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -25,6 +25,7 @@
 #include <memory>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 
 /*
@@ -789,3 +790,4 @@ To mean_all(Param<Ti> in) {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 2bb2443cb2..efe577c9ce 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -22,6 +22,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -141,25 +142,28 @@ outType convertType(inType value) {
 }
 
 template<>
-char convertType<compute_t<common::half>, char>(compute_t<common::half> value) {
+char convertType<compute_t<arrayfire::common::half>, char>(
+    compute_t<arrayfire::common::half> value) {
     return (char)((short)value);
 }
 
 template<>
-compute_t<common::half> convertType<char, compute_t<common::half>>(char value) {
-    return compute_t<common::half>(value);
+compute_t<arrayfire::common::half>
+convertType<char, compute_t<arrayfire::common::half>>(char value) {
+    return compute_t<arrayfire::common::half>(value);
 }
 
 template<>
-unsigned char convertType<compute_t<common::half>, unsigned char>(
-    compute_t<common::half> value) {
+unsigned char convertType<compute_t<arrayfire::common::half>, unsigned char>(
+    compute_t<arrayfire::common::half> value) {
     return (unsigned char)((short)value);
 }
 
 template<>
-compute_t<common::half> convertType<unsigned char, compute_t<common::half>>(
+compute_t<arrayfire::common::half>
+convertType<unsigned char, compute_t<arrayfire::common::half>>(
     unsigned char value) {
-    return compute_t<common::half>(value);
+    return compute_t<arrayfire::common::half>(value);
 }
 
 template<>
@@ -193,7 +197,7 @@ OTHER_SPECIALIZATIONS(short)
 OTHER_SPECIALIZATIONS(ushort)
 OTHER_SPECIALIZATIONS(uchar)
 OTHER_SPECIALIZATIONS(char)
-OTHER_SPECIALIZATIONS(common::half)
+OTHER_SPECIALIZATIONS(arrayfire::common::half)
 
 template<typename inType, typename outType, bool SAMEDIMS>
 class reshapeCopy {
@@ -320,3 +324,4 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/random_engine.hpp b/src/backend/oneapi/kernel/random_engine.hpp
index 8c9b5e9251..d86700a7fb 100644
--- a/src/backend/oneapi/kernel/random_engine.hpp
+++ b/src/backend/oneapi/kernel/random_engine.hpp
@@ -30,6 +30,7 @@ static const int TABLE_SIZE = 16;
 static const int MAX_BLOCKS = 32;
 static const int STATE_SIZE = (256 * 3);
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -202,3 +203,4 @@ void normalDistributionMT(Param<T> out, const size_t elements,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/random_engine_mersenne.hpp b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
index 6a429feee9..e0a0f57c8d 100644
--- a/src/backend/oneapi/kernel/random_engine_mersenne.hpp
+++ b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
@@ -44,6 +44,7 @@
 #pragma once
 #include <kernel/random_engine_write.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -367,3 +368,4 @@ class normalMersenne {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/random_engine_philox.hpp b/src/backend/oneapi/kernel/random_engine_philox.hpp
index e43cfa31e5..b5887aa16e 100644
--- a/src/backend/oneapi/kernel/random_engine_philox.hpp
+++ b/src/backend/oneapi/kernel/random_engine_philox.hpp
@@ -47,6 +47,7 @@
 #pragma once
 #include <kernel/random_engine_write.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 // Utils
@@ -196,3 +197,4 @@ class normalPhilox {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/random_engine_threefry.hpp b/src/backend/oneapi/kernel/random_engine_threefry.hpp
index 931e60ef63..2e8b6e0d16 100644
--- a/src/backend/oneapi/kernel/random_engine_threefry.hpp
+++ b/src/backend/oneapi/kernel/random_engine_threefry.hpp
@@ -47,6 +47,7 @@
 #pragma once
 #include <kernel/random_engine_write.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 // Utils
@@ -255,3 +256,4 @@ class normalThreefry {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index 824feb95b8..426b518eba 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -9,6 +9,7 @@
 #pragma once
 #include <CL/sycl.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -273,15 +274,11 @@ static void boxMullerTransform(Td *const out1, Td *const out2, const Tc &r1,
     *out2 = static_cast<Td>(r * c);
 }
 // template<>
-//__device__ void boxMullerTransform<common::half, __half>(
-//    common::half *const out1, common::half *const out2, const __half &r1,
-//    const __half &r2) {
-//    float o1, o2;
-//    float fr1 = __half2float(r1);
-//    float fr2 = __half2float(r2);
-//    boxMullerTransform(&o1, &o2, fr1, fr2);
-//    *out1 = o1;
-//    *out2 = o2;
+//__device__ void boxMullerTransform<arrayfire::common::half, __half>(
+//    arrayfire::common::half *const out1, arrayfire::common::half *const out2,
+//    const __half &r1, const __half &r2) { float o1, o2; float fr1 =
+//    __half2float(r1); float fr2 = __half2float(r2); boxMullerTransform(&o1,
+//    &o2, fr1, fr2); *out1 = o1; *out2 = o2;
 //}
 
 // Writes without boundary checking
@@ -407,7 +404,7 @@ static void writeOut128Bytes(cdouble *out, const uint &index,
     out[index] = {1.0 - getDouble01(r1, r2), 1.0 - getDouble01(r3, r4)};
 }
 
-static void writeOut128Bytes(common::half *out, const uint &index,
+static void writeOut128Bytes(arrayfire::common::half *out, const uint &index,
                              const uint groupSz, const uint &r1, const uint &r2,
                              const uint &r3, const uint &r4) {
     // out[index]               = oneMinusGetHalf01(r1);
@@ -457,10 +454,10 @@ static void boxMullerWriteOut128Bytes(cdouble *out, const uint &index,
                        getDouble01(r3, r4));
 }
 
-static void boxMullerWriteOut128Bytes(common::half *out, const uint &index,
-                                      const uint groupSz, const uint &r1,
-                                      const uint &r2, const uint &r3,
-                                      const uint &r4) {
+static void boxMullerWriteOut128Bytes(arrayfire::common::half *out,
+                                      const uint &index, const uint groupSz,
+                                      const uint &r1, const uint &r2,
+                                      const uint &r3, const uint &r4) {
     //   boxMullerTransform(&out[index], &out[index + groupSz],
     //                      getHalfNegative11(r1), getHalf01(r1 >> 16));
     //   boxMullerTransform(&out[index + 2 * groupSz],
@@ -711,10 +708,11 @@ static void partialBoxMullerWriteOut128Bytes(cdouble *out, const uint &index,
     if (index < elements) { out[index] = {n1, n2}; }
 }
 
-static void partialWriteOut128Bytes(common::half *out, const uint &index,
-                                    const uint groupSz, const uint &r1,
-                                    const uint &r2, const uint &r3,
-                                    const uint &r4, const uint &elements) {
+static void partialWriteOut128Bytes(arrayfire::common::half *out,
+                                    const uint &index, const uint groupSz,
+                                    const uint &r1, const uint &r2,
+                                    const uint &r3, const uint &r4,
+                                    const uint &elements) {
     //  if (index < elements) { out[index] = oneMinusGetHalf01(r1); }
     //  if (index + groupSz < elements) {
     //      out[index + groupSz] = oneMinusGetHalf01(r1 >> 16);
@@ -740,10 +738,13 @@ static void partialWriteOut128Bytes(common::half *out, const uint &index,
 }
 
 // Normalized writes with boundary checking
-static void partialBoxMullerWriteOut128Bytes(
-    common::half *out, const uint &index, const uint groupSz, const uint &r1,
-    const uint &r2, const uint &r3, const uint &r4, const uint &elements) {
-    //    common::half n[8];
+static void partialBoxMullerWriteOut128Bytes(arrayfire::common::half *out,
+                                             const uint &index,
+                                             const uint groupSz, const uint &r1,
+                                             const uint &r2, const uint &r3,
+                                             const uint &r4,
+                                             const uint &elements) {
+    //    arrayfire::common::half n[8];
     //    boxMullerTransform(n + 0, n + 1, getHalfNegative11(r1),
     //                       getHalf01(r1 >> 16));
     //    boxMullerTransform(n + 2, n + 3, getHalfNegative11(r2),
@@ -776,3 +777,4 @@ static void partialBoxMullerWriteOut128Bytes(
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
index d3106c5e7b..cce47881f2 100644
--- a/src/backend/oneapi/kernel/range.hpp
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -117,9 +118,10 @@ void range(Param<T> out, const int dim) {
 }
 
 template<>
-void range(Param<common::half> out, const int dim) {
-    ONEAPI_NOT_SUPPORTED("TODO: fix common::half support");
+void range(Param<arrayfire::common::half> out, const int dim) {
+    ONEAPI_NOT_SUPPORTED("TODO: fix arrayfire::common::half support");
 }
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/reduce.hpp b/src/backend/oneapi/kernel/reduce.hpp
index 6fa38e0269..6807a68396 100644
--- a/src/backend/oneapi/kernel/reduce.hpp
+++ b/src/backend/oneapi/kernel/reduce.hpp
@@ -28,6 +28,7 @@
 #include <iostream>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -110,3 +111,4 @@ void reduce_all(Param<To> out, Param<Ti> in, bool change_nan, double nanval) {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index 1a318e8bc5..eb8b206a02 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -25,6 +25,7 @@
 #include <iostream>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -289,3 +290,4 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/reduce_config.hpp b/src/backend/oneapi/kernel/reduce_config.hpp
index a7d185de75..ca892f4cc8 100644
--- a/src/backend/oneapi/kernel/reduce_config.hpp
+++ b/src/backend/oneapi/kernel/reduce_config.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -23,3 +24,4 @@ static const uint REPEAT            = 32;
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index 6efb6851b1..bfb4f808aa 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -25,6 +25,7 @@
 #include <iostream>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -243,3 +244,4 @@ void reduce_dim_default(Param<To> out, Param<Ti> in, bool change_nan,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index a4094f8cb9..94553f2b07 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -25,6 +25,7 @@
 #include <iostream>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -241,3 +242,4 @@ void reduce_first_default(Param<To> out, Param<Ti> in, bool change_nan,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/reorder.hpp b/src/backend/oneapi/kernel/reorder.hpp
old mode 100755
new mode 100644
index 2eb7484db2..6aa6cd39c0
--- a/src/backend/oneapi/kernel/reorder.hpp
+++ b/src/backend/oneapi/kernel/reorder.hpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -128,3 +129,4 @@ void reorder(Param<T> out, const Param<T> in, const dim_t* rdims) {
 }
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp
index 8c1a6e9140..eb0683791c 100644
--- a/src/backend/oneapi/kernel/scan_dim.hpp
+++ b/src/backend/oneapi/kernel/scan_dim.hpp
@@ -17,6 +17,7 @@
 #include <kernel/default_config.hpp>
 #include <memory.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -353,3 +354,4 @@ static void scan_dim(Param<To> out, Param<Ti> in, bool inclusive_scan) {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index a7fe567c75..78039dd36d 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -17,6 +17,7 @@
 #include <kernel/default_config.hpp>
 #include <memory.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -311,3 +312,4 @@ static void scan_first(Param<To> out, Param<Ti> in, bool inclusive_scan) {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index 8c7fef325f..0fac0bacb7 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -164,3 +165,4 @@ void transpose(Param<T> out, const Param<T> in, const bool conjugate,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/transpose_inplace.hpp b/src/backend/oneapi/kernel/transpose_inplace.hpp
index 108b9596f9..d397436dfc 100644
--- a/src/backend/oneapi/kernel/transpose_inplace.hpp
+++ b/src/backend/oneapi/kernel/transpose_inplace.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -195,3 +196,4 @@ void transpose_inplace(Param<T> in, const bool conjugate,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/triangle.hpp b/src/backend/oneapi/kernel/triangle.hpp
index cf9c3e22a3..96fdeb3d88 100644
--- a/src/backend/oneapi/kernel/triangle.hpp
+++ b/src/backend/oneapi/kernel/triangle.hpp
@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -120,3 +121,4 @@ void triangle(Param<T> out, const Param<T> in, bool is_upper,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/unwrap.hpp b/src/backend/oneapi/kernel/unwrap.hpp
old mode 100755
new mode 100644
index 475e55b66c..a6fa8ee64e
--- a/src/backend/oneapi/kernel/unwrap.hpp
+++ b/src/backend/oneapi/kernel/unwrap.hpp
@@ -15,6 +15,7 @@
 #include <debug_oneapi.hpp>
 #include <kernel/default_config.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -168,3 +169,4 @@ void unwrap(Param<T> out, const Param<T> in, const dim_t wx, const dim_t wy,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index 4158641dce..d9ee535eb6 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -20,6 +20,7 @@
 #include <backend.hpp>
 #include <math.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -192,3 +193,4 @@ static void where(Param<uint> &out, Param<T> in) {
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/wrap.hpp b/src/backend/oneapi/kernel/wrap.hpp
old mode 100755
new mode 100644
index 0cac661ba6..e574b4a127
--- a/src/backend/oneapi/kernel/wrap.hpp
+++ b/src/backend/oneapi/kernel/wrap.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -160,3 +161,4 @@ void wrap(Param<T> out, const Param<T> in, const dim_t wx, const dim_t wy,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/wrap_dilated.hpp b/src/backend/oneapi/kernel/wrap_dilated.hpp
old mode 100755
new mode 100644
index 12760a57c6..c479316968
--- a/src/backend/oneapi/kernel/wrap_dilated.hpp
+++ b/src/backend/oneapi/kernel/wrap_dilated.hpp
@@ -19,6 +19,7 @@
 #include <string>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
@@ -175,3 +176,4 @@ void wrap_dilated(Param<T> out, const Param<T> in, const dim_t wx,
 
 }  // namespace kernel
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/logic.hpp b/src/backend/oneapi/logic.hpp
index e1706583e2..650d079159 100644
--- a/src/backend/oneapi/logic.hpp
+++ b/src/backend/oneapi/logic.hpp
@@ -15,6 +15,7 @@
 #include <af/defines.h>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T, af_op_t op>
 Array<char> logicOp(const Array<T> &lhs, const Array<T> &rhs,
@@ -28,3 +29,4 @@ Array<T> bitOp(const Array<T> &lhs, const Array<T> &rhs,
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/lookup.cpp b/src/backend/oneapi/lookup.cpp
index 304ab9afa7..101dc90c1d 100644
--- a/src/backend/oneapi/lookup.cpp
+++ b/src/backend/oneapi/lookup.cpp
@@ -14,8 +14,9 @@
 #include <err_oneapi.hpp>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
@@ -61,3 +62,4 @@ INSTANTIATE(ushort);
 INSTANTIATE(short);
 INSTANTIATE(half);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/lookup.hpp b/src/backend/oneapi/lookup.hpp
index 2fe9b0240c..78d8da1ac1 100644
--- a/src/backend/oneapi/lookup.hpp
+++ b/src/backend/oneapi/lookup.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                    const unsigned dim);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/lu.cpp b/src/backend/oneapi/lu.cpp
index 170efca58c..b1d0b4b746 100644
--- a/src/backend/oneapi/lu.cpp
+++ b/src/backend/oneapi/lu.cpp
@@ -15,6 +15,7 @@
 #include <copy.hpp>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 Array<int> convertPivot(int *ipiv, int in_sz, int out_sz) {
@@ -50,9 +51,11 @@ INSTANTIATE_LU(double)
 INSTANTIATE_LU(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -84,5 +87,6 @@ INSTANTIATE_LU(double)
 INSTANTIATE_LU(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/lu.hpp b/src/backend/oneapi/lu.hpp
index 8ab1f25a7a..a6b1eeb982 100644
--- a/src/backend/oneapi/lu.hpp
+++ b/src/backend/oneapi/lu.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
@@ -19,3 +20,4 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot = true);
 
 bool isLAPACKAvailable();
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/match_template.cpp b/src/backend/oneapi/match_template.cpp
index 6a0182f7bd..28794ff2eb 100644
--- a/src/backend/oneapi/match_template.cpp
+++ b/src/backend/oneapi/match_template.cpp
@@ -11,6 +11,7 @@
 
 #include <err_oneapi.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename inType, typename outType>
@@ -36,3 +37,4 @@ INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/match_template.hpp b/src/backend/oneapi/match_template.hpp
index 9e79f3e19b..84ea6d337a 100644
--- a/src/backend/oneapi/match_template.hpp
+++ b/src/backend/oneapi/match_template.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename inType, typename outType>
 Array<outType> match_template(const Array<inType> &sImg,
                               const Array<inType> &tImg,
                               const af::matchType mType);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/math.cpp b/src/backend/oneapi/math.cpp
index e9c1666960..a673f9293b 100644
--- a/src/backend/oneapi/math.cpp
+++ b/src/backend/oneapi/math.cpp
@@ -10,6 +10,7 @@
 #include "math.hpp"
 #include <common/half.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 cfloat operator+(cfloat lhs, cfloat rhs) {
     // cfloat res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
@@ -51,3 +52,4 @@ cdouble division(cdouble lhs, double rhs) {
     return retVal;
 }
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp
index 584efa1d14..063d82f370 100644
--- a/src/backend/oneapi/math.hpp
+++ b/src/backend/oneapi/math.hpp
@@ -28,6 +28,7 @@
 /* Other */
 #endif
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -113,8 +114,8 @@ inline double maxval() {
 }
 
 template<>
-inline common::half maxval() {
-    return std::numeric_limits<common::half>::infinity();
+inline arrayfire::common::half maxval() {
+    return std::numeric_limits<arrayfire::common::half>::infinity();
 }
 
 template<>
@@ -127,8 +128,8 @@ inline double minval() {
     return -std::numeric_limits<double>::infinity();
 }
 template<>
-inline common::half minval() {
-    return -std::numeric_limits<common::half>::infinity();
+inline arrayfire::common::half minval() {
+    return -std::numeric_limits<arrayfire::common::half>::infinity();
 }
 
 template<typename T>
@@ -141,10 +142,13 @@ static inline T imag(T in) {
     return std::imag(in);
 }
 
-inline common::half operator+(common::half lhs, common::half rhs) noexcept {
-    return common::half(static_cast<float>(lhs) + static_cast<float>(rhs));
+inline arrayfire::common::half operator+(arrayfire::common::half lhs,
+                                         arrayfire::common::half rhs) noexcept {
+    return arrayfire::common::half(static_cast<float>(lhs) +
+                                   static_cast<float>(rhs));
 }
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #if defined(__GNUC__) || defined(__GNUG__)
 /* GCC/G++, Clang/LLVM, Intel ICC */
diff --git a/src/backend/oneapi/max.cpp b/src/backend/oneapi/max.cpp
index 4ae8efeaee..8b6ef71a10 100644
--- a/src/backend/oneapi/max.cpp
+++ b/src/backend/oneapi/max.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 // max
 INSTANTIATE(af_max_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_max_t, short, short)
 INSTANTIATE(af_max_t, ushort, ushort)
 INSTANTIATE(af_max_t, half, half)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/mean.cpp b/src/backend/oneapi/mean.cpp
index 85c4bc0576..09763bb739 100644
--- a/src/backend/oneapi/mean.cpp
+++ b/src/backend/oneapi/mean.cpp
@@ -15,9 +15,10 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 using std::swap;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in) {
@@ -78,3 +79,4 @@ INSTANTIATE_WGT(cdouble, double);
 INSTANTIATE_WGT(half, float);
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/mean.hpp b/src/backend/oneapi/mean.hpp
index c682fa8d5f..1ff66440b5 100644
--- a/src/backend/oneapi/mean.hpp
+++ b/src/backend/oneapi/mean.hpp
@@ -10,6 +10,7 @@
 #pragma once
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename Ti, typename Tw, typename To>
 To mean(const Array<Ti>& in);
@@ -24,3 +25,4 @@ template<typename T, typename Tw>
 Array<T> mean(const Array<T>& in, const Array<Tw>& wts, const int dim);
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/meanshift.cpp b/src/backend/oneapi/meanshift.cpp
index fa352ed5c1..de517e700f 100644
--- a/src/backend/oneapi/meanshift.cpp
+++ b/src/backend/oneapi/meanshift.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
@@ -46,3 +47,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/meanshift.hpp b/src/backend/oneapi/meanshift.hpp
index 014c0f2468..dbe26b4c85 100644
--- a/src/backend/oneapi/meanshift.hpp
+++ b/src/backend/oneapi/meanshift.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
                    const float &chromaticSigma, const unsigned &numIterations,
                    const bool &isColor);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/medfilt.cpp b/src/backend/oneapi/medfilt.cpp
index 1729573628..3b1ff319c5 100644
--- a/src/backend/oneapi/medfilt.cpp
+++ b/src/backend/oneapi/medfilt.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -63,3 +64,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/medfilt.hpp b/src/backend/oneapi/medfilt.hpp
index 1e356a23bb..eb459f7dd9 100644
--- a/src/backend/oneapi/medfilt.hpp
+++ b/src/backend/oneapi/medfilt.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -20,3 +21,4 @@ Array<T> medfilt2(const Array<T> &in, const int w_len, const int w_wid,
                   const af::borderType edge_pad);
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index 314e1fd0a8..e87812e5b4 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -20,13 +20,14 @@
 
 #include <utility>
 
-using common::bytesToString;
+using arrayfire::common::bytesToString;
 
 using af::dim4;
 using std::function;
 using std::move;
 using std::unique_ptr;
 
+namespace arrayfire {
 namespace oneapi {
 float getMemoryPressure() { return memoryManager().getMemoryPressure(); }
 float getMemoryPressureThreshold() {
@@ -195,7 +196,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
-INSTANTIATE(common::half)
+INSTANTIATE(arrayfire::common::half)
 
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
@@ -332,3 +333,4 @@ void AllocatorPinned::nativeFree(void *ptr) {
     // }
 }
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/memory.hpp b/src/backend/oneapi/memory.hpp
index 2ed71fdd19..bcb8c1dabf 100644
--- a/src/backend/oneapi/memory.hpp
+++ b/src/backend/oneapi/memory.hpp
@@ -16,6 +16,7 @@
 #include <memory>
 #include <vector>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 sycl::buffer<T> *bufferAlloc(const size_t &bytes);
@@ -65,7 +66,7 @@ bool jitTreeExceedsMemoryPressure(size_t bytes);
 void setMemStepSize(size_t step_bytes);
 size_t getMemStepSize(void);
 
-class Allocator final : public common::memory::AllocatorInterface {
+class Allocator final : public common::AllocatorInterface {
    public:
     Allocator();
     ~Allocator() = default;
@@ -76,7 +77,7 @@ class Allocator final : public common::memory::AllocatorInterface {
     void nativeFree(void *ptr) override;
 };
 
-class AllocatorPinned final : public common::memory::AllocatorInterface {
+class AllocatorPinned final : public common::AllocatorInterface {
    public:
     AllocatorPinned();
     ~AllocatorPinned() = default;
@@ -91,3 +92,4 @@ class AllocatorPinned final : public common::memory::AllocatorInterface {
 };
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/min.cpp b/src/backend/oneapi/min.cpp
index 3afa0d9787..ea9900543c 100644
--- a/src/backend/oneapi/min.cpp
+++ b/src/backend/oneapi/min.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 // min
 INSTANTIATE(af_min_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_min_t, short, short)
 INSTANTIATE(af_min_t, ushort, ushort)
 INSTANTIATE(af_min_t, half, half)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/moments.cpp b/src/backend/oneapi/moments.cpp
index 119e01cbc9..50efe4ccd5 100644
--- a/src/backend/oneapi/moments.cpp
+++ b/src/backend/oneapi/moments.cpp
@@ -12,6 +12,7 @@
 #include <err_oneapi.hpp>
 // #include <kernel/moments.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 static inline unsigned bitCount(unsigned v) {
@@ -54,3 +55,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/moments.hpp b/src/backend/oneapi/moments.hpp
index 6201ccb897..3dcf1e194f 100644
--- a/src/backend/oneapi/moments.hpp
+++ b/src/backend/oneapi/moments.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<float> moments(const Array<T> &in, const af_moment_type moment);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/morph.cpp b/src/backend/oneapi/morph.cpp
index adef3be8d6..44fe6a6529 100644
--- a/src/backend/oneapi/morph.cpp
+++ b/src/backend/oneapi/morph.cpp
@@ -17,6 +17,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -66,3 +67,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/morph.hpp b/src/backend/oneapi/morph.hpp
index 086baf2a90..47d3399f87 100644
--- a/src/backend/oneapi/morph.hpp
+++ b/src/backend/oneapi/morph.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
@@ -16,3 +17,4 @@ Array<T> morph(const Array<T> &in, const Array<T> &mask, bool isDilation);
 template<typename T>
 Array<T> morph3d(const Array<T> &in, const Array<T> &mask, bool isDilation);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/nearest_neighbour.cpp b/src/backend/oneapi/nearest_neighbour.cpp
index 30bc6d90d3..7a34ba0fba 100644
--- a/src/backend/oneapi/nearest_neighbour.cpp
+++ b/src/backend/oneapi/nearest_neighbour.cpp
@@ -18,6 +18,7 @@
 using af::dim4;
 // nsing cl::Device;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename To, af_match_type dist_type>
@@ -86,3 +87,4 @@ INSTANTIATE(uchar, uint)
 INSTANTIATE(uintl, uint)  // For Hamming
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/nearest_neighbour.hpp b/src/backend/oneapi/nearest_neighbour.hpp
index f16b709d8e..1af9889b00 100644
--- a/src/backend/oneapi/nearest_neighbour.hpp
+++ b/src/backend/oneapi/nearest_neighbour.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename To>
@@ -19,5 +20,5 @@ void nearest_neighbour(Array<uint>& idx, Array<To>& dist, const Array<T>& query,
                        const Array<T>& train, const uint dist_dim,
                        const uint n_dist,
                        const af_match_type dist_type = AF_SSD);
-
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/orb.cpp b/src/backend/oneapi/orb.cpp
index aaca439632..b00cf0395f 100644
--- a/src/backend/oneapi/orb.cpp
+++ b/src/backend/oneapi/orb.cpp
@@ -17,6 +17,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename convAccT>
@@ -66,3 +67,4 @@ INSTANTIATE(float, float)
 INSTANTIATE(double, double)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/orb.hpp b/src/backend/oneapi/orb.hpp
index aa1fe324bb..ab29a6813b 100644
--- a/src/backend/oneapi/orb.hpp
+++ b/src/backend/oneapi/orb.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename convAccT>
@@ -21,4 +22,5 @@ unsigned orb(Array<float> &x, Array<float> &y, Array<float> &score,
              const unsigned max_feat, const float scl_fctr,
              const unsigned levels, const bool blur_img);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index 4e22f742ae..c0f3a0d08e 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -59,12 +59,13 @@ using std::to_string;
 using std::unique_ptr;
 using std::vector;
 
-using common::getEnvVar;
-using common::ltrim;
-using common::memory::MemoryManagerBase;
-using oneapi::Allocator;
-using oneapi::AllocatorPinned;
+using arrayfire::common::getEnvVar;
+using arrayfire::common::ltrim;
+using arrayfire::common::MemoryManagerBase;
+using arrayfire::oneapi::Allocator;
+using arrayfire::oneapi::AllocatorPinned;
 
+namespace arrayfire {
 namespace oneapi {
 
 static string get_system() {
@@ -587,7 +588,7 @@ void resetMemoryManagerPinned() {
     return DeviceManager::getInstance().resetMemoryManagerPinned();
 }
 
-graphics::ForgeManager& forgeManager() {
+arrayfire::common::ForgeManager& forgeManager() {
     return *(DeviceManager::getInstance().fgMngr);
 }
 
@@ -606,6 +607,7 @@ GraphicsResourceManager& interopManager() {
 }
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 /*
 //TODO: select which external api functions to expose and add to
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index 46d24393f3..aa58ea5a7e 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -20,18 +20,16 @@ namespace spdlog {
 class logger;
 }
 
-namespace graphics {
-class ForgeManager;
-}
-
+namespace arrayfire {
 namespace common {
-namespace memory {
 class MemoryManagerBase;
-}
+class ForgeManager;
 }  // namespace common
+}  // namespace arrayfire
 
-using common::memory::MemoryManagerBase;
+using arrayfire::common::MemoryManagerBase;
 
+namespace arrayfire {
 namespace oneapi {
 
 // Forward declarations
@@ -110,7 +108,7 @@ void setMemoryManagerPinned(std::unique_ptr<MemoryManagerBase> mgr);
 
 void resetMemoryManagerPinned();
 
-graphics::ForgeManager& forgeManager();
+arrayfire::common::ForgeManager& forgeManager();
 
 GraphicsResourceManager& interopManager();
 
@@ -119,3 +117,4 @@ GraphicsResourceManager& interopManager();
 void setActiveContext(int device);
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/plot.cpp b/src/backend/oneapi/plot.cpp
index 6abf9896a3..d2fa041291 100644
--- a/src/backend/oneapi/plot.cpp
+++ b/src/backend/oneapi/plot.cpp
@@ -15,13 +15,14 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 void copy_plot(const Array<T> &P, fg_plot plot) {
     ONEAPI_NOT_SUPPORTED("copy_plot Not supported");
 
-    // ForgeModule &_ = graphics::forgePlugin();
+    // ForgeModule &_ = common::forgePlugin();
     // if (isGLSharingSupported()) {
     //     CheckGL("Begin OpenCL resource copy");
     //     const cl::Buffer *d_P = P.get();
@@ -80,3 +81,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/plot.hpp b/src/backend/oneapi/plot.hpp
index c7c922e270..ed8bd5e118 100644
--- a/src/backend/oneapi/plot.hpp
+++ b/src/backend/oneapi/plot.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 void copy_plot(const Array<T> &P, fg_plot plot);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/print.hpp b/src/backend/oneapi/print.hpp
index 787df41df2..0e487278d5 100644
--- a/src/backend/oneapi/print.hpp
+++ b/src/backend/oneapi/print.hpp
@@ -11,6 +11,7 @@
 #include <backend.hpp>
 #include <ostream>
 
+namespace arrayfire {
 namespace oneapi {
 static std::ostream& operator<<(std::ostream& out, const cfloat& var) {
     out << "(" << std::real(var) << "," << std::imag(var) << ")";
@@ -22,3 +23,4 @@ static std::ostream& operator<<(std::ostream& out, const cdouble& var) {
     return out;
 }
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/product.cpp b/src/backend/oneapi/product.cpp
index 6d449e1fa7..bc3f9421ae 100644
--- a/src/backend/oneapi/product.cpp
+++ b/src/backend/oneapi/product.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 // sum
 INSTANTIATE(af_mul_t, float, float)
@@ -28,3 +29,4 @@ INSTANTIATE(af_mul_t, short, int)
 INSTANTIATE(af_mul_t, ushort, uint)
 INSTANTIATE(af_mul_t, half, float)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/qr.cpp b/src/backend/oneapi/qr.cpp
index 80fa226994..32bf559f4c 100644
--- a/src/backend/oneapi/qr.cpp
+++ b/src/backend/oneapi/qr.cpp
@@ -23,6 +23,7 @@
 #include <magma/magma_helper.h>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -112,9 +113,11 @@ INSTANTIATE_QR(double)
 INSTANTIATE_QR(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -138,5 +141,6 @@ INSTANTIATE_QR(double)
 INSTANTIATE_QR(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/qr.hpp b/src/backend/oneapi/qr.hpp
index 3ae750cf70..ad8ed882a0 100644
--- a/src/backend/oneapi/qr.hpp
+++ b/src/backend/oneapi/qr.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &orig);
@@ -16,3 +17,4 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &orig);
 template<typename T>
 Array<T> qr_inplace(Array<T> &in);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/random_engine.cpp b/src/backend/oneapi/random_engine.cpp
index cff66a7170..7045dcc8cc 100644
--- a/src/backend/oneapi/random_engine.cpp
+++ b/src/backend/oneapi/random_engine.cpp
@@ -14,8 +14,9 @@
 #include <af/defines.h>
 #include <af/dim4.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl) {
@@ -103,3 +104,4 @@ INSTANTIATE_NORMAL(cfloat)
 INSTANTIATE_NORMAL(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/random_engine.hpp b/src/backend/oneapi/random_engine.hpp
index 0839d387b8..7738294d06 100644
--- a/src/backend/oneapi/random_engine.hpp
+++ b/src/backend/oneapi/random_engine.hpp
@@ -13,6 +13,7 @@
 #include <backend.hpp>
 #include <af/defines.h>
 
+namespace arrayfire {
 namespace oneapi {
 void initMersenneState(Array<uint> &state, const uintl seed,
                        const Array<uint> &tbl);
@@ -39,3 +40,4 @@ Array<T> normalDistribution(const af::dim4 &dims, Array<uint> pos,
                             Array<uint> recursion_table,
                             Array<uint> temper_table, Array<uint> state);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/range.cpp b/src/backend/oneapi/range.cpp
index e5498d12d8..caa8ed48bc 100644
--- a/src/backend/oneapi/range.cpp
+++ b/src/backend/oneapi/range.cpp
@@ -16,8 +16,9 @@
 
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> range(const dim4& dim, const int seq_dim) {
@@ -52,3 +53,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/range.hpp b/src/backend/oneapi/range.hpp
index 7191152fb1..6a997c6787 100644
--- a/src/backend/oneapi/range.hpp
+++ b/src/backend/oneapi/range.hpp
@@ -10,7 +10,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> range(const dim4& dim, const int seq_dim = -1);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/reduce.hpp b/src/backend/oneapi/reduce.hpp
index 668fa1ac72..6d6ab31670 100644
--- a/src/backend/oneapi/reduce.hpp
+++ b/src/backend/oneapi/reduce.hpp
@@ -11,6 +11,7 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<af_op_t op, typename Ti, typename To>
 Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan = false,
@@ -25,3 +26,4 @@ template<af_op_t op, typename Ti, typename To>
 Array<To> reduce_all(const Array<Ti> &in, bool change_nan = false,
                      double nanval = 0);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp
index 007fbccac4..898f77d006 100644
--- a/src/backend/oneapi/reduce_impl.hpp
+++ b/src/backend/oneapi/reduce_impl.hpp
@@ -18,6 +18,7 @@
 using af::dim4;
 using std::swap;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<af_op_t op, typename Ti, typename To>
@@ -45,6 +46,7 @@ Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
 }
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #define INSTANTIATE(Op, Ti, To)                                                \
     template Array<To> reduce<Op, Ti, To>(const Array<Ti> &in, const int dim,  \
diff --git a/src/backend/oneapi/regions.cpp b/src/backend/oneapi/regions.cpp
index 73ebccc46e..983b3b9000 100644
--- a/src/backend/oneapi/regions.cpp
+++ b/src/backend/oneapi/regions.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -39,3 +40,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/regions.hpp b/src/backend/oneapi/regions.hpp
index 585f7e6e14..34e90f2918 100644
--- a/src/backend/oneapi/regions.hpp
+++ b/src/backend/oneapi/regions.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 Array<T> regions(const Array<char> &in, af_connectivity connectivity);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/reorder.cpp b/src/backend/oneapi/reorder.cpp
index fc5c7f26a7..d62db984e9 100644
--- a/src/backend/oneapi/reorder.cpp
+++ b/src/backend/oneapi/reorder.cpp
@@ -14,8 +14,9 @@
 #include <reorder.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> reorder(const Array<T> &in, const af::dim4 &rdims) {
@@ -47,3 +48,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/reorder.hpp b/src/backend/oneapi/reorder.hpp
index eb2cc8ef9c..a587bc9de3 100644
--- a/src/backend/oneapi/reorder.hpp
+++ b/src/backend/oneapi/reorder.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> reorder(const Array<T> &in, const af::dim4 &rdims);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/reshape.cpp b/src/backend/oneapi/reshape.cpp
index 87a7e7d28e..768a167480 100644
--- a/src/backend/oneapi/reshape.cpp
+++ b/src/backend/oneapi/reshape.cpp
@@ -14,8 +14,9 @@
 #include <common/half.hpp>
 // #include <kernel/memcopy.hpp>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename inType, typename outType>
@@ -79,3 +80,4 @@ INSTANTIATE_COMPLEX(cfloat)
 INSTANTIATE_COMPLEX(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/resize.cpp b/src/backend/oneapi/resize.cpp
index 89bdea49b1..6d8d3307ab 100644
--- a/src/backend/oneapi/resize.cpp
+++ b/src/backend/oneapi/resize.cpp
@@ -14,6 +14,7 @@
 #include <af/dim4.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
@@ -46,3 +47,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/resize.hpp b/src/backend/oneapi/resize.hpp
index 77b5972588..4cd7aa39aa 100644
--- a/src/backend/oneapi/resize.hpp
+++ b/src/backend/oneapi/resize.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
                 const af_interp_type method);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/rotate.cpp b/src/backend/oneapi/rotate.cpp
index 37f8abbe00..b5cd2fa6e3 100644
--- a/src/backend/oneapi/rotate.cpp
+++ b/src/backend/oneapi/rotate.cpp
@@ -12,6 +12,7 @@
 
 // #include <kernel/rotate.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
@@ -56,3 +57,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/rotate.hpp b/src/backend/oneapi/rotate.hpp
index 369bbd2521..ee6114da0d 100644
--- a/src/backend/oneapi/rotate.hpp
+++ b/src/backend/oneapi/rotate.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
                 const af_interp_type method);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/scalar.hpp b/src/backend/oneapi/scalar.hpp
index fee814f9f2..9e5ac25704 100644
--- a/src/backend/oneapi/scalar.hpp
+++ b/src/backend/oneapi/scalar.hpp
@@ -12,6 +12,7 @@
 #include <math.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -21,3 +22,4 @@ Array<T> createScalarNode(const dim4 &size, const T val) {
 }
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/scan.cpp b/src/backend/oneapi/scan.cpp
index 81b7494d68..f7151ce076 100644
--- a/src/backend/oneapi/scan.cpp
+++ b/src/backend/oneapi/scan.cpp
@@ -13,6 +13,7 @@
 #include <kernel/scan_dim.hpp>
 #include <kernel/scan_first.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusiveScan) {
@@ -54,3 +55,4 @@ INSTANTIATE_SCAN_ALL(af_mul_t)
 INSTANTIATE_SCAN_ALL(af_min_t)
 INSTANTIATE_SCAN_ALL(af_max_t)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/scan.hpp b/src/backend/oneapi/scan.hpp
index 5e8508a8da..59522a8c4b 100644
--- a/src/backend/oneapi/scan.hpp
+++ b/src/backend/oneapi/scan.hpp
@@ -10,7 +10,9 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<af_op_t op, typename Ti, typename To>
 Array<To> scan(const Array<Ti>& in, const int dim, bool inclusive_scan = true);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/scan_by_key.cpp b/src/backend/oneapi/scan_by_key.cpp
index 555817819c..dabca1815a 100644
--- a/src/backend/oneapi/scan_by_key.cpp
+++ b/src/backend/oneapi/scan_by_key.cpp
@@ -16,6 +16,7 @@
 // #include <kernel/scan_dim_by_key.hpp>
 // #include <kernel/scan_first_by_key.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
@@ -64,3 +65,4 @@ INSTANTIATE_SCAN_BY_KEY_OP(af_mul_t)
 INSTANTIATE_SCAN_BY_KEY_OP(af_min_t)
 INSTANTIATE_SCAN_BY_KEY_OP(af_max_t)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/scan_by_key.hpp b/src/backend/oneapi/scan_by_key.hpp
index 556d59f922..7512f479c1 100644
--- a/src/backend/oneapi/scan_by_key.hpp
+++ b/src/backend/oneapi/scan_by_key.hpp
@@ -10,8 +10,10 @@
 #include <Array.hpp>
 #include <optypes.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<af_op_t op, typename Ti, typename Tk, typename To>
 Array<To> scan(const Array<Tk>& key, const Array<Ti>& in, const int dim,
                bool inclusive_scan = true);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/select.cpp b/src/backend/oneapi/select.cpp
index beea59a771..08458b9778 100644
--- a/src/backend/oneapi/select.cpp
+++ b/src/backend/oneapi/select.cpp
@@ -20,12 +20,13 @@
 
 using af::dim4;
 
-using common::half;
-using common::NaryNode;
+using arrayfire::common::half;
+using arrayfire::common::NaryNode;
 
 using std::make_shared;
 using std::max;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
@@ -141,3 +142,4 @@ INSTANTIATE(half);
 
 #undef INSTANTIATE
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/select.hpp b/src/backend/oneapi/select.hpp
index 00d0eb06c6..754a0ec44d 100644
--- a/src/backend/oneapi/select.hpp
+++ b/src/backend/oneapi/select.hpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
@@ -27,3 +28,4 @@ template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
                           const T &b_val, const af::dim4 &odims);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/set.cpp b/src/backend/oneapi/set.cpp
index 01fa0a6bcf..a76363f10b 100644
--- a/src/backend/oneapi/set.cpp
+++ b/src/backend/oneapi/set.cpp
@@ -15,6 +15,7 @@
 #include <sort.hpp>
 #include <af/dim4.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 using af::dim4;
 
@@ -158,3 +159,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/set.hpp b/src/backend/oneapi/set.hpp
index 7836873639..85d3386489 100644
--- a/src/backend/oneapi/set.hpp
+++ b/src/backend/oneapi/set.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> setUnique(const Array<T> &in, const bool is_sorted);
@@ -21,3 +22,4 @@ template<typename T>
 Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
                       const bool is_unique);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/shift.cpp b/src/backend/oneapi/shift.cpp
index e4ada40a5c..d72477c770 100644
--- a/src/backend/oneapi/shift.cpp
+++ b/src/backend/oneapi/shift.cpp
@@ -14,13 +14,14 @@
 #include <traits.hpp>
 
 using af::dim4;
-using common::Node_ptr;
-using common::ShiftNodeBase;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::ShiftNodeBase;
 using std::array;
 using std::make_shared;
 using std::static_pointer_cast;
 using std::string;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -71,3 +72,4 @@ INSTANTIATE(char)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/shift.hpp b/src/backend/oneapi/shift.hpp
index f236018321..1c808479d0 100644
--- a/src/backend/oneapi/shift.hpp
+++ b/src/backend/oneapi/shift.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> shift(const Array<T> &in, const int sdims[4]);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sift.cpp b/src/backend/oneapi/sift.cpp
index 9197c23d14..72dccab12d 100644
--- a/src/backend/oneapi/sift.cpp
+++ b/src/backend/oneapi/sift.cpp
@@ -16,6 +16,7 @@
 using af::dim4;
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename convAccT>
@@ -73,3 +74,4 @@ INSTANTIATE(float, float)
 INSTANTIATE(double, double)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sift.hpp b/src/backend/oneapi/sift.hpp
index 5c2a33dca6..ae656a73fd 100644
--- a/src/backend/oneapi/sift.hpp
+++ b/src/backend/oneapi/sift.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename convAccT>
@@ -23,4 +24,5 @@ unsigned sift(Array<float>& x, Array<float>& y, Array<float>& score,
               const float img_scale, const float feature_ratio,
               const bool compute_GLOH);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sobel.cpp b/src/backend/oneapi/sobel.cpp
index 7d722e7f4d..54ba117be7 100644
--- a/src/backend/oneapi/sobel.cpp
+++ b/src/backend/oneapi/sobel.cpp
@@ -15,6 +15,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename Ti, typename To>
@@ -46,3 +47,4 @@ INSTANTIATE(short, int)
 INSTANTIATE(ushort, int)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sobel.hpp b/src/backend/oneapi/sobel.hpp
index 94d3e06879..44e2356dc5 100644
--- a/src/backend/oneapi/sobel.hpp
+++ b/src/backend/oneapi/sobel.hpp
@@ -10,10 +10,12 @@
 #include <Array.hpp>
 #include <utility>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename Ti, typename To>
 std::pair<Array<To>, Array<To>> sobelDerivatives(const Array<Ti> &img,
                                                  const unsigned &ker_size);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/solve.cpp b/src/backend/oneapi/solve.cpp
index ee662de210..a4082c0d1f 100644
--- a/src/backend/oneapi/solve.cpp
+++ b/src/backend/oneapi/solve.cpp
@@ -32,6 +32,7 @@ using cl::Buffer;
 using std::min;
 using std::vector;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -331,9 +332,11 @@ INSTANTIATE_SOLVE(cfloat)
 INSTANTIATE_SOLVE(double)
 INSTANTIATE_SOLVE(cdouble)
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -361,5 +364,6 @@ INSTANTIATE_SOLVE(double)
 INSTANTIATE_SOLVE(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/solve.hpp b/src/backend/oneapi/solve.hpp
index 330605aa35..acea9327b4 100644
--- a/src/backend/oneapi/solve.hpp
+++ b/src/backend/oneapi/solve.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> solve(const Array<T> &a, const Array<T> &b,
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> solveLU(const Array<T> &a, const Array<int> &pivot, const Array<T> &b,
                  const af_mat_prop options = AF_MAT_NONE);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sort.cpp b/src/backend/oneapi/sort.cpp
index b5e0eb73fd..599d23c896 100644
--- a/src/backend/oneapi/sort.cpp
+++ b/src/backend/oneapi/sort.cpp
@@ -16,6 +16,7 @@
 #include <sort.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending) {
@@ -64,3 +65,4 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sort.hpp b/src/backend/oneapi/sort.hpp
index ae7fdc9e6a..73512ed973 100644
--- a/src/backend/oneapi/sort.hpp
+++ b/src/backend/oneapi/sort.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sort_by_key.cpp b/src/backend/oneapi/sort_by_key.cpp
index f2a140c338..f7b5beca91 100644
--- a/src/backend/oneapi/sort_by_key.cpp
+++ b/src/backend/oneapi/sort_by_key.cpp
@@ -16,6 +16,7 @@
 #include <sort_by_key.hpp>
 #include <stdexcept>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename Tk, typename Tv>
 void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
@@ -53,3 +54,4 @@ INSTANTIATE1(uchar)
 INSTANTIATE1(intl)
 INSTANTIATE1(uintl)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sort_by_key.hpp b/src/backend/oneapi/sort_by_key.hpp
index 2ba2c67ba3..665fdccaca 100644
--- a/src/backend/oneapi/sort_by_key.hpp
+++ b/src/backend/oneapi/sort_by_key.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename Tk, typename Tv>
 void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
                  const Array<Tv> &ival, const unsigned dim, bool isAscending);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sort_index.cpp b/src/backend/oneapi/sort_index.cpp
index 6600db9f7c..c0df0fb9de 100644
--- a/src/backend/oneapi/sort_index.cpp
+++ b/src/backend/oneapi/sort_index.cpp
@@ -18,8 +18,9 @@
 #include <sort_index.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void sort_index(Array<T> &okey, Array<uint> &oval, const Array<T> &in,
@@ -77,3 +78,4 @@ INSTANTIATE(uintl)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sort_index.hpp b/src/backend/oneapi/sort_index.hpp
index 2e7f262e62..30d6db07b9 100644
--- a/src/backend/oneapi/sort_index.hpp
+++ b/src/backend/oneapi/sort_index.hpp
@@ -9,8 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void sort_index(Array<T> &okey, Array<unsigned> &oval, const Array<T> &in,
                 const unsigned dim, bool isAscending);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sparse.cpp b/src/backend/oneapi/sparse.cpp
index 18e1d48e81..37e5826430 100644
--- a/src/backend/oneapi/sparse.cpp
+++ b/src/backend/oneapi/sparse.cpp
@@ -26,6 +26,7 @@
 #include <stdexcept>
 #include <string>
 
+namespace arrayfire {
 namespace oneapi {
 
 using namespace common;
@@ -225,3 +226,4 @@ INSTANTIATE_SPARSE(cdouble)
 #undef INSTANTIATE_SPARSE
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sparse.hpp b/src/backend/oneapi/sparse.hpp
index 3958dcea3b..e7440fc405 100644
--- a/src/backend/oneapi/sparse.hpp
+++ b/src/backend/oneapi/sparse.hpp
@@ -12,6 +12,7 @@
 #include <Array.hpp>
 #include <common/SparseArray.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, af_storage stype>
@@ -25,3 +26,4 @@ common::SparseArray<T> sparseConvertStorageToStorage(
     const common::SparseArray<T> &in);
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sparse_arith.cpp b/src/backend/oneapi/sparse_arith.cpp
index e39bed14e4..856d300553 100644
--- a/src/backend/oneapi/sparse_arith.cpp
+++ b/src/backend/oneapi/sparse_arith.cpp
@@ -25,6 +25,7 @@
 #include <scan.hpp>
 #include <where.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 using namespace common;
@@ -178,3 +179,4 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sparse_arith.hpp b/src/backend/oneapi/sparse_arith.hpp
index 589620c314..b35d4963e1 100644
--- a/src/backend/oneapi/sparse_arith.hpp
+++ b/src/backend/oneapi/sparse_arith.hpp
@@ -12,6 +12,7 @@
 #include <optypes.hpp>
 #include <sparse.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 // These two functions cannot be overloaded by return type.
@@ -28,3 +29,4 @@ template<typename T, af_op_t op>
 common::SparseArray<T> arithOp(const common::SparseArray<T> &lhs,
                                const common::SparseArray<T> &rhs);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sparse_blas.cpp b/src/backend/oneapi/sparse_blas.cpp
index b9fcd6fb52..6d414c8ee0 100644
--- a/src/backend/oneapi/sparse_blas.cpp
+++ b/src/backend/oneapi/sparse_blas.cpp
@@ -30,6 +30,7 @@
 #include <cpu/cpu_sparse_blas.hpp>
 #endif  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace oneapi {
 
 using namespace common;
@@ -100,3 +101,4 @@ INSTANTIATE_SPARSE(cfloat)
 INSTANTIATE_SPARSE(cdouble)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sparse_blas.hpp b/src/backend/oneapi/sparse_blas.hpp
index d187a4422a..a5acc6ffc0 100644
--- a/src/backend/oneapi/sparse_blas.hpp
+++ b/src/backend/oneapi/sparse_blas.hpp
@@ -11,10 +11,12 @@
 #include <common/SparseArray.hpp>
 #include <sparse.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhs,
                 af_mat_prop optLhs, af_mat_prop optRhs);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sum.cpp b/src/backend/oneapi/sum.cpp
index 30850564e8..fb20ce6121 100644
--- a/src/backend/oneapi/sum.cpp
+++ b/src/backend/oneapi/sum.cpp
@@ -10,8 +10,9 @@
 #include <common/half.hpp>
 #include "reduce_impl.hpp"
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 // sum
 INSTANTIATE(af_add_t, float, float)
@@ -37,3 +38,4 @@ INSTANTIATE(af_add_t, ushort, float)
 INSTANTIATE(af_add_t, half, half)
 INSTANTIATE(af_add_t, half, float)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/surface.cpp b/src/backend/oneapi/surface.cpp
index 38ad3388f5..2a8d604772 100644
--- a/src/backend/oneapi/surface.cpp
+++ b/src/backend/oneapi/surface.cpp
@@ -17,12 +17,13 @@ using af::dim4;
 // using cl::Memory;
 using std::vector;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 void copy_surface(const Array<T> &P, fg_surface surface) {
     ONEAPI_NOT_SUPPORTED("copy_surface Not supported");
-    // ForgeModule &_ = graphics::forgePlugin();
+    // ForgeModule &_ = common::forgePlugin();
     // if (isGLSharingSupported()) {
     //     CheckGL("Begin OpenCL resource copy");
     //     const cl::Buffer *d_P = P.get();
@@ -82,3 +83,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/surface.hpp b/src/backend/oneapi/surface.hpp
index 0c4110fd36..2d868301e0 100644
--- a/src/backend/oneapi/surface.hpp
+++ b/src/backend/oneapi/surface.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 void copy_surface(const Array<T> &P, fg_surface surface);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/susan.cpp b/src/backend/oneapi/susan.cpp
index 94173b3e4c..437259681c 100644
--- a/src/backend/oneapi/susan.cpp
+++ b/src/backend/oneapi/susan.cpp
@@ -17,6 +17,7 @@
 using af::features;
 using std::vector;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -74,3 +75,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/susan.hpp b/src/backend/oneapi/susan.hpp
index 8510117dea..1a0c4ffe8c 100644
--- a/src/backend/oneapi/susan.hpp
+++ b/src/backend/oneapi/susan.hpp
@@ -12,6 +12,7 @@
 
 using af::features;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -21,4 +22,5 @@ unsigned susan(Array<float> &x_out, Array<float> &y_out,
                const float geom_thr, const float feature_ratio,
                const unsigned edge);
 
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/svd.cpp b/src/backend/oneapi/svd.cpp
index 8a886983f9..fad4c2f35b 100644
--- a/src/backend/oneapi/svd.cpp
+++ b/src/backend/oneapi/svd.cpp
@@ -24,6 +24,7 @@
 #include <magma/magma_helper.h>
 #include <platform.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename Tr>
@@ -235,9 +236,11 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #else  // WITH_LINEAR_ALGEBRA
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T, typename Tr>
@@ -264,5 +267,6 @@ INSTANTIATE(cfloat, float)
 INSTANTIATE(cdouble, double)
 
 }  // namespace oneapi
+}  // namespace arrayfire
 
 #endif  // WITH_LINEAR_ALGEBRA
diff --git a/src/backend/oneapi/svd.hpp b/src/backend/oneapi/svd.hpp
index 297c899be6..4b001d2ad0 100644
--- a/src/backend/oneapi/svd.hpp
+++ b/src/backend/oneapi/svd.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T, typename Tr>
 void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
@@ -16,3 +17,4 @@ void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in);
 template<typename T, typename Tr>
 void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/tile.cpp b/src/backend/oneapi/tile.cpp
index 384c0f0710..5f2c38c475 100644
--- a/src/backend/oneapi/tile.cpp
+++ b/src/backend/oneapi/tile.cpp
@@ -14,8 +14,9 @@
 #include <common/half.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> tile(const Array<T> &in, const af::dim4 &tileDims) {
@@ -49,3 +50,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/tile.hpp b/src/backend/oneapi/tile.hpp
index 0ad5a9869a..f11e2aa711 100644
--- a/src/backend/oneapi/tile.hpp
+++ b/src/backend/oneapi/tile.hpp
@@ -9,7 +9,10 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> tile(const Array<T> &in, const af::dim4 &tileDims);
-}
+
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/topk.cpp b/src/backend/oneapi/topk.cpp
index 8d963ac4c2..35c0b66975 100644
--- a/src/backend/oneapi/topk.cpp
+++ b/src/backend/oneapi/topk.cpp
@@ -22,7 +22,7 @@
 
 // using cl::Buffer;
 // using cl::Event;
-using common::half;
+using arrayfire::common::half;
 
 using std::iota;
 using std::min;
@@ -30,6 +30,7 @@ using std::partial_sort_copy;
 using std::transform;
 using std::vector;
 
+namespace arrayfire {
 namespace oneapi {
 vector<af_index_t> indexForTopK(const int k) {
     af_index_t idx;
@@ -181,3 +182,4 @@ INSTANTIATE(long long)
 INSTANTIATE(unsigned long long)
 INSTANTIATE(half)
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/topk.hpp b/src/backend/oneapi/topk.hpp
index 8390733751..fa816b9ca7 100644
--- a/src/backend/oneapi/topk.hpp
+++ b/src/backend/oneapi/topk.hpp
@@ -7,8 +7,11 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void topk(Array<T>& keys, Array<unsigned>& vals, const Array<T>& in,
           const int k, const int dim, const af::topkFunction order);
-}
+
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/traits.hpp b/src/backend/oneapi/traits.hpp
index 61fab0663c..57e1949082 100644
--- a/src/backend/oneapi/traits.hpp
+++ b/src/backend/oneapi/traits.hpp
@@ -23,17 +23,17 @@ static bool iscplx() {
     return false;
 }
 template<>
-inline bool iscplx<oneapi::cfloat>() {
+inline bool iscplx<arrayfire::oneapi::cfloat>() {
     return true;
 }
 template<>
-inline bool iscplx<oneapi::cdouble>() {
+inline bool iscplx<arrayfire::oneapi::cdouble>() {
     return true;
 }
 
 template<typename T>
 inline std::string scalar_to_option(const T &val) {
-    using namespace common;
+    using namespace arrayfire::common;
     using namespace std;
     return to_string(+val);
 }
diff --git a/src/backend/oneapi/transform.cpp b/src/backend/oneapi/transform.cpp
index 732ba39cc0..720dfa1654 100644
--- a/src/backend/oneapi/transform.cpp
+++ b/src/backend/oneapi/transform.cpp
@@ -12,6 +12,7 @@
 // #include <kernel/transform.hpp>
 #include <err_oneapi.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -59,3 +60,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/transform.hpp b/src/backend/oneapi/transform.hpp
index 4433518055..ea62f261b0 100644
--- a/src/backend/oneapi/transform.hpp
+++ b/src/backend/oneapi/transform.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const af_interp_type method, const bool inverse,
                const bool perspective);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/transpose.cpp b/src/backend/oneapi/transpose.cpp
index cef137b561..580573125f 100644
--- a/src/backend/oneapi/transpose.cpp
+++ b/src/backend/oneapi/transpose.cpp
@@ -15,8 +15,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -50,3 +51,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/transpose.hpp b/src/backend/oneapi/transpose.hpp
index 16056bb6c5..88ca4abce0 100644
--- a/src/backend/oneapi/transpose.hpp
+++ b/src/backend/oneapi/transpose.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -18,3 +19,4 @@ template<typename T>
 void transpose_inplace(Array<T> &in, const bool conjugate);
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/transpose_inplace.cpp b/src/backend/oneapi/transpose_inplace.cpp
index 52a62d7837..ddbb14e419 100644
--- a/src/backend/oneapi/transpose_inplace.cpp
+++ b/src/backend/oneapi/transpose_inplace.cpp
@@ -15,8 +15,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -47,3 +48,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/triangle.cpp b/src/backend/oneapi/triangle.cpp
index afe0c27b7f..e418c15b93 100644
--- a/src/backend/oneapi/triangle.cpp
+++ b/src/backend/oneapi/triangle.cpp
@@ -16,8 +16,9 @@
 #include <af/dim4.hpp>
 
 using af::dim4;
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -54,3 +55,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(half)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/triangle.hpp b/src/backend/oneapi/triangle.hpp
index 0dc1a48a11..d56a26c126 100644
--- a/src/backend/oneapi/triangle.hpp
+++ b/src/backend/oneapi/triangle.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void triangle(Array<T> &out, const Array<T> &in, const bool is_upper,
@@ -18,3 +19,4 @@ template<typename T>
 Array<T> triangle(const Array<T> &in, const bool is_upper,
                   const bool is_unit_diag);
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/types.hpp b/src/backend/oneapi/types.hpp
index 10bd0e64c7..dacfd85f01 100644
--- a/src/backend/oneapi/types.hpp
+++ b/src/backend/oneapi/types.hpp
@@ -20,6 +20,7 @@
 #include <complex>
 #include <string>
 
+namespace arrayfire {
 namespace common {
 /// This is a CPU based half which need to be converted into floats before they
 /// are used
@@ -33,7 +34,9 @@ struct kernel_type<common::half> {
     using compute = float;
 };
 }  // namespace common
+}  // namespace arrayfire
 
+namespace arrayfire {
 namespace oneapi {
 using cdouble = std::complex<double>;
 using cfloat  = std::complex<float>;
@@ -130,7 +133,7 @@ inline const char *getFullName<cdouble>() {
 #if 0
 template<typename... ARGS>
 AF_CONSTEXPR const char *getTypeBuildDefinition() {
-    using common::half;
+    using arrayfire::common::half;
     using std::any_of;
     using std::array;
     using std::begin;
@@ -161,3 +164,4 @@ AF_CONSTEXPR const char *getTypeBuildDefinition() {
 #endif
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/unary.hpp b/src/backend/oneapi/unary.hpp
index 0e8a267c07..2c9ccf54ce 100644
--- a/src/backend/oneapi/unary.hpp
+++ b/src/backend/oneapi/unary.hpp
@@ -14,6 +14,7 @@
 #include <optypes.hpp>
 #include <af/traits.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<af_op_t op>
@@ -78,8 +79,8 @@ UNARY_DECL(bitnot, "__bitnot")
 
 template<typename T, af_op_t op>
 Array<T> unaryOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
-    using common::Node;
-    using common::Node_ptr;
+    using arrayfire::common::Node;
+    using arrayfire::common::Node_ptr;
     using std::array;
 
     auto createUnary = [](array<Node_ptr, 1> &operands) {
@@ -95,7 +96,7 @@ Array<T> unaryOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
 
 template<typename T, af_op_t op>
 Array<char> checkOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
-    using common::Node_ptr;
+    using arrayfire::common::Node_ptr;
 
     auto createUnary = [](std::array<Node_ptr, 1> &operands) {
         return Node_ptr(new common::UnaryNode(
@@ -109,3 +110,4 @@ Array<char> checkOp(const Array<T> &in, dim4 outDim = dim4(-1, -1, -1, -1)) {
 }
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/unwrap.cpp b/src/backend/oneapi/unwrap.cpp
index bfb21aef17..15d60afe5d 100644
--- a/src/backend/oneapi/unwrap.cpp
+++ b/src/backend/oneapi/unwrap.cpp
@@ -14,8 +14,9 @@
 #include <unwrap.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -60,3 +61,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/unwrap.hpp b/src/backend/oneapi/unwrap.hpp
index beab1dca4c..9977e99af4 100644
--- a/src/backend/oneapi/unwrap.hpp
+++ b/src/backend/oneapi/unwrap.hpp
@@ -9,9 +9,11 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> unwrap(const Array<T> &in, const dim_t wx, const dim_t wy,
                 const dim_t sx, const dim_t sy, const dim_t px, const dim_t py,
                 const dim_t dx, const dim_t dy, const bool is_column);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/vector_field.cpp b/src/backend/oneapi/vector_field.cpp
index d42c86c270..92f310698a 100644
--- a/src/backend/oneapi/vector_field.cpp
+++ b/src/backend/oneapi/vector_field.cpp
@@ -14,6 +14,7 @@
 
 using af::dim4;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -33,3 +34,4 @@ INSTANTIATE(ushort)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/vector_field.hpp b/src/backend/oneapi/vector_field.hpp
index 2c2a9b565b..b6bf83a52e 100644
--- a/src/backend/oneapi/vector_field.hpp
+++ b/src/backend/oneapi/vector_field.hpp
@@ -10,9 +10,11 @@
 #include <Array.hpp>
 #include <common/graphics_common.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 void copy_vector_field(const Array<T> &points, const Array<T> &directions,
                        fg_vector_field vfield);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/where.cpp b/src/backend/oneapi/where.cpp
index 2965cbe883..bc9e45a515 100644
--- a/src/backend/oneapi/where.cpp
+++ b/src/backend/oneapi/where.cpp
@@ -14,6 +14,7 @@
 #include <af/dim4.hpp>
 #include <complex>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -40,3 +41,4 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/where.hpp b/src/backend/oneapi/where.hpp
index a63ca73cb9..e4b1b0b87f 100644
--- a/src/backend/oneapi/where.hpp
+++ b/src/backend/oneapi/where.hpp
@@ -9,7 +9,9 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<uint> where(const Array<T>& in);
-}
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/wrap.cpp b/src/backend/oneapi/wrap.cpp
index b00b61efef..1400db07f0 100644
--- a/src/backend/oneapi/wrap.cpp
+++ b/src/backend/oneapi/wrap.cpp
@@ -17,8 +17,9 @@
 #include <wrap.hpp>
 #include <stdexcept>
 
-using common::half;
+using arrayfire::common::half;
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -73,3 +74,4 @@ INSTANTIATE(half)
 #undef INSTANTIATE
 
 }  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/wrap.hpp b/src/backend/oneapi/wrap.hpp
index ae831a9bb1..245632cbca 100644
--- a/src/backend/oneapi/wrap.hpp
+++ b/src/backend/oneapi/wrap.hpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
@@ -22,3 +23,4 @@ Array<T> wrap_dilated(const Array<T> &in, const dim_t ox, const dim_t oy,
                       const dim_t sy, const dim_t px, const dim_t py,
                       const dim_t dx, const dim_t dy, const bool is_column);
 }  // namespace oneapi
+}  // namespace arrayfire

From ab1027dbfbfbb70a1ab30c7a5b46f7d9422c95bd Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 30 Dec 2022 16:50:01 -0500
Subject: [PATCH 154/473] Fix af_spdlog target for non-header-only builds

---
 CMakeLists.txt | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 440f28ae18..96498f9a2d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -261,16 +261,20 @@ else()
   )
   add_subdirectory(${${spdlog_prefix}_SOURCE_DIR} ${${spdlog_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
 
-  target_include_directories(af_spdlog SYSTEM INTERFACE "${${spdlog_prefix}_SOURCE_DIR}/include")
   if(TARGET fmt::fmt)
     set_target_properties(af_spdlog
       PROPERTIES
         INTERFACE_COMPILE_DEFINITIONS "SPDLOG_FMT_EXTERNAL")
   endif()
+
   if(AF_WITH_SPDLOG_HEADER_ONLY)
     set_target_properties(af_spdlog
       PROPERTIES
-        INTERFACE_COMPILE_DEFINITIONS "$<TARGET_PROPERTY:af_spdlog,INTERFACE_COMPILE_DEFINITIONS>;SPDLOG_HEADER_ONLY")
+        INTERFACE_LINK_LIBRARIES "spdlog_header_only")
+  else()
+    set_target_properties(af_spdlog
+      PROPERTIES
+        INTERFACE_LINK_LIBRARIES "spdlog")
   endif()
 endif()
 

From a7d772f20fce330c00e258619c3b0b27e9bf7de7 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 30 Dec 2022 16:56:33 -0500
Subject: [PATCH 155/473] Make CUDA libraries for dynamic linking private

---
 src/backend/cuda/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 5e0119d93d..aa9f3fc037 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -588,7 +588,7 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
 
 else()
   target_link_libraries(afcuda
-    PUBLIC
+    PRIVATE
       ${CUDA_CUBLAS_LIBRARIES}
       ${CUDA_CUFFT_LIBRARIES}
       ${CUDA_cusolver_LIBRARY}

From 529e98b49c40131173f261c071e3fcdfe482742e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 31 Dec 2022 16:39:19 -0500
Subject: [PATCH 156/473] Add Version class to manage external version printing
 and conparisons

---
 src/backend/common/ArrayFireTypesIO.hpp |  52 ++++++++++++
 src/backend/common/CMakeLists.txt       |   1 +
 src/backend/common/DependencyModule.cpp |  55 +++++++-----
 src/backend/common/DependencyModule.hpp |  18 ++--
 src/backend/common/Version.hpp          |  76 +++++++++++++++++
 src/backend/common/util.cpp             |   5 --
 src/backend/common/util.hpp             |   2 -
 src/backend/cuda/convolveNN.cpp         |   4 +-
 src/backend/cuda/cudnn.cpp              |   4 +-
 src/backend/cuda/cudnnModule.cpp        | 107 ++++++++++++++----------
 src/backend/cuda/cudnnModule.hpp        |   5 +-
 src/backend/cuda/cusparseModule.cpp     |  18 ++++
 src/backend/cuda/device_manager.cpp     |  44 +++++-----
 13 files changed, 282 insertions(+), 109 deletions(-)
 create mode 100644 src/backend/common/Version.hpp

diff --git a/src/backend/common/ArrayFireTypesIO.hpp b/src/backend/common/ArrayFireTypesIO.hpp
index 234df93b43..2d6b514a3e 100644
--- a/src/backend/common/ArrayFireTypesIO.hpp
+++ b/src/backend/common/ArrayFireTypesIO.hpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #pragma once
+#include <common/Version.hpp>
 #include <spdlog/fmt/bundled/ranges.h>
 #include <spdlog/fmt/ostr.h>
 #include <af/seq.h>
@@ -35,3 +36,54 @@ struct fmt::formatter<af_seq> {
         return format_to(ctx.out(), "({} -({})-> {})", p.begin, p.step, p.end);
     }
 };
+
+template<>
+struct fmt::formatter<arrayfire::common::Version> {
+    // show major version
+    bool show_major = false;
+    // show minor version
+    bool show_minor = false;
+    // show patch version
+    bool show_patch = false;
+
+    // Parses format specifications of the form ['M' | 'm' | 'p'].
+    constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) {
+        auto it = ctx.begin(), end = ctx.end();
+        if (it == end || *it == '}') {
+            show_major = show_minor = show_patch = true;
+            return it;
+        }
+        do {
+            switch (*it) {
+                case 'M': show_major = true; break;
+                case 'm': show_minor = true; break;
+                case 'p': show_patch = true; break;
+                default: throw format_error("invalid format");
+            }
+            ++it;
+        } while (it != end && *it != '}');
+        return ctx.begin();
+    }
+
+    // Formats the point p using the parsed format specification (presentation)
+    // stored in this formatter.
+    template<typename FormatContext>
+    auto format(const arrayfire::common::Version& ver, FormatContext& ctx)
+        -> decltype(ctx.out()) {
+        // ctx.out() is an output iterator to write to.
+        // if (ver.major == -1) return format_to(ctx.out(), "N/A");
+        if (ver.minor == -1) show_minor = false;
+        if (ver.patch == -1) show_patch = false;
+        if (show_major && !show_minor && !show_patch) {
+            return format_to(ctx.out(), "{}", ver.major);
+        }
+        if (show_major && show_minor && !show_patch) {
+            return format_to(ctx.out(), "{}.{}", ver.major, ver.minor);
+        }
+        if (show_major && show_minor && show_patch) {
+            return format_to(ctx.out(), "{}.{}.{}", ver.major, ver.minor,
+                             ver.patch);
+        }
+        return ctx.out();
+    }
+};
diff --git a/src/backend/common/CMakeLists.txt b/src/backend/common/CMakeLists.txt
index 795e5df44c..b33ea2598e 100644
--- a/src/backend/common/CMakeLists.txt
+++ b/src/backend/common/CMakeLists.txt
@@ -45,6 +45,7 @@ target_sources(afcommon_interface
     ${CMAKE_CURRENT_SOURCE_DIR}/SparseArray.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/TemplateArg.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/TemplateTypename.hpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/Version.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/blas_headers.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/cast.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/cast.hpp
diff --git a/src/backend/common/DependencyModule.cpp b/src/backend/common/DependencyModule.cpp
index 6511c54e67..d8552e450d 100644
--- a/src/backend/common/DependencyModule.cpp
+++ b/src/backend/common/DependencyModule.cpp
@@ -7,8 +7,10 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <common/ArrayFireTypesIO.hpp>
 #include <common/DependencyModule.hpp>
 #include <common/Logger.hpp>
+#include <common/Version.hpp>
 #include <common/module_loading.hpp>
 
 #include <algorithm>
@@ -26,8 +28,6 @@ using std::string;
 using std::to_string;
 using std::vector;
 
-constexpr Version NullVersion{-1, -1, -1};
-
 #ifdef OS_WIN
 #include <Windows.h>
 
@@ -35,7 +35,7 @@ static const char* librarySuffix = ".dll";
 
 namespace {
 vector<string> libNames(const std::string& name, const string& suffix,
-                        const Version& ver = NullVersion) {
+                        const Version& ver = arrayfire::common::NullVersion) {
     UNUSED(ver);  // Windows DLL files are not version suffixed
     return {name + suffix + librarySuffix};
 }
@@ -48,11 +48,11 @@ static const char* libraryPrefix = "lib";
 
 namespace {
 vector<string> libNames(const std::string& name, const string& suffix,
-                        const Version& ver = NullVersion) {
+                        const Version& ver = arrayfire::common::NullVersion) {
     UNUSED(suffix);
     const string noVerName = libraryPrefix + name + librarySuffix;
-    if (ver != NullVersion) {
-        const string infix = "." + to_string(std::get<0>(ver)) + ".";
+    if (ver != arrayfire::common::NullVersion) {
+        const string infix = "." + to_string(ver.major) + ".";
         return {libraryPrefix + name + infix + librarySuffix, noVerName};
     } else {
         return {noVerName};
@@ -67,15 +67,14 @@ static const char* libraryPrefix = "lib";
 
 namespace {
 vector<string> libNames(const std::string& name, const string& suffix,
-                        const Version& ver = NullVersion) {
+                        const Version& ver = arrayfire::common::NullVersion) {
     UNUSED(suffix);
     const string noVerName = libraryPrefix + name + librarySuffix;
-    if (ver != NullVersion) {
-        const string soname("." + to_string(std::get<0>(ver)));
+    if (ver != arrayfire::common::NullVersion) {
+        const string soname("." + to_string(ver.major));
 
-        const string vsfx = "." + to_string(std::get<0>(ver)) + "." +
-                            to_string(std::get<1>(ver)) + "." +
-                            to_string(std::get<2>(ver));
+        const string vsfx = "." + to_string(ver.major) + "." +
+                            to_string(ver.minor) + "." + to_string(ver.patch);
         return {noVerName + vsfx, noVerName + soname, noVerName};
     } else {
         return {noVerName};
@@ -92,7 +91,9 @@ namespace common {
 
 DependencyModule::DependencyModule(const char* plugin_file_name,
                                    const char** paths)
-    : handle(nullptr), logger(common::loggerFactory("platform")) {
+    : handle(nullptr)
+    , logger(common::loggerFactory("platform"))
+    , version(-1, -1) {
     // TODO(umar): Implement handling of non-standard paths
     UNUSED(paths);
     if (plugin_file_name) {
@@ -107,12 +108,14 @@ DependencyModule::DependencyModule(const char* plugin_file_name,
     }
 }
 
-DependencyModule::DependencyModule(const vector<string>& plugin_base_file_name,
-                                   const vector<string>& suffixes,
-                                   const vector<string>& paths,
-                                   const size_t verListSize,
-                                   const Version* versions)
-    : handle(nullptr), logger(common::loggerFactory("platform")) {
+DependencyModule::DependencyModule(
+    const vector<string>& plugin_base_file_name, const vector<string>& suffixes,
+    const vector<string>& paths, const size_t verListSize,
+    const Version* versions,
+    std::function<Version(const LibHandle&)> versionFunction)
+    : handle(nullptr)
+    , logger(common::loggerFactory("platform"))
+    , version(-1, -1) {
     for (const string& base_name : plugin_base_file_name) {
         for (const string& path : paths) {
             UNUSED(path);
@@ -128,7 +131,12 @@ DependencyModule::DependencyModule(const vector<string>& plugin_base_file_name,
                         AF_TRACE("Attempting to load: {}", fileName);
                         handle = loadLibrary(fileName.c_str());
                         if (handle) {
-                            AF_TRACE("Found: {}", fileName);
+                            if (versionFunction) {
+                                version = versionFunction(handle);
+                                AF_TRACE("Found: {}({})", fileName, version);
+                            } else {
+                                AF_TRACE("Found: {}", fileName);
+                            }
                             return;
                         }
                     }
@@ -138,7 +146,12 @@ DependencyModule::DependencyModule(const vector<string>& plugin_base_file_name,
                 AF_TRACE("Attempting to load: {}", fileNames[0]);
                 handle = loadLibrary(fileNames[0].c_str());
                 if (handle) {
-                    AF_TRACE("Found: {}", fileNames[0]);
+                    if (versionFunction) {
+                        version = versionFunction(handle);
+                        AF_TRACE("Found: {}({})", fileNames[0], version);
+                    } else {
+                        AF_TRACE("Found: {}", fileNames[0]);
+                    }
                     return;
                 }
             }
diff --git a/src/backend/common/DependencyModule.hpp b/src/backend/common/DependencyModule.hpp
index 41cc64569e..6473a4d3bd 100644
--- a/src/backend/common/DependencyModule.hpp
+++ b/src/backend/common/DependencyModule.hpp
@@ -10,6 +10,7 @@
 #pragma once
 
 #include <common/Logger.hpp>
+#include <common/Version.hpp>
 #include <common/defines.hpp>
 #include <common/module_loading.hpp>
 
@@ -25,8 +26,6 @@ class logger;
 namespace arrayfire {
 namespace common {
 
-using Version = std::tuple<int, int, int>;  // major, minor, patch
-
 /// Allows you to create classes which dynamically load dependencies at runtime
 ///
 /// Creates a dependency module which will dynamically load a library
@@ -37,6 +36,7 @@ class DependencyModule {
     LibHandle handle;
     std::shared_ptr<spdlog::logger> logger;
     std::vector<void*> functions;
+    Version version;
 
    public:
     /// Loads the library \p plugin_file_name from the \p paths locations
@@ -47,11 +47,12 @@ class DependencyModule {
     DependencyModule(const char* plugin_file_name,
                      const char** paths = nullptr);
 
-    DependencyModule(const std::vector<std::string>& plugin_base_file_name,
-                     const std::vector<std::string>& suffixes,
-                     const std::vector<std::string>& paths,
-                     const size_t verListSize = 0,
-                     const Version* versions  = nullptr);
+    DependencyModule(
+        const std::vector<std::string>& plugin_base_file_name,
+        const std::vector<std::string>& suffixes,
+        const std::vector<std::string>& paths, const size_t verListSize = 0,
+        const Version* versions                                  = nullptr,
+        std::function<Version(const LibHandle&)> versionFunction = {});
 
     ~DependencyModule() noexcept;
 
@@ -68,6 +69,9 @@ class DependencyModule {
     /// Returns true if all of the symbols for the module were loaded
     bool symbolsLoaded() const noexcept;
 
+    /// Returns the version of the module
+    Version getVersion() const noexcept { return version; }
+
     /// Returns the last error message that occurred because of loading the
     /// library
     static std::string getErrorMessage() noexcept;
diff --git a/src/backend/common/Version.hpp b/src/backend/common/Version.hpp
new file mode 100644
index 0000000000..0b88444222
--- /dev/null
+++ b/src/backend/common/Version.hpp
@@ -0,0 +1,76 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <string>
+
+// Some compilers create these macros in the header. Causes
+// some errors in the Version struct constructor
+#ifdef major
+#undef major
+#endif
+#ifdef minor
+#undef minor
+#endif
+
+namespace arrayfire {
+namespace common {
+struct Version {
+    int major = -1;
+    int minor = -1;
+    int patch = -1;
+
+    /// Checks if the major version is defined before minor and minor is defined
+    /// before patch
+    constexpr static bool validate(int major_, int minor_,
+                                   int patch_) noexcept {
+        return !(major_ < 0 && (minor_ >= 0 || patch_ >= 0)) &&
+               !(minor_ < 0 && patch_ >= 0);
+    }
+
+    constexpr Version(const int ver_major, const int ver_minor = -1,
+                      const int ver_patch = -1) noexcept
+        : major(ver_major), minor(ver_minor), patch(ver_patch) {}
+};
+
+constexpr bool operator==(const Version& lhs, const Version& rhs) {
+    return lhs.major == rhs.major && lhs.minor == rhs.minor &&
+           lhs.patch == rhs.patch;
+}
+
+constexpr bool operator!=(const Version& lhs, const Version& rhs) {
+    return !(lhs == rhs);
+}
+
+constexpr static Version NullVersion{-1, -1, -1};
+
+constexpr bool operator<(const Version& lhs, const Version& rhs) {
+    if (lhs == NullVersion || rhs == NullVersion) return false;
+    if (lhs.major != -1 && rhs.major != -1 && lhs.major < rhs.major)
+        return true;
+    if (lhs.minor != -1 && rhs.minor != -1 && lhs.minor < rhs.minor)
+        return true;
+    if (lhs.patch != -1 && rhs.patch != -1 && lhs.patch < rhs.patch)
+        return true;
+    return false;
+}
+
+inline Version fromCudaVersion(size_t version_int) {
+    return {static_cast<int>(version_int / 1000),
+            static_cast<int>(version_int % 1000) / 10,
+            static_cast<int>(version_int % 10)};
+}
+
+inline std::string int_version_to_string(int version) {
+    return std::to_string(version / 1000) + "." +
+           std::to_string(static_cast<int>((version % 1000) / 10.));
+}
+
+}  // namespace common
+}  // namespace arrayfire
diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp
index a4cc1e2421..2d4a8e5ea0 100644
--- a/src/backend/common/util.cpp
+++ b/src/backend/common/util.cpp
@@ -136,11 +136,6 @@ void saveKernel(const string& funcName, const string& jit_ker,
     fclose(f);
 }
 
-string int_version_to_string(int version) {
-    return to_string(version / 1000) + "." +
-           to_string(static_cast<int>((version % 1000) / 10.));
-}
-
 #if defined(OS_WIN)
 string getTemporaryDirectory() {
     DWORD bufSize = 261;  // limit according to GetTempPath documentation
diff --git a/src/backend/common/util.hpp b/src/backend/common/util.hpp
index ce154775f9..8a1ad42838 100644
--- a/src/backend/common/util.hpp
+++ b/src/backend/common/util.hpp
@@ -30,8 +30,6 @@ std::string& ltrim(std::string& s);
 void saveKernel(const std::string& funcName, const std::string& jit_ker,
                 const std::string& ext);
 
-std::string int_version_to_string(int version);
-
 std::string& getCacheDirectory();
 
 bool directoryExists(const std::string& path);
diff --git a/src/backend/cuda/convolveNN.cpp b/src/backend/cuda/convolveNN.cpp
index 47dbe634cb..4988d807f3 100644
--- a/src/backend/cuda/convolveNN.cpp
+++ b/src/backend/cuda/convolveNN.cpp
@@ -70,7 +70,7 @@ pair<cudnnConvolutionFwdAlgo_t, size_t> getForwardAlgorithm(
     size_t workspace_bytes = 0;
 
     auto version = getCudnnPlugin().getVersion();
-    if (std::get<0>(version) >= 8) {
+    if (version.major >= 8) {
         int maxAlgoCount = 0;
         CUDNN_CHECK(cuda::cudnnGetConvolutionForwardAlgorithmMaxCount(
             cudnn, &maxAlgoCount));
@@ -419,7 +419,7 @@ pair<cudnnConvolutionBwdFilterAlgo_t, size_t> getBackwardFilterAlgorithm(
     size_t workspace_bytes = 0;
 
     auto version = getCudnnPlugin().getVersion();
-    if (std::get<0>(version) >= 8) {
+    if (version.major >= 8) {
         int maxAlgoCount = 0;
         CUDNN_CHECK(cuda::cudnnGetConvolutionBackwardFilterAlgorithmMaxCount(
             cudnn, &maxAlgoCount));
diff --git a/src/backend/cuda/cudnn.cpp b/src/backend/cuda/cudnn.cpp
index aa5ffd2db4..b6fd903729 100644
--- a/src/backend/cuda/cudnn.cpp
+++ b/src/backend/cuda/cudnn.cpp
@@ -238,7 +238,7 @@ cudnnStatus_t cudnnGetConvolutionForwardAlgorithm(
     cudnnConvolutionFwdPreference_t preference, size_t memoryLimitInBytes,
     cudnnConvolutionFwdAlgo_t *algo) {
     auto version = getCudnnPlugin().getVersion();
-    if (std::get<0>(version) < 8) {
+    if (version.major < 8) {
         return getCudnnPlugin().cudnnGetConvolutionForwardAlgorithm(
             handle, xDesc, wDesc, convDesc, yDesc, preference,
             memoryLimitInBytes, algo);
@@ -259,7 +259,7 @@ cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm(
     cudnnConvolutionBwdFilterPreference_t preference, size_t memoryLimitInBytes,
     cudnnConvolutionBwdFilterAlgo_t *algo) {
     auto version = getCudnnPlugin().getVersion();
-    if (std::get<0>(version) < 8) {
+    if (version.major < 8) {
         return getCudnnPlugin().cudnnGetConvolutionBackwardFilterAlgorithm(
             handle, xDesc, dyDesc, convDesc, dwDesc, preference,
             memoryLimitInBytes, algo);
diff --git a/src/backend/cuda/cudnnModule.cpp b/src/backend/cuda/cudnnModule.cpp
index 596516bbe5..657c867156 100644
--- a/src/backend/cuda/cudnnModule.cpp
+++ b/src/backend/cuda/cudnnModule.cpp
@@ -7,10 +7,12 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <cudnnModule.hpp>
+
+#include <common/ArrayFireTypesIO.hpp>
 #include <common/Logger.hpp>
 #include <common/err_common.hpp>
 #include <common/util.hpp>
-#include <cudnnModule.hpp>
 #include <device_manager.hpp>
 #include <utility.hpp>
 
@@ -18,7 +20,7 @@
 #include <string>
 #include <tuple>
 
-using arrayfire::common::int_version_to_string;
+using arrayfire::common::fromCudaVersion;
 using arrayfire::common::Version;
 using std::make_tuple;
 using std::string;
@@ -29,17 +31,17 @@ namespace cuda {
 // clang-format off
 // Latest version from each minor releases are enlisted below
 constexpr std::array<common::Version, 11> cudnnVersions = {
-    make_tuple(8, 0,  1),
-    make_tuple(7, 6,  5),
-    make_tuple(7, 5,  1),
-    make_tuple(7, 4,  2),
-    make_tuple(7, 3,  1),
-    make_tuple(7, 2,  1),
-    make_tuple(7, 1,  4),
-    make_tuple(7, 0,  5),
-    make_tuple(6, 0, 21),
-    make_tuple(5, 1, 10),
-    make_tuple(4, 0,  7)
+    Version(8, 0,  1),
+    Version(7, 6,  5),
+    Version(7, 5,  1),
+    Version(7, 4,  2),
+    Version(7, 3,  1),
+    Version(7, 2,  1),
+    Version(7, 1,  4),
+    Version(7, 0,  5),
+    Version(6, 0, 21),
+    Version(5, 1, 10),
+    Version(4, 0,  7)
 };
 // clang-format on
 
@@ -47,22 +49,32 @@ spdlog::logger* cudnnModule::getLogger() const noexcept {
     return module.getLogger();
 }
 
-auto cudnnVersionComponents(size_t version) {
-    size_t major = version / 1000;
-    size_t minor = (version - (major * 1000)) / 100;
-    size_t patch = (version - (major * 1000) - (minor * 100));
-    return make_tuple(major, minor, patch);
+Version cudnnVersionComponents(size_t version) {
+    int major = static_cast<int>(version / 1000);
+    int minor = static_cast<int>((version - (major * 1000)) / 100);
+    int patch = static_cast<int>(version - (major * 1000) - (minor * 100));
+    return {major, minor, patch};
 }
 
-auto cudaRuntimeVersionComponents(size_t version) {
-    auto major = version / 1000;
-    auto minor = (version - (major * 1000)) / 10;
-    return make_tuple(major, minor);
+Version cudaRuntimeVersionComponents(size_t version) {
+    int major = static_cast<int>(version / 1000);
+    int minor = static_cast<int>((version - (major * 1000)) / 10);
+    int patch =
+        static_cast<int>((version - (major * 1000) - (minor * 10)) / 10);
+    return {major, minor, patch};
+}
+
+Version getCudnnVersion(const LibHandle& handle) {
+    std::function<size_t()> fptr(reinterpret_cast<size_t (*)()>(
+        common::getFunctionPointer(handle, "cudnnGetVersion")));
+    size_t v = fptr();
+
+    return cudnnVersionComponents(v);
 }
 
 cudnnModule::cudnnModule()
-    : module({"cudnn"}, {"", "64_7", "64_8", "64_6", "64_5", "64_4"}, {""},
-             cudnnVersions.size(), cudnnVersions.data()) {
+    : module({"cudnn"}, {"", "64_8", "64_7", "64_6", "64_5", "64_4"}, {""},
+             cudnnVersions.size(), cudnnVersions.data(), getCudnnVersion) {
     if (!module.isLoaded()) {
         AF_TRACE(
             "WARNING: Unable to load cuDNN: {}"
@@ -77,39 +89,41 @@ cudnnModule::cudnnModule()
 
     MODULE_FUNCTION_INIT(cudnnGetVersion);
 
-    int rtmajor, rtminor;
-    size_t cudnn_version          = this->cudnnGetVersion();
-    size_t cudnn_rtversion        = 0;
-    std::tie(major, minor, patch) = cudnnVersionComponents(cudnn_version);
+    size_t cudnn_rtversion_val = 0;
 
-    if (cudnn_version >= 6000) {
-        MODULE_FUNCTION_INIT(cudnnGetCudartVersion);
-        cudnn_rtversion = this->cudnnGetCudartVersion();
-    } else {
+    Version cudnn_version = module.getVersion();
+    if (cudnn_version < Version(6)) {
         AF_TRACE(
-            "Warning: This version of cuDNN({}.{}) does not support "
+            "Warning: This version of cuDNN({}) does not support "
             "cudnnGetCudartVersion. No runtime checks performed.",
-            major, minor);
+            cudnn_version);
+    } else {
+        MODULE_FUNCTION_INIT(cudnnGetCudartVersion);
+        cudnn_rtversion_val = this->cudnnGetCudartVersion();
     }
 
-    std::tie(rtmajor, rtminor) = cudaRuntimeVersionComponents(cudnn_rtversion);
+    Version cudnn_rtversion = cudaRuntimeVersionComponents(cudnn_rtversion_val);
+
+    AF_TRACE("cuDNN Version: {} cuDNN CUDA Runtime: {}", cudnn_version,
+             cudnn_rtversion);
 
-    AF_TRACE("cuDNN Version: {}.{}.{} cuDNN CUDA Runtime: {}.{}", major, minor,
-             patch, rtmajor, rtminor);
+    Version compiled_cudnn_version = fromCudaVersion(CUDNN_VERSION);
 
     // Check to see if the version of cuDNN ArrayFire was compiled against
     // is compatible with the version loaded at runtime
-    if (CUDNN_VERSION <= 6000 && cudnn_version > CUDNN_VERSION) {
+    if (compiled_cudnn_version.major <= 6 &&
+        compiled_cudnn_version < cudnn_version) {
         string error_msg = fmt::format(
             "ArrayFire was compiled with an older version of cuDNN({}.{}) that "
             "does not support the version that was loaded at runtime({}.{}).",
-            CUDNN_MAJOR, CUDNN_MINOR, major, minor);
+            CUDNN_MAJOR, CUDNN_MINOR, cudnn_version.major, cudnn_version.minor);
         AF_ERROR(error_msg, AF_ERR_NOT_SUPPORTED);
     }
 
-    int afcuda_runtime = 0;
-    cudaRuntimeGetVersion(&afcuda_runtime);
-    if (afcuda_runtime != static_cast<int>(cudnn_rtversion)) {
+    int afcuda_runtime_version = 0;
+    cudaRuntimeGetVersion(&afcuda_runtime_version);
+    Version afcuda_runtime = fromCudaVersion(afcuda_runtime_version);
+    if (afcuda_runtime != cudnn_rtversion) {
         getLogger()->warn(
             "WARNING: ArrayFire CUDA Runtime({}) and cuDNN CUDA "
             "Runtime({}) do not match. For maximum compatibility, make sure "
@@ -117,8 +131,7 @@ cudnnModule::cudnnModule()
             // NOTE: the int version formats from CUDA and cuDNN are different
             // so we are using int_version_to_string for the ArrayFire CUDA
             // runtime
-            int_version_to_string(afcuda_runtime),
-            int_version_to_string(cudnn_rtversion));
+            afcuda_runtime, cudnn_rtversion);
     }
 
     MODULE_FUNCTION_INIT(cudnnConvolutionBackwardData);
@@ -139,14 +152,16 @@ cudnnModule::cudnnModule()
     MODULE_FUNCTION_INIT(cudnnGetConvolutionBackwardFilterWorkspaceSize);
     MODULE_FUNCTION_INIT(cudnnFindConvolutionForwardAlgorithm);
     MODULE_FUNCTION_INIT(cudnnFindConvolutionBackwardFilterAlgorithm);
-    if (major < 8) {
+    if (cudnn_version.major < 8) {
         MODULE_FUNCTION_INIT(cudnnGetConvolutionForwardAlgorithm);
         MODULE_FUNCTION_INIT(cudnnGetConvolutionBackwardFilterAlgorithm);
     }
     MODULE_FUNCTION_INIT(cudnnGetConvolutionNdForwardOutputDim);
     MODULE_FUNCTION_INIT(cudnnSetConvolution2dDescriptor);
     MODULE_FUNCTION_INIT(cudnnSetFilter4dDescriptor);
-    if (major == 4) { MODULE_FUNCTION_INIT(cudnnSetFilter4dDescriptor_v4); }
+    if (cudnn_version.major == 4) {
+        MODULE_FUNCTION_INIT(cudnnSetFilter4dDescriptor_v4);
+    }
     MODULE_FUNCTION_INIT(cudnnSetStream);
     MODULE_FUNCTION_INIT(cudnnSetTensor4dDescriptor);
 
diff --git a/src/backend/cuda/cudnnModule.hpp b/src/backend/cuda/cudnnModule.hpp
index 54c4b3b708..26856f69d7 100644
--- a/src/backend/cuda/cudnnModule.hpp
+++ b/src/backend/cuda/cudnnModule.hpp
@@ -66,7 +66,6 @@ namespace cuda {
 
 class cudnnModule {
     common::DependencyModule module;
-    int major{}, minor{}, patch{};
 
    public:
     cudnnModule();
@@ -102,9 +101,7 @@ class cudnnModule {
     spdlog::logger* getLogger() const noexcept;
 
     /// Returns the version of the cuDNN loaded at runtime
-    std::tuple<int, int, int> getVersion() const noexcept {
-        return std::make_tuple(major, minor, patch);
-    }
+    common::Version getVersion() const noexcept { return module.getVersion(); }
 
     bool isLoaded() const noexcept { return module.isLoaded(); }
 };
diff --git a/src/backend/cuda/cusparseModule.cpp b/src/backend/cuda/cusparseModule.cpp
index bc049fcb01..7d470f00e9 100644
--- a/src/backend/cuda/cusparseModule.cpp
+++ b/src/backend/cuda/cusparseModule.cpp
@@ -8,16 +8,34 @@
  ********************************************************/
 
 #include <cusparseModule.hpp>
+#include <cusparse.hpp>
 
+#include <common/Version.hpp>
 #include <common/err_common.hpp>
 #include <af/defines.h>
 
 #include <cuda.h>
 #include <string>
 
+using arrayfire::common::Version;
+
 namespace arrayfire {
 namespace cuda {
 
+common::Version getCusparseVersion(const LibHandle& handle) {
+    std::function<cusparseStatus_t(libraryPropertyType, int*)> fptr(
+        reinterpret_cast<cusparseStatus_t (*)(libraryPropertyType, int*)>(
+            common::getFunctionPointer(handle, "cusparseGetProperty")));
+
+    int major, minor, patch;
+    CUSPARSE_CHECK(fptr(MAJOR_VERSION, &major));
+    CUSPARSE_CHECK(fptr(MINOR_VERSION, &minor));
+    CUSPARSE_CHECK(fptr(PATCH_LEVEL, &patch));
+
+    Version out{major, minor, patch};
+    return out;
+}
+
 cusparseModule::cusparseModule()
     :
 #ifdef AF_cusparse_STATIC_LINKING
diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 5f79b00abf..8e7ca0e7d2 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -7,12 +7,15 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <device_manager.hpp>
+
 #if defined(OS_WIN)
 #include <windows.h>
 #endif
 
 #include <GraphicsResourceManager.hpp>
 #include <build_version.hpp>
+#include <common/ArrayFireTypesIO.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
 #include <common/MemoryManagerBase.hpp>
@@ -21,7 +24,6 @@
 #include <common/host_memory.hpp>
 #include <common/util.hpp>
 #include <cublas_v2.h>  // needed for af/cuda.h
-#include <device_manager.hpp>
 #include <driver.h>
 #include <err_cuda.hpp>
 #include <memory.hpp>
@@ -46,8 +48,8 @@
 #include <thread>
 #include <utility>
 
+using arrayfire::common::fromCudaVersion;
 using arrayfire::common::getEnvVar;
-using arrayfire::common::int_version_to_string;
 using std::begin;
 using std::end;
 using std::find;
@@ -202,7 +204,7 @@ bool checkDeviceWithRuntime(int runtime, pair<int, int> compute) {
                 "create an issue or a pull request on the ArrayFire repository "
                 "to update the Toolkit2MaxCompute array with this version of "
                 "the CUDA Runtime. Continuing.",
-                int_version_to_string(runtime));
+                fromCudaVersion(runtime));
         return true;
     }
 
@@ -264,7 +266,7 @@ void checkAndSetDevMaxCompute(pair<int, int> &computeCapability) {
                     "Please create an issue or a pull request on the ArrayFire "
                     "repository to update the Toolkit2MaxCompute array with "
                     "this version of the CUDA Runtime.",
-                    int_version_to_string(rtCudaVer), originalCompute.first,
+                    fromCudaVersion(rtCudaVer), originalCompute.first,
                     originalCompute.second, computeCapability.first,
                     computeCapability.second, computeCapability.first,
                     computeCapability.second);
@@ -451,14 +453,15 @@ void debugRuntimeCheck(spdlog::logger *logger, int runtime_version,
     // display a message in the trace. Do not throw an error unless this is
     // a debug build
     if (runtime_it == end(CudaToDriverVersion)) {
-        char buf[256];
-        char err_msg[] =
-            "CUDA runtime version(%s) not recognized. Please create an issue "
+        constexpr size_t buf_size = 256;
+        char buf[buf_size];
+        const char *err_msg =
+            "CUDA runtime version({}) not recognized. Please create an issue "
             "or a pull request on the ArrayFire repository to update the "
             "CudaToDriverVersion variable with this version of the CUDA "
             "runtime.\n";
-        snprintf(buf, 256, err_msg,
-                 int_version_to_string(runtime_version).c_str());
+        fmt::format_to_n(buf, buf_size, err_msg,
+                         fromCudaVersion(runtime_version));
         AF_TRACE("{}", buf);
 #ifndef NDEBUG
         AF_ERROR(buf, AF_ERR_RUNTIME);
@@ -471,7 +474,7 @@ void debugRuntimeCheck(spdlog::logger *logger, int runtime_version,
             "array. Please create an issue or a pull request on the ArrayFire "
             "repository to update the CudaToDriverVersion variable with this "
             "version of the CUDA runtime.\n",
-            int_version_to_string(driver_version).c_str());
+            fromCudaVersion(driver_version));
     }
 }
 
@@ -486,17 +489,17 @@ void DeviceManager::checkCudaVsDriverVersion() {
     CUDA_CHECK(cudaRuntimeGetVersion(&runtime));
 
     AF_TRACE("CUDA Driver supports up to CUDA {} ArrayFire CUDA Runtime {}",
-             int_version_to_string(driver), int_version_to_string(runtime));
+             fromCudaVersion(driver), fromCudaVersion(runtime));
 
     debugRuntimeCheck(getLogger(), runtime, driver);
 
     if (runtime > driver) {
         string msg =
-            "ArrayFire was built with CUDA %s which requires GPU driver "
-            "version %.2f or later. Please download and install the latest "
+            "ArrayFire was built with CUDA {} which requires GPU driver "
+            "version {Mm} or later. Please download and install the latest "
             "drivers from https://www.nvidia.com/drivers for your GPU. "
             "Alternatively, you could rebuild ArrayFire with CUDA Toolkit "
-            "version %s to use the current drivers.";
+            "version {} to use the current drivers.";
 
         auto runtime_it =
             find_if(begin(CudaToDriverVersion), end(CudaToDriverVersion),
@@ -504,18 +507,19 @@ void DeviceManager::checkCudaVsDriverVersion() {
                         return runtime == ver.version;
                     });
 
+        constexpr size_t buf_size = 1024;
         // If the runtime version is not part of the CudaToDriverVersion
         // array, display a message in the trace. Do not throw an error
         // unless this is a debug build
         if (runtime_it == end(CudaToDriverVersion)) {
-            char buf[1024];
+            char buf[buf_size];
             char err_msg[] =
                 "CUDA runtime version(%s) not recognized. Please create an "
                 "issue or a pull request on the ArrayFire repository to "
                 "update the CudaToDriverVersion variable with this "
                 "version of the CUDA Toolkit.";
-            snprintf(buf, 1024, err_msg,
-                     int_version_to_string(runtime).c_str());
+            snprintf(buf, buf_size, err_msg,
+                     fmt::format("{}", fromCudaVersion(runtime)).c_str());
             AF_TRACE("{}", buf);
             return;
         }
@@ -527,9 +531,9 @@ void DeviceManager::checkCudaVsDriverVersion() {
             runtime_it->unix_min_version;
 #endif
 
-        char buf[1024];
-        snprintf(buf, 1024, msg.c_str(), int_version_to_string(runtime).c_str(),
-                 minimumDriverVersion, int_version_to_string(driver).c_str());
+        char buf[buf_size];
+        fmt::format_to_n(buf, buf_size, msg, fromCudaVersion(runtime),
+                         minimumDriverVersion, fromCudaVersion(driver));
 
         AF_ERROR(buf, AF_ERR_DRIVER);
     }

From 66ca6e92adc49d76c237029592218c1de204369d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 31 Dec 2022 17:13:03 -0500
Subject: [PATCH 157/473] Update AF_ASSERT_ARRAYS_[EQ,NEAR] to accept sparse
 arrays

AF_ASSERT_ARRAY_* now accept sparse arrays and can be compared
to dense arrays now
---
 test/arrayfire_test.cpp | 282 ++++++++++++++++++++++++++++++++++++++--
 test/sparse_arith.cpp   |  68 ++--------
 test/sparse_common.hpp  |   2 +-
 test/sparse_convert.cpp |  30 +----
 4 files changed, 284 insertions(+), 98 deletions(-)

diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index b9e73b0458..a8f8a34562 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -40,6 +40,7 @@
 
 using af::af_cdouble;
 using af::af_cfloat;
+using std::vector;
 
 bool operator==(const af_half &lhs, const af_half &rhs) {
     return lhs.data_ == rhs.data_;
@@ -1390,6 +1391,116 @@ INSTANTIATE(long long);
 INSTANTIATE(unsigned long long);
 #undef INSTANTIATE
 
+template<typename T>
+struct sparseCooValue {
+    int row = 0;
+    int col = 0;
+    T value = 0;
+    sparseCooValue(int r, int c, T v) : row(r), col(c), value(v) {}
+};
+
+template<typename T>
+void swap(sparseCooValue<T> &lhs, sparseCooValue<T> &rhs) {
+    std::swap(lhs.row, rhs.row);
+    std::swap(lhs.col, rhs.col);
+    std::swap(lhs.value, rhs.value);
+}
+
+template<typename T>
+bool operator<(const sparseCooValue<T> &lhs, const sparseCooValue<T> &rhs) {
+    if (lhs.row < rhs.row) {
+        return true;
+    } else if (lhs.row == rhs.row && lhs.col < rhs.col) {
+        return true;
+    } else {
+        return false;
+    }
+}
+
+template<typename T>
+std::ostream &operator<<(std::ostream &os, const sparseCooValue<T> &val) {
+    os << "(" << val.row << ", " << val.col << "): " << val.value;
+    return os;
+}
+
+template<typename T>
+bool isZero(const sparseCooValue<T> &val) {
+    return val.value == 0.;
+}
+
+template<typename T>
+vector<sparseCooValue<T>> toCooVector(const af::array &arr) {
+    vector<sparseCooValue<T>> out;
+    if (arr.issparse()) {
+        switch (sparseGetStorage(arr)) {
+            case AF_STORAGE_COO: {
+                dim_t nnz = sparseGetNNZ(arr);
+                vector<int> row(nnz), col(nnz);
+                vector<T> values(nnz);
+                sparseGetValues(arr).host(values.data());
+                sparseGetRowIdx(arr).host(row.data());
+                sparseGetColIdx(arr).host(col.data());
+                out.reserve(nnz);
+                for (int i = 0; i < nnz; i++) {
+                    out.emplace_back(row[i], col[i], values[i]);
+                }
+            } break;
+            case AF_STORAGE_CSR: {
+                dim_t nnz = sparseGetNNZ(arr);
+                vector<int> row(arr.dims(0) + 1), col(nnz);
+                vector<T> values(nnz);
+                sparseGetValues(arr).host(values.data());
+                sparseGetRowIdx(arr).host(row.data());
+                sparseGetColIdx(arr).host(col.data());
+                out.reserve(nnz);
+                for (int i = 0; i < row.size() - 1; i++) {
+                    for (int r = row[i]; r < row[i + 1]; r++) {
+                        out.emplace_back(i, col[r], values[r]);
+                    }
+                }
+            } break;
+            case AF_STORAGE_CSC: {
+                dim_t nnz = sparseGetNNZ(arr);
+                vector<int> row(nnz), col(arr.dims(1) + 1);
+                vector<T> values(nnz);
+                sparseGetValues(arr).host(values.data());
+                sparseGetRowIdx(arr).host(row.data());
+                sparseGetColIdx(arr).host(col.data());
+                out.reserve(nnz);
+                for (int i = 0; i < col.size() - 1; i++) {
+                    for (int c = col[i]; c < col[i + 1]; c++) {
+                        out.emplace_back(row[c], i, values[c]);
+                    }
+                }
+            } break;
+            default: throw std::logic_error("NOT SUPPORTED");
+        }
+    } else {
+        vector<T> values(arr.elements());
+        arr.host(values.data());
+        int M = arr.dims(0), N = arr.dims(1);
+        for (int j = 0; j < N; j++) {
+            for (int i = 0; i < M; i++) {
+                if (std::fpclassify(real(values[j * M + i])) == FP_ZERO) {
+                    out.emplace_back(i, j, values[j * M + i]);
+                }
+            }
+        }
+    }
+
+    // Remove zero elements from result to ensure that only non-zero elements
+    // are compared
+    out.erase(std::remove_if(out.begin(), out.end(), isZero<T>), out.end());
+    std::sort(begin(out), end(out));
+    return out;
+}
+
+template<typename T>
+bool operator==(const sparseCooValue<T> &lhs, sparseCooValue<T> &rhs) {
+    return lhs.row == rhs.row && lhs.col == rhs.col &&
+           cmp(lhs.value, rhs.value);
+}
+
 template<typename T>
 std::string printContext(const std::vector<T> &hGold, std::string goldName,
                          const std::vector<T> &hOut, std::string outName,
@@ -1495,6 +1606,92 @@ std::string printContext(const std::vector<T> &hGold, std::string goldName,
     return os.str();
 }
 
+template<typename T>
+std::string printContext(const std::vector<sparseCooValue<T>> &hGold,
+                         std::string goldName,
+                         const std::vector<sparseCooValue<T>> &hOut,
+                         std::string outName, af::dim4 arrDims,
+                         af::dim4 arrStrides, dim_t idx) {
+    std::ostringstream os;
+
+    af::dim4 coords = unravelIdx(idx, arrDims, arrStrides);
+    dim_t ctxWidth  = 5;
+
+    // Coordinates that span dim0
+    af::dim4 coordsMinBound = coords;
+    coordsMinBound[0]       = 0;
+    af::dim4 coordsMaxBound = coords;
+    coordsMaxBound[0]       = arrDims[0] - 1;
+
+    // dim0 positions that can be displayed
+    dim_t dim0Start = std::max<dim_t>(0LL, idx - ctxWidth);
+    dim_t dim0End   = std::min<dim_t>(idx + ctxWidth + 1LL, hGold.size());
+
+    int setwval = 9;
+    // Linearized indices of values in vectors that can be displayed
+    dim_t vecStartIdx =
+        std::max<dim_t>(ravelIdx(coordsMinBound, arrStrides), idx - ctxWidth);
+    os << "Idx: ";
+    for (int elem = dim0Start; elem < dim0End; elem++) {
+        if (elem == idx) {
+            os << std::setw(setwval - 2) << "[" << elem << "]";
+        } else {
+            os << std::setw(setwval) << elem;
+        }
+    }
+    os << "\nRow: ";
+    for (int elem = dim0Start; elem < dim0End; elem++) {
+        if (elem == idx) {
+            os << std::setw(setwval - 2) << "[" << hGold[elem].row << "]";
+        } else {
+            os << std::setw(setwval) << hGold[elem].row;
+        }
+    }
+    os << "\n     ";
+    for (int elem = dim0Start; elem < dim0End; elem++) {
+        if (elem == idx) {
+            os << std::setw(setwval - 2) << "[" << hOut[elem].row << "]";
+        } else {
+            os << std::setw(setwval) << hOut[elem].row;
+        }
+    }
+    os << "\nCol: ";
+    for (int elem = dim0Start; elem < dim0End; elem++) {
+        if (elem == idx) {
+            os << std::setw(setwval - 2) << "[" << hGold[elem].col << "]";
+        } else {
+            os << std::setw(setwval) << hGold[elem].col;
+        }
+    }
+    os << "\n     ";
+    for (int elem = dim0Start; elem < dim0End; elem++) {
+        if (elem == idx) {
+            os << std::setw(setwval - 2) << "[" << hOut[elem].col << "]";
+        } else {
+            os << std::setw(setwval) << hOut[elem].col;
+        }
+    }
+
+    os << "\nValue: ";
+    for (int elem = dim0Start; elem < dim0End; elem++) {
+        if (elem == idx) {
+            os << std::setw(setwval - 2) << "[" << hGold[elem].value << "]";
+        } else {
+            os << std::setw(setwval) << hGold[elem].value;
+        }
+    }
+    os << "\n       ";
+    for (int elem = dim0Start; elem < dim0End; elem++) {
+        if (elem == idx) {
+            os << std::setw(setwval - 2) << "[" << hOut[elem].value << "]";
+        } else {
+            os << std::setw(setwval) << hOut[elem].value;
+        }
+    }
+
+    return os.str();
+}
+
 template<typename T>
 ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
                                       const std::vector<T> &a, af::dim4 aDims,
@@ -1502,6 +1699,7 @@ ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
                                       float maxAbsDiff, IntegerTag) {
     UNUSED(maxAbsDiff);
     typedef typename std::vector<T>::const_iterator iter;
+
     std::pair<iter, iter> mismatches =
         std::mismatch(a.begin(), a.end(), b.begin());
     iter bItr = mismatches.second;
@@ -1525,7 +1723,7 @@ struct absMatch {
     absMatch(float diff) : diff_(diff) {}
 
     template<typename T>
-    bool operator()(T lhs, T rhs) {
+    bool operator()(const T &lhs, const T &rhs) const {
         if (diff_ > 0) {
             using half_float::abs;
             using std::abs;
@@ -1537,25 +1735,26 @@ struct absMatch {
 };
 
 template<>
-bool absMatch::operator()<af::af_cfloat>(af::af_cfloat lhs, af::af_cfloat rhs) {
+bool absMatch::operator()<af::af_cfloat>(const af::af_cfloat &lhs,
+                                         const af::af_cfloat &rhs) const {
     return af::abs(rhs - lhs) <= diff_;
 }
 
 template<>
-bool absMatch::operator()<af::af_cdouble>(af::af_cdouble lhs,
-                                          af::af_cdouble rhs) {
+bool absMatch::operator()<af::af_cdouble>(const af::af_cdouble &lhs,
+                                          const af::af_cdouble &rhs) const {
     return af::abs(rhs - lhs) <= diff_;
 }
 
 template<>
-bool absMatch::operator()<std::complex<float>>(std::complex<float> lhs,
-                                               std::complex<float> rhs) {
+bool absMatch::operator()<std::complex<float>>(
+    const std::complex<float> &lhs, const std::complex<float> &rhs) const {
     return std::abs(rhs - lhs) <= diff_;
 }
 
 template<>
-bool absMatch::operator()<std::complex<double>>(std::complex<double> lhs,
-                                                std::complex<double> rhs) {
+bool absMatch::operator()<std::complex<double>>(
+    const std::complex<double> &lhs, const std::complex<double> &rhs) const {
     return std::abs(rhs - lhs) <= diff_;
 }
 
@@ -1597,6 +1796,53 @@ ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
     }
 }
 
+template<typename T>
+::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
+                                      const std::vector<sparseCooValue<T>> &a,
+                                      af::dim4 aDims,
+                                      const std::vector<sparseCooValue<T>> &b,
+                                      af::dim4 bDims, float maxAbsDiff,
+                                      IntegerTag) {
+    return ::testing::AssertionFailure() << "Unsupported sparse type\n";
+}
+template<typename T>
+::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
+                                      const std::vector<sparseCooValue<T>> &a,
+                                      af::dim4 aDims,
+                                      const std::vector<sparseCooValue<T>> &b,
+                                      af::dim4 bDims, float maxAbsDiff,
+                                      FloatTag) {
+    typedef typename std::vector<sparseCooValue<T>>::const_iterator iter;
+    // TODO(mark): Modify equality for float
+
+    const absMatch diff(maxAbsDiff);
+    std::pair<iter, iter> mismatches = std::mismatch(
+        a.begin(), a.end(), b.begin(),
+        [&diff](const sparseCooValue<T> &lhs, const sparseCooValue<T> &rhs) {
+            return lhs.row == rhs.row && lhs.col == rhs.col &&
+                   diff(lhs.value, rhs.value);
+        });
+
+    iter aItr = mismatches.first;
+    iter bItr = mismatches.second;
+
+    if (aItr == a.end()) {
+        return ::testing::AssertionSuccess();
+    } else {
+        dim_t idx       = std::distance(b.begin(), bItr);
+        af::dim4 coords = unravelIdx(idx, bDims, calcStrides(bDims));
+
+        af::dim4 aStrides = calcStrides(aDims);
+
+        ::testing::AssertionResult result =
+            ::testing::AssertionFailure()
+            << "VALUE DIFFERS at " << idx << ":\n"
+            << printContext(a, aName, b, bName, aDims, aStrides, idx);
+
+        return result;
+    }
+}
+
 template<typename T>
 ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
                                       const af::array &a, const af::array &b,
@@ -1606,13 +1852,21 @@ ::testing::AssertionResult elemWiseEq(std::string aName, std::string bName,
         FloatTag, IntegerTag>::type TagType;
     TagType tag;
 
-    std::vector<T> hA(static_cast<size_t>(a.elements()));
-    a.host(hA.data());
+    if (a.issparse() || b.issparse()) {
+        vector<sparseCooValue<T>> hA = toCooVector<T>(a);
+        vector<sparseCooValue<T>> hB = toCooVector<T>(b);
 
-    std::vector<T> hB(static_cast<size_t>(b.elements()));
-    b.host(hB.data());
-    return elemWiseEq<T>(aName, bName, hA, a.dims(), hB, b.dims(), maxAbsDiff,
-                         tag);
+        return elemWiseEq<T>(aName, bName, hA, a.dims(), hB, b.dims(),
+                             maxAbsDiff, tag);
+    } else {
+        std::vector<T> hA(static_cast<size_t>(a.elements()));
+        a.host(hA.data());
+
+        std::vector<T> hB(static_cast<size_t>(b.elements()));
+        b.host(hB.data());
+        return elemWiseEq<T>(aName, bName, hA, a.dims(), hB, b.dims(),
+                             maxAbsDiff, tag);
+    }
 }
 
 template<typename T>
diff --git a/test/sparse_arith.cpp b/test/sparse_arith.cpp
index 5f08340530..8415effed5 100644
--- a/test/sparse_arith.cpp
+++ b/test/sparse_arith.cpp
@@ -91,41 +91,6 @@ struct arith_op<af_div_t> {
     array operator()(array v1, array v2) { return v1 / v2; }
 };
 
-template<typename T>
-void sparseCompare(array A, array B, const double eps) {
-// This macro is used to check if either value is finite and then call assert
-// If neither value is finite, then they can be assumed to be equal to either
-// inf or nan
-#define ASSERT_FINITE_EQ(V1, V2)                  \
-    if (std::isfinite(V1) || std::isfinite(V2)) { \
-        ASSERT_NEAR(V1, V2, eps) << "at : " << i; \
-    }
-
-    array AValues = sparseGetValues(A);
-    array ARowIdx = sparseGetRowIdx(A);
-    array AColIdx = sparseGetColIdx(A);
-
-    array BValues = sparseGetValues(B);
-    array BRowIdx = sparseGetRowIdx(B);
-    array BColIdx = sparseGetColIdx(B);
-
-    // Verify row and col indices
-    ASSERT_EQ(0, max<int>(ARowIdx - BRowIdx));
-    ASSERT_EQ(0, max<int>(AColIdx - BColIdx));
-
-    T* ptrA = AValues.host<T>();
-    T* ptrB = BValues.host<T>();
-    for (int i = 0; i < AValues.elements(); i++) {
-        ASSERT_FINITE_EQ(real(ptrA[i]), real(ptrB[i]));
-
-        if (A.iscomplex()) { ASSERT_FINITE_EQ(imag(ptrA[i]), imag(ptrB[i])); }
-    }
-    freeHost(ptrA);
-    freeHost(ptrB);
-
-#undef ASSERT_FINITE_EQ
-}
-
 template<typename T, af_op_t op>
 void sparseArithTester(const int m, const int n, int factor, const double eps) {
     deviceGC();
@@ -154,17 +119,10 @@ void sparseArithTester(const int m, const int n, int factor, const double eps) {
     array revO = arith_op<op>()(B, OA);
     array revD = arith_op<op>()(B, A);
 
-    ASSERT_NEAR(0, sum<double>(abs(real(resR - resD))) / (m * n), eps);
-    ASSERT_NEAR(0, sum<double>(abs(imag(resR - resD))) / (m * n), eps);
-
-    ASSERT_NEAR(0, sum<double>(abs(real(resO - resD))) / (m * n), eps);
-    ASSERT_NEAR(0, sum<double>(abs(imag(resO - resD))) / (m * n), eps);
-
-    ASSERT_NEAR(0, sum<double>(abs(real(revR - revD))) / (m * n), eps);
-    ASSERT_NEAR(0, sum<double>(abs(imag(revR - revD))) / (m * n), eps);
-
-    ASSERT_NEAR(0, sum<double>(abs(real(revO - revD))) / (m * n), eps);
-    ASSERT_NEAR(0, sum<double>(abs(imag(revO - revD))) / (m * n), eps);
+    ASSERT_ARRAYS_NEAR(resD, resR, eps);
+    ASSERT_ARRAYS_NEAR(resD, resO, eps);
+    ASSERT_ARRAYS_NEAR(revD, revR, eps);
+    ASSERT_ARRAYS_NEAR(revD, revO, eps);
 }
 
 // Mul
@@ -200,11 +158,11 @@ void sparseArithTesterMul(const int m, const int n, int factor,
 
         // Check resR against conR
         array conR = sparseConvertTo(resR, AF_STORAGE_CSR);
-        sparseCompare<T>(resR, conR, eps);
+        ASSERT_ARRAYS_NEAR(resR, conR, eps);
 
         // Check resO against conO
         array conO = sparseConvertTo(resR, AF_STORAGE_COO);
-        sparseCompare<T>(resO, conO, eps);
+        ASSERT_ARRAYS_NEAR(resO, conO, eps);
     }
 
     // Reverse
@@ -219,11 +177,11 @@ void sparseArithTesterMul(const int m, const int n, int factor,
 
         // Check resR against conR
         array conR = sparseConvertTo(resR, AF_STORAGE_CSR);
-        sparseCompare<T>(resR, conR, eps);
+        ASSERT_ARRAYS_NEAR(resR, conR, eps);
 
         // Check resO against conO
         array conO = sparseConvertTo(resR, AF_STORAGE_COO);
-        sparseCompare<T>(resO, conO, eps);
+        ASSERT_ARRAYS_NEAR(resO, conO, eps);
     }
 }
 
@@ -266,11 +224,11 @@ void sparseArithTesterDiv(const int m, const int n, int factor,
 
     // Check resR against conR
     array conR = sparseConvertTo(resR, AF_STORAGE_CSR);
-    sparseCompare<T>(resR, conR, eps);
+    ASSERT_ARRAYS_EQ(resR, conR);
 
     // Check resO against conO
     array conO = sparseConvertTo(resR, AF_STORAGE_COO);
-    sparseCompare<T>(resO, conO, eps);
+    ASSERT_ARRAYS_EQ(resO, conO);
 }
 
 #define ARITH_TESTS_OPS(T, M, N, F, EPS)              \
@@ -325,11 +283,11 @@ void ssArithmetic(const int m, const int n, int factor, const double eps) {
     // Arith Op
     array resS = binOp(spA, spB);
     array resD = binOp(A, B);
+    ASSERT_ARRAYS_NEAR(resD, resS, eps);
+
     array revS = binOp(spB, spA);
     array revD = binOp(B, A);
-
-    ASSERT_ARRAYS_NEAR(resD, dense(resS), eps);
-    ASSERT_ARRAYS_NEAR(revD, dense(revS), eps);
+    ASSERT_ARRAYS_NEAR(revD, revS, eps);
 }
 
 #define SP_SP_ARITH_TEST(type, m, n, factor, eps)           \
diff --git a/test/sparse_common.hpp b/test/sparse_common.hpp
index bc95871b68..41dd3fd05d 100644
--- a/test/sparse_common.hpp
+++ b/test/sparse_common.hpp
@@ -161,7 +161,7 @@ static void convertCSR(const int M, const int N, const double ratio,
     af::array s  = af::sparse(a, AF_STORAGE_CSR);
     af::array aa = af::dense(s);
 
-    ASSERT_EQ(0, af::max<double>(af::abs(a - aa)));
+    ASSERT_ARRAYS_EQ(a, aa);
 }
 
 // This test essentially verifies that the sparse structures have the correct
diff --git a/test/sparse_convert.cpp b/test/sparse_convert.cpp
index 04599e03ca..7e8b927542 100644
--- a/test/sparse_convert.cpp
+++ b/test/sparse_convert.cpp
@@ -78,34 +78,8 @@ void sparseConvertTester(const int m, const int n, int factor) {
     // Create the dest type from dense - gold
     array dA = sparse(A, dest);
 
-    // Verify nnZ
-    dim_t dNNZ   = sparseGetNNZ(dA);
-    dim_t s2dNNZ = sparseGetNNZ(s2d);
-
-    ASSERT_EQ(dNNZ, s2dNNZ);
-
-    // Verify Types
-    af_storage dType   = sparseGetStorage(dA);
-    af_storage s2dType = sparseGetStorage(s2d);
-
-    ASSERT_EQ(dType, s2dType);
-
-    // Get the individual arrays and verify equality
-    array dValues = sparseGetValues(dA);
-    array dRowIdx = sparseGetRowIdx(dA);
-    array dColIdx = sparseGetColIdx(dA);
-
-    array s2dValues = sparseGetValues(s2d);
-    array s2dRowIdx = sparseGetRowIdx(s2d);
-    array s2dColIdx = sparseGetColIdx(s2d);
-
-    // Verify values
-    ASSERT_EQ(0, max<double>(real(dValues - s2dValues)));
-    ASSERT_EQ(0, max<double>(imag(dValues - s2dValues)));
-
-    // Verify row and col indices
-    ASSERT_EQ(0, max<int>(dRowIdx - s2dRowIdx));
-    ASSERT_EQ(0, max<int>(dColIdx - s2dColIdx));
+    ASSERT_ARRAYS_EQ(dA, s2d);
+    ASSERT_ARRAYS_EQ(A, s2d);
 }
 
 #define CONVERT_TESTS_TYPES(T, STYPE, DTYPE, SUFFIX, M, N, F) \

From 8b6a4acbbe5b983bf94bfc5b0f3ba4ee1b24e478 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 31 Dec 2022 20:20:22 -0500
Subject: [PATCH 158/473] Add support for CUDA 12

---
 src/backend/common/ArrayFireTypesIO.hpp       |   1 -
 src/backend/cuda/cusparseModule.cpp           |  72 +++++---
 src/backend/cuda/cusparseModule.hpp           |  71 ++++---
 .../cuda/cusparse_descriptor_helpers.hpp      |   9 +-
 src/backend/cuda/device_manager.cpp           |   4 +-
 src/backend/cuda/sparse.cu                    | 173 +++++++++++++-----
 src/backend/cuda/sparse_arith.cu              | 118 ++++++------
 src/backend/cuda/sparse_blas.cu               |  27 ++-
 src/backend/cuda/thrust_utils.hpp             |  18 --
 9 files changed, 316 insertions(+), 177 deletions(-)

diff --git a/src/backend/common/ArrayFireTypesIO.hpp b/src/backend/common/ArrayFireTypesIO.hpp
index 2d6b514a3e..81b73f9988 100644
--- a/src/backend/common/ArrayFireTypesIO.hpp
+++ b/src/backend/common/ArrayFireTypesIO.hpp
@@ -9,7 +9,6 @@
 
 #pragma once
 #include <common/Version.hpp>
-#include <spdlog/fmt/bundled/ranges.h>
 #include <spdlog/fmt/ostr.h>
 #include <af/seq.h>
 
diff --git a/src/backend/cuda/cusparseModule.cpp b/src/backend/cuda/cusparseModule.cpp
index 7d470f00e9..84daa25460 100644
--- a/src/backend/cuda/cusparseModule.cpp
+++ b/src/backend/cuda/cusparseModule.cpp
@@ -7,11 +7,13 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <cusparseModule.hpp>
 #include <cusparse.hpp>
+#include <cusparseModule.hpp>
 
 #include <common/Version.hpp>
 #include <common/err_common.hpp>
+#include <cusparse.hpp>
+#include <platform.hpp>
 #include <af/defines.h>
 
 #include <cuda.h>
@@ -41,7 +43,8 @@ cusparseModule::cusparseModule()
 #ifdef AF_cusparse_STATIC_LINKING
     module(nullptr, nullptr)
 #else
-    module({"cusparse"}, {"64_11", "64_10", "64_9", "64_8"}, {""})
+    module({"cusparse"}, {"64_12", "64_11", "64_10", "64_9", "64_8"}, {""}, 0,
+           nullptr, getCusparseVersion)
 #endif
 {
 #ifdef AF_cusparse_STATIC_LINKING
@@ -62,11 +65,44 @@ cusparseModule::cusparseModule()
     }
 #endif
 
+    MODULE_FUNCTION_INIT(cusparseGetVersion);
+
+#if CUSPARSE_VERSION < 11300
     MODULE_FUNCTION_INIT(cusparseCcsc2dense);
     MODULE_FUNCTION_INIT(cusparseCcsr2dense);
     MODULE_FUNCTION_INIT(cusparseCdense2csc);
     MODULE_FUNCTION_INIT(cusparseCdense2csr);
     MODULE_FUNCTION_INIT(cusparseCgthr);
+    MODULE_FUNCTION_INIT(cusparseDcsc2dense);
+    MODULE_FUNCTION_INIT(cusparseDcsr2dense);
+    MODULE_FUNCTION_INIT(cusparseDdense2csc);
+    MODULE_FUNCTION_INIT(cusparseDdense2csr);
+    MODULE_FUNCTION_INIT(cusparseDgthr);
+    MODULE_FUNCTION_INIT(cusparseScsc2dense);
+    MODULE_FUNCTION_INIT(cusparseScsr2dense);
+    MODULE_FUNCTION_INIT(cusparseSdense2csc);
+    MODULE_FUNCTION_INIT(cusparseSdense2csr);
+    MODULE_FUNCTION_INIT(cusparseSgthr);
+    MODULE_FUNCTION_INIT(cusparseZcsc2dense);
+    MODULE_FUNCTION_INIT(cusparseZcsr2dense);
+    MODULE_FUNCTION_INIT(cusparseZdense2csc);
+    MODULE_FUNCTION_INIT(cusparseZdense2csr);
+    MODULE_FUNCTION_INIT(cusparseZgthr);
+#else
+    MODULE_FUNCTION_INIT(cusparseCreateCsc);
+    MODULE_FUNCTION_INIT(cusparseSparseToDense_bufferSize);
+    MODULE_FUNCTION_INIT(cusparseSparseToDense);
+    MODULE_FUNCTION_INIT(cusparseDenseToSparse_bufferSize);
+    MODULE_FUNCTION_INIT(cusparseDenseToSparse_analysis);
+    MODULE_FUNCTION_INIT(cusparseDenseToSparse_convert);
+    MODULE_FUNCTION_INIT(cusparseSpMatGetSize);
+    MODULE_FUNCTION_INIT(cusparseCsrSetPointers);
+    MODULE_FUNCTION_INIT(cusparseCscSetPointers);
+    MODULE_FUNCTION_INIT(cusparseSetPointerMode);
+    MODULE_FUNCTION_INIT(cusparseXcsrsort_bufferSizeExt);
+    MODULE_FUNCTION_INIT(cusparseXcsrsort);
+#endif
+
     MODULE_FUNCTION_INIT(cusparseCnnz);
     MODULE_FUNCTION_INIT(cusparseCreateCsr);
     MODULE_FUNCTION_INIT(cusparseCreateDnMat);
@@ -74,25 +110,15 @@ cusparseModule::cusparseModule()
     MODULE_FUNCTION_INIT(cusparseCreateIdentityPermutation);
     MODULE_FUNCTION_INIT(cusparseCreate);
     MODULE_FUNCTION_INIT(cusparseCreateMatDescr);
-    MODULE_FUNCTION_INIT(cusparseDcsc2dense);
-    MODULE_FUNCTION_INIT(cusparseDcsr2dense);
-    MODULE_FUNCTION_INIT(cusparseDdense2csc);
-    MODULE_FUNCTION_INIT(cusparseDdense2csr);
     MODULE_FUNCTION_INIT(cusparseDestroyDnMat);
     MODULE_FUNCTION_INIT(cusparseDestroyDnVec);
     MODULE_FUNCTION_INIT(cusparseDestroy);
     MODULE_FUNCTION_INIT(cusparseDestroyMatDescr);
     MODULE_FUNCTION_INIT(cusparseDestroySpMat);
-    MODULE_FUNCTION_INIT(cusparseDgthr);
     MODULE_FUNCTION_INIT(cusparseDnnz);
-    MODULE_FUNCTION_INIT(cusparseScsc2dense);
-    MODULE_FUNCTION_INIT(cusparseScsr2dense);
-    MODULE_FUNCTION_INIT(cusparseSdense2csc);
-    MODULE_FUNCTION_INIT(cusparseSdense2csr);
     MODULE_FUNCTION_INIT(cusparseSetMatIndexBase);
     MODULE_FUNCTION_INIT(cusparseSetMatType);
     MODULE_FUNCTION_INIT(cusparseSetStream);
-    MODULE_FUNCTION_INIT(cusparseSgthr);
     MODULE_FUNCTION_INIT(cusparseSnnz);
     MODULE_FUNCTION_INIT(cusparseSpMM_bufferSize);
     MODULE_FUNCTION_INIT(cusparseSpMM);
@@ -103,14 +129,14 @@ cusparseModule::cusparseModule()
     MODULE_FUNCTION_INIT(cusparseXcoosortByColumn);
     MODULE_FUNCTION_INIT(cusparseXcoosortByRow);
     MODULE_FUNCTION_INIT(cusparseXcsr2coo);
-#if CUDA_VERSION >= 11000
-    MODULE_FUNCTION_INIT(cusparseXcsrgeam2Nnz);
-#else
+#if CUSPARSE_VERSION < 11000
     MODULE_FUNCTION_INIT(cusparseXcsrgeamNnz);
-#endif
-    MODULE_FUNCTION_INIT(cusparseZcsc2dense);
-    MODULE_FUNCTION_INIT(cusparseZcsr2dense);
-#if CUDA_VERSION >= 11000
+    MODULE_FUNCTION_INIT(cusparseScsrgeam);
+    MODULE_FUNCTION_INIT(cusparseDcsrgeam);
+    MODULE_FUNCTION_INIT(cusparseCcsrgeam);
+    MODULE_FUNCTION_INIT(cusparseZcsrgeam);
+#else
+    MODULE_FUNCTION_INIT(cusparseXcsrgeam2Nnz);
     MODULE_FUNCTION_INIT(cusparseScsrgeam2_bufferSizeExt);
     MODULE_FUNCTION_INIT(cusparseScsrgeam2);
     MODULE_FUNCTION_INIT(cusparseDcsrgeam2_bufferSizeExt);
@@ -119,15 +145,7 @@ cusparseModule::cusparseModule()
     MODULE_FUNCTION_INIT(cusparseCcsrgeam2);
     MODULE_FUNCTION_INIT(cusparseZcsrgeam2_bufferSizeExt);
     MODULE_FUNCTION_INIT(cusparseZcsrgeam2);
-#else
-    MODULE_FUNCTION_INIT(cusparseScsrgeam);
-    MODULE_FUNCTION_INIT(cusparseDcsrgeam);
-    MODULE_FUNCTION_INIT(cusparseCcsrgeam);
-    MODULE_FUNCTION_INIT(cusparseZcsrgeam);
 #endif
-    MODULE_FUNCTION_INIT(cusparseZdense2csc);
-    MODULE_FUNCTION_INIT(cusparseZdense2csr);
-    MODULE_FUNCTION_INIT(cusparseZgthr);
     MODULE_FUNCTION_INIT(cusparseZnnz);
 
 #ifndef AF_cusparse_STATIC_LINKING
diff --git a/src/backend/cuda/cusparseModule.hpp b/src/backend/cuda/cusparseModule.hpp
index ac7e826a13..5f63cec285 100644
--- a/src/backend/cuda/cusparseModule.hpp
+++ b/src/backend/cuda/cusparseModule.hpp
@@ -22,37 +22,61 @@ class cusparseModule {
     cusparseModule();
     ~cusparseModule() = default;
 
+    MODULE_MEMBER(cusparseGetVersion);
+
+#if CUSPARSE_VERSION < 11300
     MODULE_MEMBER(cusparseCcsc2dense);
     MODULE_MEMBER(cusparseCcsr2dense);
     MODULE_MEMBER(cusparseCdense2csc);
     MODULE_MEMBER(cusparseCdense2csr);
     MODULE_MEMBER(cusparseCgthr);
-    MODULE_MEMBER(cusparseCnnz);
-    MODULE_MEMBER(cusparseCreateCsr);
-    MODULE_MEMBER(cusparseCreateDnMat);
-    MODULE_MEMBER(cusparseCreateDnVec);
-    MODULE_MEMBER(cusparseCreateIdentityPermutation);
-    MODULE_MEMBER(cusparseCreate);
-    MODULE_MEMBER(cusparseCreateMatDescr);
     MODULE_MEMBER(cusparseDcsc2dense);
     MODULE_MEMBER(cusparseDcsr2dense);
     MODULE_MEMBER(cusparseDdense2csc);
     MODULE_MEMBER(cusparseDdense2csr);
+    MODULE_MEMBER(cusparseDgthr);
+    MODULE_MEMBER(cusparseScsc2dense);
+    MODULE_MEMBER(cusparseScsr2dense);
+    MODULE_MEMBER(cusparseSdense2csc);
+    MODULE_MEMBER(cusparseSdense2csr);
+    MODULE_MEMBER(cusparseSgthr);
+    MODULE_MEMBER(cusparseZcsc2dense);
+    MODULE_MEMBER(cusparseZcsr2dense);
+    MODULE_MEMBER(cusparseZdense2csc);
+    MODULE_MEMBER(cusparseZdense2csr);
+    MODULE_MEMBER(cusparseZgthr);
+#else
+    MODULE_MEMBER(cusparseCreateCsc);
+    MODULE_MEMBER(cusparseSparseToDense);
+    MODULE_MEMBER(cusparseSparseToDense_bufferSize);
+    MODULE_MEMBER(cusparseDenseToSparse_bufferSize);
+    MODULE_MEMBER(cusparseDenseToSparse_analysis);
+    MODULE_MEMBER(cusparseDenseToSparse_convert);
+    MODULE_MEMBER(cusparseSpMatGetSize);
+    MODULE_MEMBER(cusparseCsrSetPointers);
+    MODULE_MEMBER(cusparseCscSetPointers);
+    MODULE_MEMBER(cusparseGather);
+    MODULE_MEMBER(cusparseSetPointerMode);
+    MODULE_MEMBER(cusparseXcsrsort_bufferSizeExt);
+    MODULE_MEMBER(cusparseXcsrsort);
+#endif
+
+    MODULE_MEMBER(cusparseCreateCsr);
     MODULE_MEMBER(cusparseDestroyDnMat);
     MODULE_MEMBER(cusparseDestroyDnVec);
     MODULE_MEMBER(cusparseDestroy);
     MODULE_MEMBER(cusparseDestroyMatDescr);
     MODULE_MEMBER(cusparseDestroySpMat);
-    MODULE_MEMBER(cusparseDgthr);
+    MODULE_MEMBER(cusparseCnnz);
+    MODULE_MEMBER(cusparseCreateDnMat);
+    MODULE_MEMBER(cusparseCreateDnVec);
+    MODULE_MEMBER(cusparseCreateIdentityPermutation);
+    MODULE_MEMBER(cusparseCreate);
+    MODULE_MEMBER(cusparseCreateMatDescr);
     MODULE_MEMBER(cusparseDnnz);
-    MODULE_MEMBER(cusparseScsc2dense);
-    MODULE_MEMBER(cusparseScsr2dense);
-    MODULE_MEMBER(cusparseSdense2csc);
-    MODULE_MEMBER(cusparseSdense2csr);
     MODULE_MEMBER(cusparseSetMatIndexBase);
     MODULE_MEMBER(cusparseSetMatType);
     MODULE_MEMBER(cusparseSetStream);
-    MODULE_MEMBER(cusparseSgthr);
     MODULE_MEMBER(cusparseSnnz);
     MODULE_MEMBER(cusparseSpMM_bufferSize);
     MODULE_MEMBER(cusparseSpMM);
@@ -63,11 +87,14 @@ class cusparseModule {
     MODULE_MEMBER(cusparseXcoosortByColumn);
     MODULE_MEMBER(cusparseXcoosortByRow);
     MODULE_MEMBER(cusparseXcsr2coo);
-    MODULE_MEMBER(cusparseZcsc2dense);
-    MODULE_MEMBER(cusparseZcsr2dense);
 
-#if CUDA_VERSION >= 11000
-    MODULE_MEMBER(cusparseXcsrgeam2Nnz);
+#if CUSPARSE_VERSION < 11000
+    MODULE_MEMBER(cusparseCcsrgeam);
+    MODULE_MEMBER(cusparseDcsrgeam);
+    MODULE_MEMBER(cusparseScsrgeam);
+    MODULE_MEMBER(cusparseZcsrgeam);
+    MODULE_MEMBER(cusparseXcsrgeamNnz);
+#else
     MODULE_MEMBER(cusparseCcsrgeam2_bufferSizeExt);
     MODULE_MEMBER(cusparseCcsrgeam2);
     MODULE_MEMBER(cusparseDcsrgeam2_bufferSizeExt);
@@ -76,17 +103,9 @@ class cusparseModule {
     MODULE_MEMBER(cusparseScsrgeam2);
     MODULE_MEMBER(cusparseZcsrgeam2_bufferSizeExt);
     MODULE_MEMBER(cusparseZcsrgeam2);
-#else
-    MODULE_MEMBER(cusparseXcsrgeamNnz);
-    MODULE_MEMBER(cusparseCcsrgeam);
-    MODULE_MEMBER(cusparseDcsrgeam);
-    MODULE_MEMBER(cusparseScsrgeam);
-    MODULE_MEMBER(cusparseZcsrgeam);
+    MODULE_MEMBER(cusparseXcsrgeam2Nnz);
 #endif
 
-    MODULE_MEMBER(cusparseZdense2csc);
-    MODULE_MEMBER(cusparseZdense2csr);
-    MODULE_MEMBER(cusparseZgthr);
     MODULE_MEMBER(cusparseZnnz);
 
     spdlog::logger* getLogger() const noexcept;
diff --git a/src/backend/cuda/cusparse_descriptor_helpers.hpp b/src/backend/cuda/cusparse_descriptor_helpers.hpp
index 41e369b0d8..99d474cdbb 100644
--- a/src/backend/cuda/cusparse_descriptor_helpers.hpp
+++ b/src/backend/cuda/cusparse_descriptor_helpers.hpp
@@ -13,6 +13,7 @@
 // CUDA Toolkit 10.0 or later
 
 #include <common/unique_handle.hpp>
+#include <cudaDataType.hpp>
 #include <cusparse.hpp>
 
 #include <utility>
@@ -21,8 +22,9 @@ namespace arrayfire {
 namespace cuda {
 
 template<typename T>
-auto csrMatDescriptor(const common::SparseArray<T> &in) {
+auto cusparseDescriptor(const common::SparseArray<T> &in) {
     auto dims = in.dims();
+
     return common::make_handle<cusparseSpMatDescr_t>(
         dims[0], dims[1], in.getNNZ(), (void *)(in.getRowIdx().get()),
         (void *)(in.getColIdx().get()), (void *)(in.getValues().get()),
@@ -38,9 +40,10 @@ auto denVecDescriptor(const Array<T> &in) {
 
 template<typename T>
 auto denMatDescriptor(const Array<T> &in) {
-    auto dims = in.dims();
+    auto dims    = in.dims();
+    auto strides = in.strides();
     return common::make_handle<cusparseDnMatDescr_t>(
-        dims[0], dims[1], dims[0], (void *)(in.get()), getType<T>(),
+        dims[0], dims[1], strides[1], (void *)in.get(), getType<T>(),
         CUSPARSE_ORDER_COL);
 }
 
diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 8e7ca0e7d2..4f0d534b8d 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -101,6 +101,7 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {12000, 9, 0, 0},
     {11080, 9, 0, 0},
     {11070, 8, 7, 0},
     {11060, 8, 6, 0},
@@ -137,6 +138,7 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
+        {12000, 525.60f, 527.41f},
         {11080, 450.80f, 452.39f},
         {11070, 450.80f, 452.39f},
         {11060, 450.80f, 452.39f},
@@ -159,7 +161,7 @@ static const ToolkitDriverVersions
 
 // Vector of minimum supported compute versions for CUDA toolkit (i+1).*
 // where i is the index of the vector
-static const std::array<int, 11> minSV{{1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3}};
+static const std::array<int, 12> minSV{{1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 5}};
 
 static ComputeCapabilityToStreamingProcessors gpus[] = {
     {0x10, 8},   {0x11, 8},   {0x12, 8},   {0x13, 8},   {0x20, 32},
diff --git a/src/backend/cuda/sparse.cu b/src/backend/cuda/sparse.cu
index 6dec35090c..dd6d8d22b7 100644
--- a/src/backend/cuda/sparse.cu
+++ b/src/backend/cuda/sparse.cu
@@ -14,8 +14,11 @@
 #include <common/err_common.hpp>
 #include <complex.hpp>
 #include <copy.hpp>
+#include <cudaDataType.hpp>
 #include <cusparse.hpp>
 #include <cusparseModule.hpp>
+#include <cusparse_descriptor_helpers.hpp>
+#include <handle.hpp>
 #include <kernel/sparse.hpp>
 #include <lookup.hpp>
 #include <math.hpp>
@@ -129,6 +132,9 @@ struct gthr_func_def_t {
             _.cusparse##PREFIX##FUNC);                                      \
     }
 
+/// Newer versions of cusparse use matrix descriptor instead of types encoded in
+/// their names
+#if CUSPARSE_VERSION < 11300
 SPARSE_FUNC_DEF(dense2csr)
 SPARSE_FUNC(dense2csr, float, S)
 SPARSE_FUNC(dense2csr, double, D)
@@ -153,17 +159,18 @@ SPARSE_FUNC(csc2dense, double, D)
 SPARSE_FUNC(csc2dense, cfloat, C)
 SPARSE_FUNC(csc2dense, cdouble, Z)
 
-SPARSE_FUNC_DEF(nnz)
-SPARSE_FUNC(nnz, float, S)
-SPARSE_FUNC(nnz, double, D)
-SPARSE_FUNC(nnz, cfloat, C)
-SPARSE_FUNC(nnz, cdouble, Z)
-
 SPARSE_FUNC_DEF(gthr)
 SPARSE_FUNC(gthr, float, S)
 SPARSE_FUNC(gthr, double, D)
 SPARSE_FUNC(gthr, cfloat, C)
 SPARSE_FUNC(gthr, cdouble, Z)
+#endif
+
+SPARSE_FUNC_DEF(nnz)
+SPARSE_FUNC(nnz, float, S)
+SPARSE_FUNC(nnz, double, D)
+SPARSE_FUNC(nnz, cfloat, C)
+SPARSE_FUNC(nnz, cdouble, Z)
 
 #undef SPARSE_FUNC
 #undef SPARSE_FUNC_DEF
@@ -198,6 +205,7 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
     const int N = in.dims()[1];
 
     cusparseModule &_ = getCusparsePlugin();
+#if CUSPARSE_VERSION < 11300
     // Create Sparse Matrix Descriptor
     cusparseMatDescr_t descr = 0;
     CUSPARSE_CHECK(_.cusparseCreateMatDescr(&descr));
@@ -232,20 +240,97 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
     }
     Array<T> values = createEmptyArray<T>(dim4(nNZ));
 
-    if (stype == AF_STORAGE_CSR)
+    if (stype == AF_STORAGE_CSR) {
         CUSPARSE_CHECK(dense2csr_func<T>()(
             sparseHandle(), M, N, descr, in.get(), in.strides()[1],
             nnzPerDir.get(), values.get(), rowIdx.get(), colIdx.get()));
-    else
+    } else {
         CUSPARSE_CHECK(dense2csc_func<T>()(
             sparseHandle(), M, N, descr, in.get(), in.strides()[1],
             nnzPerDir.get(), values.get(), rowIdx.get(), colIdx.get()));
-
+    }
     // Destory Sparse Matrix Descriptor
     CUSPARSE_CHECK(_.cusparseDestroyMatDescr(descr));
 
     return createArrayDataSparseArray<T>(in.dims(), values, rowIdx, colIdx,
                                          stype);
+#else
+    auto matA = denMatDescriptor(in);
+    cusparseSpMatDescr_t matB;
+
+    auto d_csr_offsets = createEmptyArray<int>(M + 1);
+
+    if (stype == AF_STORAGE_CSR) {
+        // Create sparse matrix B in CSR format
+        CUSPARSE_CHECK(
+            _.cusparseCreateCsr(&matB, M, N, 0, d_csr_offsets.get(), nullptr,
+                                nullptr, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                CUSPARSE_INDEX_BASE_ZERO, getType<T>()));
+    } else {
+        CUSPARSE_CHECK(
+            _.cusparseCreateCsc(&matB, M, N, 0, d_csr_offsets.get(), nullptr,
+                                nullptr, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
+                                CUSPARSE_INDEX_BASE_ZERO, getType<T>()));
+    }
+
+    // allocate an external buffer if needed
+    size_t bufferSize;
+    CUSPARSE_CHECK(_.cusparseDenseToSparse_bufferSize(
+        sparseHandle(), matA, matB, CUSPARSE_DENSETOSPARSE_ALG_DEFAULT,
+        &bufferSize));
+
+    auto dBuffer = memAlloc<char>(bufferSize);
+
+    // execute Sparse to Dense conversion
+    CUSPARSE_CHECK(_.cusparseDenseToSparse_analysis(
+        sparseHandle(), matA, matB, CUSPARSE_DENSETOSPARSE_ALG_DEFAULT,
+        dBuffer.get()));
+    // get number of non-zero elements
+    int64_t num_rows_tmp, num_cols_tmp, nnz;
+    CUSPARSE_CHECK(
+        _.cusparseSpMatGetSize(matB, &num_rows_tmp, &num_cols_tmp, &nnz));
+
+    auto d_csr_columns = createEmptyArray<int>(nnz);
+    auto d_csr_values  = createEmptyArray<T>(nnz);
+    // allocate CSR column indices and values
+    // reset offsets, column indices, and values pointers
+    if (stype == AF_STORAGE_CSR) {
+        // Create sparse matrix B in CSR format
+        // reset offsets, column indices, and values pointers
+        CUSPARSE_CHECK(_.cusparseCsrSetPointers(matB, d_csr_offsets.get(),
+                                                d_csr_columns.get(),
+                                                d_csr_values.get()));
+
+    } else {
+        // reset offsets, column indices, and values pointers
+        CUSPARSE_CHECK(_.cusparseCscSetPointers(matB, d_csr_offsets.get(),
+                                                d_csr_columns.get(),
+                                                d_csr_values.get()));
+    }
+    // execute Sparse to Dense conversion
+    CUSPARSE_CHECK(_.cusparseDenseToSparse_convert(
+        sparseHandle(), matA, matB, CUSPARSE_DENSETOSPARSE_ALG_DEFAULT,
+        dBuffer.get()));
+
+    if (stype == AF_STORAGE_CSR) {
+        size_t pBufferSizeInBytes = 0;
+        auto desc                 = make_handle<cusparseMatDescr_t>();
+        CUSPARSE_CHECK(_.cusparseXcsrsort_bufferSizeExt(
+            sparseHandle(), M, N, nnz, d_csr_offsets.get(), d_csr_columns.get(),
+            &pBufferSizeInBytes));
+        auto pBuffer = memAlloc<char>(pBufferSizeInBytes);
+        Array<int> P = createEmptyArray<int>(nnz);
+        CUSPARSE_CHECK(
+            _.cusparseCreateIdentityPermutation(sparseHandle(), nnz, P.get()));
+        CUSPARSE_CHECK(_.cusparseXcsrsort(
+            sparseHandle(), M, N, nnz, desc, (int *)d_csr_offsets.get(),
+            (int *)d_csr_columns.get(), P.get(), pBuffer.get()));
+        d_csr_values = lookup(d_csr_values, P, 0);
+    }
+
+    return createArrayDataSparseArray<T>(in.dims(), d_csr_values, d_csr_offsets,
+                                         d_csr_columns, stype, false);
+#endif
 }
 
 // Partial template specialization of sparseConvertStorageToDense for COO
@@ -266,7 +351,8 @@ Array<T> sparseConvertCOOToDense(const SparseArray<T> &in) {
 template<typename T, af_storage stype>
 Array<T> sparseConvertStorageToDense(const SparseArray<T> &in) {
     // Create Sparse Matrix Descriptor
-    cusparseModule &_        = getCusparsePlugin();
+    cusparseModule &_ = getCusparsePlugin();
+#if CUSPARSE_VERSION < 11300
     cusparseMatDescr_t descr = 0;
     CUSPARSE_CHECK(_.cusparseCreateMatDescr(&descr));
     _.cusparseSetMatType(descr, CUSPARSE_MATRIX_TYPE_GENERAL);
@@ -277,19 +363,36 @@ Array<T> sparseConvertStorageToDense(const SparseArray<T> &in) {
     Array<T> dense = createValueArray<T>(in.dims(), scalar<T>(0));
     int d_strides1 = dense.strides()[1];
 
-    if (stype == AF_STORAGE_CSR)
+    if (stype == AF_STORAGE_CSR) {
         CUSPARSE_CHECK(
             csr2dense_func<T>()(sparseHandle(), M, N, descr,
                                 in.getValues().get(), in.getRowIdx().get(),
                                 in.getColIdx().get(), dense.get(), d_strides1));
-    else
+    } else {
         CUSPARSE_CHECK(
             csc2dense_func<T>()(sparseHandle(), M, N, descr,
                                 in.getValues().get(), in.getRowIdx().get(),
                                 in.getColIdx().get(), dense.get(), d_strides1));
+    }
 
     // Destory Sparse Matrix Descriptor
     CUSPARSE_CHECK(_.cusparseDestroyMatDescr(descr));
+#else
+    unique_handle<cusparseSpMatDescr_t> inhandle = cusparseDescriptor(in);
+
+    Array<T> dense = createEmptyArray<T>(in.dims());
+    unique_handle<cusparseDnMatDescr_t> outhandle = denMatDescriptor(dense);
+
+    size_t bufferSize = 0;
+    _.cusparseSparseToDense_bufferSize(sparseHandle(), inhandle, outhandle,
+                                       CUSPARSE_SPARSETODENSE_ALG_DEFAULT,
+                                       &bufferSize);
+
+    auto dBuffer = memAlloc<char>(bufferSize);
+    _.cusparseSparseToDense(sparseHandle(), inhandle, outhandle,
+                            CUSPARSE_SPARSETODENSE_ALG_DEFAULT, dBuffer.get());
+
+#endif
 
     return dense;
 }
@@ -321,27 +424,27 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
             sparseHandle(), in.dims()[0], in.dims()[1], nNZ,
             converted.getRowIdx().get(), converted.getColIdx().get(),
             &pBufferSizeInBytes));
-        shared_ptr<char> pBuffer(memAlloc<char>(pBufferSizeInBytes).release(),
-                                 memFree<char>);
+        auto pBuffer = memAlloc<char>(pBufferSizeInBytes);
 
-        shared_ptr<int> P(memAlloc<int>(nNZ).release(), memFree<int>);
+        // shared_ptr<int> P(memAlloc<int>(nNZ).release(), memFree<int>);
+        Array<int> P = createEmptyArray<int>(nNZ);
         CUSPARSE_CHECK(
             _.cusparseCreateIdentityPermutation(sparseHandle(), nNZ, P.get()));
 
-        CUSPARSE_CHECK(_.cusparseXcoosortByColumn(
+        CUSPARSE_CHECK(_.cusparseXcoosortByRow(
             sparseHandle(), in.dims()[0], in.dims()[1], nNZ,
             converted.getRowIdx().get(), converted.getColIdx().get(), P.get(),
-            (void *)pBuffer.get()));
+            pBuffer.get()));
 
-        CUSPARSE_CHECK(gthr_func<T>()(sparseHandle(), nNZ, in.getValues().get(),
-                                      converted.getValues().get(), P.get(),
-                                      CUSPARSE_INDEX_BASE_ZERO));
+        converted.getValues() = lookup<T, int>(in.getValues(), P, 0);
 
     } else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) {
         // The cusparse csr sort function is not behaving correctly.
         // So the work around is to convert the COO into row major and then
         // convert it to CSR
 
+        int M = in.dims()[0];
+        int N = in.dims()[1];
         // Deep copy input into temporary COO Row Major
         SparseArray<T> cooT = createArrayDataSparseArray<T>(
             in.dims(), in.getValues(), in.getRowIdx(), in.getColIdx(),
@@ -351,39 +454,27 @@ SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
         {
             size_t pBufferSizeInBytes = 0;
             CUSPARSE_CHECK(_.cusparseXcoosort_bufferSizeExt(
-                sparseHandle(), cooT.dims()[0], cooT.dims()[1], nNZ,
-                cooT.getRowIdx().get(), cooT.getColIdx().get(),
-                &pBufferSizeInBytes));
-            shared_ptr<char> pBuffer(
-                memAlloc<char>(pBufferSizeInBytes).release(), memFree<char>);
+                sparseHandle(), M, N, nNZ, cooT.getRowIdx().get(),
+                cooT.getColIdx().get(), &pBufferSizeInBytes));
+            auto pBuffer = memAlloc<char>(pBufferSizeInBytes);
 
-            shared_ptr<int> P(memAlloc<int>(nNZ).release(), memFree<int>);
+            Array<int> P = createEmptyArray<int>(nNZ);
             CUSPARSE_CHECK(_.cusparseCreateIdentityPermutation(sparseHandle(),
                                                                nNZ, P.get()));
 
             CUSPARSE_CHECK(_.cusparseXcoosortByRow(
-                sparseHandle(), cooT.dims()[0], cooT.dims()[1], nNZ,
-                cooT.getRowIdx().get(), cooT.getColIdx().get(), P.get(),
-                (void *)pBuffer.get()));
+                sparseHandle(), M, N, nNZ, cooT.getRowIdx().get(),
+                cooT.getColIdx().get(), P.get(), pBuffer.get()));
 
-            CUSPARSE_CHECK(gthr_func<T>()(
-                sparseHandle(), nNZ, in.getValues().get(),
-                cooT.getValues().get(), P.get(), CUSPARSE_INDEX_BASE_ZERO));
+            converted.getValues() = lookup<T, int>(in.getValues(), P, 0);
         }
 
         // Copy values and colIdx as is
-        CUDA_CHECK(
-            cudaMemcpyAsync(converted.getValues().get(), cooT.getValues().get(),
-                            cooT.getValues().elements() * sizeof(T),
-                            cudaMemcpyDeviceToDevice, getActiveStream()));
-        CUDA_CHECK(
-            cudaMemcpyAsync(converted.getColIdx().get(), cooT.getColIdx().get(),
-                            cooT.getColIdx().elements() * sizeof(int),
-                            cudaMemcpyDeviceToDevice, getActiveStream()));
+        copyArray<int, int>(converted.getColIdx(), cooT.getColIdx());
 
         // cusparse function to compress row from coordinate
         CUSPARSE_CHECK(_.cusparseXcoo2csr(
-            sparseHandle(), cooT.getRowIdx().get(), nNZ, cooT.dims()[0],
+            sparseHandle(), cooT.getRowIdx().get(), nNZ, M,
             converted.getRowIdx().get(), CUSPARSE_INDEX_BASE_ZERO));
 
         // No need to call CSRSORT
diff --git a/src/backend/cuda/sparse_arith.cu b/src/backend/cuda/sparse_arith.cu
index 63bda7f733..8a60aba4d3 100644
--- a/src/backend/cuda/sparse_arith.cu
+++ b/src/backend/cuda/sparse_arith.cu
@@ -7,6 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <kernel/sparse_arith.hpp>
 #include <sparse_arith.hpp>
 
 #include <arith.hpp>
@@ -16,11 +17,13 @@
 #include <complex.hpp>
 #include <copy.hpp>
 #include <cusparse.hpp>
-#include <kernel/sparse_arith.hpp>
+#include <cusparse_descriptor_helpers.hpp>
+#include <handle.hpp>
 #include <lookup.hpp>
 #include <math.hpp>
 #include <platform.hpp>
 #include <sparse.hpp>
+#include <sparse_handle.hpp>
 #include <where.hpp>
 
 #include <stdexcept>
@@ -123,10 +126,10 @@ SparseArray<T> arithOp(const SparseArray<T> &lhs, const Array<T> &rhs,
         return _.cusparse##INFIX##FUNC;          \
     }
 
-#if CUDA_VERSION >= 11000
+#if CUSPARSE_VERSION >= 11000
 
 template<typename T>
-using csrgeam2_buffer_size_def = cusparseStatus_t (*)(
+using csrgeam2_bufferSizeExt_def = cusparseStatus_t (*)(
     cusparseHandle_t, int, int, const T *, const cusparseMatDescr_t, int,
     const T *, const int *, const int *, const T *, const cusparseMatDescr_t,
     int, const T *, const int *, const int *, const cusparseMatDescr_t,
@@ -134,21 +137,21 @@ using csrgeam2_buffer_size_def = cusparseStatus_t (*)(
 
 #define SPARSE_ARITH_OP_BUFFER_SIZE_FUNC_DEF(FUNC) \
     template<typename T>                           \
-    FUNC##_buffer_size_def<T> FUNC##_buffer_size_func();
+    FUNC##_def<T> FUNC##_func();
 
-SPARSE_ARITH_OP_BUFFER_SIZE_FUNC_DEF(csrgeam2);
+SPARSE_ARITH_OP_BUFFER_SIZE_FUNC_DEF(csrgeam2_bufferSizeExt);
 
-#define SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(FUNC, TYPE, INFIX)        \
-    template<>                                                     \
-    FUNC##_buffer_size_def<TYPE> FUNC##_buffer_size_func<TYPE>() { \
-        cusparseModule &_ = getCusparsePlugin();                   \
-        return _.cusparse##INFIX##FUNC##_bufferSizeExt;            \
+#define SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(FUNC, TYPE, INFIX) \
+    template<>                                              \
+    FUNC##_def<TYPE> FUNC##_func<TYPE>() {                  \
+        cusparseModule &_ = getCusparsePlugin();            \
+        return _.cusparse##INFIX##FUNC;                     \
     }
 
-SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2, float, S);
-SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2, double, D);
-SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2, cfloat, C);
-SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2, cdouble, Z);
+SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2_bufferSizeExt, float, S);
+SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2_bufferSizeExt, double, D);
+SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2_bufferSizeExt, cfloat, C);
+SPARSE_ARITH_OP_BUFFER_SIZE_FUNC(csrgeam2_bufferSizeExt, cdouble, Z);
 
 template<typename T>
 using csrgeam2_def = cusparseStatus_t (*)(cusparseHandle_t, int, int, const T *,
@@ -188,11 +191,12 @@ SPARSE_ARITH_OP_FUNC(csrgeam, cdouble, Z);
 
 template<typename T, af_op_t op>
 SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
-    lhs.eval();
-    rhs.eval();
+    cusparseModule &_ = getCusparsePlugin();
+    af::storage sfmt  = lhs.getStorage();
+    auto ldesc        = make_handle<cusparseMatDescr_t>();
+    auto rdesc        = make_handle<cusparseMatDescr_t>();
+    auto odesc        = make_handle<cusparseMatDescr_t>();
 
-    af::storage sfmt      = lhs.getStorage();
-    auto desc             = make_handle<cusparseMatDescr_t>();
     const dim4 ldims      = lhs.dims();
     const int M           = ldims[0];
     const int N           = ldims[1];
@@ -203,59 +207,63 @@ SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
     const int *csrRowPtrB = rhs.getRowIdx().get();
     const int *csrColPtrB = rhs.getColIdx().get();
 
-    auto outRowIdx = createEmptyArray<int>(dim4(M + 1));
+    int baseC, nnzC = M + 1;
 
-    int *csrRowPtrC = outRowIdx.get();
-    int baseC, nnzC;
-    int *nnzcDevHostPtr = &nnzC;
+    auto nnzDevHostPtr = memAlloc<int>(1);
+    auto outRowIdx     = createValueArray<int>(M + 1, 0);
 
-    T alpha           = scalar<T>(1);
-    T beta            = op == af_sub_t ? scalar<T>(-1) : alpha;
-    cusparseModule &_ = getCusparsePlugin();
+    T alpha = scalar<T>(1);
+    T beta  = op == af_sub_t ? scalar<T>(-1) : scalar<T>(1);
 
-#if CUDA_VERSION >= 11000
-    size_t pBufferSize = 0;
+    T *csrValC      = nullptr;
+    int *csrColIndC = nullptr;
 
-    csrgeam2_buffer_size_func<T>()(
-        sparseHandle(), M, N, &alpha, desc, nnzA, lhs.getValues().get(),
-        csrRowPtrA, csrColPtrA, &beta, desc, nnzB, rhs.getValues().get(),
-        csrRowPtrB, csrColPtrB, desc, NULL, csrRowPtrC, NULL, &pBufferSize);
+#if CUSPARSE_VERSION < 11000
+    CUSPARSE_CHECK(_.cusparseXcsrgeamNnz(
+        sparseHandle(), M, N, ldesc, nnzA, csrRowPtrA, csrColPtrA, rdesc, nnzB,
+        csrRowPtrB, csrColPtrB, odesc, outRowIdx.get(), nnzDevHostPtr.get()));
+#else
+    size_t pBufferSize = 0;
 
-    auto tmpBuffer = createEmptyArray<char>(dim4(pBufferSize));
+    CUSPARSE_CHECK(csrgeam2_bufferSizeExt_func<T>()(
+        sparseHandle(), M, N, &alpha, ldesc, nnzA, lhs.getValues().get(),
+        csrRowPtrA, csrColPtrA, &beta, rdesc, nnzB, rhs.getValues().get(),
+        csrRowPtrB, csrColPtrB, odesc, csrValC, outRowIdx.get(), csrColIndC,
+        &pBufferSize));
 
+    auto tmpBuffer = memAlloc<char>(pBufferSize);
     CUSPARSE_CHECK(_.cusparseXcsrgeam2Nnz(
-        sparseHandle(), M, N, desc, nnzA, csrRowPtrA, csrColPtrA, desc, nnzB,
-        csrRowPtrB, csrColPtrB, desc, csrRowPtrC, nnzcDevHostPtr,
+        sparseHandle(), M, N, ldesc, nnzA, csrRowPtrA, csrColPtrA, rdesc, nnzB,
+        csrRowPtrB, csrColPtrB, odesc, outRowIdx.get(), nnzDevHostPtr.get(),
         tmpBuffer.get()));
-#else
-    CUSPARSE_CHECK(_.cusparseXcsrgeamNnz(
-        sparseHandle(), M, N, desc, nnzA, csrRowPtrA, csrColPtrA, desc, nnzB,
-        csrRowPtrB, csrColPtrB, desc, csrRowPtrC, nnzcDevHostPtr));
 #endif
-    if (NULL != nnzcDevHostPtr) {
-        nnzC = *nnzcDevHostPtr;
+    if (NULL != nnzDevHostPtr) {
+        CUDA_CHECK(cudaMemcpyAsync(&nnzC, nnzDevHostPtr.get(), sizeof(int),
+                                   cudaMemcpyDeviceToHost, getActiveStream()));
+        CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
     } else {
-        CUDA_CHECK(cudaMemcpyAsync(&nnzC, csrRowPtrC + M, sizeof(int),
+        CUDA_CHECK(cudaMemcpyAsync(&nnzC, outRowIdx.get() + M, sizeof(int),
                                    cudaMemcpyDeviceToHost, getActiveStream()));
-        CUDA_CHECK(cudaMemcpyAsync(&baseC, csrRowPtrC, sizeof(int),
+        CUDA_CHECK(cudaMemcpyAsync(&baseC, outRowIdx.get(), sizeof(int),
                                    cudaMemcpyDeviceToHost, getActiveStream()));
         CUDA_CHECK(cudaStreamSynchronize(cuda::getActiveStream()));
         nnzC -= baseC;
     }
-
-    auto outColIdx = createEmptyArray<int>(dim4(nnzC));
-    auto outValues = createEmptyArray<T>(dim4(nnzC));
-#if CUDA_VERSION >= 11000
-    csrgeam2_func<T>()(sparseHandle(), M, N, &alpha, desc, nnzA,
-                       lhs.getValues().get(), csrRowPtrA, csrColPtrA, &beta,
-                       desc, nnzB, rhs.getValues().get(), csrRowPtrB,
-                       csrColPtrB, desc, outValues.get(), csrRowPtrC,
-                       outColIdx.get(), tmpBuffer.get());
+    auto outColIdx = createEmptyArray<int>(nnzC);
+    auto outValues = createEmptyArray<T>(nnzC);
+
+#if CUSPARSE_VERSION < 11000
+    CUSPARSE_CHECK(csrgeam_func<T>()(
+        sparseHandle(), M, N, &alpha, ldesc, nnzA, lhs.getValues().get(),
+        csrRowPtrA, csrColPtrA, &beta, rdesc, nnzB, rhs.getValues().get(),
+        csrRowPtrB, csrColPtrB, odesc, outValues.get(), outRowIdx.get(),
+        outColIdx.get()));
 #else
-    csrgeam_func<T>()(sparseHandle(), M, N, &alpha, desc, nnzA,
-                      lhs.getValues().get(), csrRowPtrA, csrColPtrA, &beta,
-                      desc, nnzB, rhs.getValues().get(), csrRowPtrB, csrColPtrB,
-                      desc, outValues.get(), csrRowPtrC, outColIdx.get());
+    CUSPARSE_CHECK(csrgeam2_func<T>()(
+        sparseHandle(), M, N, &alpha, ldesc, nnzA, lhs.getValues().get(),
+        csrRowPtrA, csrColPtrA, &beta, rdesc, nnzB, rhs.getValues().get(),
+        csrRowPtrB, csrColPtrB, odesc, outValues.get(), outRowIdx.get(),
+        outColIdx.get(), tmpBuffer.get()));
 #endif
     SparseArray<T> retVal = createArrayDataSparseArray(
         ldims, outValues, outRowIdx, outColIdx, sfmt);
diff --git a/src/backend/cuda/sparse_blas.cu b/src/backend/cuda/sparse_blas.cu
index 965186a915..f0ef6a45c3 100644
--- a/src/backend/cuda/sparse_blas.cu
+++ b/src/backend/cuda/sparse_blas.cu
@@ -36,6 +36,23 @@ cusparseOperation_t toCusparseTranspose(af_mat_prop opt) {
     return out;
 }
 
+#if CUSPARSE_VERSION < 11300
+#define AF_CUSPARSE_SPMV_CSR_ALG1 CUSPARSE_CSRMV_ALG1
+#define AF_CUSPARSE_SPMV_ALG_DEFAULT CUSPARSE_MV_ALG_DEFAULT
+#define AF_CUSPARSE_SPMM_CSR_ALG1 CUSPARSE_CSRMM_ALG1
+#define AF_CUSPARSE_SPMM_CSR_ALG1 CUSPARSE_CSRMM_ALG1
+#elif CUSPARSE_VERSION < 11400
+#define AF_CUSPARSE_SPMV_CSR_ALG1 CUSPARSE_CSRMV_ALG1
+#define AF_CUSPARSE_SPMV_ALG_DEFAULT CUSPARSE_MV_ALG_DEFAULT
+#define AF_CUSPARSE_SPMM_CSR_ALG1 CUSPARSE_SPMM_CSR_ALG1
+#define AF_CUSPARSE_SPMM_CSR_ALG1 CUSPARSE_SPMM_CSR_ALG1
+#else
+#define AF_CUSPARSE_SPMV_CSR_ALG1 CUSPARSE_SPMV_CSR_ALG1
+#define AF_CUSPARSE_SPMV_ALG_DEFAULT CUSPARSE_SPMV_ALG_DEFAULT
+#define AF_CUSPARSE_SPMM_CSR_ALG1 CUSPARSE_SPMM_CSR_ALG1
+#define AF_CUSPARSE_SPMM_CSR_ALG1 CUSPARSE_SPMM_CSR_ALG1
+#endif
+
 #if defined(AF_USE_NEW_CUSPARSE_API)
 
 template<typename T>
@@ -47,7 +64,7 @@ size_t spmvBufferSize(cusparseOperation_t opA, const T *alpha,
     cusparseModule &_ = getCusparsePlugin();
     CUSPARSE_CHECK(_.cusparseSpMV_bufferSize(
         sparseHandle(), opA, alpha, matA, vecX, beta, vecY, getComputeType<T>(),
-        CUSPARSE_CSRMV_ALG1, &retVal));
+        AF_CUSPARSE_SPMV_CSR_ALG1, &retVal));
     return retVal;
 }
 
@@ -58,7 +75,7 @@ void spmv(cusparseOperation_t opA, const T *alpha,
     cusparseModule &_ = getCusparsePlugin();
     CUSPARSE_CHECK(_.cusparseSpMV(sparseHandle(), opA, alpha, matA, vecX, beta,
                                   vecY, getComputeType<T>(),
-                                  CUSPARSE_MV_ALG_DEFAULT, buffer));
+                                  AF_CUSPARSE_SPMV_ALG_DEFAULT, buffer));
 }
 
 template<typename T>
@@ -70,7 +87,7 @@ size_t spmmBufferSize(cusparseOperation_t opA, cusparseOperation_t opB,
     cusparseModule &_ = getCusparsePlugin();
     CUSPARSE_CHECK(_.cusparseSpMM_bufferSize(
         sparseHandle(), opA, opB, alpha, matA, matB, beta, matC,
-        getComputeType<T>(), CUSPARSE_CSRMM_ALG1, &retVal));
+        getComputeType<T>(), AF_CUSPARSE_SPMM_CSR_ALG1, &retVal));
     return retVal;
 }
 
@@ -81,7 +98,7 @@ void spmm(cusparseOperation_t opA, cusparseOperation_t opB, const T *alpha,
     cusparseModule &_ = getCusparsePlugin();
     CUSPARSE_CHECK(_.cusparseSpMM(sparseHandle(), opA, opB, alpha, matA, matB,
                                   beta, matC, getComputeType<T>(),
-                                  CUSPARSE_CSRMM_ALG1, buffer));
+                                  AF_CUSPARSE_SPMM_CSR_ALG1, buffer));
 }
 
 #else
@@ -158,7 +175,7 @@ Array<T> matmul(const common::SparseArray<T> &lhs, const Array<T> &rhs,
 
 #if defined(AF_USE_NEW_CUSPARSE_API)
 
-    auto spMat = csrMatDescriptor<T>(lhs);
+    auto spMat = cusparseDescriptor<T>(lhs);
 
     if (rDims[rColDim] == 1) {
         auto dnVec = denVecDescriptor<T>(rhs);
diff --git a/src/backend/cuda/thrust_utils.hpp b/src/backend/cuda/thrust_utils.hpp
index 8aafbc1752..0646b934ba 100644
--- a/src/backend/cuda/thrust_utils.hpp
+++ b/src/backend/cuda/thrust_utils.hpp
@@ -20,25 +20,7 @@ using ThrustVector = thrust::device_vector<T, ThrustAllocator<T>>;
 }  // namespace cuda
 }  // namespace arrayfire
 
-#if THRUST_MAJOR_VERSION >= 1 && THRUST_MINOR_VERSION >= 8
-
 #define THRUST_SELECT(fn, ...) \
     fn(arrayfire::cuda::ThrustArrayFirePolicy(), __VA_ARGS__)
 #define THRUST_SELECT_OUT(res, fn, ...) \
     res = fn(arrayfire::cuda::ThrustArrayFirePolicy(), __VA_ARGS__)
-
-#else
-
-#define THRUST_SELECT(fn, ...)                                                 \
-    do {                                                                       \
-        CUDA_CHECK(cudaStreamSynchronize(arrayfire::cuda::getActiveStream())); \
-        fn(__VA_ARGS__);                                                       \
-    } while (0)
-
-#define THRUST_SELECT_OUT(res, fn, ...)                                        \
-    do {                                                                       \
-        CUDA_CHECK(cudaStreamSynchronize(arrayfire::cuda::getActiveStream())); \
-        res = fn(__VA_ARGS__);                                                 \
-    } while (0)
-
-#endif

From 8263656d274125f853396b64d18f1d108969b54f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 2 Jan 2023 15:42:57 -0500
Subject: [PATCH 159/473] Enable support for p2447 style span initialization

---
 CMakeLists.txt                                |  3 ++
 src/backend/common/kernel_cache.hpp           |  2 +-
 src/backend/cuda/jit.cpp                      |  3 +-
 .../cuda/kernel/anisotropic_diffusion.hpp     |  7 ++-
 src/backend/cuda/kernel/approx.hpp            |  4 +-
 src/backend/cuda/kernel/assign.hpp            |  2 +-
 src/backend/cuda/kernel/bilateral.hpp         |  4 +-
 src/backend/cuda/kernel/canny.hpp             | 24 +++++-----
 src/backend/cuda/kernel/convolve.hpp          | 26 +++++------
 src/backend/cuda/kernel/diagonal.hpp          |  4 +-
 src/backend/cuda/kernel/diff.hpp              |  2 +-
 src/backend/cuda/kernel/exampleFunction.hpp   |  2 +-
 src/backend/cuda/kernel/fftconvolve.hpp       |  8 ++--
 src/backend/cuda/kernel/flood_fill.hpp        | 16 +++----
 src/backend/cuda/kernel/gradient.hpp          |  8 ++--
 src/backend/cuda/kernel/histogram.hpp         |  4 +-
 src/backend/cuda/kernel/hsv_rgb.hpp           |  2 +-
 src/backend/cuda/kernel/identity.hpp          |  6 +--
 src/backend/cuda/kernel/iir.hpp               |  4 +-
 src/backend/cuda/kernel/index.hpp             |  5 +--
 src/backend/cuda/kernel/iota.hpp              |  5 +--
 src/backend/cuda/kernel/ireduce.hpp           |  8 ++--
 src/backend/cuda/kernel/lookup.hpp            |  6 +--
 src/backend/cuda/kernel/lu_split.hpp          |  2 +-
 src/backend/cuda/kernel/match_template.hpp    |  2 +-
 src/backend/cuda/kernel/meanshift.hpp         |  2 +-
 src/backend/cuda/kernel/medfilt.hpp           | 18 ++++----
 src/backend/cuda/kernel/memcopy.hpp           | 15 +++----
 src/backend/cuda/kernel/moments.hpp           |  6 +--
 src/backend/cuda/kernel/morph.hpp             |  8 ++--
 src/backend/cuda/kernel/pad_array_borders.hpp |  2 +-
 src/backend/cuda/kernel/range.hpp             |  5 +--
 src/backend/cuda/kernel/reorder.hpp           |  6 +--
 src/backend/cuda/kernel/resize.hpp            |  2 +-
 src/backend/cuda/kernel/rotate.hpp            |  2 +-
 src/backend/cuda/kernel/scan_dim.hpp          |  6 +--
 .../cuda/kernel/scan_dim_by_key_impl.hpp      | 13 +++---
 src/backend/cuda/kernel/scan_first.hpp        |  6 +--
 .../cuda/kernel/scan_first_by_key_impl.hpp    | 14 +++---
 src/backend/cuda/kernel/select.hpp            |  4 +-
 src/backend/cuda/kernel/sobel.hpp             |  4 +-
 src/backend/cuda/kernel/sparse.hpp            |  4 +-
 src/backend/cuda/kernel/sparse_arith.hpp      | 16 +++----
 src/backend/cuda/kernel/susan.hpp             | 10 ++---
 src/backend/cuda/kernel/tile.hpp              |  5 +--
 src/backend/cuda/kernel/transform.hpp         |  2 +-
 src/backend/cuda/kernel/transpose.hpp         |  4 +-
 src/backend/cuda/kernel/transpose_inplace.hpp |  4 +-
 src/backend/cuda/kernel/triangle.hpp          |  2 +-
 src/backend/cuda/kernel/unwrap.hpp            |  2 +-
 src/backend/cuda/kernel/where.hpp             |  5 +--
 src/backend/cuda/kernel/wrap.hpp              |  4 +-
 src/backend/opencl/jit.cpp                    |  3 +-
 .../opencl/kernel/anisotropic_diffusion.hpp   |  6 +--
 src/backend/opencl/kernel/approx.hpp          | 10 ++---
 src/backend/opencl/kernel/assign.hpp          |  4 +-
 src/backend/opencl/kernel/bilateral.hpp       |  4 +-
 src/backend/opencl/kernel/canny.hpp           | 12 ++---
 .../opencl/kernel/convolve/conv2_impl.hpp     |  5 +--
 .../opencl/kernel/convolve/conv_common.hpp    |  5 +--
 .../opencl/kernel/convolve_separable.cpp      |  6 +--
 src/backend/opencl/kernel/cscmm.hpp           |  2 +-
 src/backend/opencl/kernel/cscmv.hpp           |  4 +-
 src/backend/opencl/kernel/csrmm.hpp           |  2 +-
 src/backend/opencl/kernel/csrmv.hpp           |  9 ++--
 src/backend/opencl/kernel/diagonal.hpp        |  6 +--
 src/backend/opencl/kernel/diff.hpp            |  4 +-
 src/backend/opencl/kernel/exampleFunction.hpp |  4 +-
 src/backend/opencl/kernel/fast.hpp            | 12 ++---
 src/backend/opencl/kernel/fftconvolve.hpp     | 18 ++++----
 src/backend/opencl/kernel/flood_fill.hpp      |  6 +--
 src/backend/opencl/kernel/gradient.hpp        |  4 +-
 src/backend/opencl/kernel/harris.hpp          | 10 ++---
 src/backend/opencl/kernel/histogram.hpp       |  4 +-
 src/backend/opencl/kernel/homography.hpp      | 20 ++++-----
 src/backend/opencl/kernel/hsv_rgb.hpp         |  4 +-
 src/backend/opencl/kernel/identity.hpp        |  4 +-
 src/backend/opencl/kernel/iir.hpp             |  3 +-
 src/backend/opencl/kernel/index.hpp           |  2 +-
 src/backend/opencl/kernel/iota.hpp            |  2 +-
 src/backend/opencl/kernel/ireduce.hpp         | 12 ++---
 src/backend/opencl/kernel/laset.hpp           |  4 +-
 src/backend/opencl/kernel/laswp.hpp           |  3 +-
 src/backend/opencl/kernel/lookup.hpp          |  4 +-
 src/backend/opencl/kernel/lu_split.hpp        |  4 +-
 src/backend/opencl/kernel/match_template.hpp  |  2 +-
 src/backend/opencl/kernel/mean.hpp            |  6 +--
 src/backend/opencl/kernel/meanshift.hpp       |  4 +-
 src/backend/opencl/kernel/medfilt.hpp         |  8 ++--
 src/backend/opencl/kernel/memcopy.hpp         |  7 ++-
 src/backend/opencl/kernel/moments.hpp         |  4 +-
 src/backend/opencl/kernel/morph.hpp           |  5 +--
 .../opencl/kernel/nearest_neighbour.hpp       |  5 +--
 src/backend/opencl/kernel/orb.hpp             | 11 ++---
 .../opencl/kernel/pad_array_borders.hpp       |  5 +--
 src/backend/opencl/kernel/random_engine.hpp   |  5 +--
 src/backend/opencl/kernel/range.hpp           |  4 +-
 src/backend/opencl/kernel/reduce.hpp          | 12 +++--
 src/backend/opencl/kernel/reduce_by_key.hpp   | 45 +++++++++----------
 src/backend/opencl/kernel/regions.hpp         |  9 ++--
 src/backend/opencl/kernel/reorder.hpp         |  4 +-
 src/backend/opencl/kernel/resize.hpp          |  4 +-
 src/backend/opencl/kernel/rotate.hpp          |  6 +--
 src/backend/opencl/kernel/scan_dim.hpp        |  4 +-
 .../opencl/kernel/scan_dim_by_key_impl.hpp    |  3 +-
 src/backend/opencl/kernel/scan_first.hpp      |  4 +-
 .../opencl/kernel/scan_first_by_key_impl.hpp  |  3 +-
 src/backend/opencl/kernel/select.hpp          |  8 ++--
 src/backend/opencl/kernel/sift.hpp            | 23 +++++-----
 src/backend/opencl/kernel/sobel.hpp           |  4 +-
 src/backend/opencl/kernel/sparse.hpp          | 20 ++++-----
 src/backend/opencl/kernel/sparse_arith.hpp    | 10 ++---
 src/backend/opencl/kernel/susan.hpp           |  8 ++--
 src/backend/opencl/kernel/swapdblk.hpp        |  4 +-
 src/backend/opencl/kernel/tile.hpp            |  3 +-
 src/backend/opencl/kernel/transform.hpp       |  6 +--
 src/backend/opencl/kernel/transpose.hpp       |  4 +-
 .../opencl/kernel/transpose_inplace.hpp       |  6 +--
 src/backend/opencl/kernel/triangle.hpp        |  2 +-
 src/backend/opencl/kernel/unwrap.hpp          |  4 +-
 src/backend/opencl/kernel/where.hpp           |  4 +-
 src/backend/opencl/kernel/wrap.hpp            |  9 ++--
 122 files changed, 378 insertions(+), 429 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 96498f9a2d..d610bba1c5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -303,6 +303,9 @@ if(NOT TARGET nonstd::span-lite)
     PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
   set_target_properties(span-lite
     PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${span_include_dir}")
+  set_target_properties(span-lite
+    PROPERTIES INTERFACE_COMPILE_DEFINITIONS "span_FEATURE_WITH_INITIALIZER_LIST_P2447=1")
+
 endif()
 
 af_dep_check_and_populate(${assets_prefix}
diff --git a/src/backend/common/kernel_cache.hpp b/src/backend/common/kernel_cache.hpp
index bef3b6b577..50602963b1 100644
--- a/src/backend/common/kernel_cache.hpp
+++ b/src/backend/common/kernel_cache.hpp
@@ -50,7 +50,7 @@ namespace common {
 ///
 /// \code
 /// auto transpose = getKernel("arrayfire::cuda::transpose",
-/// std::array{transpase_cuh_src},
+/// {{transpase_cuh_src}},
 ///         {
 ///           TemplateTypename<T>(),
 ///           TemplateArg(conjugate),
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 2ffc2f72cf..86b2b2e6a6 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -322,8 +322,7 @@ static CUfunction getKernel(const vector<Node*>& output_nodes,
         const common::Source jit_src{jitKer.c_str(), jitKer.size(),
                                      deterministicHash(jitKer)};
 
-        return common::getKernel(funcName, std::array{jit_src}, {}, {}, true)
-            .get();
+        return common::getKernel(funcName, {{jit_src}}, {}, {}, true).get();
     }
     return common::getKernel(entry, funcName, true).get();
 }
diff --git a/src/backend/cuda/kernel/anisotropic_diffusion.hpp b/src/backend/cuda/kernel/anisotropic_diffusion.hpp
index e727d7ca4c..f376b8842e 100644
--- a/src/backend/cuda/kernel/anisotropic_diffusion.hpp
+++ b/src/backend/cuda/kernel/anisotropic_diffusion.hpp
@@ -28,12 +28,11 @@ template<typename T>
 void anisotropicDiffusion(Param<T> inout, const float dt, const float mct,
                           const af::fluxFunction fftype, bool isMCDE) {
     auto diffUpdate = common::getKernel(
-        "arrayfire::cuda::diffUpdate",
-        std::array{anisotropic_diffusion_cuh_src},
+        "arrayfire::cuda::diffUpdate", {{anisotropic_diffusion_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(fftype),
                      TemplateArg(isMCDE)),
-        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y),
-                   DefineValue(YDIM_LOAD)});
+        {{DefineValue(THREADS_X), DefineValue(THREADS_Y),
+          DefineValue(YDIM_LOAD)}});
 
     dim3 threads(THREADS_X, THREADS_Y, 1);
 
diff --git a/src/backend/cuda/kernel/approx.hpp b/src/backend/cuda/kernel/approx.hpp
index db705da687..46490c06b1 100644
--- a/src/backend/cuda/kernel/approx.hpp
+++ b/src/backend/cuda/kernel/approx.hpp
@@ -29,7 +29,7 @@ void approx1(Param<Ty> yo, CParam<Ty> yi, CParam<Tp> xo, const int xdim,
              const Tp &xi_beg, const Tp &xi_step, const float offGrid,
              const af::interpType method, const int order) {
     auto approx1 = common::getKernel(
-        "arrayfire::cuda::approx1", std::array{approx1_cuh_src},
+        "arrayfire::cuda::approx1", {{approx1_cuh_src}},
         TemplateArgs(TemplateTypename<Ty>(), TemplateTypename<Tp>(),
                      TemplateArg(xdim), TemplateArg(order)));
 
@@ -57,7 +57,7 @@ void approx2(Param<Ty> zo, CParam<Ty> zi, CParam<Tp> xo, const int xdim,
              const Tp &yi_beg, const Tp &yi_step, const float offGrid,
              const af::interpType method, const int order) {
     auto approx2 = common::getKernel(
-        "arrayfire::cuda::approx2", std::array{approx2_cuh_src},
+        "arrayfire::cuda::approx2", {{approx2_cuh_src}},
         TemplateArgs(TemplateTypename<Ty>(), TemplateTypename<Tp>(),
                      TemplateArg(xdim), TemplateArg(ydim), TemplateArg(order)));
 
diff --git a/src/backend/cuda/kernel/assign.hpp b/src/backend/cuda/kernel/assign.hpp
index 75c24e874c..008de72d37 100644
--- a/src/backend/cuda/kernel/assign.hpp
+++ b/src/backend/cuda/kernel/assign.hpp
@@ -24,7 +24,7 @@ void assign(Param<T> out, CParam<T> in, const AssignKernelParam& p) {
     constexpr int THREADS_Y = 8;
 
     auto assignKer =
-        common::getKernel("arrayfire::cuda::assign", std::array{assign_cuh_src},
+        common::getKernel("arrayfire::cuda::assign", {{assign_cuh_src}},
                           TemplateArgs(TemplateTypename<T>()));
 
     const dim3 threads(THREADS_X, THREADS_Y);
diff --git a/src/backend/cuda/kernel/bilateral.hpp b/src/backend/cuda/kernel/bilateral.hpp
index cf19eeb97c..c32d946792 100644
--- a/src/backend/cuda/kernel/bilateral.hpp
+++ b/src/backend/cuda/kernel/bilateral.hpp
@@ -24,9 +24,9 @@ template<typename inType, typename outType>
 void bilateral(Param<outType> out, CParam<inType> in, float s_sigma,
                float c_sigma) {
     auto bilateral = common::getKernel(
-        "arrayfire::cuda::bilateral", std::array{bilateral_cuh_src},
+        "arrayfire::cuda::bilateral", {{bilateral_cuh_src}},
         TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>()),
-        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        {{DefineValue(THREADS_X), DefineValue(THREADS_Y)}});
 
     dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/canny.hpp b/src/backend/cuda/kernel/canny.hpp
index 61af04ba6c..ef3dc6c40c 100644
--- a/src/backend/cuda/kernel/canny.hpp
+++ b/src/backend/cuda/kernel/canny.hpp
@@ -28,10 +28,10 @@ template<typename T>
 void nonMaxSuppression(Param<T> output, CParam<T> magnitude, CParam<T> dx,
                        CParam<T> dy) {
     auto nonMaxSuppress = common::getKernel(
-        "arrayfire::cuda::nonMaxSuppression", std::array{canny_cuh_src},
+        "arrayfire::cuda::nonMaxSuppression", {{canny_cuh_src}},
         TemplateArgs(TemplateTypename<T>()),
-        std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
-                   DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        {{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
+          DefineValue(THREADS_X), DefineValue(THREADS_Y)}});
 
     dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
@@ -50,20 +50,20 @@ void nonMaxSuppression(Param<T> output, CParam<T> magnitude, CParam<T> dx,
 template<typename T>
 void edgeTrackingHysteresis(Param<T> output, CParam<T> strong, CParam<T> weak) {
     auto initEdgeOut = common::getKernel(
-        "arrayfire::cuda::initEdgeOut", std::array{canny_cuh_src},
+        "arrayfire::cuda::initEdgeOut", {{canny_cuh_src}},
         TemplateArgs(TemplateTypename<T>()),
-        std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
-                   DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        {{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
+          DefineValue(THREADS_X), DefineValue(THREADS_Y)}});
     auto edgeTrack = common::getKernel(
-        "arrayfire::cuda::edgeTrack", std::array{canny_cuh_src},
+        "arrayfire::cuda::edgeTrack", {{canny_cuh_src}},
         TemplateArgs(TemplateTypename<T>()),
-        std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
-                   DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        {{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
+          DefineValue(THREADS_X), DefineValue(THREADS_Y)}});
     auto suppressLeftOver = common::getKernel(
-        "arrayfire::cuda::suppressLeftOver", std::array{canny_cuh_src},
+        "arrayfire::cuda::suppressLeftOver", {{canny_cuh_src}},
         TemplateArgs(TemplateTypename<T>()),
-        std::array{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
-                   DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        {{DefineValue(STRONG), DefineValue(WEAK), DefineValue(NOEDGE),
+          DefineValue(THREADS_X), DefineValue(THREADS_Y)}});
 
     dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/convolve.hpp b/src/backend/cuda/kernel/convolve.hpp
index 8183805e7c..38339f2de2 100644
--- a/src/backend/cuda/kernel/convolve.hpp
+++ b/src/backend/cuda/kernel/convolve.hpp
@@ -101,11 +101,10 @@ template<typename T, typename aT>
 void convolve_1d(conv_kparam_t& p, Param<T> out, CParam<T> sig, CParam<aT> filt,
                  const bool expand) {
     auto convolve1 = common::getKernel(
-        "arrayfire::cuda::convolve1", std::array{convolve1_cuh_src},
+        "arrayfire::cuda::convolve1", {{convolve1_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
                      TemplateArg(expand)),
-        std::array{DefineValue(MAX_CONV1_FILTER_LEN),
-                   DefineValue(CONV_THREADS)});
+        {{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS)}});
 
     prepareKernelArgs<T>(p, out.dims, filt.dims, 1);
 
@@ -158,11 +157,11 @@ void conv2Helper(const conv_kparam_t& p, Param<T> out, CParam<T> sig,
     }
 
     auto convolve2 = common::getKernel(
-        "arrayfire::cuda::convolve2", std::array{convolve2_cuh_src},
+        "arrayfire::cuda::convolve2", {{convolve2_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
                      TemplateArg(expand), TemplateArg(f0), TemplateArg(f1)),
-        std::array{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
-                   DefineValue(CONV2_THREADS_X), DefineValue(CONV2_THREADS_Y)});
+        {{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
+          DefineValue(CONV2_THREADS_X), DefineValue(CONV2_THREADS_Y)}});
 
     // FIXME: case where filter array is strided
     auto constMemPtr = convolve2.getDevPtr(conv_c_name);
@@ -203,12 +202,12 @@ template<typename T, typename aT>
 void convolve_3d(conv_kparam_t& p, Param<T> out, CParam<T> sig, CParam<aT> filt,
                  const bool expand) {
     auto convolve3 = common::getKernel(
-        "arrayfire::cuda::convolve3", std::array{convolve3_cuh_src},
+        "arrayfire::cuda::convolve3", {{convolve3_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
                      TemplateArg(expand)),
-        std::array{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
-                   DefineValue(CONV3_CUBE_X), DefineValue(CONV3_CUBE_Y),
-                   DefineValue(CONV3_CUBE_Z)});
+        {{DefineValue(MAX_CONV1_FILTER_LEN), DefineValue(CONV_THREADS),
+          DefineValue(CONV3_CUBE_X), DefineValue(CONV3_CUBE_Y),
+          DefineValue(CONV3_CUBE_Z)}});
 
     prepareKernelArgs<T>(p, out.dims, filt.dims, 3);
 
@@ -308,13 +307,12 @@ void convolve2(Param<T> out, CParam<T> signal, CParam<aT> filter, int conv_dim,
     }
 
     auto convolve2_separable = common::getKernel(
-        "arrayfire::cuda::convolve2_separable",
-        std::array{convolve_separable_cuh_src},
+        "arrayfire::cuda::convolve2_separable", {{convolve_separable_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<aT>(),
                      TemplateArg(conv_dim), TemplateArg(expand),
                      TemplateArg(fLen)),
-        std::array{DefineValue(MAX_SCONV_FILTER_LEN),
-                   DefineValue(SCONV_THREADS_X), DefineValue(SCONV_THREADS_Y)});
+        {{DefineValue(MAX_SCONV_FILTER_LEN), DefineValue(SCONV_THREADS_X),
+          DefineValue(SCONV_THREADS_Y)}});
 
     dim3 threads(SCONV_THREADS_X, SCONV_THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/diagonal.hpp b/src/backend/cuda/kernel/diagonal.hpp
index 4ffb6fa4ff..40b25e159e 100644
--- a/src/backend/cuda/kernel/diagonal.hpp
+++ b/src/backend/cuda/kernel/diagonal.hpp
@@ -22,7 +22,7 @@ namespace kernel {
 template<typename T>
 void diagCreate(Param<T> out, CParam<T> in, int num) {
     auto genDiagMat = common::getKernel("arrayfire::cuda::createDiagonalMat",
-                                        std::array{diagonal_cuh_src},
+                                        {{diagonal_cuh_src}},
                                         TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(32, 8);
@@ -47,7 +47,7 @@ void diagCreate(Param<T> out, CParam<T> in, int num) {
 template<typename T>
 void diagExtract(Param<T> out, CParam<T> in, int num) {
     auto extractDiag = common::getKernel("arrayfire::cuda::extractDiagonal",
-                                         std::array{diagonal_cuh_src},
+                                         {{diagonal_cuh_src}},
                                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(256, 1);
diff --git a/src/backend/cuda/kernel/diff.hpp b/src/backend/cuda/kernel/diff.hpp
index c547e0e933..cdce6eaf8f 100644
--- a/src/backend/cuda/kernel/diff.hpp
+++ b/src/backend/cuda/kernel/diff.hpp
@@ -26,7 +26,7 @@ void diff(Param<T> out, CParam<T> in, const int indims, const unsigned dim,
     constexpr unsigned TY = 16;
 
     auto diff =
-        common::getKernel("arrayfire::cuda::diff", std::array{diff_cuh_src},
+        common::getKernel("arrayfire::cuda::diff", {{diff_cuh_src}},
                           TemplateArgs(TemplateTypename<T>(), TemplateArg(dim),
                                        TemplateArg(isDiff2)));
 
diff --git a/src/backend/cuda/kernel/exampleFunction.hpp b/src/backend/cuda/kernel/exampleFunction.hpp
index 730c309a86..4f037eb771 100644
--- a/src/backend/cuda/kernel/exampleFunction.hpp
+++ b/src/backend/cuda/kernel/exampleFunction.hpp
@@ -29,7 +29,7 @@ static const unsigned TY = 16;  // Kernel Launch Config Values
 template<typename T>  // CUDA kernel wrapper function
 void exampleFunc(Param<T> c, CParam<T> a, CParam<T> b, const af_someenum_t p) {
     auto exampleFunc = common::getKernel("arrayfire::cuda::exampleFunc",
-                                         std::array{exampleFunction_cuh_src},
+                                         {{exampleFunction_cuh_src}},
                                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(TX, TY, 1);  // set your cuda launch config for blocks
diff --git a/src/backend/cuda/kernel/fftconvolve.hpp b/src/backend/cuda/kernel/fftconvolve.hpp
index cf45bc18a4..da3657d4de 100644
--- a/src/backend/cuda/kernel/fftconvolve.hpp
+++ b/src/backend/cuda/kernel/fftconvolve.hpp
@@ -25,10 +25,10 @@ template<typename convT, typename T>
 void packDataHelper(Param<convT> sig_packed, Param<convT> filter_packed,
                     CParam<T> sig, CParam<T> filter) {
     auto packData = common::getKernel(
-        "arrayfire::cuda::packData", std::array{fftconvolve_cuh_src},
+        "arrayfire::cuda::packData", {{fftconvolve_cuh_src}},
         TemplateArgs(TemplateTypename<convT>(), TemplateTypename<T>()));
     auto padArray = common::getKernel(
-        "arrayfire::cuda::padArray", std::array{fftconvolve_cuh_src},
+        "arrayfire::cuda::padArray", {{fftconvolve_cuh_src}},
         TemplateArgs(TemplateTypename<convT>(), TemplateTypename<T>()));
 
     dim_t *sd = sig.dims;
@@ -69,7 +69,7 @@ template<typename T, typename convT>
 void complexMultiplyHelper(Param<convT> sig_packed, Param<convT> filter_packed,
                            AF_BATCH_KIND kind) {
     auto cplxMul = common::getKernel(
-        "arrayfire::cuda::complexMultiply", std::array{fftconvolve_cuh_src},
+        "arrayfire::cuda::complexMultiply", {{fftconvolve_cuh_src}},
         TemplateArgs(TemplateTypename<convT>(), TemplateArg(kind)));
 
     int sig_packed_elem    = 1;
@@ -102,7 +102,7 @@ void reorderOutputHelper(Param<T> out, Param<convT> packed, CParam<T> sig,
     constexpr bool RoundResult = std::is_integral<T>::value;
 
     auto reorderOut = common::getKernel(
-        "arrayfire::cuda::reorderOutput", std::array{fftconvolve_cuh_src},
+        "arrayfire::cuda::reorderOutput", {{fftconvolve_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateTypename<convT>(),
                      TemplateArg(expand), TemplateArg(RoundResult)));
 
diff --git a/src/backend/cuda/kernel/flood_fill.hpp b/src/backend/cuda/kernel/flood_fill.hpp
index 29f5741a04..03e3fd8fea 100644
--- a/src/backend/cuda/kernel/flood_fill.hpp
+++ b/src/backend/cuda/kernel/flood_fill.hpp
@@ -46,15 +46,15 @@ void floodFill(Param<T> out, CParam<T> image, CParam<uint> seedsx,
         CUDA_NOT_SUPPORTED(errMessage);
     }
 
-    auto initSeeds = common::getKernel("arrayfire::cuda::initSeeds",
-                                       std::array{flood_fill_cuh_src},
-                                       TemplateArgs(TemplateTypename<T>()));
-    auto floodStep = common::getKernel(
-        "arrayfire::cuda::floodStep", std::array{flood_fill_cuh_src},
-        TemplateArgs(TemplateTypename<T>()),
-        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+    auto initSeeds =
+        common::getKernel("arrayfire::cuda::initSeeds", {{flood_fill_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>()));
+    auto floodStep =
+        common::getKernel("arrayfire::cuda::floodStep", {{flood_fill_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>()),
+                          {{DefineValue(THREADS_X), DefineValue(THREADS_Y)}});
     auto finalizeOutput = common::getKernel(
-        "arrayfire::cuda::finalizeOutput", std::array{flood_fill_cuh_src},
+        "arrayfire::cuda::finalizeOutput", {{flood_fill_cuh_src}},
         TemplateArgs(TemplateTypename<T>()));
 
     EnqueueArgs qArgs(dim3(divup(seedsx.elements(), THREADS)), dim3(THREADS),
diff --git a/src/backend/cuda/kernel/gradient.hpp b/src/backend/cuda/kernel/gradient.hpp
index a6f2a8a6b9..3aaf250e60 100644
--- a/src/backend/cuda/kernel/gradient.hpp
+++ b/src/backend/cuda/kernel/gradient.hpp
@@ -26,10 +26,10 @@ void gradient(Param<T> grad0, Param<T> grad1, CParam<T> in) {
     constexpr unsigned TX = 32;
     constexpr unsigned TY = 8;
 
-    auto gradient = common::getKernel(
-        "arrayfire::cuda::gradient", std::array{gradient_cuh_src},
-        TemplateArgs(TemplateTypename<T>()),
-        std::array{DefineValue(TX), DefineValue(TY)});
+    auto gradient =
+        common::getKernel("arrayfire::cuda::gradient", {{gradient_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>()),
+                          {{DefineValue(TX), DefineValue(TY)}});
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/histogram.hpp b/src/backend/cuda/kernel/histogram.hpp
index b9a9945c99..ddc0d7fae0 100644
--- a/src/backend/cuda/kernel/histogram.hpp
+++ b/src/backend/cuda/kernel/histogram.hpp
@@ -25,9 +25,9 @@ template<typename T>
 void histogram(Param<uint> out, CParam<T> in, int nbins, float minval,
                float maxval, bool isLinear) {
     auto histogram = common::getKernel(
-        "arrayfire::cuda::histogram", std::array{histogram_cuh_src},
+        "arrayfire::cuda::histogram", {{histogram_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(isLinear)),
-        std::array{DefineValue(MAX_BINS), DefineValue(THRD_LOAD)});
+        {{DefineValue(MAX_BINS), DefineValue(THRD_LOAD)}});
 
     dim3 threads(kernel::THREADS_X, 1);
 
diff --git a/src/backend/cuda/kernel/hsv_rgb.hpp b/src/backend/cuda/kernel/hsv_rgb.hpp
index fe89bb34cb..83cae19e33 100644
--- a/src/backend/cuda/kernel/hsv_rgb.hpp
+++ b/src/backend/cuda/kernel/hsv_rgb.hpp
@@ -23,7 +23,7 @@ static const int THREADS_Y = 16;
 template<typename T>
 void hsv2rgb_convert(Param<T> out, CParam<T> in, bool isHSV2RGB) {
     auto hsvrgbConverter = common::getKernel(
-        "arrayfire::cuda::hsvrgbConverter", std::array{hsv_rgb_cuh_src},
+        "arrayfire::cuda::hsvrgbConverter", {{hsv_rgb_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(isHSV2RGB)));
 
     const dim3 threads(THREADS_X, THREADS_Y);
diff --git a/src/backend/cuda/kernel/identity.hpp b/src/backend/cuda/kernel/identity.hpp
index 42fe1707e8..c3aea2dc8b 100644
--- a/src/backend/cuda/kernel/identity.hpp
+++ b/src/backend/cuda/kernel/identity.hpp
@@ -21,9 +21,9 @@ namespace kernel {
 
 template<typename T>
 void identity(Param<T> out) {
-    auto identity = common::getKernel("arrayfire::cuda::identity",
-                                      std::array{identity_cuh_src},
-                                      TemplateArgs(TemplateTypename<T>()));
+    auto identity =
+        common::getKernel("arrayfire::cuda::identity", {{identity_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(32, 8);
     int blocks_x = divup(out.dims[0], threads.x);
diff --git a/src/backend/cuda/kernel/iir.hpp b/src/backend/cuda/kernel/iir.hpp
index f0f58512d8..a17d205fd8 100644
--- a/src/backend/cuda/kernel/iir.hpp
+++ b/src/backend/cuda/kernel/iir.hpp
@@ -24,9 +24,9 @@ void iir(Param<T> y, CParam<T> c, CParam<T> a) {
     constexpr int MAX_A_SIZE = 1024;
 
     auto iir = common::getKernel(
-        "arrayfire::cuda::iir", std::array{iir_cuh_src},
+        "arrayfire::cuda::iir", {{iir_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(batch_a)),
-        std::array{DefineValue(MAX_A_SIZE)});
+        {{DefineValue(MAX_A_SIZE)}});
 
     const int blocks_y = y.dims[1];
     const int blocks_x = y.dims[2];
diff --git a/src/backend/cuda/kernel/index.hpp b/src/backend/cuda/kernel/index.hpp
index 63d318408e..d2a4d06d37 100644
--- a/src/backend/cuda/kernel/index.hpp
+++ b/src/backend/cuda/kernel/index.hpp
@@ -22,9 +22,8 @@ namespace kernel {
 
 template<typename T>
 void index(Param<T> out, CParam<T> in, const IndexKernelParam& p) {
-    auto index =
-        common::getKernel("arrayfire::cuda::index", std::array{index_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto index = common::getKernel("arrayfire::cuda::index", {{index_cuh_src}},
+                                   TemplateArgs(TemplateTypename<T>()));
     dim3 threads;
     switch (out.dims[1]) {
         case 1: threads.y = 1; break;
diff --git a/src/backend/cuda/kernel/iota.hpp b/src/backend/cuda/kernel/iota.hpp
index 7624f68559..1007ec2f1e 100644
--- a/src/backend/cuda/kernel/iota.hpp
+++ b/src/backend/cuda/kernel/iota.hpp
@@ -27,9 +27,8 @@ void iota(Param<T> out, const af::dim4 &sdims) {
     constexpr unsigned TILEX   = 512;
     constexpr unsigned TILEY   = 32;
 
-    auto iota =
-        common::getKernel("arrayfire::cuda::iota", std::array{iota_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto iota = common::getKernel("arrayfire::cuda::iota", {{iota_cuh_src}},
+                                  TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(IOTA_TX, IOTA_TY, 1);
 
diff --git a/src/backend/cuda/kernel/ireduce.hpp b/src/backend/cuda/kernel/ireduce.hpp
index 91539469eb..c394c01f83 100644
--- a/src/backend/cuda/kernel/ireduce.hpp
+++ b/src/backend/cuda/kernel/ireduce.hpp
@@ -37,10 +37,10 @@ void ireduce_dim_launcher(Param<T> out, uint *olptr, CParam<T> in,
     blocks.y             = divup(blocks.y, blocks.z);
 
     auto ireduceDim = common::getKernel(
-        "arrayfire::cuda::ireduceDim", std::array{ireduce_cuh_src},
+        "arrayfire::cuda::ireduceDim", {{ireduce_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(op), TemplateArg(dim),
                      TemplateArg(is_first), TemplateArg(threads_y)),
-        std::array{DefineValue(THREADS_X)});
+        {{DefineValue(THREADS_X)}});
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -104,10 +104,10 @@ void ireduce_first_launcher(Param<T> out, uint *olptr, CParam<T> in,
 
     // threads_x can take values 32, 64, 128, 256
     auto ireduceFirst = common::getKernel(
-        "arrayfire::cuda::ireduceFirst", std::array{ireduce_cuh_src},
+        "arrayfire::cuda::ireduceFirst", {{ireduce_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(op),
                      TemplateArg(is_first), TemplateArg(threads_x)),
-        std::array{DefineValue(THREADS_PER_BLOCK)});
+        {{DefineValue(THREADS_PER_BLOCK)}});
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
diff --git a/src/backend/cuda/kernel/lookup.hpp b/src/backend/cuda/kernel/lookup.hpp
index b4395980f0..4d23596d6c 100644
--- a/src/backend/cuda/kernel/lookup.hpp
+++ b/src/backend/cuda/kernel/lookup.hpp
@@ -44,9 +44,9 @@ void lookup(Param<in_t> out, CParam<in_t> in, CParam<idx_t> indices, int nDims,
         dim3 blocks(blks, 1);
 
         auto lookup1d = common::getKernel(
-            "arrayfire::cuda::lookup1D", std::array{lookup_cuh_src},
+            "arrayfire::cuda::lookup1D", {{lookup_cuh_src}},
             TemplateArgs(TemplateTypename<in_t>(), TemplateTypename<idx_t>()),
-            std::array{DefineValue(THREADS), DefineValue(THRD_LOAD)});
+            {{DefineValue(THREADS), DefineValue(THRD_LOAD)}});
 
         EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
@@ -65,7 +65,7 @@ void lookup(Param<in_t> out, CParam<in_t> in, CParam<idx_t> indices, int nDims,
         blocks.y = divup(blocks.y, blocks.z);
 
         auto lookupnd = common::getKernel(
-            "arrayfire::cuda::lookupND", std::array{lookup_cuh_src},
+            "arrayfire::cuda::lookupND", {{lookup_cuh_src}},
             TemplateArgs(TemplateTypename<in_t>(), TemplateTypename<idx_t>(),
                          TemplateArg(dim)));
         EnqueueArgs qArgs(blocks, threads, getActiveStream());
diff --git a/src/backend/cuda/kernel/lu_split.hpp b/src/backend/cuda/kernel/lu_split.hpp
index 1d2a185276..467173c218 100644
--- a/src/backend/cuda/kernel/lu_split.hpp
+++ b/src/backend/cuda/kernel/lu_split.hpp
@@ -32,7 +32,7 @@ void lu_split(Param<T> lower, Param<T> upper, Param<T> in) {
         lower.dims[0] == in.dims[0] && lower.dims[1] == in.dims[1];
 
     auto luSplit = common::getKernel(
-        "arrayfire::cuda::luSplit", std::array{lu_split_cuh_src},
+        "arrayfire::cuda::luSplit", {{lu_split_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(sameDims)));
 
     dim3 threads(TX, TY, 1);
diff --git a/src/backend/cuda/kernel/match_template.hpp b/src/backend/cuda/kernel/match_template.hpp
index c9754473ae..a605eabab5 100644
--- a/src/backend/cuda/kernel/match_template.hpp
+++ b/src/backend/cuda/kernel/match_template.hpp
@@ -26,7 +26,7 @@ void matchTemplate(Param<outType> out, CParam<inType> srch,
                    CParam<inType> tmplt, const af::matchType mType,
                    bool needMean) {
     auto matchTemplate = common::getKernel(
-        "arrayfire::cuda::matchTemplate", std::array{match_template_cuh_src},
+        "arrayfire::cuda::matchTemplate", {{match_template_cuh_src}},
         TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>(),
                      TemplateArg(mType), TemplateArg(needMean)));
 
diff --git a/src/backend/cuda/kernel/meanshift.hpp b/src/backend/cuda/kernel/meanshift.hpp
index c1882c91fc..600f456fb9 100644
--- a/src/backend/cuda/kernel/meanshift.hpp
+++ b/src/backend/cuda/kernel/meanshift.hpp
@@ -29,7 +29,7 @@ void meanshift(Param<T> out, CParam<T> in, const float spatialSigma,
     typedef typename std::conditional<std::is_same<T, double>::value, double,
                                       float>::type AccType;
     auto meanshift = common::getKernel(
-        "arrayfire::cuda::meanshift", std::array{meanshift_cuh_src},
+        "arrayfire::cuda::meanshift", {{meanshift_cuh_src}},
         TemplateArgs(TemplateTypename<AccType>(), TemplateTypename<T>(),
                      TemplateArg((IsColor ? 3 : 1))  // channels
                      ));
diff --git a/src/backend/cuda/kernel/medfilt.hpp b/src/backend/cuda/kernel/medfilt.hpp
index 69920b5ac0..20f3514ec6 100644
--- a/src/backend/cuda/kernel/medfilt.hpp
+++ b/src/backend/cuda/kernel/medfilt.hpp
@@ -27,11 +27,11 @@ template<typename T>
 void medfilt2(Param<T> out, CParam<T> in, const af::borderType pad, int w_len,
               int w_wid) {
     UNUSED(w_wid);
-    auto medfilt2 = common::getKernel(
-        "arrayfire::cuda::medfilt2", std::array{medfilt_cuh_src},
-        TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
-                     TemplateArg(w_len), TemplateArg(w_wid)),
-        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+    auto medfilt2 =
+        common::getKernel("arrayfire::cuda::medfilt2", {{medfilt_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
+                                       TemplateArg(w_len), TemplateArg(w_wid)),
+                          {{DefineValue(THREADS_X), DefineValue(THREADS_Y)}});
 
     const dim3 threads(THREADS_X, THREADS_Y);
 
@@ -47,10 +47,10 @@ void medfilt2(Param<T> out, CParam<T> in, const af::borderType pad, int w_len,
 
 template<typename T>
 void medfilt1(Param<T> out, CParam<T> in, const af::borderType pad, int w_wid) {
-    auto medfilt1 = common::getKernel(
-        "arrayfire::cuda::medfilt1", std::array{medfilt_cuh_src},
-        TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
-                     TemplateArg(w_wid)));
+    auto medfilt1 =
+        common::getKernel("arrayfire::cuda::medfilt1", {{medfilt_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>(), TemplateArg(pad),
+                                       TemplateArg(w_wid)));
 
     const dim3 threads(THREADS_X);
 
diff --git a/src/backend/cuda/kernel/memcopy.hpp b/src/backend/cuda/kernel/memcopy.hpp
index b75cc39c86..f4d39e6c64 100644
--- a/src/backend/cuda/kernel/memcopy.hpp
+++ b/src/backend/cuda/kernel/memcopy.hpp
@@ -128,23 +128,20 @@ void memcopy(Param<T> out, CParam<T> in, dim_t indims) {
     // Conversion to cuda base vector types.
     switch (sizeofNewT) {
         case 1: {
-            auto memCopy{common::getKernel(kernelName,
-                                           std::array{memcopy_cuh_src},
+            auto memCopy{common::getKernel(kernelName, {{memcopy_cuh_src}},
                                            TemplateArgs(TemplateArg("char")))};
             memCopy(qArgs, Param<char>((char *)out.ptr, out.dims, out.strides),
                     CParam<char>((const char *)in.ptr, in.dims, in.strides));
         } break;
         case 2: {
-            auto memCopy{common::getKernel(kernelName,
-                                           std::array{memcopy_cuh_src},
+            auto memCopy{common::getKernel(kernelName, {{memcopy_cuh_src}},
                                            TemplateArgs(TemplateArg("short")))};
             memCopy(qArgs,
                     Param<short>((short *)out.ptr, out.dims, out.strides),
                     CParam<short>((const short *)in.ptr, in.dims, in.strides));
         } break;
         case 4: {
-            auto memCopy{common::getKernel(kernelName,
-                                           std::array{memcopy_cuh_src},
+            auto memCopy{common::getKernel(kernelName, {{memcopy_cuh_src}},
                                            TemplateArgs(TemplateArg("float")))};
             memCopy(qArgs,
                     Param<float>((float *)out.ptr, out.dims, out.strides),
@@ -152,7 +149,7 @@ void memcopy(Param<T> out, CParam<T> in, dim_t indims) {
         } break;
         case 8: {
             auto memCopy{
-                common::getKernel(kernelName, std::array{memcopy_cuh_src},
+                common::getKernel(kernelName, {{memcopy_cuh_src}},
                                   TemplateArgs(TemplateArg("float2")))};
             memCopy(
                 qArgs, Param<float2>((float2 *)out.ptr, out.dims, out.strides),
@@ -160,7 +157,7 @@ void memcopy(Param<T> out, CParam<T> in, dim_t indims) {
         } break;
         case 16: {
             auto memCopy{
-                common::getKernel(kernelName, std::array{memcopy_cuh_src},
+                common::getKernel(kernelName, {{memcopy_cuh_src}},
                                   TemplateArgs(TemplateArg("float4")))};
             memCopy(
                 qArgs, Param<float4>((float4 *)out.ptr, out.dims, out.strides),
@@ -200,7 +197,7 @@ void copy(Param<outType> dst, CParam<inType> src, dim_t ondims,
         : (th.loop2 || th.loop3) ? "arrayfire::cuda::scaledCopyLoop123"
         : th.loop1               ? "arrayfire::cuda::scaledCopyLoop1"
                                  : "arrayfire::cuda::scaledCopy",
-        std::array{copy_cuh_src},
+        {{copy_cuh_src}},
         TemplateArgs(TemplateTypename<inType>(), TemplateTypename<outType>(),
                      TemplateArg(same_dims), TemplateArg(factor != 1.0)))};
 
diff --git a/src/backend/cuda/kernel/moments.hpp b/src/backend/cuda/kernel/moments.hpp
index ece6627c71..dcc1161b23 100644
--- a/src/backend/cuda/kernel/moments.hpp
+++ b/src/backend/cuda/kernel/moments.hpp
@@ -22,9 +22,9 @@ static const int THREADS = 128;
 
 template<typename T>
 void moments(Param<float> out, CParam<T> in, const af::momentType moment) {
-    auto moments = common::getKernel("arrayfire::cuda::moments",
-                                     std::array{moments_cuh_src},
-                                     TemplateArgs(TemplateTypename<T>()));
+    auto moments =
+        common::getKernel("arrayfire::cuda::moments", {{moments_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(THREADS, 1, 1);
     dim3 blocks(in.dims[1], in.dims[2] * in.dims[3]);
diff --git a/src/backend/cuda/kernel/morph.hpp b/src/backend/cuda/kernel/morph.hpp
index 4936d659b4..0aff8ff639 100644
--- a/src/backend/cuda/kernel/morph.hpp
+++ b/src/backend/cuda/kernel/morph.hpp
@@ -32,10 +32,10 @@ void morph(Param<T> out, CParam<T> in, CParam<T> mask, bool isDilation) {
     const int SeLength = (windLen <= 10 ? windLen : 0);
 
     auto morph = common::getKernel(
-        "arrayfire::cuda::morph", std::array{morph_cuh_src},
+        "arrayfire::cuda::morph", {{morph_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(isDilation),
                      TemplateArg(SeLength)),
-        std::array{DefineValue(MAX_MORPH_FILTER_LEN)});
+        {{DefineValue(MAX_MORPH_FILTER_LEN)}});
 
     morph.copyToReadOnly(morph.getDevPtr("cFilter"),
                          reinterpret_cast<CUdeviceptr>(mask.ptr),
@@ -68,10 +68,10 @@ void morph3d(Param<T> out, CParam<T> in, CParam<T> mask, bool isDilation) {
     }
 
     auto morph3D = common::getKernel(
-        "arrayfire::cuda::morph3D", std::array{morph_cuh_src},
+        "arrayfire::cuda::morph3D", {{morph_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(isDilation),
                      TemplateArg(windLen)),
-        std::array{DefineValue(MAX_MORPH_FILTER_LEN)});
+        {{DefineValue(MAX_MORPH_FILTER_LEN)}});
 
     morph3D.copyToReadOnly(
         morph3D.getDevPtr("cFilter"), reinterpret_cast<CUdeviceptr>(mask.ptr),
diff --git a/src/backend/cuda/kernel/pad_array_borders.hpp b/src/backend/cuda/kernel/pad_array_borders.hpp
index 85acaabb26..b52fcf1401 100644
--- a/src/backend/cuda/kernel/pad_array_borders.hpp
+++ b/src/backend/cuda/kernel/pad_array_borders.hpp
@@ -29,7 +29,7 @@ template<typename T>
 void padBorders(Param<T> out, CParam<T> in, dim4 const lBoundPadding,
                 const af::borderType btype) {
     auto padBorders = common::getKernel(
-        "arrayfire::cuda::padBorders", std::array{pad_array_borders_cuh_src},
+        "arrayfire::cuda::padBorders", {{pad_array_borders_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(btype)));
 
     dim3 threads(kernel::PADB_THREADS_X, kernel::PADB_THREADS_Y);
diff --git a/src/backend/cuda/kernel/range.hpp b/src/backend/cuda/kernel/range.hpp
index 2e222f6e21..9b75276dc4 100644
--- a/src/backend/cuda/kernel/range.hpp
+++ b/src/backend/cuda/kernel/range.hpp
@@ -26,9 +26,8 @@ void range(Param<T> out, const int dim) {
     constexpr unsigned RANGE_TILEX = 512;
     constexpr unsigned RANGE_TILEY = 32;
 
-    auto range =
-        common::getKernel("arrayfire::cuda::range", std::array{range_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto range = common::getKernel("arrayfire::cuda::range", {{range_cuh_src}},
+                                   TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(RANGE_TX, RANGE_TY, 1);
 
diff --git a/src/backend/cuda/kernel/reorder.hpp b/src/backend/cuda/kernel/reorder.hpp
index e2b83e4ab8..e54ebcf417 100644
--- a/src/backend/cuda/kernel/reorder.hpp
+++ b/src/backend/cuda/kernel/reorder.hpp
@@ -26,9 +26,9 @@ void reorder(Param<T> out, CParam<T> in, const dim_t *rdims) {
     constexpr unsigned TILEX = 512;
     constexpr unsigned TILEY = 32;
 
-    auto reorder = common::getKernel("arrayfire::cuda::reorder",
-                                     std::array{reorder_cuh_src},
-                                     TemplateArgs(TemplateTypename<T>()));
+    auto reorder =
+        common::getKernel("arrayfire::cuda::reorder", {{reorder_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/resize.hpp b/src/backend/cuda/kernel/resize.hpp
index 254e23e7d3..6129fe1e64 100644
--- a/src/backend/cuda/kernel/resize.hpp
+++ b/src/backend/cuda/kernel/resize.hpp
@@ -25,7 +25,7 @@ static const unsigned TY = 16;
 template<typename T>
 void resize(Param<T> out, CParam<T> in, af_interp_type method) {
     auto resize = common::getKernel(
-        "arrayfire::cuda::resize", std::array{resize_cuh_src},
+        "arrayfire::cuda::resize", {{resize_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(method)));
 
     dim3 threads(TX, TY, 1);
diff --git a/src/backend/cuda/kernel/rotate.hpp b/src/backend/cuda/kernel/rotate.hpp
index b31218047c..f1aa40585a 100644
--- a/src/backend/cuda/kernel/rotate.hpp
+++ b/src/backend/cuda/kernel/rotate.hpp
@@ -34,7 +34,7 @@ template<typename T>
 void rotate(Param<T> out, CParam<T> in, const float theta,
             const af::interpType method, const int order) {
     auto rotate = common::getKernel(
-        "arrayfire::cuda::rotate", std::array{rotate_cuh_src},
+        "arrayfire::cuda::rotate", {{rotate_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(order)));
 
     const float c = cos(-theta), s = sin(-theta);
diff --git a/src/backend/cuda/kernel/scan_dim.hpp b/src/backend/cuda/kernel/scan_dim.hpp
index a85c15a5ed..9fc32c61e9 100644
--- a/src/backend/cuda/kernel/scan_dim.hpp
+++ b/src/backend/cuda/kernel/scan_dim.hpp
@@ -26,12 +26,12 @@ static void scan_dim_launcher(Param<To> out, Param<To> tmp, CParam<Ti> in,
                               const uint threads_y, const dim_t blocks_all[4],
                               int dim, bool isFinalPass, bool inclusive_scan) {
     auto scan_dim = common::getKernel(
-        "arrayfire::cuda::scan_dim", std::array{scan_dim_cuh_src},
+        "arrayfire::cuda::scan_dim", {{scan_dim_cuh_src}},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>(),
                      TemplateArg(op), TemplateArg(dim),
                      TemplateArg(isFinalPass), TemplateArg(threads_y),
                      TemplateArg(inclusive_scan)),
-        std::array{DefineValue(THREADS_X)});
+        {{DefineValue(THREADS_X)}});
 
     dim3 threads(THREADS_X, threads_y);
 
@@ -54,7 +54,7 @@ static void bcast_dim_launcher(Param<To> out, CParam<To> tmp,
                                const uint threads_y, const dim_t blocks_all[4],
                                int dim, bool inclusive_scan) {
     auto scan_dim_bcast = common::getKernel(
-        "arrayfire::cuda::scan_dim_bcast", std::array{scan_dim_cuh_src},
+        "arrayfire::cuda::scan_dim_bcast", {{scan_dim_cuh_src}},
         TemplateArgs(TemplateTypename<To>(), TemplateArg(op),
                      TemplateArg(dim)));
 
diff --git a/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp b/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
index 0dda0b872f..0a07b7fa1e 100644
--- a/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
+++ b/src/backend/cuda/kernel/scan_dim_by_key_impl.hpp
@@ -33,11 +33,10 @@ static void scan_dim_nonfinal_launcher(Param<To> out, Param<To> tmp,
                                        const dim_t blocks_all[4],
                                        bool inclusive_scan) {
     auto scanbykey_dim_nonfinal = common::getKernel(
-        "arrayfire::cuda::scanbykey_dim_nonfinal",
-        std::array{scan_dim_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_dim_nonfinal", {{scan_dim_by_key_cuh_src}},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
                      TemplateTypename<To>(), TemplateArg(op)),
-        std::array{DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)});
+        {{DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)}});
 
     dim3 threads(THREADS_X, threads_y);
 
@@ -58,11 +57,10 @@ static void scan_dim_final_launcher(Param<To> out, CParam<Ti> in,
                                     const dim_t blocks_all[4],
                                     bool calculateFlags, bool inclusive_scan) {
     auto scanbykey_dim_final = common::getKernel(
-        "arrayfire::cuda::scanbykey_dim_final",
-        std::array{scan_dim_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_dim_final", {{scan_dim_by_key_cuh_src}},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
                      TemplateTypename<To>(), TemplateArg(op)),
-        std::array{DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)});
+        {{DefineValue(THREADS_X), DefineKeyValue(DIMY, threads_y)}});
 
     dim3 threads(THREADS_X, threads_y);
 
@@ -81,8 +79,7 @@ static void bcast_dim_launcher(Param<To> out, CParam<To> tmp, Param<int> tlid,
                                const int dim, const uint threads_y,
                                const dim_t blocks_all[4]) {
     auto scanbykey_dim_bcast = common::getKernel(
-        "arrayfire::cuda::scanbykey_dim_bcast",
-        std::array{scan_dim_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_dim_bcast", {{scan_dim_by_key_cuh_src}},
         TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
     dim3 threads(THREADS_X, threads_y);
     dim3 blocks(blocks_all[0] * blocks_all[2], blocks_all[1] * blocks_all[3]);
diff --git a/src/backend/cuda/kernel/scan_first.hpp b/src/backend/cuda/kernel/scan_first.hpp
index fec9d4be7a..868816f4ed 100644
--- a/src/backend/cuda/kernel/scan_first.hpp
+++ b/src/backend/cuda/kernel/scan_first.hpp
@@ -27,11 +27,11 @@ static void scan_first_launcher(Param<To> out, Param<To> tmp, CParam<Ti> in,
                                 const uint threads_x, bool isFinalPass,
                                 bool inclusive_scan) {
     auto scan_first = common::getKernel(
-        "arrayfire::cuda::scan_first", std::array{scan_first_cuh_src},
+        "arrayfire::cuda::scan_first", {{scan_first_cuh_src}},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>(),
                      TemplateArg(op), TemplateArg(isFinalPass),
                      TemplateArg(threads_x), TemplateArg(inclusive_scan)),
-        std::array{DefineValue(THREADS_PER_BLOCK)});
+        {{DefineValue(THREADS_PER_BLOCK)}});
 
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
@@ -52,7 +52,7 @@ static void bcast_first_launcher(Param<To> out, CParam<To> tmp,
                                  const uint blocks_x, const uint blocks_y,
                                  const uint threads_x, bool inclusive_scan) {
     auto scan_first_bcast = common::getKernel(
-        "arrayfire::cuda::scan_first_bcast", std::array{scan_first_cuh_src},
+        "arrayfire::cuda::scan_first_bcast", {{scan_first_cuh_src}},
         TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
 
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
diff --git a/src/backend/cuda/kernel/scan_first_by_key_impl.hpp b/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
index 16abf56b3e..bf873fdd3d 100644
--- a/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
+++ b/src/backend/cuda/kernel/scan_first_by_key_impl.hpp
@@ -32,11 +32,10 @@ static void scan_nonfinal_launcher(Param<To> out, Param<To> tmp,
                                    const uint threads_x, bool inclusive_scan) {
     auto scanbykey_first_nonfinal = common::getKernel(
         "arrayfire::cuda::scanbykey_first_nonfinal",
-        std::array{scan_first_by_key_cuh_src},
+        {{scan_first_by_key_cuh_src}},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
                      TemplateTypename<To>(), TemplateArg(op)),
-        std::array{DefineValue(THREADS_PER_BLOCK),
-                   DefineKeyValue(DIMX, threads_x)});
+        {{DefineValue(THREADS_PER_BLOCK), DefineKeyValue(DIMX, threads_x)}});
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
@@ -54,12 +53,10 @@ static void scan_final_launcher(Param<To> out, CParam<Ti> in, CParam<Tk> key,
                                 const uint threads_x, bool calculateFlags,
                                 bool inclusive_scan) {
     auto scanbykey_first_final = common::getKernel(
-        "arrayfire::cuda::scanbykey_first_final",
-        std::array{scan_first_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_first_final", {{scan_first_by_key_cuh_src}},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<Tk>(),
                      TemplateTypename<To>(), TemplateArg(op)),
-        std::array{DefineValue(THREADS_PER_BLOCK),
-                   DefineKeyValue(DIMX, threads_x)});
+        {{DefineValue(THREADS_PER_BLOCK), DefineKeyValue(DIMX, threads_x)}});
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
 
@@ -76,8 +73,7 @@ static void bcast_first_launcher(Param<To> out, Param<To> tmp, Param<int> tlid,
                                  const dim_t blocks_x, const dim_t blocks_y,
                                  const uint threads_x) {
     auto scanbykey_first_bcast = common::getKernel(
-        "arrayfire::cuda::scanbykey_first_bcast",
-        std::array{scan_first_by_key_cuh_src},
+        "arrayfire::cuda::scanbykey_first_bcast", {{scan_first_by_key_cuh_src}},
         TemplateArgs(TemplateTypename<To>(), TemplateArg(op)));
     dim3 threads(threads_x, THREADS_PER_BLOCK / threads_x);
     dim3 blocks(blocks_x * out.dims[2], blocks_y * out.dims[3]);
diff --git a/src/backend/cuda/kernel/select.hpp b/src/backend/cuda/kernel/select.hpp
index 1b6d78fa8f..4df1d3da83 100644
--- a/src/backend/cuda/kernel/select.hpp
+++ b/src/backend/cuda/kernel/select.hpp
@@ -31,7 +31,7 @@ void select(Param<T> out, CParam<char> cond, CParam<T> a, CParam<T> b,
     for (int i = 0; i < 4; i++) { is_same &= (a.dims[i] == b.dims[i]); }
 
     auto select = common::getKernel(
-        "arrayfire::cuda::select", std::array{select_cuh_src},
+        "arrayfire::cuda::select", {{select_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_same)));
 
     dim3 threads(DIMX, DIMY);
@@ -60,7 +60,7 @@ template<typename T>
 void select_scalar(Param<T> out, CParam<char> cond, CParam<T> a, const T b,
                    int ndims, bool flip) {
     auto selectScalar = common::getKernel(
-        "arrayfire::cuda::selectScalar", std::array{select_cuh_src},
+        "arrayfire::cuda::selectScalar", {{select_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(flip)));
 
     dim3 threads(DIMX, DIMY);
diff --git a/src/backend/cuda/kernel/sobel.hpp b/src/backend/cuda/kernel/sobel.hpp
index 130625c11b..710b930404 100644
--- a/src/backend/cuda/kernel/sobel.hpp
+++ b/src/backend/cuda/kernel/sobel.hpp
@@ -28,9 +28,9 @@ void sobel(Param<To> dx, Param<To> dy, CParam<Ti> in,
     UNUSED(ker_size);
 
     auto sobel3x3 = common::getKernel(
-        "arrayfire::cuda::sobel3x3", std::array{sobel_cuh_src},
+        "arrayfire::cuda::sobel3x3", {{sobel_cuh_src}},
         TemplateArgs(TemplateTypename<Ti>(), TemplateTypename<To>()),
-        std::array{DefineValue(THREADS_X), DefineValue(THREADS_Y)});
+        {{DefineValue(THREADS_X), DefineValue(THREADS_Y)}});
 
     const dim3 threads(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/sparse.hpp b/src/backend/cuda/kernel/sparse.hpp
index efed1ed6d7..6629d0fec6 100644
--- a/src/backend/cuda/kernel/sparse.hpp
+++ b/src/backend/cuda/kernel/sparse.hpp
@@ -25,8 +25,8 @@ void coo2dense(Param<T> output, CParam<T> values, CParam<int> rowIdx,
     constexpr int reps = 4;
 
     auto coo2Dense = common::getKernel(
-        "arrayfire::cuda::coo2Dense", std::array{sparse_cuh_src},
-        TemplateArgs(TemplateTypename<T>()), std::array{DefineValue(reps)});
+        "arrayfire::cuda::coo2Dense", {{sparse_cuh_src}},
+        TemplateArgs(TemplateTypename<T>()), {{DefineValue(reps)}});
 
     dim3 threads(256, 1, 1);
 
diff --git a/src/backend/cuda/kernel/sparse_arith.hpp b/src/backend/cuda/kernel/sparse_arith.hpp
index 13dd5ddb7e..b21d2130e5 100644
--- a/src/backend/cuda/kernel/sparse_arith.hpp
+++ b/src/backend/cuda/kernel/sparse_arith.hpp
@@ -28,9 +28,9 @@ template<typename T, af_op_t op>
 void sparseArithOpCSR(Param<T> out, CParam<T> values, CParam<int> rowIdx,
                       CParam<int> colIdx, CParam<T> rhs, const bool reverse) {
     auto csrArithDSD = common::getKernel(
-        "arrayfire::cuda::csrArithDSD", std::array{sparse_arith_cuh_src},
+        "arrayfire::cuda::csrArithDSD", {{sparse_arith_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
-        std::array{DefineValue(TX), DefineValue(TY)});
+        {{DefineValue(TX), DefineValue(TY)}});
 
     // Each Y for threads does one row
     dim3 threads(TX, TY, 1);
@@ -48,9 +48,9 @@ template<typename T, af_op_t op>
 void sparseArithOpCOO(Param<T> out, CParam<T> values, CParam<int> rowIdx,
                       CParam<int> colIdx, CParam<T> rhs, const bool reverse) {
     auto cooArithDSD = common::getKernel(
-        "arrayfire::cuda::cooArithDSD", std::array{sparse_arith_cuh_src},
+        "arrayfire::cuda::cooArithDSD", {{sparse_arith_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
-        std::array{DefineValue(THREADS)});
+        {{DefineValue(THREADS)}});
 
     // Linear indexing with one elements per thread
     dim3 threads(THREADS, 1, 1);
@@ -68,9 +68,9 @@ template<typename T, af_op_t op>
 void sparseArithOpCSR(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
                       CParam<T> rhs, const bool reverse) {
     auto csrArithSSD = common::getKernel(
-        "arrayfire::cuda::csrArithSSD", std::array{sparse_arith_cuh_src},
+        "arrayfire::cuda::csrArithSSD", {{sparse_arith_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
-        std::array{DefineValue(TX), DefineValue(TY)});
+        {{DefineValue(TX), DefineValue(TY)}});
 
     // Each Y for threads does one row
     dim3 threads(TX, TY, 1);
@@ -88,9 +88,9 @@ template<typename T, af_op_t op>
 void sparseArithOpCOO(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
                       CParam<T> rhs, const bool reverse) {
     auto cooArithSSD = common::getKernel(
-        "arrayfire::cuda::cooArithSSD", std::array{sparse_arith_cuh_src},
+        "arrayfire::cuda::cooArithSSD", {{sparse_arith_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(op)),
-        std::array{DefineValue(THREADS)});
+        {{DefineValue(THREADS)}});
 
     // Linear indexing with one elements per thread
     dim3 threads(THREADS, 1, 1);
diff --git a/src/backend/cuda/kernel/susan.hpp b/src/backend/cuda/kernel/susan.hpp
index 42082bd221..28a96a1e6d 100644
--- a/src/backend/cuda/kernel/susan.hpp
+++ b/src/backend/cuda/kernel/susan.hpp
@@ -26,10 +26,10 @@ template<typename T>
 void susan_responses(T* out, const T* in, const unsigned idim0,
                      const unsigned idim1, const int radius, const float t,
                      const float g, const unsigned edge) {
-    auto susan = common::getKernel(
-        "arrayfire::cuda::susan", std::array{susan_cuh_src},
-        TemplateArgs(TemplateTypename<T>()),
-        std::array{DefineValue(BLOCK_X), DefineValue(BLOCK_Y)});
+    auto susan =
+        common::getKernel("arrayfire::cuda::susan", {{susan_cuh_src}},
+                          TemplateArgs(TemplateTypename<T>()),
+                          {{DefineValue(BLOCK_X), DefineValue(BLOCK_Y)}});
 
     dim3 threads(BLOCK_X, BLOCK_Y);
     dim3 blocks(divup(idim0 - edge * 2, BLOCK_X),
@@ -48,7 +48,7 @@ void nonMaximal(float* x_out, float* y_out, float* resp_out, unsigned* count,
                 const unsigned idim0, const unsigned idim1, const T* resp_in,
                 const unsigned edge, const unsigned max_corners) {
     auto nonMax =
-        common::getKernel("arrayfire::cuda::nonMax", std::array{susan_cuh_src},
+        common::getKernel("arrayfire::cuda::nonMax", {{susan_cuh_src}},
                           TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(BLOCK_X, BLOCK_Y);
diff --git a/src/backend/cuda/kernel/tile.hpp b/src/backend/cuda/kernel/tile.hpp
index 035cc39437..e25bdce4b7 100644
--- a/src/backend/cuda/kernel/tile.hpp
+++ b/src/backend/cuda/kernel/tile.hpp
@@ -26,9 +26,8 @@ void tile(Param<T> out, CParam<T> in) {
     constexpr unsigned TILEX = 512;
     constexpr unsigned TILEY = 32;
 
-    auto tile =
-        common::getKernel("arrayfire::cuda::tile", std::array{tile_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto tile = common::getKernel("arrayfire::cuda::tile", {{tile_cuh_src}},
+                                  TemplateArgs(TemplateTypename<T>()));
 
     dim3 threads(TX, TY, 1);
 
diff --git a/src/backend/cuda/kernel/transform.hpp b/src/backend/cuda/kernel/transform.hpp
index 4ed94d7949..5405fcc9cc 100644
--- a/src/backend/cuda/kernel/transform.hpp
+++ b/src/backend/cuda/kernel/transform.hpp
@@ -32,7 +32,7 @@ template<typename T>
 void transform(Param<T> out, CParam<T> in, CParam<float> tf, const bool inverse,
                const bool perspective, const af::interpType method, int order) {
     auto transform = common::getKernel(
-        "arrayfire::cuda::transform", std::array{transform_cuh_src},
+        "arrayfire::cuda::transform", {{transform_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(inverse),
                      TemplateArg(order)));
 
diff --git a/src/backend/cuda/kernel/transpose.hpp b/src/backend/cuda/kernel/transpose.hpp
index 7ec97b7127..f84ff89b96 100644
--- a/src/backend/cuda/kernel/transpose.hpp
+++ b/src/backend/cuda/kernel/transpose.hpp
@@ -27,10 +27,10 @@ template<typename T>
 void transpose(Param<T> out, CParam<T> in, const bool conjugate,
                const bool is32multiple) {
     auto transpose = common::getKernel(
-        "arrayfire::cuda::transpose", std::array{transpose_cuh_src},
+        "arrayfire::cuda::transpose", {{transpose_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(conjugate),
                      TemplateArg(is32multiple)),
-        std::array{DefineValue(TILE_DIM), DefineValue(THREADS_Y)});
+        {{DefineValue(TILE_DIM), DefineValue(THREADS_Y)}});
 
     dim3 threads(kernel::THREADS_X, kernel::THREADS_Y);
 
diff --git a/src/backend/cuda/kernel/transpose_inplace.hpp b/src/backend/cuda/kernel/transpose_inplace.hpp
index b5374b6025..5ff28020c4 100644
--- a/src/backend/cuda/kernel/transpose_inplace.hpp
+++ b/src/backend/cuda/kernel/transpose_inplace.hpp
@@ -27,10 +27,10 @@ template<typename T>
 void transpose_inplace(Param<T> in, const bool conjugate,
                        const bool is32multiple) {
     auto transposeIP = common::getKernel(
-        "arrayfire::cuda::transposeIP", std::array{transpose_inplace_cuh_src},
+        "arrayfire::cuda::transposeIP", {{transpose_inplace_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(conjugate),
                      TemplateArg(is32multiple)),
-        std::array{DefineValue(TILE_DIM), DefineValue(THREADS_Y)});
+        {{DefineValue(TILE_DIM), DefineValue(THREADS_Y)}});
 
     // dimensions passed to this function should be input dimensions
     // any necessary transformations and dimension related calculations are
diff --git a/src/backend/cuda/kernel/triangle.hpp b/src/backend/cuda/kernel/triangle.hpp
index 3c1841a324..ba922a3115 100644
--- a/src/backend/cuda/kernel/triangle.hpp
+++ b/src/backend/cuda/kernel/triangle.hpp
@@ -27,7 +27,7 @@ void triangle(Param<T> r, CParam<T> in, bool is_upper, bool is_unit_diag) {
     constexpr unsigned TILEY = 32;
 
     auto triangle = common::getKernel(
-        "arrayfire::cuda::triangle", std::array{triangle_cuh_src},
+        "arrayfire::cuda::triangle", {{triangle_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_upper),
                      TemplateArg(is_unit_diag)));
 
diff --git a/src/backend/cuda/kernel/unwrap.hpp b/src/backend/cuda/kernel/unwrap.hpp
index 6105b8b0a1..20ad8e67e3 100644
--- a/src/backend/cuda/kernel/unwrap.hpp
+++ b/src/backend/cuda/kernel/unwrap.hpp
@@ -25,7 +25,7 @@ void unwrap(Param<T> out, CParam<T> in, const int wx, const int wy,
             const int sx, const int sy, const int px, const int py,
             const int dx, const int dy, const int nx, const bool is_column) {
     auto unwrap = common::getKernel(
-        "arrayfire::cuda::unwrap", std::array{unwrap_cuh_src},
+        "arrayfire::cuda::unwrap", {{unwrap_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     dim3 threads, blocks;
diff --git a/src/backend/cuda/kernel/where.hpp b/src/backend/cuda/kernel/where.hpp
index 0dddc456b9..0b500d4628 100644
--- a/src/backend/cuda/kernel/where.hpp
+++ b/src/backend/cuda/kernel/where.hpp
@@ -24,9 +24,8 @@ namespace kernel {
 
 template<typename T>
 static void where(Param<uint> &out, CParam<T> in) {
-    auto where =
-        common::getKernel("arrayfire::cuda::where", std::array{where_cuh_src},
-                          TemplateArgs(TemplateTypename<T>()));
+    auto where = common::getKernel("arrayfire::cuda::where", {{where_cuh_src}},
+                                   TemplateArgs(TemplateTypename<T>()));
 
     uint threads_x = nextpow2(std::max(32u, (uint)in.dims[0]));
     threads_x      = std::min(threads_x, THREADS_PER_BLOCK);
diff --git a/src/backend/cuda/kernel/wrap.hpp b/src/backend/cuda/kernel/wrap.hpp
index 37b9e97cf9..e95db0f3f3 100644
--- a/src/backend/cuda/kernel/wrap.hpp
+++ b/src/backend/cuda/kernel/wrap.hpp
@@ -24,7 +24,7 @@ template<typename T>
 void wrap(Param<T> out, CParam<T> in, const int wx, const int wy, const int sx,
           const int sy, const int px, const int py, const bool is_column) {
     auto wrap = common::getKernel(
-        "arrayfire::cuda::wrap", std::array{wrap_cuh_src},
+        "arrayfire::cuda::wrap", {{wrap_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     int nx = (out.dims[0] + 2 * px - wx) / sx + 1;
@@ -52,7 +52,7 @@ void wrap_dilated(Param<T> out, CParam<T> in, const dim_t wx, const dim_t wy,
                   const dim_t py, const dim_t dx, const dim_t dy,
                   const bool is_column) {
     auto wrap = common::getKernel(
-        "arrayfire::cuda::wrap_dilated", std::array{wrap_cuh_src},
+        "arrayfire::cuda::wrap_dilated", {{wrap_cuh_src}},
         TemplateArgs(TemplateTypename<T>(), TemplateArg(is_column)));
 
     int nx = 1 + (out.dims[0] + 2 * px - (((wx - 1) * dx) + 1)) / sx;
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 30a942d2dd..f7ba973032 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -281,8 +281,7 @@ cl::Kernel getKernel(const vector<Node*>& output_nodes,
         if (isHalfSupported(device)) {
             options.emplace_back(DefineKey(USE_HALF));
         }
-        return common::getKernel(funcName,
-                                 std::array{jit_cl_src, jitKer_cl_src}, {},
+        return common::getKernel(funcName, {{jit_cl_src, jitKer_cl_src}}, {},
                                  options, true)
             .get();
     }
diff --git a/src/backend/opencl/kernel/anisotropic_diffusion.hpp b/src/backend/opencl/kernel/anisotropic_diffusion.hpp
index bf13bb4cd5..a8655be95e 100644
--- a/src/backend/opencl/kernel/anisotropic_diffusion.hpp
+++ b/src/backend/opencl/kernel/anisotropic_diffusion.hpp
@@ -50,9 +50,9 @@ void anisotropicDiffusion(Param inout, const float dt, const float mct,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto diffUpdate = common::getKernel(
-        "aisoDiffUpdate", std::array{anisotropic_diffusion_cl_src}, tmpltArgs,
-        compileOpts);
+    auto diffUpdate =
+        common::getKernel("aisoDiffUpdate", {{anisotropic_diffusion_cl_src}},
+                          tmpltArgs, compileOpts);
 
     NDRange local(THREADS_X, THREADS_Y, 1);
 
diff --git a/src/backend/opencl/kernel/approx.hpp b/src/backend/opencl/kernel/approx.hpp
index 797ac19d4b..d23a590e7f 100644
--- a/src/backend/opencl/kernel/approx.hpp
+++ b/src/backend/opencl/kernel/approx.hpp
@@ -73,9 +73,8 @@ void approx1(Param yo, const Param yi, const Param xo, const int xdim,
     };
     auto compileOpts = genCompileOptions<Ty, Tp>(order, xdim);
 
-    auto approx1 =
-        common::getKernel("approx1", std::array{interp_cl_src, approx1_cl_src},
-                          tmpltArgs, compileOpts);
+    auto approx1 = common::getKernel(
+        "approx1", {{interp_cl_src, approx1_cl_src}}, tmpltArgs, compileOpts);
 
     NDRange local(THREADS, 1, 1);
     dim_t blocksPerMat = divup(yo.info.dims[0], local[0]);
@@ -112,9 +111,8 @@ void approx2(Param zo, const Param zi, const Param xo, const int xdim,
     };
     auto compileOpts = genCompileOptions<Ty, Tp>(order, xdim, ydim);
 
-    auto approx2 =
-        common::getKernel("approx2", std::array{interp_cl_src, approx2_cl_src},
-                          tmpltArgs, compileOpts);
+    auto approx2 = common::getKernel(
+        "approx2", {{interp_cl_src, approx2_cl_src}}, tmpltArgs, compileOpts);
 
     NDRange local(TX, TY, 1);
     dim_t blocksPerMatX = divup(zo.info.dims[0], local[0]);
diff --git a/src/backend/opencl/kernel/assign.hpp b/src/backend/opencl/kernel/assign.hpp
index 447e4e8c60..b7cd779027 100644
--- a/src/backend/opencl/kernel/assign.hpp
+++ b/src/backend/opencl/kernel/assign.hpp
@@ -42,8 +42,8 @@ void assign(Param out, const Param in, const AssignKernelParam_t& p,
         DefineKeyValue(T, dtype_traits<T>::getName()),
         getTypeBuildDefinition<T>()};
 
-    auto assign = common::getKernel("assignKernel", std::array{assign_cl_src},
-                                    targs, options);
+    auto assign =
+        common::getKernel("assignKernel", {{assign_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/bilateral.hpp b/src/backend/opencl/kernel/bilateral.hpp
index 832611dcdb..eba0f2bb10 100644
--- a/src/backend/opencl/kernel/bilateral.hpp
+++ b/src/backend/opencl/kernel/bilateral.hpp
@@ -44,8 +44,8 @@ void bilateral(Param out, const Param in, const float s_sigma,
     if (UseNativeExp) { options.emplace_back(DefineKey(USE_NATIVE_EXP)); }
     options.emplace_back(getTypeBuildDefinition<inType>());
 
-    auto bilateralOp = common::getKernel(
-        "bilateral", std::array{bilateral_cl_src}, targs, options);
+    auto bilateralOp =
+        common::getKernel("bilateral", {{bilateral_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/canny.hpp b/src/backend/opencl/kernel/canny.hpp
index 3659e1fb4b..bcc850e6ba 100644
--- a/src/backend/opencl/kernel/canny.hpp
+++ b/src/backend/opencl/kernel/canny.hpp
@@ -43,7 +43,7 @@ void nonMaxSuppression(Param output, const Param magnitude, const Param dx,
     options.emplace_back(getTypeBuildDefinition<T>());
 
     auto nonMaxOp = common::getKernel(
-        "nonMaxSuppressionKernel", std::array{nonmax_suppression_cl_src},
+        "nonMaxSuppressionKernel", {{nonmax_suppression_cl_src}},
         TemplateArgs(TemplateTypename<T>()), options);
 
     NDRange threads(kernel::THREADS_X, kernel::THREADS_Y, 1);
@@ -76,7 +76,7 @@ void initEdgeOut(Param output, const Param strong, const Param weak) {
     options.emplace_back(getTypeBuildDefinition<T>());
 
     auto initOp =
-        common::getKernel("initEdgeOutKernel", std::array{trace_edge_cl_src},
+        common::getKernel("initEdgeOutKernel", {{trace_edge_cl_src}},
                           TemplateArgs(TemplateTypename<T>()), options);
 
     NDRange threads(kernel::THREADS_X, kernel::THREADS_Y, 1);
@@ -108,9 +108,9 @@ void suppressLeftOver(Param output) {
     };
     options.emplace_back(getTypeBuildDefinition<T>());
 
-    auto finalOp = common::getKernel(
-        "suppressLeftOverKernel", std::array{trace_edge_cl_src},
-        TemplateArgs(TemplateTypename<T>()), options);
+    auto finalOp =
+        common::getKernel("suppressLeftOverKernel", {{trace_edge_cl_src}},
+                          TemplateArgs(TemplateTypename<T>()), options);
 
     NDRange threads(kernel::THREADS_X, kernel::THREADS_Y, 1);
 
@@ -145,7 +145,7 @@ void edgeTrackingHysteresis(Param output, const Param strong,
     options.emplace_back(getTypeBuildDefinition<T>());
 
     auto edgeTraceOp =
-        common::getKernel("edgeTrackKernel", std::array{trace_edge_cl_src},
+        common::getKernel("edgeTrackKernel", {{trace_edge_cl_src}},
                           TemplateArgs(TemplateTypename<T>()), options);
 
     NDRange threads(kernel::THREADS_X, kernel::THREADS_Y);
diff --git a/src/backend/opencl/kernel/convolve/conv2_impl.hpp b/src/backend/opencl/kernel/convolve/conv2_impl.hpp
index 59f0523de8..9798714750 100644
--- a/src/backend/opencl/kernel/convolve/conv2_impl.hpp
+++ b/src/backend/opencl/kernel/convolve/conv2_impl.hpp
@@ -51,9 +51,8 @@ void conv2Helper(const conv_kparam_t& param, Param out, const Param signal,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto convolve =
-        common::getKernel("convolve", std::array{ops_cl_src, convolve_cl_src},
-                          tmpltArgs, compileOpts);
+    auto convolve = common::getKernel(
+        "convolve", {{ops_cl_src, convolve_cl_src}}, tmpltArgs, compileOpts);
 
     convolve(EnqueueArgs(getQueue(), param.global, param.local), *out.data,
              out.info, *signal.data, signal.info, *param.impulse, filter.info,
diff --git a/src/backend/opencl/kernel/convolve/conv_common.hpp b/src/backend/opencl/kernel/convolve/conv_common.hpp
index 93c4781976..bd93419c7c 100644
--- a/src/backend/opencl/kernel/convolve/conv_common.hpp
+++ b/src/backend/opencl/kernel/convolve/conv_common.hpp
@@ -114,9 +114,8 @@ void convNHelper(const conv_kparam_t& param, Param& out, const Param& signal,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto convolve =
-        common::getKernel("convolve", std::array{ops_cl_src, convolve_cl_src},
-                          tmpltArgs, compileOpts);
+    auto convolve = common::getKernel(
+        "convolve", {{ops_cl_src, convolve_cl_src}}, tmpltArgs, compileOpts);
 
     convolve(EnqueueArgs(getQueue(), param.global, param.local), *out.data,
              out.info, *signal.data, signal.info, cl::Local(param.loc_size),
diff --git a/src/backend/opencl/kernel/convolve_separable.cpp b/src/backend/opencl/kernel/convolve_separable.cpp
index 6f7611428b..41bfa55dde 100644
--- a/src/backend/opencl/kernel/convolve_separable.cpp
+++ b/src/backend/opencl/kernel/convolve_separable.cpp
@@ -63,9 +63,9 @@ void convSep(Param out, const Param signal, const Param filter,
         DefineKeyValue(LOCAL_MEM_SIZE, locSize),
         getTypeBuildDefinition<T>()};
 
-    auto conv = common::getKernel(
-        "convolve", std::array{ops_cl_src, convolve_separable_cl_src},
-        tmpltArgs, compileOpts);
+    auto conv =
+        common::getKernel("convolve", {{ops_cl_src, convolve_separable_cl_src}},
+                          tmpltArgs, compileOpts);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/cscmm.hpp b/src/backend/opencl/kernel/cscmm.hpp
index 4fb0cc3479..a668025726 100644
--- a/src/backend/opencl/kernel/cscmm.hpp
+++ b/src/backend/opencl/kernel/cscmm.hpp
@@ -57,7 +57,7 @@ void cscmm_nn(Param out, const Param &values, const Param &colIdx,
         getTypeBuildDefinition<T>()};
 
     auto cscmmNN =
-        common::getKernel("cscmm_nn", std::array{cscmm_cl_src}, targs, options);
+        common::getKernel("cscmm_nn", {{cscmm_cl_src}}, targs, options);
 
     cl::NDRange local(threads, 1);
     int M = out.info.dims[0];
diff --git a/src/backend/opencl/kernel/cscmv.hpp b/src/backend/opencl/kernel/cscmv.hpp
index 675176e393..88008480f8 100644
--- a/src/backend/opencl/kernel/cscmv.hpp
+++ b/src/backend/opencl/kernel/cscmv.hpp
@@ -54,8 +54,8 @@ void cscmv(Param out, const Param &values, const Param &colIdx,
         DefineKeyValue(IS_CPLX, (iscplx<T>() ? 1 : 0)),
         getTypeBuildDefinition<T>()};
 
-    auto cscmvBlock = common::getKernel("cscmv_block", std::array{cscmv_cl_src},
-                                        targs, options);
+    auto cscmvBlock =
+        common::getKernel("cscmv_block", {{cscmv_cl_src}}, targs, options);
 
     int K        = colIdx.info.dims[0] - 1;
     int M        = out.info.dims[0];
diff --git a/src/backend/opencl/kernel/csrmm.hpp b/src/backend/opencl/kernel/csrmm.hpp
index a786f7cafb..60499bf877 100644
--- a/src/backend/opencl/kernel/csrmm.hpp
+++ b/src/backend/opencl/kernel/csrmm.hpp
@@ -56,7 +56,7 @@ void csrmm_nt(Param out, const Param &values, const Param &rowIdx,
 
     // FIXME: Switch to perf (thread vs block) baesd kernel
     auto csrmm_nt_func =
-        common::getKernel("csrmm_nt", std::array{csrmm_cl_src}, targs, options);
+        common::getKernel("csrmm_nt", {{csrmm_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_PER_GROUP, 1);
     int M = rowIdx.info.dims[0] - 1;
diff --git a/src/backend/opencl/kernel/csrmv.hpp b/src/backend/opencl/kernel/csrmv.hpp
index 3c948f0177..ca39ae4d32 100644
--- a/src/backend/opencl/kernel/csrmv.hpp
+++ b/src/backend/opencl/kernel/csrmv.hpp
@@ -58,11 +58,10 @@ void csrmv(Param out, const Param &values, const Param &rowIdx,
         getTypeBuildDefinition<T>()};
 
     auto csrmv =
-        (is_csrmv_block
-             ? common::getKernel("csrmv_thread", std::array{csrmv_cl_src},
-                                 targs, options)
-             : common::getKernel("csrmv_block", std::array{csrmv_cl_src}, targs,
-                                 options));
+        (is_csrmv_block ? common::getKernel("csrmv_thread", {{csrmv_cl_src}},
+                                            targs, options)
+                        : common::getKernel("csrmv_block", {{csrmv_cl_src}},
+                                            targs, options));
 
     int M = rowIdx.info.dims[0] - 1;
 
diff --git a/src/backend/opencl/kernel/diagonal.hpp b/src/backend/opencl/kernel/diagonal.hpp
index 9f2ded02c7..e8340fba03 100644
--- a/src/backend/opencl/kernel/diagonal.hpp
+++ b/src/backend/opencl/kernel/diagonal.hpp
@@ -36,8 +36,8 @@ static void diagCreate(Param out, Param in, int num) {
         DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
         getTypeBuildDefinition<T>()};
 
-    auto diagCreate = common::getKernel(
-        "diagCreateKernel", std::array{diag_create_cl_src}, targs, options);
+    auto diagCreate = common::getKernel("diagCreateKernel",
+                                        {{diag_create_cl_src}}, targs, options);
 
     cl::NDRange local(32, 8);
     int groups_x = divup(out.info.dims[0], local[0]);
@@ -61,7 +61,7 @@ static void diagExtract(Param out, Param in, int num) {
         getTypeBuildDefinition<T>()};
 
     auto diagExtract = common::getKernel(
-        "diagExtractKernel", std::array{diag_extract_cl_src}, targs, options);
+        "diagExtractKernel", {{diag_extract_cl_src}}, targs, options);
 
     cl::NDRange local(256, 1);
     int groups_x = divup(out.info.dims[0], local[0]);
diff --git a/src/backend/opencl/kernel/diff.hpp b/src/backend/opencl/kernel/diff.hpp
index 817bd92bac..33ccbbfca8 100644
--- a/src/backend/opencl/kernel/diff.hpp
+++ b/src/backend/opencl/kernel/diff.hpp
@@ -39,8 +39,8 @@ void diff(Param out, const Param in, const unsigned indims, const unsigned dim,
         DefineKeyValue(isDiff2, (isDiff2 ? 1 : 0)),
         getTypeBuildDefinition<T>()};
 
-    auto diffOp = common::getKernel("diff_kernel", std::array{diff_cl_src},
-                                    targs, options);
+    auto diffOp =
+        common::getKernel("diff_kernel", {{diff_cl_src}}, targs, options);
 
     cl::NDRange local(TX, TY, 1);
     if (dim == 0 && indims == 1) { local = cl::NDRange(TX * TY, 1, 1); }
diff --git a/src/backend/opencl/kernel/exampleFunction.hpp b/src/backend/opencl/kernel/exampleFunction.hpp
index 8de171e908..794c34670c 100644
--- a/src/backend/opencl/kernel/exampleFunction.hpp
+++ b/src/backend/opencl/kernel/exampleFunction.hpp
@@ -61,8 +61,8 @@ void exampleFunc(Param c, const Param a, const Param b, const af_someenum_t p) {
 
     // Fetch the Kernel functor, go to common/kernel_cache.hpp
     // to find details of this function
-    auto exOp = common::getKernel("example", std::array{example_cl_src}, targs,
-                                  options);
+    auto exOp =
+        common::getKernel("example", {{example_cl_src}}, targs, options);
 
     // configure work group parameters
     cl::NDRange local(THREADS_X, THREADS_Y);
diff --git a/src/backend/opencl/kernel/fast.hpp b/src/backend/opencl/kernel/fast.hpp
index 5e75bd1995..73351803b6 100644
--- a/src/backend/opencl/kernel/fast.hpp
+++ b/src/backend/opencl/kernel/fast.hpp
@@ -45,12 +45,12 @@ void fast(const unsigned arc_length, unsigned *out_feat, Param &x_out,
         DefineKeyValue(NONMAX, static_cast<unsigned>(nonmax)),
         getTypeBuildDefinition<T>()};
 
-    auto locate  = common::getKernel("locate_features", std::array{fast_cl_src},
-                                     targs, options);
-    auto nonMax  = common::getKernel("non_max_counts", std::array{fast_cl_src},
-                                     targs, options);
-    auto getFeat = common::getKernel("get_features", std::array{fast_cl_src},
-                                     targs, options);
+    auto locate =
+        common::getKernel("locate_features", {{fast_cl_src}}, targs, options);
+    auto nonMax =
+        common::getKernel("non_max_counts", {{fast_cl_src}}, targs, options);
+    auto getFeat =
+        common::getKernel("get_features", {{fast_cl_src}}, targs, options);
 
     const unsigned max_feat =
         ceil(in.info.dims[0] * in.info.dims[1] * feature_ratio);
diff --git a/src/backend/opencl/kernel/fftconvolve.hpp b/src/backend/opencl/kernel/fftconvolve.hpp
index c43e750a89..ab6fc944e7 100644
--- a/src/backend/opencl/kernel/fftconvolve.hpp
+++ b/src/backend/opencl/kernel/fftconvolve.hpp
@@ -85,10 +85,10 @@ void packDataHelper(Param packed, Param sig, Param filter, const int rank,
         options.emplace_back(DefineKeyValue(CONVT, "double"));
     }
 
-    auto packData = common::getKernel(
-        "pack_data", std::array{fftconvolve_pack_cl_src}, targs, options);
-    auto padArray = common::getKernel(
-        "pad_array", std::array{fftconvolve_pack_cl_src}, targs, options);
+    auto packData = common::getKernel("pack_data", {{fftconvolve_pack_cl_src}},
+                                      targs, options);
+    auto padArray = common::getKernel("pad_array", {{fftconvolve_pack_cl_src}},
+                                      targs, options);
 
     Param sig_tmp, filter_tmp;
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
@@ -147,9 +147,8 @@ void complexMultiplyHelper(Param packed, Param sig, Param filter,
         options.emplace_back(DefineKeyValue(CONVT, "double"));
     }
 
-    auto cplxMul = common::getKernel("complex_multiply",
-                                     std::array{fftconvolve_multiply_cl_src},
-                                     targs, options);
+    auto cplxMul = common::getKernel(
+        "complex_multiply", {{fftconvolve_multiply_cl_src}}, targs, options);
 
     Param sig_tmp, filter_tmp;
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
@@ -195,9 +194,8 @@ void reorderOutputHelper(Param out, Param packed, Param sig, Param filter,
         options.emplace_back(DefineKeyValue(CONVT, "double"));
     }
 
-    auto reorder = common::getKernel("reorder_output",
-                                     std::array{fftconvolve_reorder_cl_src},
-                                     targs, options);
+    auto reorder = common::getKernel(
+        "reorder_output", {{fftconvolve_reorder_cl_src}}, targs, options);
 
     int fftScale = 1;
 
diff --git a/src/backend/opencl/kernel/flood_fill.hpp b/src/backend/opencl/kernel/flood_fill.hpp
index d0af9aa7c9..0b0b29fefe 100644
--- a/src/backend/opencl/kernel/flood_fill.hpp
+++ b/src/backend/opencl/kernel/flood_fill.hpp
@@ -39,7 +39,7 @@ void initSeeds(Param out, const Param seedsx, const Param seedsy) {
         DefineKey(INIT_SEEDS), getTypeBuildDefinition<T>()};
 
     auto initSeeds =
-        common::getKernel("init_seeds", std::array{flood_fill_cl_src},
+        common::getKernel("init_seeds", {{flood_fill_cl_src}},
                           TemplateArgs(TemplateTypename<T>()), options);
     cl::NDRange local(kernel::THREADS, 1, 1);
     cl::NDRange global(divup(seedsx.info.dims[0], local[0]) * local[0], 1, 1);
@@ -57,7 +57,7 @@ void finalizeOutput(Param out, const T newValue) {
         getTypeBuildDefinition<T>()};
 
     auto finalizeOut =
-        common::getKernel("finalize_output", std::array{flood_fill_cl_src},
+        common::getKernel("finalize_output", {{flood_fill_cl_src}},
                           TemplateArgs(TemplateTypename<T>()), options);
     cl::NDRange local(kernel::THREADS_X, kernel::THREADS_Y, 1);
     cl::NDRange global(divup(out.info.dims[0], local[0]) * local[0],
@@ -89,7 +89,7 @@ void floodFill(Param out, const Param image, const Param seedsx,
         getTypeBuildDefinition<T>()};
 
     auto floodStep =
-        common::getKernel("flood_step", std::array{flood_fill_cl_src},
+        common::getKernel("flood_step", {{flood_fill_cl_src}},
                           TemplateArgs(TemplateTypename<T>()), options);
     cl::NDRange local(kernel::THREADS_X, kernel::THREADS_Y, 1);
     cl::NDRange global(divup(out.info.dims[0], local[0]) * local[0],
diff --git a/src/backend/opencl/kernel/gradient.hpp b/src/backend/opencl/kernel/gradient.hpp
index cab0a98abf..6809f10c19 100644
--- a/src/backend/opencl/kernel/gradient.hpp
+++ b/src/backend/opencl/kernel/gradient.hpp
@@ -41,8 +41,8 @@ void gradient(Param grad0, Param grad1, const Param in) {
         DefineKeyValue(CPLX, static_cast<int>(iscplx<T>())),
         getTypeBuildDefinition<T>()};
 
-    auto gradOp = common::getKernel("gradient", std::array{gradient_cl_src},
-                                    targs, options);
+    auto gradOp =
+        common::getKernel("gradient", {{gradient_cl_src}}, targs, options);
 
     cl::NDRange local(TX, TY, 1);
 
diff --git a/src/backend/opencl/kernel/harris.hpp b/src/backend/opencl/kernel/harris.hpp
index 942fb44d1b..835c20c745 100644
--- a/src/backend/opencl/kernel/harris.hpp
+++ b/src/backend/opencl/kernel/harris.hpp
@@ -71,14 +71,12 @@ std::array<Kernel, 4> getHarrisKernels() {
         getTypeBuildDefinition<T>()};
 
     return {
-        common::getKernel("second_order_deriv", std::array{harris_cl_src},
-                          targs, options),
-        common::getKernel("keep_corners", std::array{harris_cl_src}, targs,
+        common::getKernel("second_order_deriv", {{harris_cl_src}}, targs,
                           options),
-        common::getKernel("harris_responses", std::array{harris_cl_src}, targs,
-                          options),
-        common::getKernel("non_maximal", std::array{harris_cl_src}, targs,
+        common::getKernel("keep_corners", {{harris_cl_src}}, targs, options),
+        common::getKernel("harris_responses", {{harris_cl_src}}, targs,
                           options),
+        common::getKernel("non_maximal", {{harris_cl_src}}, targs, options),
     };
 }
 
diff --git a/src/backend/opencl/kernel/histogram.hpp b/src/backend/opencl/kernel/histogram.hpp
index a05bad05f6..d138202240 100644
--- a/src/backend/opencl/kernel/histogram.hpp
+++ b/src/backend/opencl/kernel/histogram.hpp
@@ -42,8 +42,8 @@ void histogram(Param out, const Param in, int nbins, float minval, float maxval,
     options.emplace_back(getTypeBuildDefinition<T>());
     if (isLinear) { options.emplace_back(DefineKey(IS_LINEAR)); }
 
-    auto histogram = common::getKernel(
-        "histogram", std::array{histogram_cl_src}, targs, options);
+    auto histogram =
+        common::getKernel("histogram", {{histogram_cl_src}}, targs, options);
 
     int nElems  = in.info.dims[0] * in.info.dims[1];
     int blk_x   = divup(nElems, THRD_LOAD * THREADS_X);
diff --git a/src/backend/opencl/kernel/homography.hpp b/src/backend/opencl/kernel/homography.hpp
index 328f39d753..4c785b57a1 100644
--- a/src/backend/opencl/kernel/homography.hpp
+++ b/src/backend/opencl/kernel/homography.hpp
@@ -50,16 +50,16 @@ std::array<Kernel, 5> getHomographyKernels(const af_homography_type htype) {
         options.emplace_back(DefineKey(IS_CPU));
     }
     return {
-        common::getKernel("compute_homography", std::array{homography_cl_src},
-                          targs, options),
-        common::getKernel("eval_homography", std::array{homography_cl_src},
-                          targs, options),
-        common::getKernel("compute_median", std::array{homography_cl_src},
-                          targs, options),
-        common::getKernel("find_min_median", std::array{homography_cl_src},
-                          targs, options),
-        common::getKernel("compute_lmeds_inliers",
-                          std::array{homography_cl_src}, targs, options),
+        common::getKernel("compute_homography", {{homography_cl_src}}, targs,
+                          options),
+        common::getKernel("eval_homography", {{homography_cl_src}}, targs,
+                          options),
+        common::getKernel("compute_median", {{homography_cl_src}}, targs,
+                          options),
+        common::getKernel("find_min_median", {{homography_cl_src}}, targs,
+                          options),
+        common::getKernel("compute_lmeds_inliers", {{homography_cl_src}}, targs,
+                          options),
     };
 }
 
diff --git a/src/backend/opencl/kernel/hsv_rgb.hpp b/src/backend/opencl/kernel/hsv_rgb.hpp
index 1f46cc5085..4ca85a4f74 100644
--- a/src/backend/opencl/kernel/hsv_rgb.hpp
+++ b/src/backend/opencl/kernel/hsv_rgb.hpp
@@ -37,8 +37,8 @@ void hsv2rgb_convert(Param out, const Param in, bool isHSV2RGB) {
         getTypeBuildDefinition<T>()};
     if (isHSV2RGB) { options.emplace_back(DefineKey(isHSV2RGB)); }
 
-    auto convert = common::getKernel(
-        "hsvrgbConvert", std::array{hsv_rgb_cl_src}, targs, options);
+    auto convert =
+        common::getKernel("hsvrgbConvert", {{hsv_rgb_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/identity.hpp b/src/backend/opencl/kernel/identity.hpp
index 19afcdaea7..32186164ef 100644
--- a/src/backend/opencl/kernel/identity.hpp
+++ b/src/backend/opencl/kernel/identity.hpp
@@ -37,8 +37,8 @@ static void identity(Param out) {
         DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
         getTypeBuildDefinition<T>()};
 
-    auto identityOp = common::getKernel(
-        "identity_kernel", std::array{identity_cl_src}, targs, options);
+    auto identityOp = common::getKernel("identity_kernel", {{identity_cl_src}},
+                                        targs, options);
 
     cl::NDRange local(32, 8);
     int groups_x = divup(out.info.dims[0], local[0]);
diff --git a/src/backend/opencl/kernel/iir.hpp b/src/backend/opencl/kernel/iir.hpp
index 7786197da4..34f9d2c0bf 100644
--- a/src/backend/opencl/kernel/iir.hpp
+++ b/src/backend/opencl/kernel/iir.hpp
@@ -40,8 +40,7 @@ void iir(Param y, Param c, Param a) {
         DefineKeyValue(ZERO, scalar_to_option(scalar<T>(0))),
         getTypeBuildDefinition<T>()};
 
-    auto iir =
-        common::getKernel("iir_kernel", std::array{iir_cl_src}, targs, options);
+    auto iir = common::getKernel("iir_kernel", {{iir_cl_src}}, targs, options);
 
     const int groups_y = y.info.dims[1];
     const int groups_x = y.info.dims[2];
diff --git a/src/backend/opencl/kernel/index.hpp b/src/backend/opencl/kernel/index.hpp
index 6a496d1ade..9433893b96 100644
--- a/src/backend/opencl/kernel/index.hpp
+++ b/src/backend/opencl/kernel/index.hpp
@@ -37,7 +37,7 @@ void index(Param out, const Param in, const IndexKernelParam_t& p,
         getTypeBuildDefinition<T>()};
 
     auto index =
-        common::getKernel("indexKernel", std::array{index_cl_src},
+        common::getKernel("indexKernel", {{index_cl_src}},
                           TemplateArgs(TemplateTypename<T>()), options);
     int threads_x = 256;
     int threads_y = 1;
diff --git a/src/backend/opencl/kernel/iota.hpp b/src/backend/opencl/kernel/iota.hpp
index 3308ee23e1..24d5ad7924 100644
--- a/src/backend/opencl/kernel/iota.hpp
+++ b/src/backend/opencl/kernel/iota.hpp
@@ -36,7 +36,7 @@ void iota(Param out, const af::dim4& sdims) {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         getTypeBuildDefinition<T>()};
 
-    auto iota = common::getKernel("iota_kernel", std::array{iota_cl_src},
+    auto iota = common::getKernel("iota_kernel", {{iota_cl_src}},
                                   TemplateArgs(TemplateTypename<T>()), options);
     cl::NDRange local(IOTA_TX, IOTA_TY, 1);
 
diff --git a/src/backend/opencl/kernel/ireduce.hpp b/src/backend/opencl/kernel/ireduce.hpp
index 775ee044d7..1bbcf08d2b 100644
--- a/src/backend/opencl/kernel/ireduce.hpp
+++ b/src/backend/opencl/kernel/ireduce.hpp
@@ -49,9 +49,9 @@ void ireduceDimLauncher(Param out, cl::Buffer *oidx, Param in, cl::Buffer *iidx,
         DefineKeyValue(IS_FIRST, is_first),
         getTypeBuildDefinition<T>()};
 
-    auto ireduceDim = common::getKernel(
-        "ireduce_dim_kernel", std::array{iops_cl_src, ireduce_dim_cl_src},
-        targs, options);
+    auto ireduceDim =
+        common::getKernel("ireduce_dim_kernel",
+                          {{iops_cl_src, ireduce_dim_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, threads_y);
     cl::NDRange global(groups_all[0] * groups_all[2] * local[0],
@@ -125,9 +125,9 @@ void ireduceFirstLauncher(Param out, cl::Buffer *oidx, Param in,
         DefineKeyValue(IS_FIRST, is_first),
         getTypeBuildDefinition<T>()};
 
-    auto ireduceFirst = common::getKernel(
-        "ireduce_first_kernel", std::array{iops_cl_src, ireduce_first_cl_src},
-        targs, options);
+    auto ireduceFirst = common::getKernel("ireduce_first_kernel",
+                                          {{iops_cl_src, ireduce_first_cl_src}},
+                                          targs, options);
 
     cl::NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     cl::NDRange global(groups_x * in.info.dims[2] * local[0],
diff --git a/src/backend/opencl/kernel/laset.hpp b/src/backend/opencl/kernel/laset.hpp
index 504cf9244f..63e9a66526 100644
--- a/src/backend/opencl/kernel/laset.hpp
+++ b/src/backend/opencl/kernel/laset.hpp
@@ -57,8 +57,8 @@ void laset(int m, int n, T offdiag, T diag, cl_mem dA, size_t dA_offset,
         DefineKeyValue(IS_CPLX, static_cast<int>(iscplx<T>())),
         getTypeBuildDefinition<T>()};
 
-    auto lasetOp = common::getKernel(laset_name<uplo>(),
-                                     std::array{laset_cl_src}, targs, options);
+    auto lasetOp =
+        common::getKernel(laset_name<uplo>(), {{laset_cl_src}}, targs, options);
 
     int groups_x = (m - 1) / BLK_X + 1;
     int groups_y = (n - 1) / BLK_Y + 1;
diff --git a/src/backend/opencl/kernel/laswp.hpp b/src/backend/opencl/kernel/laswp.hpp
index 5db0b388ff..7439f3680e 100644
--- a/src/backend/opencl/kernel/laswp.hpp
+++ b/src/backend/opencl/kernel/laswp.hpp
@@ -42,8 +42,7 @@ void laswp(int n, cl_mem in, size_t offset, int ldda, int k1, int k2,
         DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(MAX_PIVOTS),
         getTypeBuildDefinition<T>()};
 
-    auto laswpOp =
-        common::getKernel("laswp", std::array{laswp_cl_src}, targs, options);
+    auto laswpOp = common::getKernel("laswp", {{laswp_cl_src}}, targs, options);
 
     int groups = divup(n, NTHREADS);
     cl::NDRange local(NTHREADS);
diff --git a/src/backend/opencl/kernel/lookup.hpp b/src/backend/opencl/kernel/lookup.hpp
index 1e99e82780..3410c65266 100644
--- a/src/backend/opencl/kernel/lookup.hpp
+++ b/src/backend/opencl/kernel/lookup.hpp
@@ -48,8 +48,8 @@ void lookup(Param out, const Param in, const Param indices,
     cl::NDRange global(blk_x * out.info.dims[2] * THREADS_X,
                        blk_y * out.info.dims[3] * THREADS_Y);
 
-    auto arrIdxOp = common::getKernel("lookupND", std::array{lookup_cl_src},
-                                      targs, options);
+    auto arrIdxOp =
+        common::getKernel("lookupND", {{lookup_cl_src}}, targs, options);
 
     arrIdxOp(cl::EnqueueArgs(getQueue(), global, local), *out.data, out.info,
              *in.data, in.info, *indices.data, indices.info, blk_x, blk_y);
diff --git a/src/backend/opencl/kernel/lu_split.hpp b/src/backend/opencl/kernel/lu_split.hpp
index 65fc511415..019e02528b 100644
--- a/src/backend/opencl/kernel/lu_split.hpp
+++ b/src/backend/opencl/kernel/lu_split.hpp
@@ -41,8 +41,8 @@ void luSplitLauncher(Param lower, Param upper, const Param in, bool same_dims) {
         DefineKeyValue(ONE, scalar_to_option(scalar<T>(1))),
         getTypeBuildDefinition<T>()};
 
-    auto luSplit = common::getKernel("luSplit", std::array{lu_split_cl_src},
-                                     targs, options);
+    auto luSplit =
+        common::getKernel("luSplit", {{lu_split_cl_src}}, targs, options);
 
     cl::NDRange local(TX, TY);
 
diff --git a/src/backend/opencl/kernel/match_template.hpp b/src/backend/opencl/kernel/match_template.hpp
index 21041eb73b..8f43c99174 100644
--- a/src/backend/opencl/kernel/match_template.hpp
+++ b/src/backend/opencl/kernel/match_template.hpp
@@ -52,7 +52,7 @@ void matchTemplate(Param out, const Param srch, const Param tmplt,
         getTypeBuildDefinition<outType>()};
 
     auto matchImgOp = common::getKernel(
-        "matchTemplate", std::array{matchTemplate_cl_src}, targs, options);
+        "matchTemplate", {{matchTemplate_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/mean.hpp b/src/backend/opencl/kernel/mean.hpp
index 13f74453a8..bc80a23be9 100644
--- a/src/backend/opencl/kernel/mean.hpp
+++ b/src/backend/opencl/kernel/mean.hpp
@@ -130,8 +130,7 @@ void meanDimLauncher(Param out, Param owt, Param in, Param inWeight,
     if (output_weight) { options.emplace_back(DefineKey(OUTPUT_WEIGHT)); }
 
     auto meanOp = common::getKernel(
-        "meanDim", std::array{mean_ops_cl_src, mean_dim_cl_src}, targs,
-        options);
+        "meanDim", {{mean_ops_cl_src, mean_dim_cl_src}}, targs, options);
 
     NDRange local(THREADS_X, threads_y);
     NDRange global(groups_all[0] * groups_all[2] * local[0],
@@ -223,8 +222,7 @@ void meanFirstLauncher(Param out, Param owt, Param in, Param inWeight,
     if (output_weight) { options.emplace_back(DefineKey(OUTPUT_WEIGHT)); }
 
     auto meanOp = common::getKernel(
-        "meanFirst", std::array{mean_ops_cl_src, mean_first_cl_src}, targs,
-        options);
+        "meanFirst", {{mean_ops_cl_src, mean_first_cl_src}}, targs, options);
 
     NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     NDRange global(groups_x * in.info.dims[2] * local[0],
diff --git a/src/backend/opencl/kernel/meanshift.hpp b/src/backend/opencl/kernel/meanshift.hpp
index 24fa61374d..752e507262 100644
--- a/src/backend/opencl/kernel/meanshift.hpp
+++ b/src/backend/opencl/kernel/meanshift.hpp
@@ -43,8 +43,8 @@ void meanshift(Param out, const Param in, const float spatialSigma,
         DefineKeyValue(MAX_CHANNELS, (is_color ? 3 : 1)),
         getTypeBuildDefinition<T>()};
 
-    auto meanshiftOp = common::getKernel(
-        "meanshift", std::array{meanshift_cl_src}, targs, options);
+    auto meanshiftOp =
+        common::getKernel("meanshift", {{meanshift_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/medfilt.hpp b/src/backend/opencl/kernel/medfilt.hpp
index d38943e50d..abbd0ea5c7 100644
--- a/src/backend/opencl/kernel/medfilt.hpp
+++ b/src/backend/opencl/kernel/medfilt.hpp
@@ -49,8 +49,8 @@ void medfilt1(Param out, const Param in, const unsigned w_wid,
         DefineValue(w_wid),
         getTypeBuildDefinition<T>()};
 
-    auto medfiltOp = common::getKernel("medfilt1", std::array{medfilt1_cl_src},
-                                       targs, options);
+    auto medfiltOp =
+        common::getKernel("medfilt1", {{medfilt1_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, 1, 1);
 
@@ -87,8 +87,8 @@ void medfilt2(Param out, const Param in, const af_border_type pad,
         DefineValue(w_len),
         getTypeBuildDefinition<T>()};
 
-    auto medfiltOp = common::getKernel("medfilt2", std::array{medfilt2_cl_src},
-                                       targs, options);
+    auto medfiltOp =
+        common::getKernel("medfilt2", {{medfilt2_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/memcopy.hpp b/src/backend/opencl/kernel/memcopy.hpp
index d9fe825107..c27d8c39b6 100644
--- a/src/backend/opencl/kernel/memcopy.hpp
+++ b/src/backend/opencl/kernel/memcopy.hpp
@@ -156,9 +156,8 @@ void memcopy(const cl::Buffer& b_out, const dim4& ostrides,
         : sizeofNewT == 16
             ? "float4"
             : "type is larger than 16 bytes, which is unsupported"};
-    auto memCopy{common::getKernel(kernelName, std::array{memcopy_cl_src},
-                                   std::array{tArg},
-                                   std::array{DefineKeyValue(T, tArg)})};
+    auto memCopy{common::getKernel(kernelName, {{memcopy_cl_src}}, {{tArg}},
+                                   {{DefineKeyValue(T, tArg)}})};
     const cl::NDRange local{th.genLocal(memCopy.get())};
     const cl::NDRange global{th.genGlobal(local)};
 
@@ -230,7 +229,7 @@ void copy(const Param out, const Param in, dim_t ondims,
                                   : th.loop3 ? "scaledCopyLoop13"
                                   : th.loop1 ? "scaledCopyLoop1"
                                              : "scaledCopy",
-                                  std::array{copy_cl_src}, targs, options);
+                                  {{copy_cl_src}}, targs, options);
     const cl::NDRange local{th.genLocal(copy.get())};
     const cl::NDRange global{th.genGlobal(local)};
 
diff --git a/src/backend/opencl/kernel/moments.hpp b/src/backend/opencl/kernel/moments.hpp
index 3f269686c3..2ab1185516 100644
--- a/src/backend/opencl/kernel/moments.hpp
+++ b/src/backend/opencl/kernel/moments.hpp
@@ -38,8 +38,8 @@ void moments(Param out, const Param in, af_moment_type moment) {
         DefineKeyValue(MOMENTS_SZ, out.info.dims[0]),
         getTypeBuildDefinition<T>()};
 
-    auto momentsOp = common::getKernel("moments", std::array{moments_cl_src},
-                                       targs, options);
+    auto momentsOp =
+        common::getKernel("moments", {{moments_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS, 1, 1);
     cl::NDRange global(in.info.dims[1] * local[0],
diff --git a/src/backend/opencl/kernel/morph.hpp b/src/backend/opencl/kernel/morph.hpp
index 730a424eed..473de659f2 100644
--- a/src/backend/opencl/kernel/morph.hpp
+++ b/src/backend/opencl/kernel/morph.hpp
@@ -56,8 +56,7 @@ void morph(Param out, const Param in, const Param mask, bool isDilation) {
     };
     options.emplace_back(getTypeBuildDefinition<T>());
 
-    auto morphOp =
-        common::getKernel("morph", std::array{morph_cl_src}, targs, options);
+    auto morphOp = common::getKernel("morph", {{morph_cl_src}}, targs, options);
 
     NDRange local(THREADS_X, THREADS_Y);
 
@@ -117,7 +116,7 @@ void morph3d(Param out, const Param in, const Param mask, bool isDilation) {
     options.emplace_back(getTypeBuildDefinition<T>());
 
     auto morphOp =
-        common::getKernel("morph3d", std::array{morph_cl_src}, targs, options);
+        common::getKernel("morph3d", {{morph_cl_src}}, targs, options);
 
     NDRange local(CUBE_X, CUBE_Y, CUBE_Z);
 
diff --git a/src/backend/opencl/kernel/nearest_neighbour.hpp b/src/backend/opencl/kernel/nearest_neighbour.hpp
index b4f7e5fa36..cac36cab33 100644
--- a/src/backend/opencl/kernel/nearest_neighbour.hpp
+++ b/src/backend/opencl/kernel/nearest_neighbour.hpp
@@ -71,9 +71,8 @@ void allDistances(Param dist, Param query, Param train, const dim_t dist_dim,
         options.emplace_back(DefineKeyValue(DISTOP, "_shd_"));
         options.emplace_back(DefineKey(__SHD__));
     }
-    auto hmOp =
-        common::getKernel("knnAllDistances",
-                          std::array{nearest_neighbour_cl_src}, targs, options);
+    auto hmOp = common::getKernel("knnAllDistances",
+                                  {{nearest_neighbour_cl_src}}, targs, options);
 
     const dim_t sample_dim = (dist_dim == 0) ? 1 : 0;
 
diff --git a/src/backend/opencl/kernel/orb.hpp b/src/backend/opencl/kernel/orb.hpp
index b3e4014d05..5d4f523f16 100644
--- a/src/backend/opencl/kernel/orb.hpp
+++ b/src/backend/opencl/kernel/orb.hpp
@@ -88,14 +88,11 @@ std::array<Kernel, 4> getOrbKernels() {
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
     return {
-        common::getKernel("harris_response", std::array{orb_cl_src}, targs,
-                          compileOpts),
-        common::getKernel("keep_features", std::array{orb_cl_src}, targs,
-                          compileOpts),
-        common::getKernel("centroid_angle", std::array{orb_cl_src}, targs,
-                          compileOpts),
-        common::getKernel("extract_orb", std::array{orb_cl_src}, targs,
+        common::getKernel("harris_response", {{orb_cl_src}}, targs,
                           compileOpts),
+        common::getKernel("keep_features", {{orb_cl_src}}, targs, compileOpts),
+        common::getKernel("centroid_angle", {{orb_cl_src}}, targs, compileOpts),
+        common::getKernel("extract_orb", {{orb_cl_src}}, targs, compileOpts),
     };
 }
 
diff --git a/src/backend/opencl/kernel/pad_array_borders.hpp b/src/backend/opencl/kernel/pad_array_borders.hpp
index 8e75e5fbd5..53ee36d8d8 100644
--- a/src/backend/opencl/kernel/pad_array_borders.hpp
+++ b/src/backend/opencl/kernel/pad_array_borders.hpp
@@ -46,9 +46,8 @@ void padBorders(Param out, const Param in, dim4 const& lBPadding,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto pad =
-        common::getKernel("padBorders", std::array{pad_array_borders_cl_src},
-                          tmpltArgs, compileOpts);
+    auto pad = common::getKernel("padBorders", {{pad_array_borders_cl_src}},
+                                 tmpltArgs, compileOpts);
 
     NDRange local(PADB_THREADS_X, PADB_THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/random_engine.hpp b/src/backend/opencl/kernel/random_engine.hpp
index 96c230f133..390be184eb 100644
--- a/src/backend/opencl/kernel/random_engine.hpp
+++ b/src/backend/opencl/kernel/random_engine.hpp
@@ -163,9 +163,8 @@ void initMersenneState(cl::Buffer state, cl::Buffer table, const uintl &seed) {
     cl::NDRange local(THREADS_PER_GROUP, 1);
     cl::NDRange global(local[0] * MAX_BLOCKS, 1);
 
-    auto initOp =
-        common::getKernel("mersenneInitState",
-                          std::array{random_engine_mersenne_init_cl_src}, {});
+    auto initOp = common::getKernel("mersenneInitState",
+                                    {{random_engine_mersenne_init_cl_src}}, {});
     initOp(cl::EnqueueArgs(getQueue(), global, local), state, table, seed);
     CL_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/opencl/kernel/range.hpp b/src/backend/opencl/kernel/range.hpp
index ddb946d307..3fb58a65ce 100644
--- a/src/backend/opencl/kernel/range.hpp
+++ b/src/backend/opencl/kernel/range.hpp
@@ -36,8 +36,8 @@ void range(Param out, const int dim) {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         getTypeBuildDefinition<T>()};
 
-    auto rangeOp = common::getKernel("range_kernel", std::array{range_cl_src},
-                                     targs, options);
+    auto rangeOp =
+        common::getKernel("range_kernel", {{range_cl_src}}, targs, options);
 
     cl::NDRange local(RANGE_TX, RANGE_TY, 1);
 
diff --git a/src/backend/opencl/kernel/reduce.hpp b/src/backend/opencl/kernel/reduce.hpp
index 21db6e2edc..98982fe8f3 100644
--- a/src/backend/opencl/kernel/reduce.hpp
+++ b/src/backend/opencl/kernel/reduce.hpp
@@ -56,8 +56,7 @@ void reduceDimLauncher(Param out, Param in, const int dim, const uint threads_y,
         getTypeBuildDefinition<Ti, To>()};
 
     auto reduceDim = common::getKernel(
-        "reduce_dim_kernel", std::array{ops_cl_src, reduce_dim_cl_src}, targs,
-        options);
+        "reduce_dim_kernel", {{ops_cl_src, reduce_dim_cl_src}}, targs, options);
 
     cl::NDRange local(THREADS_X, threads_y);
     cl::NDRange global(groups_all[0] * groups_all[2] * local[0],
@@ -134,8 +133,7 @@ void reduceAllLauncher(Param out, Param in, const uint groups_x,
         getTypeBuildDefinition<Ti, To>()};
 
     auto reduceAll = common::getKernel(
-        "reduce_all_kernel", std::array{ops_cl_src, reduce_all_cl_src}, targs,
-        options);
+        "reduce_all_kernel", {{ops_cl_src, reduce_all_cl_src}}, targs, options);
 
     cl::NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     cl::NDRange global(groups_x * in.info.dims[2] * local[0],
@@ -181,9 +179,9 @@ void reduceFirstLauncher(Param out, Param in, const uint groups_x,
         DefineKeyValue(CPLX, iscplx<Ti>()),
         getTypeBuildDefinition<Ti, To>()};
 
-    auto reduceFirst = common::getKernel(
-        "reduce_first_kernel", std::array{ops_cl_src, reduce_first_cl_src},
-        targs, options);
+    auto reduceFirst =
+        common::getKernel("reduce_first_kernel",
+                          {{ops_cl_src, reduce_first_cl_src}}, targs, options);
 
     cl::NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     cl::NDRange global(groups_x * in.info.dims[2] * local[0],
diff --git a/src/backend/opencl/kernel/reduce_by_key.hpp b/src/backend/opencl/kernel/reduce_by_key.hpp
index eeb0e119df..e80e3603c6 100644
--- a/src/backend/opencl/kernel/reduce_by_key.hpp
+++ b/src/backend/opencl/kernel/reduce_by_key.hpp
@@ -64,10 +64,10 @@ void reduceBlocksByKeyDim(cl::Buffer *reduced_block_sizes, Param keys_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    auto reduceBlocksByKeyDim = common::getKernel(
-        "reduce_blocks_by_key_dim",
-        std::array{ops_cl_src, reduce_blocks_by_key_dim_cl_src}, tmpltArgs,
-        compileOpts);
+    auto reduceBlocksByKeyDim =
+        common::getKernel("reduce_blocks_by_key_dim",
+                          {{ops_cl_src, reduce_blocks_by_key_dim_cl_src}},
+                          tmpltArgs, compileOpts);
     int numBlocks = divup(n, threads_x);
 
     cl::NDRange local(threads_x);
@@ -107,10 +107,10 @@ void reduceBlocksByKey(cl::Buffer *reduced_block_sizes, Param keys_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    auto reduceBlocksByKeyFirst = common::getKernel(
-        "reduce_blocks_by_key_first",
-        std::array{ops_cl_src, reduce_blocks_by_key_first_cl_src}, tmpltArgs,
-        compileOpts);
+    auto reduceBlocksByKeyFirst =
+        common::getKernel("reduce_blocks_by_key_first",
+                          {{ops_cl_src, reduce_blocks_by_key_first_cl_src}},
+                          tmpltArgs, compileOpts);
     int numBlocks = divup(n, threads_x);
 
     cl::NDRange local(threads_x);
@@ -148,10 +148,9 @@ void finalBoundaryReduce(cl::Buffer *reduced_block_sizes, Param keys_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
-    auto finalBoundaryReduce =
-        common::getKernel("final_boundary_reduce",
-                          std::array{ops_cl_src, reduce_by_key_boundary_cl_src},
-                          tmpltArgs, compileOpts);
+    auto finalBoundaryReduce = common::getKernel(
+        "final_boundary_reduce", {{ops_cl_src, reduce_by_key_boundary_cl_src}},
+        tmpltArgs, compileOpts);
 
     cl::NDRange local(threads_x);
     cl::NDRange global(threads_x * numBlocks);
@@ -187,10 +186,10 @@ void finalBoundaryReduceDim(cl::Buffer *reduced_block_sizes, Param keys_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
-    auto finalBoundaryReduceDim = common::getKernel(
-        "final_boundary_reduce_dim",
-        std::array{ops_cl_src, reduce_by_key_boundary_dim_cl_src}, tmpltArgs,
-        compileOpts);
+    auto finalBoundaryReduceDim =
+        common::getKernel("final_boundary_reduce_dim",
+                          {{ops_cl_src, reduce_by_key_boundary_dim_cl_src}},
+                          tmpltArgs, compileOpts);
 
     cl::NDRange local(threads_x);
     cl::NDRange global(threads_x * numBlocks,
@@ -224,8 +223,8 @@ void compact(cl::Buffer *reduced_block_sizes, Param keys_out, Param vals_out,
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
     auto compact = common::getKernel(
-        "compact", std::array{ops_cl_src, reduce_by_key_compact_cl_src},
-        tmpltArgs, compileOpts);
+        "compact", {{ops_cl_src, reduce_by_key_compact_cl_src}}, tmpltArgs,
+        compileOpts);
 
     cl::NDRange local(threads_x);
     cl::NDRange global(threads_x * numBlocks, vals_out.info.dims[1],
@@ -259,7 +258,7 @@ void compactDim(cl::Buffer *reduced_block_sizes, Param keys_out, Param vals_out,
     compileOpts.emplace_back(getTypeBuildDefinition<To>());
 
     auto compactDim = common::getKernel(
-        "compact_dim", std::array{ops_cl_src, reduce_by_key_compact_dim_cl_src},
+        "compact_dim", {{ops_cl_src, reduce_by_key_compact_dim_cl_src}},
         tmpltArgs, compileOpts);
 
     cl::NDRange local(threads_x);
@@ -288,10 +287,10 @@ void testNeedsReduction(cl::Buffer needs_reduction, cl::Buffer needs_boundary,
         DefineKeyValue(DIMX, threads_x),
     };
 
-    auto testIfNeedsReduction = common::getKernel(
-        "test_needs_reduction",
-        std::array{ops_cl_src, reduce_by_key_needs_reduction_cl_src}, tmpltArgs,
-        compileOpts);
+    auto testIfNeedsReduction =
+        common::getKernel("test_needs_reduction",
+                          {{ops_cl_src, reduce_by_key_needs_reduction_cl_src}},
+                          tmpltArgs, compileOpts);
 
     cl::NDRange local(threads_x);
     cl::NDRange global(threads_x * numBlocks);
diff --git a/src/backend/opencl/kernel/regions.hpp b/src/backend/opencl/kernel/regions.hpp
index 63716ba8ea..a082d165af 100644
--- a/src/backend/opencl/kernel/regions.hpp
+++ b/src/backend/opencl/kernel/regions.hpp
@@ -67,12 +67,9 @@ std::array<Kernel, 3> getRegionsKernels(const bool full_conn,
     options.emplace_back(getTypeBuildDefinition<T>());
 
     return {
-        common::getKernel("initial_label", std::array{regions_cl_src}, targs,
-                          options),
-        common::getKernel("final_relabel", std::array{regions_cl_src}, targs,
-                          options),
-        common::getKernel("update_equiv", std::array{regions_cl_src}, targs,
-                          options),
+        common::getKernel("initial_label", {{regions_cl_src}}, targs, options),
+        common::getKernel("final_relabel", {{regions_cl_src}}, targs, options),
+        common::getKernel("update_equiv", {{regions_cl_src}}, targs, options),
     };
 }
 
diff --git a/src/backend/opencl/kernel/reorder.hpp b/src/backend/opencl/kernel/reorder.hpp
index 9322647cd2..469e8b77c3 100644
--- a/src/backend/opencl/kernel/reorder.hpp
+++ b/src/backend/opencl/kernel/reorder.hpp
@@ -36,8 +36,8 @@ void reorder(Param out, const Param in, const dim_t* rdims) {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         getTypeBuildDefinition<T>()};
 
-    auto reorderOp = common::getKernel(
-        "reorder_kernel", std::array{reorder_cl_src}, targs, options);
+    auto reorderOp =
+        common::getKernel("reorder_kernel", {{reorder_cl_src}}, targs, options);
 
     cl::NDRange local(TX, TY, 1);
 
diff --git a/src/backend/opencl/kernel/resize.hpp b/src/backend/opencl/kernel/resize.hpp
index bc813393c5..f201427ddf 100644
--- a/src/backend/opencl/kernel/resize.hpp
+++ b/src/backend/opencl/kernel/resize.hpp
@@ -67,8 +67,8 @@ void resize(Param out, const Param in, const af_interp_type method) {
         default: break;
     }
 
-    auto resizeOp = common::getKernel(
-        "resize_kernel", std::array{resize_cl_src}, targs, options);
+    auto resizeOp =
+        common::getKernel("resize_kernel", {{resize_cl_src}}, targs, options);
 
     cl::NDRange local(RESIZE_TX, RESIZE_TY, 1);
 
diff --git a/src/backend/opencl/kernel/rotate.hpp b/src/backend/opencl/kernel/rotate.hpp
index dec52c8962..a3d3f41cba 100644
--- a/src/backend/opencl/kernel/rotate.hpp
+++ b/src/backend/opencl/kernel/rotate.hpp
@@ -80,9 +80,9 @@ void rotate(Param out, const Param in, const float theta, af_interp_type method,
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
     addInterpEnumOptions(compileOpts);
 
-    auto rotate = common::getKernel("rotateKernel",
-                                    std::array{interp_cl_src, rotate_cl_src},
-                                    tmpltArgs, compileOpts);
+    auto rotate =
+        common::getKernel("rotateKernel", {{interp_cl_src, rotate_cl_src}},
+                          tmpltArgs, compileOpts);
 
     const float c = cos(-theta), s = sin(-theta);
     float tx, ty;
diff --git a/src/backend/opencl/kernel/scan_dim.hpp b/src/backend/opencl/kernel/scan_dim.hpp
index 2edc7f68c0..f9820f47cf 100644
--- a/src/backend/opencl/kernel/scan_dim.hpp
+++ b/src/backend/opencl/kernel/scan_dim.hpp
@@ -58,8 +58,8 @@ static opencl::Kernel getScanDimKernel(const std::string key, int dim,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    return common::getKernel(key, std::array{ops_cl_src, scan_dim_cl_src},
-                             tmpltArgs, compileOpts);
+    return common::getKernel(key, {{ops_cl_src, scan_dim_cl_src}}, tmpltArgs,
+                             compileOpts);
 }
 
 template<typename Ti, typename To, af_op_t op>
diff --git a/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp b/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
index 3d9745923c..c4cc7959ff 100644
--- a/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/scan_dim_by_key_impl.hpp
@@ -58,8 +58,7 @@ static opencl::Kernel getScanDimKernel(const std::string key, int dim,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    return common::getKernel(key,
-                             std::array{ops_cl_src, scan_dim_by_key_cl_src},
+    return common::getKernel(key, {{ops_cl_src, scan_dim_by_key_cl_src}},
                              tmpltArgs, compileOpts);
 }
 
diff --git a/src/backend/opencl/kernel/scan_first.hpp b/src/backend/opencl/kernel/scan_first.hpp
index 4354d27b49..569c361ef8 100644
--- a/src/backend/opencl/kernel/scan_first.hpp
+++ b/src/backend/opencl/kernel/scan_first.hpp
@@ -59,8 +59,8 @@ static opencl::Kernel getScanFirstKernel(const std::string key,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    return common::getKernel(key, std::array{ops_cl_src, scan_first_cl_src},
-                             tmpltArgs, compileOpts);
+    return common::getKernel(key, {{ops_cl_src, scan_first_cl_src}}, tmpltArgs,
+                             compileOpts);
 }
 
 template<typename Ti, typename To, af_op_t op>
diff --git a/src/backend/opencl/kernel/scan_first_by_key_impl.hpp b/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
index d0351add52..82674db44d 100644
--- a/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/scan_first_by_key_impl.hpp
@@ -62,8 +62,7 @@ static opencl::Kernel getScanFirstKernel(const std::string key,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    return common::getKernel(key,
-                             std::array{ops_cl_src, scan_first_by_key_cl_src},
+    return common::getKernel(key, {{ops_cl_src, scan_first_by_key_cl_src}},
                              tmpltArgs, compileOpts);
 }
 
diff --git a/src/backend/opencl/kernel/select.hpp b/src/backend/opencl/kernel/select.hpp
index fc37e6cb86..6de96e2cd6 100644
--- a/src/backend/opencl/kernel/select.hpp
+++ b/src/backend/opencl/kernel/select.hpp
@@ -38,8 +38,8 @@ void selectLauncher(Param out, Param cond, Param a, Param b, const int ndims,
         DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(is_same),
         getTypeBuildDefinition<T>()};
 
-    auto selectOp = common::getKernel(
-        "select_kernel", std::array{select_cl_src}, targs, options);
+    auto selectOp =
+        common::getKernel("select_kernel", {{select_cl_src}}, targs, options);
 
     int threads[] = {DIMX, DIMY};
 
@@ -81,8 +81,8 @@ void select_scalar(Param out, Param cond, Param a, const T b, const int ndims,
         DefineKeyValue(T, dtype_traits<T>::getName()), DefineValue(flip),
         getTypeBuildDefinition<T>()};
 
-    auto selectOp = common::getKernel(
-        "select_scalar_kernel", std::array{select_cl_src}, targs, options);
+    auto selectOp = common::getKernel("select_scalar_kernel", {{select_cl_src}},
+                                      targs, options);
 
     int threads[] = {DIMX, DIMY};
 
diff --git a/src/backend/opencl/kernel/sift.hpp b/src/backend/opencl/kernel/sift.hpp
index d5b248f007..01bfaa3926 100644
--- a/src/backend/opencl/kernel/sift.hpp
+++ b/src/backend/opencl/kernel/sift.hpp
@@ -356,20 +356,19 @@ std::array<Kernel, 7> getSiftKernels() {
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
     return {
-        common::getKernel("sub", std::array{sift_nonfree_cl_src}, targs,
+        common::getKernel("sub", {{sift_nonfree_cl_src}}, targs, compileOpts),
+        common::getKernel("detectExtrema", {{sift_nonfree_cl_src}}, targs,
                           compileOpts),
-        common::getKernel("detectExtrema", std::array{sift_nonfree_cl_src},
-                          targs, compileOpts),
-        common::getKernel("interpolateExtrema", std::array{sift_nonfree_cl_src},
-                          targs, compileOpts),
-        common::getKernel("calcOrientation", std::array{sift_nonfree_cl_src},
-                          targs, compileOpts),
-        common::getKernel("removeDuplicates", std::array{sift_nonfree_cl_src},
-                          targs, compileOpts),
-        common::getKernel("computeDescriptor", std::array{sift_nonfree_cl_src},
+        common::getKernel("interpolateExtrema", {{sift_nonfree_cl_src}}, targs,
+                          compileOpts),
+        common::getKernel("calcOrientation", {{sift_nonfree_cl_src}}, targs,
+                          compileOpts),
+        common::getKernel("removeDuplicates", {{sift_nonfree_cl_src}}, targs,
+                          compileOpts),
+        common::getKernel("computeDescriptor", {{sift_nonfree_cl_src}}, targs,
+                          compileOpts),
+        common::getKernel("computeGLOHDescriptor", {{sift_nonfree_cl_src}},
                           targs, compileOpts),
-        common::getKernel("computeGLOHDescriptor",
-                          std::array{sift_nonfree_cl_src}, targs, compileOpts),
     };
 }
 
diff --git a/src/backend/opencl/kernel/sobel.hpp b/src/backend/opencl/kernel/sobel.hpp
index 9e92213adf..9e7138f69d 100644
--- a/src/backend/opencl/kernel/sobel.hpp
+++ b/src/backend/opencl/kernel/sobel.hpp
@@ -39,8 +39,8 @@ void sobel(Param dx, Param dy, const Param in) {
     };
     compileOpts.emplace_back(getTypeBuildDefinition<Ti>());
 
-    auto sobel = common::getKernel("sobel3x3", std::array{sobel_cl_src}, targs,
-                                   compileOpts);
+    auto sobel =
+        common::getKernel("sobel3x3", {{sobel_cl_src}}, targs, compileOpts);
 
     cl::NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/sparse.hpp b/src/backend/opencl/kernel/sparse.hpp
index f7ef69e248..e1b29c986c 100644
--- a/src/backend/opencl/kernel/sparse.hpp
+++ b/src/backend/opencl/kernel/sparse.hpp
@@ -43,8 +43,8 @@ void coo2dense(Param out, const Param values, const Param rowIdx,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto coo2dense = common::getKernel(
-        "coo2Dense", std::array{coo2dense_cl_src}, tmpltArgs, compileOpts);
+    auto coo2dense = common::getKernel("coo2Dense", {{coo2dense_cl_src}},
+                                       tmpltArgs, compileOpts);
 
     cl::NDRange local(THREADS_PER_GROUP, 1, 1);
 
@@ -76,8 +76,8 @@ void csr2dense(Param output, const Param values, const Param rowIdx,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto csr2dense = common::getKernel(
-        "csr2Dense", std::array{csr2dense_cl_src}, tmpltArgs, compileOpts);
+    auto csr2dense = common::getKernel("csr2Dense", {{csr2dense_cl_src}},
+                                       tmpltArgs, compileOpts);
 
     cl::NDRange local(threads, 1);
     int groups_x = std::min((int)(divup(M, local[0])), MAX_GROUPS);
@@ -102,8 +102,8 @@ void dense2csr(Param values, Param rowIdx, Param colIdx, const Param dense) {
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto dense2Csr = common::getKernel(
-        "dense2Csr", std::array{dense2csr_cl_src}, tmpltArgs, compileOpts);
+    auto dense2Csr = common::getKernel("dense2Csr", {{dense2csr_cl_src}},
+                                       tmpltArgs, compileOpts);
 
     int num_rows = dense.info.dims[0];
     int num_cols = dense.info.dims[1];
@@ -147,7 +147,7 @@ void swapIndex(Param ovalues, Param oindex, const Param ivalues,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto swapIndex = common::getKernel("swapIndex", std::array{csr2coo_cl_src},
+    auto swapIndex = common::getKernel("swapIndex", {{csr2coo_cl_src}},
                                        tmpltArgs, compileOpts);
 
     cl::NDRange global(ovalues.info.dims[0], 1, 1);
@@ -169,8 +169,8 @@ void csr2coo(Param ovalues, Param orowIdx, Param ocolIdx, const Param ivalues,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto csr2coo = common::getKernel("csr2Coo", std::array{csr2coo_cl_src},
-                                     tmpltArgs, compileOpts);
+    auto csr2coo = common::getKernel("csr2Coo", {{csr2coo_cl_src}}, tmpltArgs,
+                                     compileOpts);
 
     const int MAX_GROUPS = 4096;
     int M                = irowIdx.info.dims[0] - 1;
@@ -209,7 +209,7 @@ void coo2csr(Param ovalues, Param orowIdx, Param ocolIdx, const Param ivalues,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto csrReduce = common::getKernel("csrReduce", std::array{csr2coo_cl_src},
+    auto csrReduce = common::getKernel("csrReduce", {{csr2coo_cl_src}},
                                        tmpltArgs, compileOpts);
 
     // Now we need to sort this into column major
diff --git a/src/backend/opencl/kernel/sparse_arith.hpp b/src/backend/opencl/kernel/sparse_arith.hpp
index 048a6d4876..313fa902d2 100644
--- a/src/backend/opencl/kernel/sparse_arith.hpp
+++ b/src/backend/opencl/kernel/sparse_arith.hpp
@@ -63,9 +63,8 @@ auto fetchKernel(const std::string key, const common::Source &additionalSrc,
     options.emplace_back(getTypeBuildDefinition<T>());
     options.insert(std::end(options), std::begin(additionalOptions),
                    std::end(additionalOptions));
-    return common::getKernel(
-        key, std::array{sparse_arith_common_cl_src, additionalSrc}, tmpltArgs,
-        options);
+    return common::getKernel(key, {{sparse_arith_common_cl_src, additionalSrc}},
+                             tmpltArgs, options);
 }
 
 template<typename T, af_op_t op>
@@ -144,9 +143,8 @@ static void csrCalcOutNNZ(Param outRowIdx, unsigned &nnzC, const uint M,
         TemplateTypename<uint>(),
     };
 
-    auto calcNNZ = common::getKernel("csr_calc_out_nnz",
-                                     std::array{ssarith_calc_out_nnz_cl_src},
-                                     tmpltArgs, {});
+    auto calcNNZ = common::getKernel(
+        "csr_calc_out_nnz", {{ssarith_calc_out_nnz_cl_src}}, tmpltArgs, {});
 
     cl::NDRange local(256, 1);
     cl::NDRange global(divup(M, local[0]) * local[0], 1, 1);
diff --git a/src/backend/opencl/kernel/susan.hpp b/src/backend/opencl/kernel/susan.hpp
index d407755f31..4b87b43a85 100644
--- a/src/backend/opencl/kernel/susan.hpp
+++ b/src/backend/opencl/kernel/susan.hpp
@@ -49,8 +49,8 @@ void susan(cl::Buffer* out, const cl::Buffer* in, const unsigned in_off,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto susan = common::getKernel("susan_responses", std::array{susan_cl_src},
-                                   targs, compileOpts);
+    auto susan = common::getKernel("susan_responses", {{susan_cl_src}}, targs,
+                                   compileOpts);
 
     cl::NDRange local(SUSAN_THREADS_X, SUSAN_THREADS_Y);
     cl::NDRange global(divup(idim0 - 2 * edge, local[0]) * local[0],
@@ -75,8 +75,8 @@ unsigned nonMaximal(cl::Buffer* x_out, cl::Buffer* y_out, cl::Buffer* resp_out,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto nonMax = common::getKernel("non_maximal", std::array{susan_cl_src},
-                                    targs, compileOpts);
+    auto nonMax =
+        common::getKernel("non_maximal", {{susan_cl_src}}, targs, compileOpts);
 
     unsigned corners_found = 0;
     auto d_corners_found   = memAlloc<unsigned>(1);
diff --git a/src/backend/opencl/kernel/swapdblk.hpp b/src/backend/opencl/kernel/swapdblk.hpp
index 820db15094..0b8b43fb72 100644
--- a/src/backend/opencl/kernel/swapdblk.hpp
+++ b/src/backend/opencl/kernel/swapdblk.hpp
@@ -42,8 +42,8 @@ void swapdblk(int n, int nb, cl_mem dA, size_t dA_offset, int ldda, int inca,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto swapdblk = common::getKernel("swapdblk", std::array{swapdblk_cl_src},
-                                      targs, compileOpts);
+    auto swapdblk =
+        common::getKernel("swapdblk", {{swapdblk_cl_src}}, targs, compileOpts);
 
     int nblocks = n / nb;
 
diff --git a/src/backend/opencl/kernel/tile.hpp b/src/backend/opencl/kernel/tile.hpp
index fa097ba58f..7c9b042372 100644
--- a/src/backend/opencl/kernel/tile.hpp
+++ b/src/backend/opencl/kernel/tile.hpp
@@ -42,8 +42,7 @@ void tile(Param out, const Param in) {
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto tile =
-        common::getKernel("tile", std::array{tile_cl_src}, targs, compileOpts);
+    auto tile = common::getKernel("tile", {{tile_cl_src}}, targs, compileOpts);
 
     NDRange local(TX, TY, 1);
 
diff --git a/src/backend/opencl/kernel/transform.hpp b/src/backend/opencl/kernel/transform.hpp
index a3f81fd75b..76a2dafa43 100644
--- a/src/backend/opencl/kernel/transform.hpp
+++ b/src/backend/opencl/kernel/transform.hpp
@@ -80,9 +80,9 @@ void transform(Param out, const Param in, const Param tf, bool isInverse,
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
     addInterpEnumOptions(compileOpts);
 
-    auto transform = common::getKernel(
-        "transformKernel", std::array{interp_cl_src, transform_cl_src},
-        tmpltArgs, compileOpts);
+    auto transform = common::getKernel("transformKernel",
+                                       {{interp_cl_src, transform_cl_src}},
+                                       tmpltArgs, compileOpts);
 
     const int nImg2 = in.info.dims[2];
     const int nImg3 = in.info.dims[3];
diff --git a/src/backend/opencl/kernel/transpose.hpp b/src/backend/opencl/kernel/transpose.hpp
index 3397596179..b6979cf6d5 100644
--- a/src/backend/opencl/kernel/transpose.hpp
+++ b/src/backend/opencl/kernel/transpose.hpp
@@ -49,8 +49,8 @@ void transpose(Param out, const Param in, cl::CommandQueue queue,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto transpose = common::getKernel(
-        "transpose", std::array{transpose_cl_src}, tmpltArgs, compileOpts);
+    auto transpose = common::getKernel("transpose", {{transpose_cl_src}},
+                                       tmpltArgs, compileOpts);
 
     NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/transpose_inplace.hpp b/src/backend/opencl/kernel/transpose_inplace.hpp
index b55f2e4d43..6ed5c1e5c4 100644
--- a/src/backend/opencl/kernel/transpose_inplace.hpp
+++ b/src/backend/opencl/kernel/transpose_inplace.hpp
@@ -49,9 +49,9 @@ void transpose_inplace(Param in, cl::CommandQueue& queue, const bool conjugate,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto transpose = common::getKernel("transpose_inplace",
-                                       std::array{transpose_inplace_cl_src},
-                                       tmpltArgs, compileOpts);
+    auto transpose =
+        common::getKernel("transpose_inplace", {{transpose_inplace_cl_src}},
+                          tmpltArgs, compileOpts);
 
     NDRange local(THREADS_X, THREADS_Y);
 
diff --git a/src/backend/opencl/kernel/triangle.hpp b/src/backend/opencl/kernel/triangle.hpp
index c0be0de33f..888ac21909 100644
--- a/src/backend/opencl/kernel/triangle.hpp
+++ b/src/backend/opencl/kernel/triangle.hpp
@@ -52,7 +52,7 @@ void triangle(Param out, const Param in, bool is_upper, bool is_unit_diag) {
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto triangle = common::getKernel("triangle", std::array{triangle_cl_src},
+    auto triangle = common::getKernel("triangle", {{triangle_cl_src}},
                                       tmpltArgs, compileOpts);
 
     NDRange local(TX, TY);
diff --git a/src/backend/opencl/kernel/unwrap.hpp b/src/backend/opencl/kernel/unwrap.hpp
index 08e535f713..7c3d71bb37 100644
--- a/src/backend/opencl/kernel/unwrap.hpp
+++ b/src/backend/opencl/kernel/unwrap.hpp
@@ -47,8 +47,8 @@ void unwrap(Param out, const Param in, const dim_t wx, const dim_t wy,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto unwrap = common::getKernel("unwrap", std::array{unwrap_cl_src},
-                                    tmpltArgs, compileOpts);
+    auto unwrap =
+        common::getKernel("unwrap", {{unwrap_cl_src}}, tmpltArgs, compileOpts);
 
     dim_t TX = 1, TY = 1;
     dim_t BX       = 1;
diff --git a/src/backend/opencl/kernel/where.hpp b/src/backend/opencl/kernel/where.hpp
index 88e89fd26b..980cdfe13f 100644
--- a/src/backend/opencl/kernel/where.hpp
+++ b/src/backend/opencl/kernel/where.hpp
@@ -46,8 +46,8 @@ static void get_out_idx(cl::Buffer *out_data, Param &otmp, Param &rtmp,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto getIdx = common::getKernel("get_out_idx", std::array{where_cl_src},
-                                    tmpltArgs, compileOpts);
+    auto getIdx = common::getKernel("get_out_idx", {{where_cl_src}}, tmpltArgs,
+                                    compileOpts);
 
     NDRange local(threads_x, THREADS_PER_GROUP / threads_x);
     NDRange global(local[0] * groups_x * in.info.dims[2],
diff --git a/src/backend/opencl/kernel/wrap.hpp b/src/backend/opencl/kernel/wrap.hpp
index b527cd8bce..e664c7b472 100644
--- a/src/backend/opencl/kernel/wrap.hpp
+++ b/src/backend/opencl/kernel/wrap.hpp
@@ -47,8 +47,8 @@ void wrap(Param out, const Param in, const dim_t wx, const dim_t wy,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto wrap = common::getKernel("wrap", std::array{wrap_cl_src}, tmpltArgs,
-                                  compileOpts);
+    auto wrap =
+        common::getKernel("wrap", {{wrap_cl_src}}, tmpltArgs, compileOpts);
 
     dim_t nx = (out.info.dims[0] + 2 * px - wx) / sx + 1;
     dim_t ny = (out.info.dims[1] + 2 * py - wy) / sy + 1;
@@ -92,9 +92,8 @@ void wrap_dilated(Param out, const Param in, const dim_t wx, const dim_t wy,
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 
-    auto dilatedWrap =
-        common::getKernel("wrap_dilated", std::array{wrap_dilated_cl_src},
-                          tmpltArgs, compileOpts);
+    auto dilatedWrap = common::getKernel(
+        "wrap_dilated", {{wrap_dilated_cl_src}}, tmpltArgs, compileOpts);
 
     dim_t nx = 1 + (out.info.dims[0] + 2 * px - (((wx - 1) * dx) + 1)) / sx;
     dim_t ny = 1 + (out.info.dims[1] + 2 * py - (((wy - 1) * dy) + 1)) / sy;

From 262eb94a112c28ad844b271b1db3e8f8a7bb8f4d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 2 Jan 2023 16:27:07 -0500
Subject: [PATCH 160/473] Update compiers.h header to add if constexpr macro

---
 CMakeModules/InternalUtils.cmake |   5 ++
 CMakeModules/compilers.h         | 129 +++++++++++++++++++------------
 src/backend/common/half.hpp      | 120 ++++++++++++++--------------
 src/backend/cuda/math.hpp        |  10 +--
 4 files changed, 147 insertions(+), 117 deletions(-)

diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index c698e3d290..1d1c387245 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -187,6 +187,11 @@ macro(arrayfire_set_cmake_default_variables)
   #  #else
   #  #define AF_CONSTEXPR
   #  #endif
+  #  #if __cpp_if_constexpr || __cplusplus >= 201606L
+  #  #define AF_IF_CONSTEXPR if constexpr
+  #  #else
+  #  #define AF_IF_CONSTEXPR if
+  #  #endif
   #  ]=])
   #  include(WriteCompilerDetectionHeader)
   #  write_compiler_detection_header(
diff --git a/CMakeModules/compilers.h b/CMakeModules/compilers.h
index c247005c80..60480d86ee 100644
--- a/CMakeModules/compilers.h
+++ b/CMakeModules/compilers.h
@@ -16,19 +16,24 @@
 # define AF_COMPILER_IS_HP 0
 # define AF_COMPILER_IS_Compaq 0
 # define AF_COMPILER_IS_zOS 0
+# define AF_COMPILER_IS_IBMClang 0
 # define AF_COMPILER_IS_XLClang 0
 # define AF_COMPILER_IS_XL 0
 # define AF_COMPILER_IS_VisualAge 0
+# define AF_COMPILER_IS_NVHPC 0
 # define AF_COMPILER_IS_PGI 0
 # define AF_COMPILER_IS_Cray 0
 # define AF_COMPILER_IS_TI 0
+# define AF_COMPILER_IS_FujitsuClang 0
 # define AF_COMPILER_IS_Fujitsu 0
 # define AF_COMPILER_IS_GHS 0
+# define AF_COMPILER_IS_Tasking 0
 # define AF_COMPILER_IS_SCO 0
 # define AF_COMPILER_IS_ARMCC 0
 # define AF_COMPILER_IS_AppleClang 0
 # define AF_COMPILER_IS_ARMClang 0
 # define AF_COMPILER_IS_Clang 0
+# define AF_COMPILER_IS_LCC 0
 # define AF_COMPILER_IS_GNU 0
 # define AF_COMPILER_IS_MSVC 0
 # define AF_COMPILER_IS_ADSP 0
@@ -79,6 +84,10 @@
 # undef AF_COMPILER_IS_zOS
 # define AF_COMPILER_IS_zOS 1
 
+#elif defined(__open_xl__) && defined(__clang__)
+# undef AF_COMPILER_IS_IBMClang
+# define AF_COMPILER_IS_IBMClang 1
+
 #elif defined(__ibmxl__) && defined(__clang__)
 # undef AF_COMPILER_IS_XLClang
 # define AF_COMPILER_IS_XLClang 1
@@ -91,6 +100,10 @@
 # undef AF_COMPILER_IS_VisualAge
 # define AF_COMPILER_IS_VisualAge 1
 
+#elif defined(__NVCOMPILER)
+# undef AF_COMPILER_IS_NVHPC
+# define AF_COMPILER_IS_NVHPC 1
+
 #elif defined(__PGI)
 # undef AF_COMPILER_IS_PGI
 # define AF_COMPILER_IS_PGI 1
@@ -103,7 +116,11 @@
 # undef AF_COMPILER_IS_TI
 # define AF_COMPILER_IS_TI 1
 
-#elif defined(__FUJITSU) || defined(__FCC_VERSION) || defined(__fcc_version)
+#elif defined(__CLANG_FUJITSU)
+# undef AF_COMPILER_IS_FujitsuClang
+# define AF_COMPILER_IS_FujitsuClang 1
+
+#elif defined(__FUJITSU)
 # undef AF_COMPILER_IS_Fujitsu
 # define AF_COMPILER_IS_Fujitsu 1
 
@@ -111,6 +128,10 @@
 # undef AF_COMPILER_IS_GHS
 # define AF_COMPILER_IS_GHS 1
 
+#elif defined(__TASKING__)
+# undef AF_COMPILER_IS_Tasking
+# define AF_COMPILER_IS_Tasking 1
+
 #elif defined(__SCO_VERSION__)
 # undef AF_COMPILER_IS_SCO
 # define AF_COMPILER_IS_SCO 1
@@ -131,6 +152,10 @@
 # undef AF_COMPILER_IS_Clang
 # define AF_COMPILER_IS_Clang 1
 
+#elif defined(__LCC__) && (defined(__GNUC__) || defined(__GNUG__) || defined(__MCST__))
+# undef AF_COMPILER_IS_LCC
+# define AF_COMPILER_IS_LCC 1
+
 #elif defined(__GNUC__) || defined(__GNUG__)
 # undef AF_COMPILER_IS_GNU
 # define AF_COMPILER_IS_GNU 1
@@ -139,7 +164,7 @@
 # undef AF_COMPILER_IS_MSVC
 # define AF_COMPILER_IS_MSVC 1
 
-#elif defined(__VISUALDSPVERSION__) || defined(__ADSPBLACKFIN__) || defined(__ADSPTS__) || defined(__ADSP21000__)
+#elif defined(_ADI_COMPILER)
 # undef AF_COMPILER_IS_ADSP
 # define AF_COMPILER_IS_ADSP 1
 
@@ -202,12 +227,11 @@
 #      define AF_COMPILER_CXX_GENERALIZED_INITIALIZERS 0
 #    endif
 
-#if ((__clang_major__ * 100) + __clang_minor__) >= 400 &&                      \
-    __has_feature(cxx_relaxed_constexpr)
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
-#else
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
-#endif
+#    if ((__clang_major__ * 100) + __clang_minor__) >= 400 && __has_feature(cxx_relaxed_constexpr)
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
+#    else
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
+#    endif
 
 #  elif AF_COMPILER_IS_Clang
 
@@ -260,12 +284,11 @@
 #      define AF_COMPILER_CXX_GENERALIZED_INITIALIZERS 0
 #    endif
 
-#if ((__clang_major__ * 100) + __clang_minor__) >= 301 &&                      \
-    __has_feature(cxx_relaxed_constexpr)
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
-#else
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
-#endif
+#    if ((__clang_major__ * 100) + __clang_minor__) >= 301 && __has_feature(cxx_relaxed_constexpr)
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
+#    else
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
+#    endif
 
 #  elif AF_COMPILER_IS_GNU
 
@@ -321,11 +344,11 @@
 #      define AF_COMPILER_CXX_GENERALIZED_INITIALIZERS 0
 #    endif
 
-#if (__GNUC__ * 100 + __GNUC_MINOR__) >= 500 && __cplusplus >= 201402L
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
-#else
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
-#endif
+#    if (__GNUC__ * 100 + __GNUC_MINOR__) >= 500 && __cplusplus >= 201402L
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
+#    else
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
+#    endif
 
 #  elif AF_COMPILER_IS_Intel
 
@@ -333,16 +356,25 @@
 #      error Unsupported compiler version
 #    endif
 
-  /* __INTEL_COMPILER = VRP */
-# define AF_COMPILER_VERSION_MAJOR (__INTEL_COMPILER/100)
-# define AF_COMPILER_VERSION_MINOR (__INTEL_COMPILER/10 % 10)
-# if defined(__INTEL_COMPILER_UPDATE)
-#  define AF_COMPILER_VERSION_PATCH (__INTEL_COMPILER_UPDATE)
+  /* __INTEL_COMPILER = VRP prior to 2021, and then VVVV for 2021 and later,
+     except that a few beta releases use the old format with V=2021.  */
+# if __INTEL_COMPILER < 2021 || __INTEL_COMPILER == 202110 || __INTEL_COMPILER == 202111
+#  define AF_COMPILER_VERSION_MAJOR (__INTEL_COMPILER/100)
+#  define AF_COMPILER_VERSION_MINOR (__INTEL_COMPILER/10 % 10)
+#  if defined(__INTEL_COMPILER_UPDATE)
+#   define AF_COMPILER_VERSION_PATCH (__INTEL_COMPILER_UPDATE)
+#  else
+#   define AF_COMPILER_VERSION_PATCH (__INTEL_COMPILER   % 10)
+#  endif
 # else
-#  define AF_COMPILER_VERSION_PATCH (__INTEL_COMPILER   % 10)
+#  define AF_COMPILER_VERSION_MAJOR (__INTEL_COMPILER)
+#  define AF_COMPILER_VERSION_MINOR (__INTEL_COMPILER_UPDATE)
+   /* The third version component from --version is an update index,
+      but no macro is provided for it.  */
+#  define AF_COMPILER_VERSION_PATCH (0)
 # endif
 # if defined(__INTEL_COMPILER_BUILD_DATE)
-  /* __INTEL_COMPILER_BUILD_DATE = YYYYMMDD */
+   /* __INTEL_COMPILER_BUILD_DATE = YYYYMMDD */
 #  define AF_COMPILER_VERSION_TWEAK (__INTEL_COMPILER_BUILD_DATE)
 # endif
 # if defined(_MSC_VER)
@@ -398,19 +430,11 @@
 #      define AF_COMPILER_CXX_GENERALIZED_INITIALIZERS 0
 #    endif
 
-#if __cpp_constexpr >= 201304 ||                                               \
-    (__INTEL_COMPILER >= 1700 &&                                               \
-     ((__cplusplus >= 201300L) ||                                              \
-      ((__cplusplus == 201103L) && !defined(__INTEL_CXX11_MODE__)) ||          \
-      ((((__INTEL_COMPILER == 1500) && (__INTEL_COMPILER_UPDATE == 1))) &&     \
-       defined(__GXX_EXPERIMENTAL_CXX0X__) &&                                  \
-       !defined(__INTEL_CXX11_MODE__)) ||                                      \
-      (defined(__INTEL_CXX11_MODE__) && defined(__cpp_aggregate_nsdmi))) &&    \
-     !defined(_MSC_VER))
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
-#else
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
-#endif
+#    if __cpp_constexpr >= 201304 || (__INTEL_COMPILER >= 1700 && ((__cplusplus >= 201300L) || ((__cplusplus == 201103L) && !defined(__INTEL_CXX11_MODE__)) || ((((__INTEL_COMPILER == 1500) && (__INTEL_COMPILER_UPDATE == 1))) && defined(__GXX_EXPERIMENTAL_CXX0X__) && !defined(__INTEL_CXX11_MODE__) ) || (defined(__INTEL_CXX11_MODE__) && defined(__cpp_aggregate_nsdmi)) ) && !defined(_MSC_VER))
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
+#    else
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
+#    endif
 
 #  elif AF_COMPILER_IS_MSVC
 
@@ -470,11 +494,11 @@
 #      define AF_COMPILER_CXX_GENERALIZED_INITIALIZERS 0
 #    endif
 
-#if _MSC_VER >= 1911
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
-#else
-#define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
-#endif
+#    if _MSC_VER >= 1911
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 1
+#    else
+#      define AF_COMPILER_CXX_RELAXED_CONSTEXPR 0
+#    endif
 
 #  endif
 
@@ -511,11 +535,16 @@ template<> struct AFStaticAssert<true>{};
 
 #endif
 
-#if defined(AF_COMPILER_CXX_RELAXED_CONSTEXPR) &&                              \
-    AF_COMPILER_CXX_RELAXED_CONSTEXPR
-#define AF_CONSTEXPR constexpr
-#else
-#define AF_CONSTEXPR
-#endif
+  #if defined(AF_COMPILER_CXX_RELAXED_CONSTEXPR) && AF_COMPILER_CXX_RELAXED_CONSTEXPR
+  #define AF_CONSTEXPR constexpr
+  #else
+  #define AF_CONSTEXPR
+  #endif
+  #if defined(__cpp_if_constexpr) || __cplusplus >= 201606L
+  #define AF_IF_CONSTEXPR if constexpr
+  #else
+  #define AF_IF_CONSTEXPR if
+  #endif
+
 
 #endif
diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index 8080dcffa1..f653024fb1 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -129,12 +129,11 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(T value) noexcept {
     if (S) value = -value;
     uint16_t bits = S << 15;
     if (value > 0xFFFF) {
-        if constexpr (R == std::round_toward_infinity)
-            bits |= (0x7C00 - S);
-        else if constexpr (R == std::round_toward_neg_infinity)
-            bits |= (0x7BFF + S);
-        else
-            bits |= (0x7BFF + (R != std::round_toward_zero));
+        AF_IF_CONSTEXPR(R == std::round_toward_infinity)
+        bits |= (0x7C00 - S);
+        else AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity) bits |=
+            (0x7BFF + S);
+        else bits |= (0x7BFF + (R != std::round_toward_zero));
     } else if (value) {
         uint32_t m = value, exp = 24;
         for (; m < 0x400; m <<= 1, --exp)
@@ -143,16 +142,16 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(T value) noexcept {
             ;
         bits |= (exp << 10) + m;
         if (exp > 24) {
-            if constexpr (R == std::round_to_nearest)
-                bits += (value >> (exp - 25)) & 1
+            AF_IF_CONSTEXPR(R == std::round_to_nearest)
+            bits += (value >> (exp - 25)) & 1
 #if HALF_ROUND_TIES_TO_EVEN
-                        & (((((1 << (exp - 25)) - 1) & value) != 0) | bits)
+                    & (((((1 << (exp - 25)) - 1) & value) != 0) | bits)
 #endif
-                    ;
-            else if constexpr (R == std::round_toward_infinity)
-                bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & !S;
-            else if constexpr (R == std::round_toward_neg_infinity)
-                bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & S;
+                ;
+            else AF_IF_CONSTEXPR(R == std::round_toward_infinity) bits +=
+                ((value & ((1 << (exp - 24)) - 1)) != 0) & !S;
+            else AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity) bits +=
+                ((value & ((1 << (exp - 24)) - 1)) != 0) & S;
         }
     }
     return bits;
@@ -279,34 +278,33 @@ __DH__ native_half_t float2half_impl(float value) noexcept {
     uint16_t hbits =
         base_table[bits >> 23] +
         static_cast<uint16_t>((bits & 0x7FFFFF) >> shift_table[bits >> 23]);
-    if constexpr (R == std::round_to_nearest)
-        hbits +=
-            (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) |
-             (((bits >> 23) & 0xFF) == 102)) &
-            ((hbits & 0x7C00) != 0x7C00)
+    AF_IF_CONSTEXPR(R == std::round_to_nearest)
+    hbits +=
+        (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) |
+         (((bits >> 23) & 0xFF) == 102)) &
+        ((hbits & 0x7C00) != 0x7C00)
 #if HALF_ROUND_TIES_TO_EVEN
-            &
-            (((((static_cast<uint32>(1) << (shift_table[bits >> 23] - 1)) - 1) &
-               bits) != 0) |
-             hbits)
+        & (((((static_cast<uint32>(1) << (shift_table[bits >> 23] - 1)) - 1) &
+             bits) != 0) |
+           hbits)
 #endif
-            ;
-    else if constexpr (R == std::round_toward_zero)
-        hbits -= ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23];
-    else if constexpr (R == std::round_toward_infinity)
-        hbits += ((((bits & 0x7FFFFF &
-                     ((static_cast<uint32_t>(1) << (shift_table[bits >> 23])) -
-                      1)) != 0) |
-                   (((bits >> 23) <= 102) & ((bits >> 23) != 0))) &
-                  (hbits < 0x7C00)) -
-                 ((hbits == 0xFC00) & ((bits >> 23) != 511));
-    else if constexpr (R == std::round_toward_neg_infinity)
-        hbits += ((((bits & 0x7FFFFF &
-                     ((static_cast<uint32_t>(1) << (shift_table[bits >> 23])) -
-                      1)) != 0) |
-                   (((bits >> 23) <= 358) & ((bits >> 23) != 256))) &
-                  (hbits < 0xFC00) & (hbits >> 15)) -
-                 ((hbits == 0x7C00) & ((bits >> 23) != 255));
+        ;
+    else AF_IF_CONSTEXPR(R == std::round_toward_zero) hbits -=
+        ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23];
+    else AF_IF_CONSTEXPR(R == std::round_toward_infinity) hbits +=
+        ((((bits & 0x7FFFFF &
+            ((static_cast<uint32_t>(1) << (shift_table[bits >> 23])) - 1)) !=
+           0) |
+          (((bits >> 23) <= 102) & ((bits >> 23) != 0))) &
+         (hbits < 0x7C00)) -
+        ((hbits == 0xFC00) & ((bits >> 23) != 511));
+    else AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity) hbits +=
+        ((((bits & 0x7FFFFF &
+            ((static_cast<uint32_t>(1) << (shift_table[bits >> 23])) - 1)) !=
+           0) |
+          (((bits >> 23) <= 358) & ((bits >> 23) != 256))) &
+         (hbits < 0xFC00) & (hbits >> 15)) -
+        ((hbits == 0x7C00) & ((bits >> 23) != 255));
     return hbits;
 }
 
@@ -330,10 +328,10 @@ __DH__ native_half_t float2half_impl(double value) {
         return hbits | 0x7C00 |
                (0x3FF & -static_cast<unsigned>((bits & 0xFFFFFFFFFFFFF) != 0));
     if (exp > 1038) {
-        if constexpr (R == std::round_toward_infinity)
-            return hbits | (0x7C00 - (hbits >> 15));
-        if constexpr (R == std::round_toward_neg_infinity)
-            return hbits | (0x7BFF + (hbits >> 15));
+        AF_IF_CONSTEXPR(R == std::round_toward_infinity)
+        return hbits | (0x7C00 - (hbits >> 15));
+        AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity)
+        return hbits | (0x7BFF + (hbits >> 15));
         return hbits | (0x7BFF + (R != std::round_toward_zero));
     }
     int g = 0, s = lo != 0;
@@ -350,16 +348,16 @@ __DH__ native_half_t float2half_impl(double value) {
     } else {
         s |= hi != 0;
     }
-    if constexpr (R == std::round_to_nearest)
+    AF_IF_CONSTEXPR(R == std::round_to_nearest)
 #if HALF_ROUND_TIES_TO_EVEN
-        hbits += g & (s | hbits);
+    hbits += g & (s | hbits);
 #else
-        hbits += g;
+    hbits += g;
 #endif
-    else if constexpr (R == std::round_toward_infinity)
-        hbits += ~(hbits >> 15) & (s | g);
-    else if constexpr (R == std::round_toward_neg_infinity)
-        hbits += (hbits >> 15) & (g | s);
+    else AF_IF_CONSTEXPR(R == std::round_toward_infinity) hbits +=
+        ~(hbits >> 15) & (s | g);
+    else AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity) hbits +=
+        (hbits >> 15) & (g | s);
     return hbits;
 }
 
@@ -775,21 +773,21 @@ AF_CONSTEXPR T half2int(native_half_t value) {
         return (value & 0x8000) ? std::numeric_limits<T>::min()
                                 : std::numeric_limits<T>::max();
     if (e < 0x3800) {
-        if constexpr (R == std::round_toward_infinity)
-            return T(~(value >> 15) & (e != 0));
-        else if constexpr (R == std::round_toward_neg_infinity)
-            return -T(value > 0x8000);
+        AF_IF_CONSTEXPR(R == std::round_toward_infinity)
+        return T(~(value >> 15) & (e != 0));
+        else AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity) return -T(
+            value > 0x8000);
         return T();
     }
     unsigned int m = (value & 0x3FF) | 0x400;
     e >>= 10;
     if (e < 25) {
-        if constexpr (R == std::round_to_nearest)
-            m += (1 << (24 - e)) - (~(m >> (25 - e)) & E);
-        else if constexpr (R == std::round_toward_infinity)
-            m += ((value >> 15) - 1) & ((1 << (25 - e)) - 1U);
-        else if constexpr (R == std::round_toward_neg_infinity)
-            m += -(value >> 15) & ((1 << (25 - e)) - 1U);
+        AF_IF_CONSTEXPR(R == std::round_to_nearest)
+        m += (1 << (24 - e)) - (~(m >> (25 - e)) & E);
+        else AF_IF_CONSTEXPR(R == std::round_toward_infinity) m +=
+            ((value >> 15) - 1) & ((1 << (25 - e)) - 1U);
+        else AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity) m +=
+            -(value >> 15) & ((1 << (25 - e)) - 1U);
         m >>= 25 - e;
     } else
         m <<= e - 25;
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index 4c48e6990f..31d7e5b51b 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -148,19 +148,17 @@ __DH__ static To scalar(Ti real, Ti imag) {
 
 template<typename T>
 inline T maxval() {
-    if constexpr (std::is_floating_point_v<T> && !fast_math) {
+    AF_IF_CONSTEXPR(std::is_floating_point_v<T> && !fast_math) {
         return std::numeric_limits<T>::infinity();
-    } else {
-        return std::numeric_limits<T>::max();
     }
+    else { return std::numeric_limits<T>::max(); }
 }
 template<typename T>
 inline T minval() {
-    if constexpr (std::is_floating_point_v<T> && !fast_math) {
+    AF_IF_CONSTEXPR(std::is_floating_point_v<T> && !fast_math) {
         return -std::numeric_limits<T>::infinity();
-    } else {
-        return std::numeric_limits<T>::lowest();
     }
+    else { return std::numeric_limits<T>::lowest(); }
 }
 #else
 template<typename T>

From 202cc76801db76b7608329d565729c694d102f4e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 2 Jan 2023 16:29:53 -0500
Subject: [PATCH 161/473] Revert CUDA C++ standard to 14 to support older CUDA
 toolkits

---
 src/backend/cuda/CMakeLists.txt | 30 ++++++++++++++++++++++--------
 src/backend/cuda/math.hpp       |  4 ++--
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index aa9f3fc037..c6617ffac5 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -598,15 +598,29 @@ endif()
 
 
 af_detect_and_set_cuda_architectures(afcuda)
-if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
-  set_target_properties(afcuda
-    PROPERTIES
-      CUDA_STANDARD 17
-      CUDA_STANDARD_REQUIRED ON)
+
+if(CUDA_VERSION VERSION_LESS 11.0)
+  if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
+    set_target_properties(afcuda
+      PROPERTIES
+        CUDA_STANDARD 14
+        CUDA_STANDARD_REQUIRED ON)
+  else()
+    target_compile_options(afcuda
+      PRIVATE
+        $<$<COMPILE_LANGUAGE:CUDA>:--std=c++14>)
+  endif()
 else()
-  target_compile_options(afcuda
-    PRIVATE
-      $<$<COMPILE_LANGUAGE:CUDA>:--std=c++17>)
+  if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18")
+    set_target_properties(afcuda
+      PROPERTIES
+        CUDA_STANDARD 17
+        CUDA_STANDARD_REQUIRED ON)
+  else()
+    target_compile_options(afcuda
+      PRIVATE
+        $<$<COMPILE_LANGUAGE:CUDA>:--std=c++17>)
+  endif()
 endif()
 
 target_compile_definitions(afcuda
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index 31d7e5b51b..f988372d27 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -148,14 +148,14 @@ __DH__ static To scalar(Ti real, Ti imag) {
 
 template<typename T>
 inline T maxval() {
-    AF_IF_CONSTEXPR(std::is_floating_point_v<T> && !fast_math) {
+    AF_IF_CONSTEXPR(std::is_floating_point<T>::value && !fast_math) {
         return std::numeric_limits<T>::infinity();
     }
     else { return std::numeric_limits<T>::max(); }
 }
 template<typename T>
 inline T minval() {
-    AF_IF_CONSTEXPR(std::is_floating_point_v<T> && !fast_math) {
+    AF_IF_CONSTEXPR(std::is_floating_point<T>::value && !fast_math) {
         return -std::numeric_limits<T>::infinity();
     }
     else { return std::numeric_limits<T>::lowest(); }

From 5c79a61f798b961da7045950d2dfcfee8e0e2385 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 2 Jan 2023 16:37:04 -0500
Subject: [PATCH 162/473] Fix CUB include paths when CUDA 10.2 and lower
 toolkits are used

---
 src/backend/cuda/CMakeLists.txt | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index c6617ffac5..c031deebd9 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -114,11 +114,15 @@ mark_as_advanced(
     CUDA_architecture_build_targets)
 
 if(CUDA_VERSION_MAJOR VERSION_LESS 11)
-  af_dep_check_and_populate(${cub_prefix}
-    URI https://github.com/NVIDIA/cub.git
-    REF 1.10.0
-  )
-  cuda_include_directories(${${cub_prefix}_SOURCE_DIR})
+  find_package(CUB)
+  if(NOT TARGET CUB::CUB)
+    af_dep_check_and_populate(${cub_prefix}
+      URI https://github.com/NVIDIA/cub.git
+      REF 1.10.0
+    )
+    find_package(CUB REQUIRED
+      PATHS ${${cub_prefix}_SOURCE_DIR})
+  endif()
 endif()
 
 file(GLOB jit_src "kernel/jit.cuh")
@@ -596,6 +600,12 @@ else()
   )
 endif()
 
+if(CUDA_VERSION_MAJOR VERSION_LESS 11)
+  target_link_libraries(afcuda
+    PRIVATE
+      CUB::CUB
+  )
+endif()
 
 af_detect_and_set_cuda_architectures(afcuda)
 

From 65b7d11e1714bce66c1305a7957f5a20fa104a35 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 2 Jan 2023 16:37:52 -0500
Subject: [PATCH 163/473] Set minimum toolkit version to 10.2

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d610bba1c5..ae7a3742a6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,7 +48,7 @@ endif()
 #Set Intel OpenMP as default MKL thread layer
 set(MKL_THREAD_LAYER "Intel OpenMP" CACHE STRING "The thread layer to choose for MKL")
 
-find_package(CUDA 9.0)
+find_package(CUDA 10.2)
 find_package(cuDNN 4.0)
 find_package(OpenCL 1.2)
 find_package(OpenGL)

From f917e3f038ddf8aec27871a04c3583e6a02df74f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 2 Jan 2023 19:47:34 -0500
Subject: [PATCH 164/473] Fix errors in the fmt library when printing const
 dim3 values

---
 src/backend/cuda/kernel/regions.hpp | 4 ++--
 src/backend/cuda/kernel/topk.hpp    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/backend/cuda/kernel/regions.hpp b/src/backend/cuda/kernel/regions.hpp
index b1fe3f7c8d..d03aed4517 100644
--- a/src/backend/cuda/kernel/regions.hpp
+++ b/src/backend/cuda/kernel/regions.hpp
@@ -351,12 +351,12 @@ template<typename T, bool full_conn, int n_per_thread>
 void regions(arrayfire::cuda::Param<T> out, arrayfire::cuda::CParam<char> in,
              cudaTextureObject_t tex) {
     using arrayfire::cuda::getActiveStream;
-    const dim3 threads(THREADS_X, THREADS_Y);
+    dim3 threads(THREADS_X, THREADS_Y);
 
     const int blk_x = divup(in.dims[0], threads.x * 2);
     const int blk_y = divup(in.dims[1], threads.y * 2);
 
-    const dim3 blocks(blk_x, blk_y);
+    dim3 blocks(blk_x, blk_y);
 
     CUDA_LAUNCH((initial_label<T, n_per_thread>), blocks, threads, out, in);
 
diff --git a/src/backend/cuda/kernel/topk.hpp b/src/backend/cuda/kernel/topk.hpp
index 9418a9162d..22f7c34f93 100644
--- a/src/backend/cuda/kernel/topk.hpp
+++ b/src/backend/cuda/kernel/topk.hpp
@@ -120,7 +120,7 @@ static __global__ void kerTopkDim0(Param<T> ovals, Param<uint> oidxs,
 template<typename T>
 void topkDim0(Param<T> ovals, Param<uint> oidxs, CParam<T> ivals, const int k,
               const af::topkFunction order) {
-    const dim3 threads(TOPK_THRDS_PER_BLK, 1);
+    dim3 threads(TOPK_THRDS_PER_BLK, 1);
     const int thrdLoad = TOPK_IDX_THRD_LOAD;
 
     int numBlocksX = divup(ivals.dims[0], threads.x * thrdLoad);

From c3dd2369766a36b5ade11e547f0abe4b70e0a3ac Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 3 Jan 2023 15:54:21 -0500
Subject: [PATCH 165/473] Fix warning when building spdlog. Caused errors on
 GitHub actions

---
 CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ae7a3742a6..a966d75c41 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -272,6 +272,9 @@ else()
       PROPERTIES
         INTERFACE_LINK_LIBRARIES "spdlog_header_only")
   else()
+    target_compile_options(spdlog
+      PRIVATE
+        $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>)
     set_target_properties(af_spdlog
       PROPERTIES
         INTERFACE_LINK_LIBRARIES "spdlog")

From fa44d4a729dfd88f43220181db36e2afa75e7b71 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 3 Jan 2023 15:55:17 -0500
Subject: [PATCH 166/473] Cleanup advanced CMake variables and Gtest compile
 flags and def

---
 CMakeLists.txt      | 94 ++++++++++++++++++++++++++++++---------------
 test/CMakeLists.txt | 22 +++--------
 2 files changed, 68 insertions(+), 48 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a966d75c41..8985c797ff 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -173,37 +173,6 @@ if(DEFINED USE_CPU_MKL OR DEFINED USE_OPENCL_MKL)
   endif()
 endif()
 
-mark_as_advanced(
-  AF_BUILD_FRAMEWORK
-  AF_CACHE_KERNELS_TO_DISK
-  AF_INSTALL_STANDALONE
-  AF_WITH_CPUID
-  AF_WITH_LOGGING
-  AF_WITH_STACKTRACE
-  AF_WITH_STATIC_FREEIMAGE
-  AF_WITH_NONFREE
-  AF_WITH_IMAGEIO
-  AF_WITH_RELATIVE_TEST_DIR
-  AF_TEST_WITH_MTX_FILES
-  ArrayFire_DIR
-  Boost_INCLUDE_DIR
-  CLEAR CUDA_VERSION
-  CUDA_HOST_COMPILER
-  CUDA_SDK_ROOT_DIR
-  CUDA_USE_STATIC_CUDA_RUNTIME
-  CUDA_rt_LIBRARY
-  SPDLOG_BUILD_EXAMPLES
-  SPDLOG_BUILD_TESTING
-  ADDR2LINE_PROGRAM
-  Backtrace_LIBRARY
-  AF_WITH_STATIC_MKL
-  GIT
-  Forge_DIR
-  glad_DIR
-  spdlog_DIR
-  FG_BUILD_OFFLINE
-  )
-
 if(AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL")
   set(BLA_VENDOR "Intel10_64lp")
   if(MKL_THREAD_LAYER STREQUAL "Sequential")
@@ -603,6 +572,19 @@ include(CPackConfig)
 # for ArrayFire Development. They are marked hidden.
 # If VCPKG is not used, marking them is not harmful
 mark_as_advanced(
+  AF_BUILD_FRAMEWORK
+  AF_CACHE_KERNELS_TO_DISK
+  AF_INSTALL_STANDALONE
+  AF_WITH_CPUID
+  AF_WITH_LOGGING
+  AF_WITH_STACKTRACE
+  AF_WITH_STATIC_FREEIMAGE
+  AF_WITH_NONFREE
+  AF_WITH_IMAGEIO
+  AF_WITH_RELATIVE_TEST_DIR
+  AF_TEST_WITH_MTX_FILES
+  ArrayFire_DIR
+
   VCPKG_APPLOCAL_DEPS
   VCPKG_BOOTSTRAP_OPTIONS
   VCPKG_INSTALL_OPTIONS
@@ -618,4 +600,54 @@ mark_as_advanced(
   Z_VCPKG_PWSH_PATH
   Z_VCPKG_CL
   _VCPKG_INSTALLED_DIR
+
+  Boost_INCLUDE_DIR
+  CLEAR CUDA_VERSION
+  CUDA_HOST_COMPILER
+  CUDA_SDK_ROOT_DIR
+  CUDA_USE_STATIC_CUDA_RUNTIME
+  CUDA_rt_LIBRARY
+  SPDLOG_BUILD_EXAMPLES
+  SPDLOG_BUILD_TESTING
+  ADDR2LINE_PROGRAM
+  Backtrace_LIBRARY
+  AF_WITH_STATIC_MKL
+  GIT
+  Forge_DIR
+  glad_DIR
+  spdlog_DIR
+  FG_BUILD_OFFLINE
+  SPAN_LITE_COLOURISE_TEST
+  SPAN_LITE_EXPORT_PACKAGE
+  SPAN_LITE_OPT_BUILD_EXAMPLES
+  SPAN_LITE_OPT_BUILD_TESTS
+  SPAN_LITE_OPT_SELECT_NONSTD
+  SPAN_LITE_OPT_SELECT_STD
+  FETCHCONTENT_SOURCE_DIR_SPAN-LITE
+  SPDLOG_BUILD_ALL
+  SPDLOG_BUILD_BENCH
+  SPDLOG_BUILD_EXAMPLE
+  SPDLOG_BUILD_EXAMPLE_HO
+  SPDLOG_BUILD_SHARED
+  SPDLOG_BUILD_TESTS
+  SPDLOG_BUILD_TESTS_HO
+  SPDLOG_BUILD_WARNINGS
+  SPDLOG_CLOCK_COARSE
+  SPDLOG_DISABLE_DEFAULT_LOGGER
+  SPDLOG_ENABLE_PCH
+  SPDLOG_FMT_EXTERNAL
+  SPDLOG_FMT_EXTERNAL_HO
+  SPDLOG_INSTALL
+  SPDLOG_NO_ATOMIC_LEVELS
+  SPDLOG_NO_EXCEPTIONS
+  SPDLOG_NO_THREAD_ID
+  SPDLOG_NO_TLS
+  SPDLOG_PREVENT_CHILD_FD
+  SPDLOG_SANITIZE_ADDRESS
+  SPDLOG_TIDY
+  SPDLOG_WCHAR_FILENAMES
+  SPDLOG_WCHAR_SUPPORT
+  cub_include_dir
+  fmt_DIR
+  span-lite_DIR
   )
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 1ff1d94041..16ba6f71ec 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -22,33 +22,21 @@ elseif(NOT TARGET GTest::gtest)
     URI https://github.com/google/googletest.git
     REF release-1.12.1
   )
-
-  # gtest targets cmake version 2.6 which throws warnings for policy CMP0042 on
-  # newer cmakes. This sets the default global setting for that policy.
-  set(CMAKE_POLICY_DEFAULT_CMP0042 NEW)
   if(WIN32)
     set(gtest_force_shared_crt ON
         CACHE INTERNAL "Required so that the libs Runtime is not set to MT DLL")
     set(BUILD_SHARED_LIBS OFF)
   endif()
 
-  add_definitions(-DGTEST_HAS_SEH=OFF)
   add_subdirectory(${${gtest_prefix}_SOURCE_DIR} ${${gtest_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
-  set_target_properties(gtest gtest_main
+  target_compile_definitions(gtest PRIVATE GTEST_HAS_SEH=OFF)
+  set_target_properties(gtest
     PROPERTIES
       FOLDER "ExternalProjectTargets/gtest")
+  target_compile_options(gtest
+    PRIVATE
+      $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>)
   add_library(GTest::gtest ALIAS gtest)
-  if(UNIX)
-    if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND
-      CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "10.3.0")
-      target_compile_options(gtest PRIVATE -Wno-maybe-uninitialized)
-      target_compile_options(gtest_main PRIVATE -Wno-maybe-uninitialized)
-    endif()
-  endif()
-  if(WIN32)
-      target_compile_options(gtest PRIVATE -Wno-error=ignored-attributes)
-  endif()
-
   # Hide gtest project variables
   mark_as_advanced(
     BUILD_SHARED_LIBS

From 08e7b64d954ad94b86f9bda4611a5708c168f013 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 9 Jan 2023 16:12:11 -0500
Subject: [PATCH 167/473] Fix unused cusparse CSC code path in the CUDA backend

---
 src/backend/cuda/cusparse.hpp                 | 53 +++++++++++++++++--
 src/backend/cuda/cusparseModule.cpp           |  1 +
 src/backend/cuda/cusparseModule.hpp           |  1 +
 .../cuda/cusparse_descriptor_helpers.hpp      |  6 +--
 src/backend/cuda/sparse.cu                    | 38 ++++++-------
 test/sparse_common.hpp                        | 19 +++++++
 6 files changed, 92 insertions(+), 26 deletions(-)

diff --git a/src/backend/cuda/cusparse.hpp b/src/backend/cuda/cusparse.hpp
index 467b2a82ec..e7b5a51e33 100644
--- a/src/backend/cuda/cusparse.hpp
+++ b/src/backend/cuda/cusparse.hpp
@@ -9,17 +9,64 @@
 
 #pragma once
 
+#include <common/SparseArray.hpp>
 #include <common/defines.hpp>
-#include <common/err_common.hpp>
 #include <common/unique_handle.hpp>
+#include <cudaDataType.hpp>
 #include <cusparseModule.hpp>
 #include <cusparse_v2.h>
+#include <err_cuda.hpp>
+
+#if defined(AF_USE_NEW_CUSPARSE_API)
+namespace arrayfire {
+namespace cuda {
+
+template<typename T>
+cusparseStatus_t createSpMatDescr(
+    cusparseSpMatDescr_t *out, const arrayfire::common::SparseArray<T> &arr) {
+    auto &_ = arrayfire::cuda::getCusparsePlugin();
+    switch (arr.getStorage()) {
+        case AF_STORAGE_CSR: {
+            return _.cusparseCreateCsr(
+                out, arr.dims()[0], arr.dims()[1], arr.getNNZ(),
+                (void *)arr.getRowIdx().get(), (void *)arr.getColIdx().get(),
+                (void *)arr.getValues().get(), CUSPARSE_INDEX_32I,
+                CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, getType<T>());
+        }
+#if CUSPARSE_VERSION >= 11300
+        case AF_STORAGE_CSC: {
+            return _.cusparseCreateCsc(
+                out, arr.dims()[0], arr.dims()[1], arr.getNNZ(),
+                (void *)arr.getColIdx().get(), (void *)arr.getRowIdx().get(),
+                (void *)arr.getValues().get(), CUSPARSE_INDEX_32I,
+                CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, getType<T>());
+        }
+#else
+        case AF_STORAGE_CSC:
+            CUDA_NOT_SUPPORTED(
+                "Sparse not supported for CSC on this version of the CUDA "
+                "Toolkit");
+#endif
+        case AF_STORAGE_COO: {
+            return _.cusparseCreateCoo(
+                out, arr.dims()[0], arr.dims()[1], arr.getNNZ(),
+                (void *)arr.getColIdx().get(), (void *)arr.getRowIdx().get(),
+                (void *)arr.getValues().get(), CUSPARSE_INDEX_32I,
+                CUSPARSE_INDEX_BASE_ZERO, getType<T>());
+        }
+    }
+    return CUSPARSE_STATUS_SUCCESS;
+}
+
+}  // namespace cuda
+}  // namespace arrayfire
+#endif
 
 // clang-format off
 DEFINE_HANDLER(cusparseHandle_t, arrayfire::cuda::getCusparsePlugin().cusparseCreate, arrayfire::cuda::getCusparsePlugin().cusparseDestroy);
 DEFINE_HANDLER(cusparseMatDescr_t, arrayfire::cuda::getCusparsePlugin().cusparseCreateMatDescr, arrayfire::cuda::getCusparsePlugin().cusparseDestroyMatDescr);
 #if defined(AF_USE_NEW_CUSPARSE_API)
-DEFINE_HANDLER(cusparseSpMatDescr_t, arrayfire::cuda::getCusparsePlugin().cusparseCreateCsr, arrayfire::cuda::getCusparsePlugin().cusparseDestroySpMat);
+DEFINE_HANDLER(cusparseSpMatDescr_t, arrayfire::cuda::createSpMatDescr, arrayfire::cuda::getCusparsePlugin().cusparseDestroySpMat);
 DEFINE_HANDLER(cusparseDnVecDescr_t, arrayfire::cuda::getCusparsePlugin().cusparseCreateDnVec, arrayfire::cuda::getCusparsePlugin().cusparseDestroyDnVec);
 DEFINE_HANDLER(cusparseDnMatDescr_t, arrayfire::cuda::getCusparsePlugin().cusparseCreateDnMat, arrayfire::cuda::getCusparsePlugin().cusparseDestroyDnMat);
 #endif
@@ -28,7 +75,7 @@ DEFINE_HANDLER(cusparseDnMatDescr_t, arrayfire::cuda::getCusparsePlugin().cuspar
 namespace arrayfire {
 namespace cuda {
 
-const char* errorString(cusparseStatus_t err);
+const char *errorString(cusparseStatus_t err);
 
 #define CUSPARSE_CHECK(fn)                                                    \
     do {                                                                      \
diff --git a/src/backend/cuda/cusparseModule.cpp b/src/backend/cuda/cusparseModule.cpp
index 84daa25460..a7dba5dc77 100644
--- a/src/backend/cuda/cusparseModule.cpp
+++ b/src/backend/cuda/cusparseModule.cpp
@@ -105,6 +105,7 @@ cusparseModule::cusparseModule()
 
     MODULE_FUNCTION_INIT(cusparseCnnz);
     MODULE_FUNCTION_INIT(cusparseCreateCsr);
+    MODULE_FUNCTION_INIT(cusparseCreateCoo);
     MODULE_FUNCTION_INIT(cusparseCreateDnMat);
     MODULE_FUNCTION_INIT(cusparseCreateDnVec);
     MODULE_FUNCTION_INIT(cusparseCreateIdentityPermutation);
diff --git a/src/backend/cuda/cusparseModule.hpp b/src/backend/cuda/cusparseModule.hpp
index 5f63cec285..fc3bb09b76 100644
--- a/src/backend/cuda/cusparseModule.hpp
+++ b/src/backend/cuda/cusparseModule.hpp
@@ -61,6 +61,7 @@ class cusparseModule {
     MODULE_MEMBER(cusparseXcsrsort);
 #endif
 
+    MODULE_MEMBER(cusparseCreateCoo);
     MODULE_MEMBER(cusparseCreateCsr);
     MODULE_MEMBER(cusparseDestroyDnMat);
     MODULE_MEMBER(cusparseDestroyDnVec);
diff --git a/src/backend/cuda/cusparse_descriptor_helpers.hpp b/src/backend/cuda/cusparse_descriptor_helpers.hpp
index 99d474cdbb..340a049b11 100644
--- a/src/backend/cuda/cusparse_descriptor_helpers.hpp
+++ b/src/backend/cuda/cusparse_descriptor_helpers.hpp
@@ -25,11 +25,7 @@ template<typename T>
 auto cusparseDescriptor(const common::SparseArray<T> &in) {
     auto dims = in.dims();
 
-    return common::make_handle<cusparseSpMatDescr_t>(
-        dims[0], dims[1], in.getNNZ(), (void *)(in.getRowIdx().get()),
-        (void *)(in.getColIdx().get()), (void *)(in.getValues().get()),
-        CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO,
-        getType<T>());
+    return common::make_handle<cusparseSpMatDescr_t>(in);
 }
 
 template<typename T>
diff --git a/src/backend/cuda/sparse.cu b/src/backend/cuda/sparse.cu
index dd6d8d22b7..3c39c72695 100644
--- a/src/backend/cuda/sparse.cu
+++ b/src/backend/cuda/sparse.cu
@@ -258,17 +258,19 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
     auto matA = denMatDescriptor(in);
     cusparseSpMatDescr_t matB;
 
-    auto d_csr_offsets = createEmptyArray<int>(M + 1);
+    Array<int> d_offsets = createEmptyArray<int>(0);
 
     if (stype == AF_STORAGE_CSR) {
+        d_offsets = createEmptyArray<int>(M + 1);
         // Create sparse matrix B in CSR format
         CUSPARSE_CHECK(
-            _.cusparseCreateCsr(&matB, M, N, 0, d_csr_offsets.get(), nullptr,
+            _.cusparseCreateCsr(&matB, M, N, 0, d_offsets.get(), nullptr,
                                 nullptr, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
                                 CUSPARSE_INDEX_BASE_ZERO, getType<T>()));
     } else {
+        d_offsets = createEmptyArray<int>(N + 1);
         CUSPARSE_CHECK(
-            _.cusparseCreateCsc(&matB, M, N, 0, d_csr_offsets.get(), nullptr,
+            _.cusparseCreateCsc(&matB, M, N, 0, d_offsets.get(), nullptr,
                                 nullptr, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I,
                                 CUSPARSE_INDEX_BASE_ZERO, getType<T>()));
     }
@@ -290,22 +292,20 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
     CUSPARSE_CHECK(
         _.cusparseSpMatGetSize(matB, &num_rows_tmp, &num_cols_tmp, &nnz));
 
-    auto d_csr_columns = createEmptyArray<int>(nnz);
-    auto d_csr_values  = createEmptyArray<T>(nnz);
+    auto d_ind    = createEmptyArray<int>(nnz);
+    auto d_values = createEmptyArray<T>(nnz);
     // allocate CSR column indices and values
     // reset offsets, column indices, and values pointers
     if (stype == AF_STORAGE_CSR) {
         // Create sparse matrix B in CSR format
         // reset offsets, column indices, and values pointers
-        CUSPARSE_CHECK(_.cusparseCsrSetPointers(matB, d_csr_offsets.get(),
-                                                d_csr_columns.get(),
-                                                d_csr_values.get()));
+        CUSPARSE_CHECK(_.cusparseCsrSetPointers(matB, d_offsets.get(),
+                                                d_ind.get(), d_values.get()));
 
     } else {
         // reset offsets, column indices, and values pointers
-        CUSPARSE_CHECK(_.cusparseCscSetPointers(matB, d_csr_offsets.get(),
-                                                d_csr_columns.get(),
-                                                d_csr_values.get()));
+        CUSPARSE_CHECK(_.cusparseCscSetPointers(matB, d_offsets.get(),
+                                                d_ind.get(), d_values.get()));
     }
     // execute Sparse to Dense conversion
     CUSPARSE_CHECK(_.cusparseDenseToSparse_convert(
@@ -316,20 +316,22 @@ SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in) {
         size_t pBufferSizeInBytes = 0;
         auto desc                 = make_handle<cusparseMatDescr_t>();
         CUSPARSE_CHECK(_.cusparseXcsrsort_bufferSizeExt(
-            sparseHandle(), M, N, nnz, d_csr_offsets.get(), d_csr_columns.get(),
+            sparseHandle(), M, N, nnz, d_offsets.get(), d_ind.get(),
             &pBufferSizeInBytes));
         auto pBuffer = memAlloc<char>(pBufferSizeInBytes);
         Array<int> P = createEmptyArray<int>(nnz);
         CUSPARSE_CHECK(
             _.cusparseCreateIdentityPermutation(sparseHandle(), nnz, P.get()));
         CUSPARSE_CHECK(_.cusparseXcsrsort(
-            sparseHandle(), M, N, nnz, desc, (int *)d_csr_offsets.get(),
-            (int *)d_csr_columns.get(), P.get(), pBuffer.get()));
-        d_csr_values = lookup(d_csr_values, P, 0);
+            sparseHandle(), M, N, nnz, desc, (int *)d_offsets.get(),
+            (int *)d_ind.get(), P.get(), pBuffer.get()));
+        d_values = lookup(d_values, P, 0);
+        return createArrayDataSparseArray<T>(in.dims(), d_values, d_offsets,
+                                             d_ind, stype, false);
+    } else {
+        return createArrayDataSparseArray<T>(in.dims(), d_values, d_ind,
+                                             d_offsets, stype, false);
     }
-
-    return createArrayDataSparseArray<T>(in.dims(), d_csr_values, d_csr_offsets,
-                                         d_csr_columns, stype, false);
 #endif
 }
 
diff --git a/test/sparse_common.hpp b/test/sparse_common.hpp
index 41dd3fd05d..5884871388 100644
--- a/test/sparse_common.hpp
+++ b/test/sparse_common.hpp
@@ -164,6 +164,25 @@ static void convertCSR(const int M, const int N, const double ratio,
     ASSERT_ARRAYS_EQ(a, aa);
 }
 
+template<typename T>
+static void convertCSC(const int M, const int N, const double ratio,
+                       int targetDevice = -1) {
+    if (targetDevice >= 0) af::setDevice(targetDevice);
+
+    SUPPORTED_TYPE_CHECK(T);
+#if 1
+    af::array a = cpu_randu<T>(af::dim4(M, N));
+#else
+    af::array a = af::randu(M, N);
+#endif
+    a = a * (a > ratio);
+
+    af::array s  = af::sparse(a, AF_STORAGE_CSC);
+    af::array aa = af::dense(s);
+
+    ASSERT_ARRAYS_EQ(a, aa);
+}
+
 // This test essentially verifies that the sparse structures have the correct
 // dimensions and indices using a very basic test
 template<af_storage stype>

From d453523443d3b304095b3699143784524509cab2 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 9 Jan 2023 18:59:52 -0500
Subject: [PATCH 168/473] Check the result of cuda error functions before using
 results

---
 src/backend/cuda/CMakeLists.txt     |  1 -
 src/backend/cuda/Kernel.hpp         |  2 +-
 src/backend/cuda/Module.hpp         |  2 +-
 src/backend/cuda/cu_check_macro.hpp | 30 -----------------------------
 src/backend/cuda/err_cuda.hpp       | 19 ++++++++++++++++++
 5 files changed, 21 insertions(+), 33 deletions(-)
 delete mode 100644 src/backend/cuda/cu_check_macro.hpp

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index c031deebd9..0dc208fd8b 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -406,7 +406,6 @@ add_library(afcuda
     convolveNN.cpp
     copy.cpp
     copy.hpp
-    cu_check_macro.hpp
     cublas.cpp
     cublas.hpp
 
diff --git a/src/backend/cuda/Kernel.hpp b/src/backend/cuda/Kernel.hpp
index b5375f6ad2..2199292080 100644
--- a/src/backend/cuda/Kernel.hpp
+++ b/src/backend/cuda/Kernel.hpp
@@ -14,7 +14,7 @@
 
 #include <EnqueueArgs.hpp>
 #include <backend.hpp>
-#include <cu_check_macro.hpp>
+#include <err_cuda.hpp>
 #include <cstdlib>
 #include <string>
 
diff --git a/src/backend/cuda/Module.hpp b/src/backend/cuda/Module.hpp
index b5eb028765..88881611fc 100644
--- a/src/backend/cuda/Module.hpp
+++ b/src/backend/cuda/Module.hpp
@@ -10,7 +10,7 @@
 #pragma once
 
 #include <common/ModuleInterface.hpp>
-#include <cu_check_macro.hpp>
+#include <err_cuda.hpp>
 
 #include <cuda.h>
 
diff --git a/src/backend/cuda/cu_check_macro.hpp b/src/backend/cuda/cu_check_macro.hpp
deleted file mode 100644
index a6b8d3f3e1..0000000000
--- a/src/backend/cuda/cu_check_macro.hpp
+++ /dev/null
@@ -1,30 +0,0 @@
-/*******************************************************
- * Copyright (c) 2020, ArrayFire
- * All rights reserved.
- *
- * This file is distributed under 3-clause BSD license.
- * The complete license agreement can be obtained at:
- * http://arrayfire.com/licenses/BSD-3-Clause
- ********************************************************/
-
-#pragma once
-
-#include <common/err_common.hpp>
-
-#include <cuda.h>
-
-#include <cstdio>
-
-#define CU_CHECK(fn)                                                      \
-    do {                                                                  \
-        CUresult res = fn;                                                \
-        if (res == CUDA_SUCCESS) break;                                   \
-        char cu_err_msg[1024];                                            \
-        const char* cu_err_name;                                          \
-        const char* cu_err_string;                                        \
-        cuGetErrorName(res, &cu_err_name);                                \
-        cuGetErrorString(res, &cu_err_string);                            \
-        snprintf(cu_err_msg, sizeof(cu_err_msg), "CU Error %s(%d): %s\n", \
-                 cu_err_name, (int)(res), cu_err_string);                 \
-        AF_ERROR(cu_err_msg, AF_ERR_INTERNAL);                            \
-    } while (0)
diff --git a/src/backend/cuda/err_cuda.hpp b/src/backend/cuda/err_cuda.hpp
index 091b848283..77926cdd79 100644
--- a/src/backend/cuda/err_cuda.hpp
+++ b/src/backend/cuda/err_cuda.hpp
@@ -18,6 +18,25 @@
                            boost::stacktrace::stacktrace());                \
     } while (0)
 
+#define CU_CHECK(fn)                                                          \
+    do {                                                                      \
+        CUresult res = fn;                                                    \
+        if (res == CUDA_SUCCESS) break;                                       \
+        char cu_err_msg[1024];                                                \
+        const char* cu_err_name;                                              \
+        const char* cu_err_string;                                            \
+        CUresult nameErr, strErr;                                             \
+        nameErr = cuGetErrorName(res, &cu_err_name);                          \
+        strErr  = cuGetErrorString(res, &cu_err_string);                      \
+        if (nameErr == CUDA_SUCCESS && strErr == CUDA_SUCCESS) {              \
+            snprintf(cu_err_msg, sizeof(cu_err_msg), "CU Error %s(%d): %s\n", \
+                     cu_err_name, (int)(res), cu_err_string);                 \
+            AF_ERROR(cu_err_msg, AF_ERR_INTERNAL);                            \
+        } else {                                                              \
+            AF_ERROR("CU Unknown error.\n", AF_ERR_INTERNAL);                 \
+        }                                                                     \
+    } while (0)
+
 #define CUDA_CHECK(fn)                                               \
     do {                                                             \
         cudaError_t _cuda_error = fn;                                \

From b6f234e76812053d2b541c7085ec0ba7627ca463 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 9 Jan 2023 19:00:22 -0500
Subject: [PATCH 169/473] Clear the thread_local vectors and stringstream in
 case of exception

---
 src/backend/cuda/jit.cpp | 391 +++++++++++++++++++++------------------
 1 file changed, 209 insertions(+), 182 deletions(-)

diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 86b2b2e6a6..33a80adb50 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -226,67 +226,79 @@ struct Param {
     thread_local stringstream inOffsetsStream;
     thread_local stringstream opsStream;
     thread_local stringstream outrefStream;
+    thread_local stringstream kerStream;
 
-    int oid{0};
-    for (size_t i{0}; i < full_nodes.size(); i++) {
-        const auto& node{full_nodes[i]};
-        const auto& ids_curr{full_ids[i]};
-        // Generate input parameters, only needs current id
-        node->genParams(inParamStream, ids_curr.id, is_linear);
-        // Generate input offsets, only needs current id
-        node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
-        // Generate the core function body, needs children ids as well
-        node->genFuncs(opsStream, ids_curr);
-        for (auto outIt{begin(output_ids)}, endIt{end(output_ids)};
-             (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) {
-            // Generate also output parameters
-            outParamStream << (oid == 0 ? "" : ",\n") << "Param<"
-                           << full_nodes[ids_curr.id]->getTypeStr() << "> out"
-                           << oid;
-            // Generate code to write the output (offset already in ptr)
-            opsStream << "out" << oid << ".ptr[idx] = val" << ids_curr.id
-                      << ";\n";
-            ++oid;
+    string ret;
+    try {
+        int oid{0};
+        for (size_t i{0}; i < full_nodes.size(); i++) {
+            const auto& node{full_nodes[i]};
+            const auto& ids_curr{full_ids[i]};
+            // Generate input parameters, only needs current id
+            node->genParams(inParamStream, ids_curr.id, is_linear);
+            // Generate input offsets, only needs current id
+            node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
+            // Generate the core function body, needs children ids as well
+            node->genFuncs(opsStream, ids_curr);
+            for (auto outIt{begin(output_ids)}, endIt{end(output_ids)};
+                 (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) {
+                // Generate also output parameters
+                outParamStream << (oid == 0 ? "" : ",\n") << "Param<"
+                               << full_nodes[ids_curr.id]->getTypeStr()
+                               << "> out" << oid;
+                // Generate code to write the output (offset already in ptr)
+                opsStream << "out" << oid << ".ptr[idx] = val" << ids_curr.id
+                          << ";\n";
+                ++oid;
+            }
         }
-    }
-
-    outrefStream << "\n    const Param<"
-                 << full_nodes[output_ids[0]]->getTypeStr()
-                 << "> &outref = out0;";
 
-    // Put various blocks into a single stream
-    thread_local stringstream kerStream;
-    kerStream << typedefStr << includeFileStr << "\n\n"
-              << paramTStr << '\n'
-              << kernelVoid << funcName << "(\n"
-              << inParamStream.str() << outParamStream.str() << dimParams << ')'
-              << blockStart << outrefStream.str();
-    if (is_linear) {
-        kerStream << linearInit;
-        if (loop0) kerStream << linearLoop0Start;
-        kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
-        if (loop0) kerStream << linearLoop0End;
-        kerStream << linearEnd;
-    } else {
-        if (loop0) {
-            kerStream << stridedLoop0Init << stridedLoop0Start;
+        outrefStream << "\n    const Param<"
+                     << full_nodes[output_ids[0]]->getTypeStr()
+                     << "> &outref = out0;";
+
+        // Put various blocks into a single stream
+        kerStream << typedefStr << includeFileStr << "\n\n"
+                  << paramTStr << '\n'
+                  << kernelVoid << funcName << "(\n"
+                  << inParamStream.str() << outParamStream.str() << dimParams
+                  << ')' << blockStart << outrefStream.str();
+        if (is_linear) {
+            kerStream << linearInit;
+            if (loop0) kerStream << linearLoop0Start;
+            kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
+            if (loop0) kerStream << linearLoop0End;
+            kerStream << linearEnd;
         } else {
-            kerStream << stridedLoopNInit;
-            if (loop3) kerStream << stridedLoop3Init;
-            if (loop2) kerStream << stridedLoop2Init;
-            if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
-            if (loop2) kerStream << stridedLoop2Start;
-            if (loop3) kerStream << stridedLoop3Start;
+            if (loop0) {
+                kerStream << stridedLoop0Init << stridedLoop0Start;
+            } else {
+                kerStream << stridedLoopNInit;
+                if (loop3) kerStream << stridedLoop3Init;
+                if (loop2) kerStream << stridedLoop2Init;
+                if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
+                if (loop2) kerStream << stridedLoop2Start;
+                if (loop3) kerStream << stridedLoop3Start;
+            }
+            kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
+            if (loop3) kerStream << stridedLoop3End;
+            if (loop2) kerStream << stridedLoop2End;
+            if (loop1) kerStream << stridedLoop1End;
+            if (loop0) kerStream << stridedLoop0End;
+            kerStream << stridedEnd;
         }
-        kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
-        if (loop3) kerStream << stridedLoop3End;
-        if (loop2) kerStream << stridedLoop2End;
-        if (loop1) kerStream << stridedLoop1End;
-        if (loop0) kerStream << stridedLoop0End;
-        kerStream << stridedEnd;
+        kerStream << blockEnd;
+        ret = kerStream.str();
+    } catch (...) {
+        // Prepare for next round
+        inParamStream.str("");
+        outParamStream.str("");
+        inOffsetsStream.str("");
+        opsStream.str("");
+        outrefStream.str("");
+        kerStream.str("");
+        throw;
     }
-    kerStream << blockEnd;
-    const string ret{kerStream.str()};
 
     // Prepare for next round
     inParamStream.str("");
@@ -364,150 +376,165 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
     thread_local vector<Node_ids> full_ids;
     thread_local vector<int> output_ids;
 
-    // Reserve some space to improve performance at smaller
-    // sizes
-    constexpr size_t CAP{1024};
-    if (full_nodes.capacity() < CAP) {
-        nodes.reserve(CAP);
-        output_ids.reserve(10);
-        full_nodes.reserve(CAP);
-        full_ids.reserve(CAP);
-    }
-
-    const af::dtype outputType{output_nodes[0]->getType()};
-    const size_t outputSizeofType{size_of(outputType)};
-    for (Node* node : output_nodes) {
-        assert(node->getType() == outputType);
-        const int id = node->getNodesMap(nodes, full_nodes, full_ids);
-        output_ids.push_back(id);
-    }
-
-    size_t inputSize{0};
-    unsigned nrInputs{0};
-    bool moddimsFound{false};
-    for (const Node* node : full_nodes) {
-        is_linear &= node->isLinear(outDims);
-        moddimsFound |= (node->getOp() == af_moddims_t);
-        if (node->isBuffer()) {
-            ++nrInputs;
-            inputSize += node->getBytes();
+    try {
+        // Reserve some space to improve performance at smaller
+        // sizes
+        constexpr size_t CAP{1024};
+        if (full_nodes.capacity() < CAP) {
+            nodes.reserve(CAP);
+            output_ids.reserve(10);
+            full_nodes.reserve(CAP);
+            full_ids.reserve(CAP);
         }
-    }
-    const size_t outputSize{numOutElems * outputSizeofType * nrOutputs};
-    const size_t totalSize{inputSize + outputSize};
-
-    bool emptyColumnsFound{false};
-    if (is_linear) {
-        outDims[0]    = numOutElems;
-        outDims[1]    = 1;
-        outDims[2]    = 1;
-        outDims[3]    = 1;
-        outStrides[0] = 1;
-        outStrides[1] = numOutElems;
-        outStrides[2] = numOutElems;
-        outStrides[3] = numOutElems;
-        ndims         = 1;
-    } else {
-        emptyColumnsFound = ndims > (outDims[0] == 1   ? 1
-                                     : outDims[1] == 1 ? 2
-                                     : outDims[2] == 1 ? 3
-                                                       : 4);
-    }
 
-    // Keep node_clones in scope, so that the nodes remain active for later
-    // referral in case moddims or Column elimination operations have to take
-    // place
-    vector<Node_ptr> node_clones;
-    if (moddimsFound | emptyColumnsFound) {
-        node_clones.reserve(full_nodes.size());
-        for (Node* node : full_nodes) {
-            node_clones.emplace_back(node->clone());
+        const af::dtype outputType{output_nodes[0]->getType()};
+        const size_t outputSizeofType{size_of(outputType)};
+        for (Node* node : output_nodes) {
+            assert(node->getType() == outputType);
+            const int id = node->getNodesMap(nodes, full_nodes, full_ids);
+            output_ids.push_back(id);
         }
 
-        for (const Node_ids& ids : full_ids) {
-            auto& children{node_clones[ids.id]->m_children};
-            for (int i{0}; i < Node::kMaxChildren && children[i] != nullptr;
-                 i++) {
-                children[i] = node_clones[ids.child_ids[i]];
+        size_t inputSize{0};
+        unsigned nrInputs{0};
+        bool moddimsFound{false};
+        for (const Node* node : full_nodes) {
+            is_linear &= node->isLinear(outDims);
+            moddimsFound |= (node->getOp() == af_moddims_t);
+            if (node->isBuffer()) {
+                ++nrInputs;
+                inputSize += node->getBytes();
             }
         }
+        const size_t outputSize{numOutElems * outputSizeofType * nrOutputs};
+        const size_t totalSize{inputSize + outputSize};
+
+        bool emptyColumnsFound{false};
+        if (is_linear) {
+            outDims[0]    = numOutElems;
+            outDims[1]    = 1;
+            outDims[2]    = 1;
+            outDims[3]    = 1;
+            outStrides[0] = 1;
+            outStrides[1] = numOutElems;
+            outStrides[2] = numOutElems;
+            outStrides[3] = numOutElems;
+            ndims         = 1;
+        } else {
+            emptyColumnsFound = ndims > (outDims[0] == 1   ? 1
+                                         : outDims[1] == 1 ? 2
+                                         : outDims[2] == 1 ? 3
+                                                           : 4);
+        }
 
-        if (moddimsFound) {
-            const auto isModdim{[](const Node_ptr& node) {
-                return node->getOp() == af_moddims_t;
-            }};
-            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
-                 (nodeIt = find_if(nodeIt, endIt, isModdim)) != endIt;
-                 ++nodeIt) {
-                const ModdimNode* mn{static_cast<ModdimNode*>(nodeIt->get())};
+        // Keep node_clones in scope, so that the nodes remain active for later
+        // referral in case moddims or Column elimination operations have to
+        // take place
+        vector<Node_ptr> node_clones;
+        if (moddimsFound | emptyColumnsFound) {
+            node_clones.reserve(full_nodes.size());
+            for (Node* node : full_nodes) {
+                node_clones.emplace_back(node->clone());
+            }
 
-                const auto new_strides{calcStrides(mn->m_new_shape)};
+            for (const Node_ids& ids : full_ids) {
+                auto& children{node_clones[ids.id]->m_children};
+                for (int i{0}; i < Node::kMaxChildren && children[i] != nullptr;
+                     i++) {
+                    children[i] = node_clones[ids.child_ids[i]];
+                }
+            }
+
+            if (moddimsFound) {
+                const auto isModdim{[](const Node_ptr& node) {
+                    return node->getOp() == af_moddims_t;
+                }};
+                for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
+                     (nodeIt = find_if(nodeIt, endIt, isModdim)) != endIt;
+                     ++nodeIt) {
+                    const ModdimNode* mn{
+                        static_cast<ModdimNode*>(nodeIt->get())};
+
+                    const auto new_strides{calcStrides(mn->m_new_shape)};
+                    const auto isBuffer{
+                        [](const Node& ptr) { return ptr.isBuffer(); }};
+                    for (NodeIterator<> it{nodeIt->get()},
+                         end{NodeIterator<>()};
+                         (it = find_if(it, end, isBuffer)) != end; ++it) {
+                        BufferNode<T>* buf{static_cast<BufferNode<T>*>(&(*it))};
+                        buf->m_param.dims[0]    = mn->m_new_shape[0];
+                        buf->m_param.dims[1]    = mn->m_new_shape[1];
+                        buf->m_param.dims[2]    = mn->m_new_shape[2];
+                        buf->m_param.dims[3]    = mn->m_new_shape[3];
+                        buf->m_param.strides[0] = new_strides[0];
+                        buf->m_param.strides[1] = new_strides[1];
+                        buf->m_param.strides[2] = new_strides[2];
+                        buf->m_param.strides[3] = new_strides[3];
+                    }
+                }
+            }
+            if (emptyColumnsFound) {
                 const auto isBuffer{
-                    [](const Node& ptr) { return ptr.isBuffer(); }};
-                for (NodeIterator<> it{nodeIt->get()}, end{NodeIterator<>()};
-                     (it = find_if(it, end, isBuffer)) != end; ++it) {
-                    BufferNode<T>* buf{static_cast<BufferNode<T>*>(&(*it))};
-                    buf->m_param.dims[0]    = mn->m_new_shape[0];
-                    buf->m_param.dims[1]    = mn->m_new_shape[1];
-                    buf->m_param.dims[2]    = mn->m_new_shape[2];
-                    buf->m_param.dims[3]    = mn->m_new_shape[3];
-                    buf->m_param.strides[0] = new_strides[0];
-                    buf->m_param.strides[1] = new_strides[1];
-                    buf->m_param.strides[2] = new_strides[2];
-                    buf->m_param.strides[3] = new_strides[3];
+                    [](const Node_ptr& node) { return node->isBuffer(); }};
+                for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
+                     (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt;
+                     ++nodeIt) {
+                    BufferNode<T>* buf{
+                        static_cast<BufferNode<T>*>(nodeIt->get())};
+                    removeEmptyColumns(outDims, ndims, buf->m_param.dims,
+                                       buf->m_param.strides);
                 }
+                for_each(++begin(outputs), end(outputs),
+                         [outDims, ndims](Param<T>& output) {
+                             removeEmptyColumns(outDims, ndims, output.dims,
+                                                output.strides);
+                         });
+                ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
             }
-        }
-        if (emptyColumnsFound) {
-            const auto isBuffer{
-                [](const Node_ptr& node) { return node->isBuffer(); }};
-            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
-                 (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt;
-                 ++nodeIt) {
-                BufferNode<T>* buf{static_cast<BufferNode<T>*>(nodeIt->get())};
-                removeEmptyColumns(outDims, ndims, buf->m_param.dims,
-                                   buf->m_param.strides);
+
+            full_nodes.clear();
+            for (Node_ptr& node : node_clones) {
+                full_nodes.push_back(node.get());
             }
-            for_each(++begin(outputs), end(outputs),
-                     [outDims, ndims](Param<T>& output) {
-                         removeEmptyColumns(outDims, ndims, output.dims,
-                                            output.strides);
-                     });
-            ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
         }
 
-        full_nodes.clear();
-        for (Node_ptr& node : node_clones) { full_nodes.push_back(node.get()); }
-    }
-
-    threadsMgt<dim_t> th(outDims, ndims);
-    const dim3 threads{th.genThreads()};
-    const dim3 blocks{th.genBlocks(threads, nrInputs, nrOutputs, totalSize,
-                                   outputSizeofType)};
-    auto ker = getKernel(output_nodes, output_ids, full_nodes, full_ids,
-                         is_linear, th.loop0, th.loop1, th.loop2, th.loop3);
-
-    vector<void*> args;
-    for (const Node* node : full_nodes) {
-        node->setArgs(0, is_linear,
-                      [&](int /*id*/, const void* ptr, size_t /*size*/) {
-                          args.push_back(const_cast<void*>(ptr));
-                      });
-    }
+        threadsMgt<dim_t> th(outDims, ndims);
+        const dim3 threads{th.genThreads()};
+        const dim3 blocks{th.genBlocks(threads, nrInputs, nrOutputs, totalSize,
+                                       outputSizeofType)};
+        auto ker = getKernel(output_nodes, output_ids, full_nodes, full_ids,
+                             is_linear, th.loop0, th.loop1, th.loop2, th.loop3);
+
+        vector<void*> args;
+        for (const Node* node : full_nodes) {
+            node->setArgs(0, is_linear,
+                          [&](int /*id*/, const void* ptr, size_t /*size*/) {
+                              args.push_back(const_cast<void*>(ptr));
+                          });
+        }
 
-    for (auto& out : outputs) { args.push_back(static_cast<void*>(&out)); }
+        for (auto& out : outputs) { args.push_back(static_cast<void*>(&out)); }
 
-    {
-        using namespace arrayfire::cuda::kernel_logger;
-        AF_TRACE(
-            "Launching : Dims: [{},{},{},{}] Blocks: [{}] "
-            "Threads: [{}] threads: {}",
-            outDims[0], outDims[1], outDims[2], outDims[3], blocks, threads,
-            blocks.x * threads.x * blocks.y * threads.y * blocks.z * threads.z);
+        {
+            using namespace arrayfire::cuda::kernel_logger;
+            AF_TRACE(
+                "Launching : Dims: [{},{},{},{}] Blocks: [{}] "
+                "Threads: [{}] threads: {}",
+                outDims[0], outDims[1], outDims[2], outDims[3], blocks, threads,
+                blocks.x * threads.x * blocks.y * threads.y * blocks.z *
+                    threads.z);
+        }
+        CU_CHECK(cuLaunchKernel(ker, blocks.x, blocks.y, blocks.z, threads.x,
+                                threads.y, threads.z, 0, getActiveStream(),
+                                args.data(), NULL));
+    } catch (...) {
+        // Reset the thread local vectors
+        nodes.clear();
+        output_ids.clear();
+        full_nodes.clear();
+        full_ids.clear();
+        throw;
     }
-    CU_CHECK(cuLaunchKernel(ker, blocks.x, blocks.y, blocks.z, threads.x,
-                            threads.y, threads.z, 0, getActiveStream(),
-                            args.data(), NULL));
 
     // Reset the thread local vectors
     nodes.clear();

From c40eec3e8f7e724bfe1d1ff9f624af36d6a091d9 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 28 Nov 2022 00:36:18 -0500
Subject: [PATCH 170/473] Add type checks for tests and different types

---
 test/approx1.cpp        |  1 +
 test/approx2.cpp        |  2 ++
 test/arrayfire_test.cpp |  8 +++++++
 test/arrayio.cpp        |  3 +++
 test/basic.cpp          |  1 +
 test/binary.cpp         | 47 +++++++++++++++++++++++------------------
 test/canny.cpp          |  2 +-
 test/cast.cpp           | 10 +++++++++
 test/clamp.cpp          | 19 ++++++++++++-----
 test/fft.cpp            |  3 +++
 test/half.cpp           |  4 ++++
 test/replace.cpp        | 17 ++++++++++-----
 test/rng_quality.cpp    |  3 +--
 test/topk.cpp           |  2 +-
 test/wrap.cpp           |  1 +
 15 files changed, 88 insertions(+), 35 deletions(-)

diff --git a/test/approx1.cpp b/test/approx1.cpp
index 143f66bd71..af719d8c4d 100644
--- a/test/approx1.cpp
+++ b/test/approx1.cpp
@@ -968,6 +968,7 @@ template<typename T>
 class Approx1V2Simple : public Approx1V2<T> {
    protected:
     void SetUp() {
+        SUPPORTED_TYPE_CHECK(T);
         SimpleTestData data;
         this->setTestData(&data.h_gold.front(), data.gold_dims,
                           &data.h_in.front(), data.in_dims, &data.h_pos.front(),
diff --git a/test/approx2.cpp b/test/approx2.cpp
index 1b7901bf8d..bec8bd75cf 100644
--- a/test/approx2.cpp
+++ b/test/approx2.cpp
@@ -45,6 +45,7 @@ template<typename T>
 class Approx2 : public ::testing::Test {
    public:
     virtual void SetUp() {
+        SUPPORTED_TYPE_CHECK(T);
         subMat0.push_back(af_make_seq(0, 4, 1));
         subMat0.push_back(af_make_seq(2, 6, 1));
         subMat0.push_back(af_make_seq(0, 2, 1));
@@ -903,6 +904,7 @@ template<typename T>
 class Approx2V2Simple : public Approx2V2<T> {
    protected:
     void SetUp() {
+        SUPPORTED_TYPE_CHECK(T);
         SimpleTestData data;
         this->setTestData(&data.h_gold.front(), data.gold_dims,
                           &data.h_in.front(), data.in_dims,
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index a8f8a34562..cf776b6e2b 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -260,8 +260,16 @@ ::testing::AssertionResult assertImageEq(std::string aName, std::string bName,
     switch (arrDtype) {
         case u8: return imageEq<unsigned char>(aName, bName, a, b, maxAbsDiff);
         case b8: return imageEq<char>(aName, bName, a, b, maxAbsDiff);
+        case s32: return imageEq<int>(aName, bName, a, b, maxAbsDiff);
+        case u32: return imageEq<unsigned int>(aName, bName, a, b, maxAbsDiff);
         case f32: return imageEq<float>(aName, bName, a, b, maxAbsDiff);
         case f64: return imageEq<double>(aName, bName, a, b, maxAbsDiff);
+        case s16: return imageEq<short>(aName, bName, a, b, maxAbsDiff);
+        case u16:
+            return imageEq<unsigned short>(aName, bName, a, b, maxAbsDiff);
+        case u64:
+            return imageEq<unsigned long long>(aName, bName, a, b, maxAbsDiff);
+        case s64: return imageEq<long long>(aName, bName, a, b, maxAbsDiff);
         default: throw(AF_ERR_NOT_SUPPORTED);
     }
     return ::testing::AssertionSuccess();
diff --git a/test/arrayio.cpp b/test/arrayio.cpp
index 7a578b612a..00d907a568 100644
--- a/test/arrayio.cpp
+++ b/test/arrayio.cpp
@@ -56,6 +56,7 @@ INSTANTIATE_TEST_SUITE_P(
 
 TEST_P(ArrayIOType, ReadType) {
     type_params p = GetParam();
+    if (noDoubleTests(p.type)) GTEST_SKIP() << "No double support.";
     array arr =
         readArray((string(TEST_DIR) + "/arrayio/" + p.name + ".arr").c_str(),
                   p.name.c_str());
@@ -65,6 +66,7 @@ TEST_P(ArrayIOType, ReadType) {
 
 TEST_P(ArrayIOType, ReadSize) {
     type_params p = GetParam();
+    if (noDoubleTests(p.type)) GTEST_SKIP() << "No double support.";
     array arr =
         readArray((string(TEST_DIR) + "/arrayio/" + p.name + ".arr").c_str(),
                   p.name.c_str());
@@ -89,6 +91,7 @@ void checkVals(array arr, double r, double i, af_dtype t) {
 
 TEST_P(ArrayIOType, ReadContent) {
     type_params p = GetParam();
+    if (noDoubleTests(p.type)) GTEST_SKIP() << "No double support.";
     array arr =
         readArray((string(TEST_DIR) + "/arrayio/" + p.name + ".arr").c_str(),
                   p.name.c_str());
diff --git a/test/basic.cpp b/test/basic.cpp
index c39e800408..ebb211c7b7 100644
--- a/test/basic.cpp
+++ b/test/basic.cpp
@@ -314,6 +314,7 @@ TEST(Assert, TestEqualsC) {
 }
 
 TEST(Assert, TestEqualsDiffTypes) {
+    SUPPORTED_TYPE_CHECK(double);
     array gold = constant(1, 10, 10, f64);
     array out  = constant(1, 10, 10);
 
diff --git a/test/binary.cpp b/test/binary.cpp
index b0c04a4c30..f6f9a8928f 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -360,20 +360,27 @@ TEST(BinaryTests, ISSUE_1762) {
 }
 
 template<typename T>
-class PowPrecisionTest : public ::testing::TestWithParam<T> {};
-
-#define DEF_TEST(Sx, T)                                      \
-    using PowPrecisionTest##Sx = PowPrecisionTest<T>;        \
-    TEST_P(PowPrecisionTest##Sx, Issue2304) {                \
-        T param     = GetParam();                            \
-        auto dtype  = (af_dtype)dtype_traits<T>::af_type;    \
-        af::array A = af::constant(param, 1, dtype);         \
-        af::array B = af::pow(A, 2);                         \
-        vector<T> hres(1, 0);                                \
-        B.host(&hres[0]);                                    \
-        std::fesetround(FE_TONEAREST);                       \
-        T gold = (T)std::rint(std::pow((double)param, 2.0)); \
-        ASSERT_EQ(hres[0], gold);                            \
+class PowPrecisionTest : public ::testing::TestWithParam<T> {
+    void SetUp() { SUPPORTED_TYPE_CHECK(T); }
+};
+
+#define DEF_TEST(Sx, T)                                                    \
+    using PowPrecisionTest##Sx = PowPrecisionTest<T>;                      \
+    TEST_P(PowPrecisionTest##Sx, Issue2304) {                              \
+        T param    = GetParam();                                           \
+        auto dtype = (af_dtype)dtype_traits<T>::af_type;                   \
+        if (noDoubleTests(dtype)) {                                        \
+            if (std::abs((double)param) > 10000)                           \
+                GTEST_SKIP()                                               \
+                    << "Skip larger values because double not supported."; \
+        }                                                                  \
+        af::array A = af::constant(param, 1, dtype);                       \
+        af::array B = af::pow(A, 2);                                       \
+        vector<T> hres(1, 0);                                              \
+        B.host(&hres[0]);                                                  \
+        std::fesetround(FE_TONEAREST);                                     \
+        T gold = (T)std::rint(std::pow((double)param, 2.0));               \
+        ASSERT_EQ(hres[0], gold);                                          \
     }
 
 DEF_TEST(ULong, unsigned long long)
@@ -429,15 +436,17 @@ class ResultType : public testing::TestWithParam<result_type_param> {
     af::array lhs;
     af::array rhs;
     af_dtype gold;
-    bool skip;
 
     void SetUp() {
         result_type_param params = GetParam();
         gold                     = params.result_;
-        skip                     = false;
         if (noHalfTests(params.result_) || noHalfTests(params.lhs_) ||
             noHalfTests(params.rhs_)) {
-            skip = true;
+            GTEST_SKIP() << "Half not supported on this device";
+            return;
+        } else if (noDoubleTests(params.result_) ||
+                   noDoubleTests(params.lhs_) || noDoubleTests(params.rhs_)) {
+            GTEST_SKIP() << "Double not supported on this device";
             return;
         }
         lhs = af::array(10, params.lhs_);
@@ -513,19 +522,15 @@ INSTANTIATE_TEST_SUITE_P(
 
 // clang-format off
 TEST_P(ResultType, Addition)       {
-    if (skip) return;
     ASSERT_EQ(gold, (lhs + rhs).type());
 }
 TEST_P(ResultType, Subtraction)    {
-    if (skip) return;
     ASSERT_EQ(gold, (lhs - rhs).type());
 }
 TEST_P(ResultType, Multiplication) {
-    if (skip) return;
     ASSERT_EQ(gold, (lhs * rhs).type());
 }
 TEST_P(ResultType, Division)       {
-    if (skip) return;
     ASSERT_EQ(gold, (lhs / rhs).type());
 }
 // clang-format on
diff --git a/test/canny.cpp b/test/canny.cpp
index 7e72d4e356..b34a4923b4 100644
--- a/test/canny.cpp
+++ b/test/canny.cpp
@@ -251,7 +251,7 @@ void cannyImageOtsuBatchTest(string pTestFile, const dim_t targetBatchCount) {
             canny(inputIm, AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3, false);
         outIm *= 255.0;
 
-        ASSERT_IMAGES_NEAR(outIm.as(u8), goldIm, 1.0e-3);
+        ASSERT_IMAGES_NEAR(goldIm, outIm.as(u8), 1.0e-3);
     }
 }
 
diff --git a/test/cast.cpp b/test/cast.cpp
index 96178a470c..cb1f4e3f42 100644
--- a/test/cast.cpp
+++ b/test/cast.cpp
@@ -95,6 +95,8 @@ void cast_test_complex_real() {
 
 #define COMPLEX_REAL_TESTS(Ti, To)                      \
     TEST(CAST_TEST, Test_Complex_To_Real_##Ti##_##To) { \
+        SUPPORTED_TYPE_CHECK(Ti);                       \
+        SUPPORTED_TYPE_CHECK(To);                       \
         cast_test_complex_real<Ti, To>();               \
     }
 
@@ -106,6 +108,7 @@ COMPLEX_REAL_TESTS(cdouble, double)
 TEST(CAST_TEST, Test_JIT_DuplicateCastNoop) {
     // Does a trivial cast - check JIT kernel trace to ensure a __noop is
     // generated since we don't have a way to test it directly
+    SUPPORTED_TYPE_CHECK(double);
     af_dtype ta = (af_dtype)dtype_traits<float>::af_type;
     af_dtype tb = (af_dtype)dtype_traits<double>::af_type;
     dim4 dims(num, 1, 1, 1);
@@ -129,6 +132,7 @@ TEST(CAST_TEST, Test_JIT_DuplicateCastNoop) {
 
 TEST(Cast, ImplicitCast) {
     using namespace af;
+    SUPPORTED_TYPE_CHECK(double);
     array a = randu(100, 100, f64);
     array b = a.as(f32);
 
@@ -138,6 +142,7 @@ TEST(Cast, ImplicitCast) {
 
 TEST(Cast, ConstantCast) {
     using namespace af;
+    SUPPORTED_TYPE_CHECK(double);
     array a = constant(1, 100, f64);
     array b = a.as(f32);
 
@@ -147,6 +152,7 @@ TEST(Cast, ConstantCast) {
 
 TEST(Cast, OpCast) {
     using namespace af;
+    SUPPORTED_TYPE_CHECK(double);
     array a = constant(1, 100, f64);
     a       = a + a;
     array b = a.as(f32);
@@ -156,6 +162,7 @@ TEST(Cast, OpCast) {
 }
 TEST(Cast, ImplicitCastIndexed) {
     using namespace af;
+    SUPPORTED_TYPE_CHECK(double);
     array a = randu(100, 100, f64);
     array b = a(span, 1).as(f32);
     array c = max(abs(a(span, 1) - b));
@@ -164,6 +171,7 @@ TEST(Cast, ImplicitCastIndexed) {
 
 TEST(Cast, ImplicitCastIndexedNonLinear) {
     using namespace af;
+    SUPPORTED_TYPE_CHECK(double);
     array a = randu(100, 100, f64);
     array b = a(seq(10, 20, 2), 1).as(f32);
     array c = max(abs(a(seq(10, 20, 2), 1) - b));
@@ -172,6 +180,7 @@ TEST(Cast, ImplicitCastIndexedNonLinear) {
 
 TEST(Cast, ImplicitCastIndexedNonLinearArray) {
     using namespace af;
+    SUPPORTED_TYPE_CHECK(double);
     array a   = randu(100, 100, f64);
     array idx = seq(10, 20, 2);
     array b   = a(idx, 1).as(f32);
@@ -181,6 +190,7 @@ TEST(Cast, ImplicitCastIndexedNonLinearArray) {
 
 TEST(Cast, ImplicitCastIndexedAndScoped) {
     using namespace af;
+    SUPPORTED_TYPE_CHECK(double);
     array c;
     {
         array a = randu(100, 100, f64);
diff --git a/test/clamp.cpp b/test/clamp.cpp
index d27ad3a16d..1e0b04b7c2 100644
--- a/test/clamp.cpp
+++ b/test/clamp.cpp
@@ -51,8 +51,19 @@ class Clamp : public ::testing::TestWithParam<clamp_params> {
    public:
     void SetUp() {
         clamp_params params = GetParam();
-        if (noDoubleTests(params.in_type_)) return;
-        if (noHalfTests(params.in_type_)) return;
+        SUPPORTED_TYPE_CHECK(double);
+        if (noDoubleTests(params.in_type_))
+            GTEST_SKIP() << "Double not supported on this device";
+        if (noHalfTests(params.in_type_))
+            GTEST_SKIP() << "Half not supported on this device";
+        if (noDoubleTests(params.hi_type_))
+            GTEST_SKIP() << "Double not supported on this device";
+        if (noHalfTests(params.hi_type_))
+            GTEST_SKIP() << "Half not supported on this device";
+        if (noDoubleTests(params.lo_type_))
+            GTEST_SKIP() << "Double not supported on this device";
+        if (noHalfTests(params.lo_type_))
+            GTEST_SKIP() << "Half not supported on this device";
 
         in_ = randu(params.size_, params.in_type_);
         lo_ = randu(params.size_, params.lo_type_) / T(10);
@@ -138,9 +149,7 @@ INSTANTIATE_TEST_SUITE_P(
 
 TEST_P(ClampFloatingPoint, Basic) {
     clamp_params params = GetParam();
-    if (noDoubleTests(params.in_type_)) return;
-    if (noHalfTests(params.in_type_)) return;
-    array out = clamp(in_, lo_, hi_);
+    array out           = clamp(in_, lo_, hi_);
     ASSERT_ARRAYS_NEAR(gold_, out, 1e-5);
 }
 
diff --git a/test/fft.cpp b/test/fft.cpp
index 49176ca522..0af43dca2b 100644
--- a/test/fft.cpp
+++ b/test/fft.cpp
@@ -816,6 +816,7 @@ TEST_P(FFT2D, Real32ToComplexInputsPreserved) {
 }
 
 TEST_P(FFT2D, Real64ToComplexInputsPreserved) {
+    SUPPORTED_TYPE_CHECK(double);
     fft_params params = GetParam();
     af::array a       = af::randu(params.input_dims_, f64);
     af::array a_copy  = a.copy();
@@ -834,6 +835,7 @@ TEST_P(FFTC2R, Complex32ToRInputsPreserved) {
 }
 
 TEST_P(FFTC2R, Complex64ToRInputsPreserved) {
+    SUPPORTED_TYPE_CHECK(double);
     fft_params params = GetParam();
     af::array a       = af::randu(params.input_dims_, c64);
     af::array a_copy  = a.copy();
@@ -852,6 +854,7 @@ TEST_P(FFTND, Real32ToComplexInputsPreserved) {
 }
 
 TEST_P(FFTND, Real64ToComplexInputsPreserved) {
+    SUPPORTED_TYPE_CHECK(double);
     fft_params params = GetParam();
     af::array a       = af::randu(params.input_dims_, f64);
     af::array a_copy  = a.copy();
diff --git a/test/half.cpp b/test/half.cpp
index 33ae4eae4a..7f85950170 100644
--- a/test/half.cpp
+++ b/test/half.cpp
@@ -63,6 +63,10 @@ INSTANTIATE_TEST_SUITE_P(FromF16, HalfConvert,
 TEST_P(HalfConvert, convert) {
     SUPPORTED_TYPE_CHECK(af_half);
     convert_params params = GetParam();
+    if (noDoubleTests(params.to))
+        GTEST_SKIP() << "Double not supported on this device";
+    if (noDoubleTests(params.from))
+        GTEST_SKIP() << "Double not supported on this device";
 
     array from = af::constant(params.value, 3, 3, params.from);
     array to   = from.as(params.to);
diff --git a/test/replace.cpp b/test/replace.cpp
index 14e679436b..6d72cf7fc9 100644
--- a/test/replace.cpp
+++ b/test/replace.cpp
@@ -142,7 +142,8 @@ TEST(Replace, ISSUE_1249) {
     array a    = randu(dims);
     array b    = a.copy();
     replace(b, !cond, a - a * 0.9);
-    array c = a - a * cond * 0.9;
+    array c  = (a - a * 0.9);
+    c(!cond) = a(!cond);
 
     int num = (int)dims.elements();
     vector<float> hb(num);
@@ -151,7 +152,9 @@ TEST(Replace, ISSUE_1249) {
     b.host(&hb[0]);
     c.host(&hc[0]);
 
-    for (int i = 0; i < num; i++) { ASSERT_EQ(hc[i], hb[i]) << "at " << i; }
+    for (int i = 0; i < num; i++) {
+        ASSERT_FLOAT_EQ(hc[i], hb[i]) << "at " << i;
+    }
 }
 
 TEST(Replace, 4D) {
@@ -169,7 +172,9 @@ TEST(Replace, 4D) {
     b.host(&hb[0]);
     c.host(&hc[0]);
 
-    for (int i = 0; i < num; i++) { ASSERT_EQ(hc[i], hb[i]) << "at " << i; }
+    for (int i = 0; i < num; i++) {
+        ASSERT_FLOAT_EQ(hc[i], hb[i]) << "at " << i;
+    }
 }
 
 TEST(Replace, ISSUE_1683) {
@@ -187,12 +192,14 @@ TEST(Replace, ISSUE_1683) {
     B.host(hb.data());
 
     // Ensures A is not modified by replace
-    for (int i = 0; i < (int)A.elements(); i++) { ASSERT_EQ(ha1[i], ha2[i]); }
+    for (int i = 0; i < (int)A.elements(); i++) {
+        ASSERT_FLOAT_EQ(ha1[i], ha2[i]);
+    }
 
     // Ensures replace on B works as expected
     for (int i = 0; i < (int)B.elements(); i++) {
         float val = ha1[i * A.dims(0)];
         val       = val < 0.5 ? 0 : val;
-        ASSERT_EQ(val, hb[i]);
+        ASSERT_FLOAT_EQ(val, hb[i]);
     }
 }
diff --git a/test/rng_quality.cpp b/test/rng_quality.cpp
index 8274b1dfa9..92c264dfbb 100644
--- a/test/rng_quality.cpp
+++ b/test/rng_quality.cpp
@@ -20,6 +20,7 @@ class RandomEngine : public ::testing::Test {
     virtual void SetUp() {
         // Ensure all unlocked buffers are freed
         deviceGC();
+        SUPPORTED_TYPE_CHECK(T);
     }
 };
 
@@ -30,7 +31,6 @@ TYPED_TEST_SUITE(RandomEngine, TestTypesEngine);
 
 template<typename T>
 void testRandomEnginePeriod(randomEngineType type) {
-    SUPPORTED_TYPE_CHECK(T);
     dtype ty = (dtype)dtype_traits<T>::af_type;
 
     int elem  = 1024 * 1024;
@@ -88,7 +88,6 @@ double chi2_statistic<half_float::half>(array input, array expected,
 
 template<typename T>
 void testRandomEngineUniformChi2(randomEngineType type) {
-    SUPPORTED_TYPE_CHECK(T);
     dtype ty = (dtype)dtype_traits<T>::af_type;
 
     int elem  = 256 * 1024 * 1024;
diff --git a/test/topk.cpp b/test/topk.cpp
index 86cf1287f9..58319f25e8 100644
--- a/test/topk.cpp
+++ b/test/topk.cpp
@@ -149,7 +149,7 @@ void topkTest(const int ndims, const dim_t* dims, const unsigned k,
             case f32:
                 EXPECT_FLOAT_EQ(outData[i], hovals[i]) << "at: " << i;
                 break;
-            default: EXPECT_EQ(outData[i], hovals[i]); break;
+            default: EXPECT_EQ(outData[i], hovals[i]) << "at: " << i; break;
         }
         ASSERT_EQ(outIdxs[i], hoidxs[i]) << "at: " << i;
     }
diff --git a/test/wrap.cpp b/test/wrap.cpp
index 91b57c4bc0..baff77c5b1 100644
--- a/test/wrap.cpp
+++ b/test/wrap.cpp
@@ -360,6 +360,7 @@ template<typename T>
 class WrapV2Simple : public WrapV2<T> {
    protected:
     void SetUp() {
+        SUPPORTED_TYPE_CHECK(T);
         this->releaseArrays();
         this->in_   = 0;
         this->gold_ = 0;

From f72233168f432d93c964f428e9dc894f93944085 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 29 Nov 2022 00:03:05 -0500
Subject: [PATCH 171/473] Fix fftconvolve so that floats are used for complex
 float values

---
 src/api/c/fftconvolve.cpp          | 9 +++++----
 src/backend/cuda/fftconvolve.cpp   | 9 +++++----
 src/backend/opencl/fftconvolve.cpp | 9 +++++----
 3 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/src/api/c/fftconvolve.cpp b/src/api/c/fftconvolve.cpp
index bbcb2d2a1d..f92a3fc655 100644
--- a/src/api/c/fftconvolve.cpp
+++ b/src/api/c/fftconvolve.cpp
@@ -49,10 +49,11 @@ using std::vector;
 template<typename T>
 af_array fftconvolve_fallback(const af_array signal, const af_array filter,
                               const bool expand, const int baseDim) {
-    using convT =
-        typename conditional<is_integral<T>::value || is_same<T, float>::value,
-                             float, double>::type;
-    using cT = typename conditional<is_same<convT, float>::value, cfloat,
+    using convT = typename conditional<is_integral<T>::value ||
+                                           is_same<T, float>::value ||
+                                           is_same<T, cfloat>::value,
+                                       float, double>::type;
+    using cT    = typename conditional<is_same<convT, float>::value, cfloat,
                                     cdouble>::type;
 
     const Array<cT> S = castArray<cT>(signal);
diff --git a/src/backend/cuda/fftconvolve.cpp b/src/backend/cuda/fftconvolve.cpp
index 7c50c0838c..ed22d0ea85 100644
--- a/src/backend/cuda/fftconvolve.cpp
+++ b/src/backend/cuda/fftconvolve.cpp
@@ -50,10 +50,11 @@ dim4 calcPackedSize(Array<T> const& i1, Array<T> const& i2, const int rank) {
 template<typename T>
 Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
                      const bool expand, AF_BATCH_KIND kind, const int rank) {
-    using convT =
-        typename conditional<is_integral<T>::value || is_same<T, float>::value,
-                             float, double>::type;
-    using cT = typename conditional<is_same<convT, float>::value, cfloat,
+    using convT = typename conditional<is_integral<T>::value ||
+                                           is_same<T, float>::value ||
+                                           is_same<T, cfloat>::value,
+                                       float, double>::type;
+    using cT    = typename conditional<is_same<convT, float>::value, cfloat,
                                     cdouble>::type;
 
     const dim4& sDims = signal.dims();
diff --git a/src/backend/opencl/fftconvolve.cpp b/src/backend/opencl/fftconvolve.cpp
index a4f8b1f1f1..f6b243baac 100644
--- a/src/backend/opencl/fftconvolve.cpp
+++ b/src/backend/opencl/fftconvolve.cpp
@@ -58,10 +58,11 @@ dim4 calcPackedSize(Array<T> const& i1, Array<T> const& i2, const dim_t rank) {
 template<typename T>
 Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
                      const bool expand, AF_BATCH_KIND kind, const int rank) {
-    using convT =
-        typename conditional<is_integral<T>::value || is_same<T, float>::value,
-                             float, double>::type;
-    using cT = typename conditional<is_same<convT, float>::value, cfloat,
+    using convT = typename conditional<is_integral<T>::value ||
+                                           is_same<T, float>::value ||
+                                           is_same<T, cfloat>::value,
+                                       float, double>::type;
+    using cT    = typename conditional<is_same<convT, float>::value, cfloat,
                                     cdouble>::type;
 
     const dim4& sDims = signal.dims();

From 000d4311ac5104ffeab679dda9acccbfff6da3d2 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 29 Nov 2022 00:05:05 -0500
Subject: [PATCH 172/473] Add ifdef check around powll and powul functions in
 jit.cl

---
 src/backend/opencl/kernel/jit.cl | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/backend/opencl/kernel/jit.cl b/src/backend/opencl/kernel/jit.cl
index c9c3b7eb8c..a0486106e2 100644
--- a/src/backend/opencl/kernel/jit.cl
+++ b/src/backend/opencl/kernel/jit.cl
@@ -107,12 +107,19 @@ float2 __cdivf(float2 lhs, float2 rhs) {
 #define __rem(lhs, rhs) ((lhs) % (rhs))
 #define __mod(lhs, rhs) ((lhs) % (rhs))
 
-#define __pow(lhs, rhs) \
+#define __pow(lhs, rhs)                                                 \
     convert_int_rte(pow(convert_float_rte(lhs), convert_float_rte(rhs)))
+#ifdef USE_DOUBLE
 #define __powll(lhs, rhs) \
     convert_long_rte(pow(convert_double_rte(lhs), convert_double_rte(rhs)))
 #define __powul(lhs, rhs) \
     convert_ulong_rte(pow(convert_double_rte(lhs), convert_double_rte(rhs)))
+#else
+#define __powll(lhs, rhs) \
+    convert_long_rte(pow(convert_float_rte(lhs), convert_float_rte(rhs)))
+#define __powul(lhs, rhs) \
+    convert_ulong_rte(pow(convert_float_rte(lhs), convert_float_rte(rhs)))
+#endif
 
 #ifdef USE_DOUBLE
 #define __powui(lhs, rhs) \

From a07246e41497d3eb87df1efe8b40373caaaf4ebf Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 29 Nov 2022 02:13:15 -0500
Subject: [PATCH 173/473] Update cl2hpp tag and disable building cl2hpp if
 found on system

---
 CMakeModules/build_cl2hpp.cmake | 44 +++++++++++++++++----------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/CMakeModules/build_cl2hpp.cmake b/CMakeModules/build_cl2hpp.cmake
index 14c2646c2e..0a3fef2de0 100644
--- a/CMakeModules/build_cl2hpp.cmake
+++ b/CMakeModules/build_cl2hpp.cmake
@@ -13,28 +13,30 @@
 
 find_package(OpenCL)
 
-find_path(cl2hpp_header_file_path
-  NAMES CL/cl2.hpp
-  PATHS ${OpenCL_INCLUDE_PATHS})
-
-if(cl2hpp_header_file_path)
-  add_library(cl2hpp IMPORTED INTERFACE GLOBAL)
-  add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
-
-  set_target_properties(cl2hpp OpenCL::cl2hpp PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES ${cl2hpp_header_file_path})
-elseif (NOT TARGET OpenCL::cl2hpp OR NOT TARGET cl2hpp)
-  af_dep_check_and_populate(${cl2hpp_prefix}
-    URI https://github.com/KhronosGroup/OpenCL-CLHPP.git
-    REF v2.0.12)
-
-  find_path(cl2hpp_var
+if(NOT TARGET OpenCL::cl2hpp)
+  find_path(cl2hpp_header_file_path
     NAMES CL/cl2.hpp
-    PATHS ${ArrayFire_BINARY_DIR}/extern/${cl2hpp_prefix}-src/include)
+    PATHS ${OpenCL_INCLUDE_PATHS})
 
-  add_library(cl2hpp IMPORTED INTERFACE GLOBAL)
-  add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
+  if(cl2hpp_header_file_path)
+    add_library(cl2hpp IMPORTED INTERFACE GLOBAL)
+    add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
 
-  set_target_properties(cl2hpp OpenCL::cl2hpp PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES ${cl2hpp_var})
+    set_target_properties(cl2hpp OpenCL::cl2hpp PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES ${cl2hpp_header_file_path})
+  elseif (NOT TARGET OpenCL::cl2hpp OR NOT TARGET cl2hpp)
+    af_dep_check_and_populate(${cl2hpp_prefix}
+      URI https://github.com/KhronosGroup/OpenCL-CLHPP.git
+      REF v2022.09.30)
+
+    find_path(cl2hpp_var
+      NAMES CL/cl2.hpp
+      PATHS ${ArrayFire_BINARY_DIR}/extern/${cl2hpp_prefix}-src/include)
+
+    add_library(cl2hpp IMPORTED INTERFACE GLOBAL)
+    add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
+
+    set_target_properties(cl2hpp OpenCL::cl2hpp PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES ${cl2hpp_var})
+  endif()
 endif()

From 1e210f4da349e77bcf546a4083931353b5a03edc Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 30 Nov 2022 02:25:49 -0500
Subject: [PATCH 174/473] Update the minimum required OpenCL version to 3.0

---
 CMakeLists.txt                                |   2 +-
 CMakeModules/FindOpenCL.cmake                 | 101 ++++++++++-----
 src/backend/opencl/CMakeLists.txt             | 116 +++++++++++++++++-
 src/backend/opencl/compile_module.cpp         |  16 +--
 src/backend/opencl/device_manager.cpp         |  27 ++--
 src/backend/opencl/device_manager.hpp         |   3 +
 .../kernel/reduce_blocks_by_key_first.cl      |  12 +-
 src/backend/opencl/platform.cpp               |  35 ++++--
 src/backend/opencl/platform.hpp               |   6 +-
 9 files changed, 240 insertions(+), 78 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8985c797ff..2fb83beed9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -50,7 +50,7 @@ set(MKL_THREAD_LAYER "Intel OpenMP" CACHE STRING "The thread layer to choose for
 
 find_package(CUDA 10.2)
 find_package(cuDNN 4.0)
-find_package(OpenCL 1.2)
+find_package(OpenCL 3.0)
 find_package(OpenGL)
 find_package(glad CONFIG QUIET)
 find_package(FreeImage)
diff --git a/CMakeModules/FindOpenCL.cmake b/CMakeModules/FindOpenCL.cmake
index cdaeba20cc..3ac45a4a12 100644
--- a/CMakeModules/FindOpenCL.cmake
+++ b/CMakeModules/FindOpenCL.cmake
@@ -1,35 +1,43 @@
 # Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
 # file Copyright.txt or https://cmake.org/licensing for details.
 
-#.rst:
-# FindOpenCL
-# ----------
-#
-# Try to find OpenCL
-#
-# IMPORTED Targets
-# ^^^^^^^^^^^^^^^^
-#
-# This module defines :prop_tgt:`IMPORTED` target ``OpenCL::OpenCL``, if
-# OpenCL has been found.
-#
-# Result Variables
-# ^^^^^^^^^^^^^^^^
-#
-# This module defines the following variables::
-#
-#   OpenCL_FOUND          - True if OpenCL was found
-#   OpenCL_INCLUDE_DIRS   - include directories for OpenCL
-#   OpenCL_LIBRARIES      - link against this library to use OpenCL
-#   OpenCL_VERSION_STRING - Highest supported OpenCL version (eg. 1.2)
-#   OpenCL_VERSION_MAJOR  - The major version of the OpenCL implementation
-#   OpenCL_VERSION_MINOR  - The minor version of the OpenCL implementation
-#
-# The module will also define two cache variables::
-#
-#   OpenCL_INCLUDE_DIR    - the OpenCL include directory
-#   OpenCL_LIBRARY        - the path to the OpenCL library
-#
+#[=======================================================================[.rst:
+FindOpenCL
+----------
+
+.. versionadded:: 3.1
+
+Finds Open Computing Language (OpenCL)
+
+.. versionadded:: 3.10
+  Detection of OpenCL 2.1 and 2.2.
+
+IMPORTED Targets
+^^^^^^^^^^^^^^^^
+
+.. versionadded:: 3.7
+
+This module defines :prop_tgt:`IMPORTED` target ``OpenCL::OpenCL``, if
+OpenCL has been found.
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+This module defines the following variables::
+
+  OpenCL_FOUND          - True if OpenCL was found
+  OpenCL_INCLUDE_DIRS   - include directories for OpenCL
+  OpenCL_LIBRARIES      - link against this library to use OpenCL
+  OpenCL_VERSION_STRING - Highest supported OpenCL version (eg. 1.2)
+  OpenCL_VERSION_MAJOR  - The major version of the OpenCL implementation
+  OpenCL_VERSION_MINOR  - The minor version of the OpenCL implementation
+
+The module will also define two cache variables::
+
+  OpenCL_INCLUDE_DIR    - the OpenCL include directory
+  OpenCL_LIBRARY        - the path to the OpenCL library
+
+#]=======================================================================]
 
 function(_FIND_OPENCL_VERSION)
   include(CheckSymbolExists)
@@ -37,7 +45,7 @@ function(_FIND_OPENCL_VERSION)
   set(CMAKE_REQUIRED_QUIET ${OpenCL_FIND_QUIETLY})
 
   CMAKE_PUSH_CHECK_STATE()
-  foreach(VERSION "2_0" "1_2" "1_1" "1_0")
+  foreach(VERSION "3_0" "2_2" "2_1" "2_0" "1_2" "1_1" "1_0")
     set(CMAKE_REQUIRED_INCLUDES "${OpenCL_INCLUDE_DIR}")
 
     if(APPLE)
@@ -76,6 +84,9 @@ find_path(OpenCL_INCLUDE_DIR
     ENV NVSDKCOMPUTE_ROOT
     ENV CUDA_PATH
     ENV ATISTREAMSDKROOT
+    ENV OCL_ROOT
+    /usr/local/cuda
+    /opt/cuda
   PATH_SUFFIXES
     include
     OpenCL/common/inc
@@ -94,6 +105,7 @@ if(WIN32)
         ENV CUDA_PATH
         ENV NVSDKCOMPUTE_ROOT
         ENV ATISTREAMSDKROOT
+        ENV OCL_ROOT
       PATH_SUFFIXES
         "AMD APP/lib/x86"
         lib/x86
@@ -109,6 +121,7 @@ if(WIN32)
         ENV CUDA_PATH
         ENV NVSDKCOMPUTE_ROOT
         ENV ATISTREAMSDKROOT
+        ENV OCL_ROOT
       PATH_SUFFIXES
         "AMD APP/lib/x86_64"
         lib/x86_64
@@ -116,9 +129,31 @@ if(WIN32)
         OpenCL/common/lib/x64)
   endif()
 else()
-  find_library(OpenCL_LIBRARY
-    NAMES OpenCL
-    PATH_SUFFIXES lib64/)
+  if(CMAKE_SIZEOF_VOID_P EQUAL 4)
+    find_library(OpenCL_LIBRARY
+      NAMES OpenCL
+      PATHS
+        ENV AMDAPPSDKROOT
+        ENV CUDA_PATH
+        /usr/local/cuda
+        /opt/cuda
+      PATH_SUFFIXES
+        lib/x86
+        lib)
+  elseif(CMAKE_SIZEOF_VOID_P EQUAL 8)
+    find_library(OpenCL_LIBRARY
+      NAMES OpenCL
+      PATHS
+        ENV AMDAPPSDKROOT
+        ENV CUDA_PATH
+        /usr/local/cuda
+        /opt/cuda
+      PATH_SUFFIXES
+        lib/x86_64
+        lib/x64
+        lib
+        lib64)
+  endif()
 endif()
 
 set(OpenCL_LIBRARIES ${OpenCL_LIBRARY})
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index cf31204415..d82e00d7d5 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -18,7 +18,112 @@ generate_product_version(af_opencl_ver_res_file
   FILE_DESCRIPTION "OpenCL Backend Dynamic-link library"
 )
 
-file(GLOB kernel_src kernel/*.cl kernel/KParam.hpp)
+set(kernel_src
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/KParam.hpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/anisotropic_diffusion.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/approx1.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/approx2.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/assign.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/bilateral.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/convolve.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/convolve_separable.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/coo2dense.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/copy.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/cscmm.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/cscmv.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/csr2coo.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/csr2dense.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/csrmm.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/csrmv.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/dense2csr.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/diag_create.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/diag_extract.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/diff.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/example.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/fast.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/fftconvolve_multiply.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/fftconvolve_pack.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/fftconvolve_reorder.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/flood_fill.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/gradient.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/harris.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/histogram.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/homography.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/hsv_rgb.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/identity.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/iir.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/index.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/interp.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/iops.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/iota.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/ireduce_dim.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/ireduce_first.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/jit.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/laset_band.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/laset.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/laswp.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/lookup.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/lu_split.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/matchTemplate.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/mean_dim.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/mean_first.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/mean_ops.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/meanshift.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/medfilt1.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/medfilt2.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/memcopy.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/moments.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/morph.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/nearest_neighbour.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/nonmax_suppression.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/ops.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/orb.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/pad_array_borders.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/random_engine_mersenne.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/random_engine_mersenne_init.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/random_engine_philox.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/random_engine_threefry.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/random_engine_write.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/range.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_all.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_blocks_by_key_dim.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_blocks_by_key_first.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_by_key_boundary.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_by_key_boundary_dim.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_by_key_compact.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_by_key_compact_dim.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_by_key_needs_reduction.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_dim.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reduce_first.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/regions.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/reorder.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/resize.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/rotate.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/scan_dim_by_key.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/scan_dim.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/scan_first_by_key.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/scan_first.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/select.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/sift_nonfree.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/sobel.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/sparse_arith_common.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/sparse_arith_coo.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/sparse_arith_csr.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/sp_sp_arith_csr.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/ssarith_calc_out_nnz.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/susan.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/swapdblk.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/tile.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/trace_edge.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/transform.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/transpose.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/transpose_inplace.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/triangle.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/unwrap.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/where.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/wrap.cl
+  ${CMAKE_CURRENT_SOURCE_DIR}/kernel/wrap_dilated.cl
+)
 
 set( kernel_headers_dir "kernel_headers")
 
@@ -32,11 +137,10 @@ file_to_string(
     )
 
 set(opencl_compile_definitions
-        CL_TARGET_OPENCL_VERSION=120
-        CL_HPP_TARGET_OPENCL_VERSION=120
-        CL_HPP_MINIMUM_OPENCL_VERSION=120
-        CL_HPP_ENABLE_EXCEPTIONS
-        CL_USE_DEPRECATED_OPENCL_1_2_APIS)
+        CL_TARGET_OPENCL_VERSION=300
+        CL_HPP_TARGET_OPENCL_VERSION=300
+        CL_HPP_MINIMUM_OPENCL_VERSION=300
+        CL_HPP_ENABLE_EXCEPTIONS)
 
 include(kernel/scan_by_key/CMakeLists.txt)
 include(kernel/sort_by_key/CMakeLists.txt)
diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp
index 32ea5809f5..03fd41a196 100644
--- a/src/backend/opencl/compile_module.cpp
+++ b/src/backend/opencl/compile_module.cpp
@@ -108,15 +108,7 @@ Program buildProgram(span<const string> kernelSources,
                      span<const string> compileOpts) {
     Program retVal;
     try {
-        static const string defaults =
-            string(" -D dim_t=") + string(dtype_traits<dim_t>::getName());
-
         auto device = getDevice();
-
-        const string cl_std =
-            string(" -cl-std=CL") +
-            device.getInfo<CL_DEVICE_OPENCL_C_VERSION>().substr(9, 3);
-
         Program::Sources sources;
         sources.emplace_back(DEFAULT_MACROS_STR);
         sources.emplace_back(KParam_hpp, KParam_hpp_len);
@@ -126,12 +118,8 @@ Program buildProgram(span<const string> kernelSources,
 
         ostringstream options;
         for (auto &opt : compileOpts) { options << opt; }
-
-#ifdef AF_WITH_FAST_MATH
-        options << " -cl-fast-relaxed-math -DAF_WITH_FAST_MATH";
-#endif
-
-        retVal.build({device}, (cl_std + defaults + options.str()).c_str());
+        options << getActiveDeviceBaseBuildFlags();
+        retVal.build({device}, (options.str()).c_str());
     } catch (Error &err) {
         if (err.err() == CL_BUILD_PROGRAM_FAILURE) {
             THROW_BUILD_LOG_EXCEPTION(retVal);
diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index c1fa920a97..2befa70744 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -49,6 +49,8 @@ using std::begin;
 using std::end;
 using std::find;
 using std::make_unique;
+using std::ostringstream;
+using std::sort;
 using std::string;
 using std::stringstream;
 using std::unique_ptr;
@@ -99,13 +101,6 @@ static inline bool compare_default(const unique_ptr<Device>& ldev,
         if (!is_l_curr_type && is_r_curr_type) { return false; }
     }
 
-    // For GPUs, this ensures discrete > integrated
-    auto is_l_integrated = ldev->getInfo<CL_DEVICE_HOST_UNIFIED_MEMORY>();
-    auto is_r_integrated = rdev->getInfo<CL_DEVICE_HOST_UNIFIED_MEMORY>();
-
-    if (!is_l_integrated && is_r_integrated) { return true; }
-    if (is_l_integrated && !is_r_integrated) { return false; }
-
     // At this point, the devices are of same type.
     // Sort based on emperical evidence of preferred platforms
 
@@ -263,6 +258,24 @@ DeviceManager::DeviceManager()
             mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
             mPlatforms.push_back(getPlatformEnum(*devices[i]));
             mDevices.emplace_back(std::move(devices[i]));
+
+            auto device_versions =
+                mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
+            sort(begin(device_versions), end(device_versions),
+                 [](const auto& lhs, const auto& rhs) {
+                     return lhs.version < rhs.version;
+                 });
+            cl_name_version max_version = device_versions.back();
+            ostringstream options;
+            options << fmt::format(" -cl-std=CL{}.{}",
+                                   CL_VERSION_MAJOR(max_version.version),
+                                   CL_VERSION_MINOR(max_version.version))
+                    << fmt::format(" -D dim_t={}",
+                                   dtype_traits<dim_t>::getName());
+#ifdef AF_WITH_FAST_MATH
+            options << " -cl-fast-relaxed-math";
+#endif
+            mBaseBuildFlags.push_back(options.str());
         } catch (const cl::Error& err) {
             AF_TRACE("Error creating context for device {} with error {}\n",
                      devices[i]->getInfo<CL_DEVICE_NAME>(), err.what());
diff --git a/src/backend/opencl/device_manager.hpp b/src/backend/opencl/device_manager.hpp
index 8789675fe2..cce238533c 100644
--- a/src/backend/opencl/device_manager.hpp
+++ b/src/backend/opencl/device_manager.hpp
@@ -107,6 +107,8 @@ class DeviceManager {
 
     friend const cl::Device& getDevice(int id);
 
+    friend const std::string& getActiveDeviceBaseBuildFlags();
+
     friend size_t getDeviceMemorySize(int device);
 
     friend bool isGLSharingSupported();
@@ -161,6 +163,7 @@ class DeviceManager {
     std::vector<std::unique_ptr<cl::Context>> mContexts;
     std::vector<std::unique_ptr<cl::CommandQueue>> mQueues;
     std::vector<bool> mIsGLSharingOn;
+    std::vector<std::string> mBaseBuildFlags;
     std::vector<int> mDeviceTypes;
     std::vector<int> mPlatforms;
     unsigned mUserDeviceOffset;
diff --git a/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl b/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
index 5889288f82..e473244152 100644
--- a/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
+++ b/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
@@ -9,7 +9,7 @@
 
 // Starting from OpenCL 2.0, core profile includes work group level
 // inclusive scan operations, hence skip defining custom one
-#if __OPENCL_VERSION__ < 200
+#if !__opencl_c_work_group_collective_functions
 int work_group_scan_inclusive_add(local int *wg_temp, __local int *arr) {
     local int *active_buf;
 
@@ -29,7 +29,7 @@ int work_group_scan_inclusive_add(local int *wg_temp, __local int *arr) {
     int res = active_buf[lid];
     return res;
 }
-#endif  // __OPENCL_VERSION__ < 200
+#endif
 
 kernel void reduce_blocks_by_key_first(
     global int *reduced_block_sizes, __global Tk *oKeys, KParam oKInfo,
@@ -48,7 +48,7 @@ kernel void reduce_blocks_by_key_first(
     local Tk reduced_keys[DIMX];
     local To reduced_vals[DIMX];
     local int unique_ids[DIMX];
-#if __OPENCL_VERSION__ < 200
+#if !__opencl_c_work_group_collective_functions
     local int wg_temp[DIMX];
     local int unique_flags[DIMX];
 #endif
@@ -84,11 +84,11 @@ kernel void reduce_blocks_by_key_first(
     int eq_check    = (lid > 0) ? (k != reduced_keys[lid - 1]) : 0;
     int unique_flag = (eq_check || (lid == 0)) && (gid < n);
 
-#if __OPENCL_VERSION__ < 200
+#if __opencl_c_work_group_collective_functions
+    int unique_id = work_group_scan_inclusive_add(unique_flag);
+#else
     unique_flags[lid] = unique_flag;
     int unique_id     = work_group_scan_inclusive_add(wg_temp, unique_flags);
-#else
-    int unique_id = work_group_scan_inclusive_add(unique_flag);
 #endif
     unique_ids[lid] = unique_id;
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index c040c04b09..26476b2057 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -174,8 +174,6 @@ string getDeviceInfo() noexcept {
                             0
                         ? "True"
                         : "False");
-            info << " -- Unified Memory ("
-                 << (isHostUnifiedMemory(*device) ? "True" : "False") << ")";
 #endif
             info << endl;
 
@@ -297,6 +295,14 @@ const cl::Device& getDevice(int id) {
     return *(devMngr.mDevices[id]);
 }
 
+const std::string& getActiveDeviceBaseBuildFlags() {
+    device_id_t& devId     = tlocalActiveDeviceId();
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+    return devMngr.mBaseBuildFlags[get<1>(devId)];
+}
+
 size_t getDeviceMemorySize(int device) {
     DeviceManager& devMngr = DeviceManager::getInstance();
 
@@ -321,7 +327,7 @@ cl_device_type getDeviceType() {
 bool OpenCLCPUOffload(bool forceOffloadOSX) {
     static const bool offloadEnv = getEnvVar("AF_OPENCL_CPU_OFFLOAD") != "0";
     bool offload                 = false;
-    if (offloadEnv) { offload = isHostUnifiedMemory(getDevice()); }
+    if (offloadEnv) { offload = getDeviceType() == CL_DEVICE_TYPE_CPU; }
 #if OS_MAC
     // FORCED OFFLOAD FOR LAPACK FUNCTIONS ON OSX UNIFIED MEMORY DEVICES
     //
@@ -331,11 +337,9 @@ bool OpenCLCPUOffload(bool forceOffloadOSX) {
     // variable inconsequential to the returned result.
     //
     // Issue https://github.com/arrayfire/arrayfire/issues/662
-    //
-    // Make sure device has unified memory
-    bool osx_offload = isHostUnifiedMemory(getDevice());
     // Force condition
-    offload = osx_offload && (offload || forceOffloadOSX);
+    bool osx_offload = getDeviceType() == CL_DEVICE_TYPE_CPU;
+    offload          = osx_offload && (offload || forceOffloadOSX);
 #else
     UNUSED(forceOffloadOSX);
 #endif
@@ -475,6 +479,23 @@ void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que) {
         devMngr.mQueues.push_back(move(tQueue));
         nDevices = static_cast<int>(devMngr.mDevices.size()) - 1;
 
+        auto device_versions =
+            devMngr.mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
+        sort(begin(device_versions), end(device_versions),
+             [](const auto& lhs, const auto& rhs) {
+                 return lhs.version < rhs.version;
+             });
+        cl_name_version max_version = device_versions.back();
+        ostringstream options;
+        options << fmt::format(" -cl-std=CL{}.{}",
+                               CL_VERSION_MAJOR(max_version.version),
+                               CL_VERSION_MINOR(max_version.version))
+                << fmt::format(" -D dim_t={}", dtype_traits<dim_t>::getName());
+#ifdef AF_WITH_FAST_MATH
+        options << " -cl-fast-relaxed-math";
+#endif
+        devMngr.mBaseBuildFlags.push_back(options.str());
+
         // cache the boost program_cache object, clean up done on program exit
         // not during removeDeviceContext
         namespace compute = boost::compute;
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 07eca8f856..dba60388f7 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -67,6 +67,8 @@ cl::CommandQueue& getQueue();
 
 const cl::Device& getDevice(int id = -1);
 
+const std::string& getActiveDeviceBaseBuildFlags();
+
 size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();
@@ -108,10 +110,6 @@ inline unsigned getMaxParallelThreads(const cl::Device& device) {
 
 cl_device_type getDeviceType();
 
-inline bool isHostUnifiedMemory(const cl::Device& device) {
-    return device.getInfo<CL_DEVICE_HOST_UNIFIED_MEMORY>();
-}
-
 bool OpenCLCPUOffload(bool forceOffloadOSX = true);
 
 bool isGLSharingSupported();

From 61cd88345fdac94eadedf18962ffada8593d00b1 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 30 Nov 2022 03:43:16 -0500
Subject: [PATCH 175/473] Fix some errors due to pow in CUDA code with fast
 math

---
 .../cuda/kernel/anisotropic_diffusion.cuh     | 35 ++++++++++---------
 src/backend/cuda/kernel/jit.cuh               |  2 +-
 src/backend/cuda/kernel/susan.cuh             |  2 +-
 src/backend/cuda/math.hpp                     | 13 +++++++
 4 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/src/backend/cuda/kernel/anisotropic_diffusion.cuh b/src/backend/cuda/kernel/anisotropic_diffusion.cuh
index cd393474aa..8b108b434d 100644
--- a/src/backend/cuda/kernel/anisotropic_diffusion.cuh
+++ b/src/backend/cuda/kernel/anisotropic_diffusion.cuh
@@ -19,7 +19,8 @@ __forceinline__ __device__ int index(const int x, const int y, const int dim0,
     return clamp(x, 0, dim0 - 1) * stride0 + clamp(y, 0, dim1 - 1) * stride1;
 }
 
-__device__ float quadratic(const float value) { return 1.0 / (1.0 + value); }
+__device__
+float quadratic(const float value) { return 1.0f / (1.0f + value); }
 
 template<af_flux_function FluxEnum>
 __device__ float gradientUpdate(const float mct, const float C, const float S,
@@ -39,13 +40,13 @@ __device__ float gradientUpdate(const float mct, const float C, const float S,
     db = C - W;
 
     if (FluxEnum == AF_FLUX_EXPONENTIAL) {
-        cx  = expf((df * df + 0.25f * powf(dy + 0.5f * (SE - NE), 2)) * mct);
-        cxd = expf((db * db + 0.25f * powf(dy + 0.5f * (SW - NW), 2)) * mct);
+        cx  = expf((df * df + 0.25f * afpowf(dy + 0.5f * (SE - NE), 2)) * mct);
+        cxd = expf((db * db + 0.25f * afpowf(dy + 0.5f * (SW - NW), 2)) * mct);
     } else {
         cx =
-            quadratic((df * df + 0.25f * powf(dy + 0.5f * (SE - NE), 2)) * mct);
+            quadratic((df * df + 0.25f * afpowf(dy + 0.5f * (SE - NE), 2)) * mct);
         cxd =
-            quadratic((db * db + 0.25f * powf(dy + 0.5f * (SW - NW), 2)) * mct);
+            quadratic((db * db + 0.25f * afpowf(dy + 0.5f * (SW - NW), 2)) * mct);
     }
     delta += (cx * df - cxd * db);
 
@@ -54,13 +55,13 @@ __device__ float gradientUpdate(const float mct, const float C, const float S,
     db = C - N;
 
     if (FluxEnum == AF_FLUX_EXPONENTIAL) {
-        cx  = expf((df * df + 0.25f * powf(dx + 0.5f * (SE - SW), 2)) * mct);
-        cxd = expf((db * db + 0.25f * powf(dx + 0.5f * (NE - NW), 2)) * mct);
+        cx  = expf((df * df + 0.25f * afpowf(dx + 0.5f * (SE - SW), 2)) * mct);
+        cxd = expf((db * db + 0.25f * afpowf(dx + 0.5f * (NE - NW), 2)) * mct);
     } else {
         cx =
-            quadratic((df * df + 0.25f * powf(dx + 0.5f * (SE - SW), 2)) * mct);
+            quadratic((df * df + 0.25f * afpowf(dx + 0.5f * (SE - SW), 2)) * mct);
         cxd =
-            quadratic((db * db + 0.25f * powf(dx + 0.5f * (NE - NW), 2)) * mct);
+            quadratic((db * db + 0.25f * afpowf(dx + 0.5f * (NE - NW), 2)) * mct);
     }
     delta += (cx * df - cxd * db);
 
@@ -87,8 +88,8 @@ __device__ float curvatureUpdate(const float mct, const float C, const float S,
     df0 = df;
     db0 = db;
 
-    gmsqf = (df * df + 0.25f * powf(dy + 0.5f * (SE - NE), 2));
-    gmsqb = (db * db + 0.25f * powf(dy + 0.5f * (SW - NW), 2));
+    gmsqf = (df * df + 0.25f * afpowf(dy + 0.5f * (SE - NE), 2));
+    gmsqb = (db * db + 0.25f * afpowf(dy + 0.5f * (SW - NW), 2));
 
     gmf = sqrtf(1.0e-10 + gmsqf);
     gmb = sqrtf(1.0e-10 + gmsqb);
@@ -102,8 +103,8 @@ __device__ float curvatureUpdate(const float mct, const float C, const float S,
     df = S - C;
     db = C - N;
 
-    gmsqf = (df * df + 0.25f * powf(dx + 0.5f * (SE - SW), 2));
-    gmsqb = (db * db + 0.25f * powf(dx + 0.5f * (NE - NW), 2));
+    gmsqf = (df * df + 0.25f * afpowf(dx + 0.5f * (SE - SW), 2));
+    gmsqb = (db * db + 0.25f * afpowf(dx + 0.5f * (NE - NW), 2));
     gmf   = sqrtf(1.0e-10 + gmsqf);
     gmb   = sqrtf(1.0e-10 + gmsqb);
 
@@ -114,14 +115,14 @@ __device__ float curvatureUpdate(const float mct, const float C, const float S,
 
     if (delta > 0) {
         prop_grad +=
-            (powf(fminf(db0, 0.0f), 2.0f) + powf(fmaxf(df0, 0.0f), 2.0f));
+            (afpowf(fminf(db0, 0.0f), 2.0f) + afpowf(fmaxf(df0, 0.0f), 2.0f));
         prop_grad +=
-            (powf(fminf(db, 0.0f), 2.0f) + powf(fmaxf(df, 0.0f), 2.0f));
+            (afpowf(fminf(db, 0.0f), 2.0f) + afpowf(fmaxf(df, 0.0f), 2.0f));
     } else {
         prop_grad +=
-            (powf(fmaxf(db0, 0.0f), 2.0f) + powf(fminf(df0, 0.0f), 2.0f));
+            (afpowf(fmaxf(db0, 0.0f), 2.0f) + afpowf(fminf(df0, 0.0f), 2.0f));
         prop_grad +=
-            (powf(fmaxf(db, 0.0f), 2.0f) + powf(fminf(df, 0.0f), 2.0f));
+            (afpowf(fmaxf(db, 0.0f), 2.0f) + afpowf(fminf(df, 0.0f), 2.0f));
     }
 
     return sqrtf(prop_grad) * delta;
diff --git a/src/backend/cuda/kernel/jit.cuh b/src/backend/cuda/kernel/jit.cuh
index 3d66c02f24..cfb5837719 100644
--- a/src/backend/cuda/kernel/jit.cuh
+++ b/src/backend/cuda/kernel/jit.cuh
@@ -65,7 +65,7 @@ typedef cuDoubleComplex cdouble;
         pow(static_cast<double>(lhs), static_cast<double>(rhs)));
 #else
 #define __pow(lhs, rhs) \
-    __float2int_rn(pow(__int2float_rn((int)lhs), __int2float_rn((int)rhs)))
+    __float2int_rn(powf(__int2float_rn((int)lhs), __int2float_rn((int)rhs)))
 #endif
 #define __powll(lhs, rhs) \
     __double2ll_rn(pow(__ll2double_rn(lhs), __ll2double_rn(rhs)))
diff --git a/src/backend/cuda/kernel/susan.cuh b/src/backend/cuda/kernel/susan.cuh
index e2a706e000..5bb7f28805 100644
--- a/src/backend/cuda/kernel/susan.cuh
+++ b/src/backend/cuda/kernel/susan.cuh
@@ -73,7 +73,7 @@ __global__ void susan(T* out, const T* in, const unsigned idim0,
                 if (i * i + j * j < rSqrd) {
                     float c       = m_0;
                     float m       = shrdMem[b * shrdLen + a];
-                    float exp_pow = powf((m - c) / t, 6.0f);
+                    float exp_pow = afpowf((m - c) / t, 6.0f);
                     float cM      = expf(-exp_pow);
                     nM += cM;
                 }
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index f988372d27..3562565a86 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -392,6 +392,19 @@ template<typename T>
 constexpr const __DH__ T clamp(const T value, const T lo, const T hi) {
     return clamp(value, lo, hi, [](auto lhs, auto rhs) { return lhs < rhs; });
 }
+
+#ifdef AF_WITH_FAST_MATH
+/// The pow function with fast math is constantly wrong with fast math
+/// so this function converts the operation to double when fast-math
+/// is used
+__device__ inline double afpowf(double x, double y) { return pow(x, y); }
+#else
+/// The pow function with fast math is constantly wrong with fast math
+/// so this function converts the operation to double when fast-math
+/// is used
+__device__ inline float afpowf(float x, float y) { return powf(x, y); }
+#endif
+
 }  // namespace cuda
 }  // namespace arrayfire
 

From 49a73f3e6fa021548a403e2d5fe2a30433643ca0 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 30 Nov 2022 21:00:47 -0500
Subject: [PATCH 176/473] Add OpenCL version def to af/opencl.h. Remove
 FindOpenCL from test

---
 include/af/opencl.h                |   3 +
 test/CMakeModules/FindOpenCL.cmake | 190 -----------------------------
 2 files changed, 3 insertions(+), 190 deletions(-)
 delete mode 100644 test/CMakeModules/FindOpenCL.cmake

diff --git a/include/af/opencl.h b/include/af/opencl.h
index 27cc73e181..d055804d6d 100644
--- a/include/af/opencl.h
+++ b/include/af/opencl.h
@@ -8,6 +8,9 @@
  ********************************************************/
 
 #pragma once
+#ifndef CL_TARGET_OPENCL_VERSION
+#define CL_TARGET_OPENCL_VERSION 120
+#endif
 #if defined(__APPLE__) || defined(__MACOSX)
 #include <OpenCL/cl.h>
 #else
diff --git a/test/CMakeModules/FindOpenCL.cmake b/test/CMakeModules/FindOpenCL.cmake
deleted file mode 100644
index 4d4ef57bc3..0000000000
--- a/test/CMakeModules/FindOpenCL.cmake
+++ /dev/null
@@ -1,190 +0,0 @@
-#.rst:
-# FindOpenCL
-# ----------
-#
-# Try to find OpenCL
-#
-# Once done this will define::
-#
-#   OpenCL_FOUND          - True if OpenCL was found
-#   OpenCL_INCLUDE_DIRS   - include directories for OpenCL
-#   OpenCL_LIBRARIES      - link against this library to use OpenCL
-#   OpenCL_VERSION_STRING - Highest supported OpenCL version (eg. 1.2)
-#   OpenCL_VERSION_MAJOR  - The major version of the OpenCL implementation
-#   OpenCL_VERSION_MINOR  - The minor version of the OpenCL implementation
-#
-# The module will also define two cache variables::
-#
-#   OpenCL_INCLUDE_DIR    - the OpenCL include directory
-#   OpenCL_LIBRARY        - the path to the OpenCL library
-#
-
-#=============================================================================
-# From CMake 3.2
-# Copyright 2014 Matthaeus G. Chajdas
-#
-# Distributed under the OSI-approved BSD License (the "License");
-# see accompanying file Copyright.txt for details.
-#
-# This software is distributed WITHOUT ANY WARRANTY; without even the
-# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the License for more information.
-
-# CMake - Cross Platform Makefile Generator
-# Copyright 2000-2014 Kitware, Inc.
-# Copyright 2000-2011 Insight Software Consortium
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# * Neither the names of Kitware, Inc., the Insight Software Consortium,
-# nor the names of their contributors may be used to endorse or promote
-# products derived from this software without specific prior written
-# permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#=============================================================================
-
-function(_FIND_OPENCL_VERSION)
-  include(CheckSymbolExists)
-  include(CMakePushCheckState)
-  set(CMAKE_REQUIRED_QUIET ${OpenCL_FIND_QUIETLY})
-
-  CMAKE_PUSH_CHECK_STATE()
-  foreach(VERSION "2_0" "1_2" "1_1" "1_0")
-    set(CMAKE_REQUIRED_INCLUDES "${OpenCL_INCLUDE_DIR}")
-    if(APPLE)
-      CHECK_SYMBOL_EXISTS(
-        CL_VERSION_${VERSION}
-        "${OpenCL_INCLUDE_DIR}/OpenCL/cl.h"
-        OPENCL_VERSION_${VERSION})
-    else()
-      CHECK_SYMBOL_EXISTS(
-        CL_VERSION_${VERSION}
-        "${OpenCL_INCLUDE_DIR}/CL/cl.h"
-        OPENCL_VERSION_${VERSION})
-    endif()
-
-    if(OPENCL_VERSION_${VERSION})
-      string(REPLACE "_" "." VERSION "${VERSION}")
-      set(OpenCL_VERSION_STRING ${VERSION} PARENT_SCOPE)
-      string(REGEX MATCHALL "[0-9]+" version_components "${VERSION}")
-      list(GET version_components 0 major_version)
-      list(GET version_components 1 minor_version)
-      set(OpenCL_VERSION_MAJOR ${major_version} PARENT_SCOPE)
-      set(OpenCL_VERSION_MINOR ${minor_version} PARENT_SCOPE)
-      break()
-    endif()
-  endforeach()
-  CMAKE_POP_CHECK_STATE()
-endfunction()
-
-find_path(OpenCL_INCLUDE_DIR
-  NAMES
-    CL/cl.h OpenCL/cl.h
-  PATHS
-    ENV "PROGRAMFILES(X86)"
-    ENV NVSDKCOMPUTE_ROOT
-    ENV CUDA_PATH
-    ENV AMDAPPSDKROOT
-    ENV INTELOCLSDKROOT
-    ENV ATISTREAMSDKROOT
-  PATH_SUFFIXES
-    include
-    OpenCL/common/inc
-    "AMD APP/include")
-
-_FIND_OPENCL_VERSION()
-
-if(WIN32)
-  if(CMAKE_SIZEOF_VOID_P EQUAL 4)
-    find_library(OpenCL_LIBRARY
-      NAMES OpenCL
-      PATHS
-        ENV "PROGRAMFILES(X86)"
-        ENV CUDA_PATH
-        ENV NVSDKCOMPUTE_ROOT
-        ENV AMDAPPSDKROOT
-        ENV INTELOCLSDKROOT
-        ENV ATISTREAMSDKROOT
-      PATH_SUFFIXES
-        "AMD APP/lib/x86"
-        lib/x86
-        lib/Win32
-        OpenCL/common/lib/Win32)
-  elseif(CMAKE_SIZEOF_VOID_P EQUAL 8)
-    find_library(OpenCL_LIBRARY
-      NAMES OpenCL
-      PATHS
-        ENV "PROGRAMFILES(X86)"
-        ENV CUDA_PATH
-        ENV NVSDKCOMPUTE_ROOT
-        ENV AMDAPPSDKROOT
-        ENV INTELOCLSDKROOT
-        ENV ATISTREAMSDKROOT
-      PATH_SUFFIXES
-        "AMD APP/lib/x86_64"
-        lib/x86_64
-        lib/x64
-        OpenCL/common/lib/x64)
-  endif()
-else()
-  find_library(OpenCL_LIBRARY
-    NAMES OpenCL
-    PATHS
-        ENV LD_LIBRARY_PATH
-        ENV AMDAPPSDKROOT
-        ENV INTELOCLSDKROOT
-        ENV CUDA_PATH
-        ENV NVSDKCOMPUTE_ROOT
-        ENV ATISTREAMSDKROOT
-        /usr/lib64
-        /usr/lib
-        /usr/local/lib64
-        /usr/local/lib
-        /sw/lib
-        /opt/local/lib
-    PATH_SUFFIXES
-        "AMD APP/lib/x86_64"
-        lib/x86_64
-        lib/x64
-        lib/
-        lib64/
-        x86_64-linux-gnu
-        arm-linux-gnueabihf
-    )
-endif()
-
-set(OpenCL_LIBRARIES ${OpenCL_LIBRARY})
-set(OpenCL_INCLUDE_DIRS ${OpenCL_INCLUDE_DIR})
-
-#include(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake)
-find_package_handle_standard_args(
-  OpenCL
-  FOUND_VAR OpenCL_FOUND
-  REQUIRED_VARS OpenCL_LIBRARY OpenCL_INCLUDE_DIR
-  VERSION_VAR OpenCL_VERSION_STRING)
-
-mark_as_advanced(
-  OpenCL_INCLUDE_DIR
-  OpenCL_LIBRARY)
-

From 433348e598296018605f27ea5e43c31448d8f9f8 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 2 Dec 2022 18:37:50 -0500
Subject: [PATCH 177/473] Allow OpenCL C device version checks on older
 platforms

---
 CMakeLists.txt                           |  2 +-
 src/backend/opencl/Array.cpp             |  2 +-
 src/backend/opencl/CMakeLists.txt        |  2 +-
 src/backend/opencl/device_manager.cpp    | 36 +++++++++++------
 src/backend/opencl/device_manager.hpp    |  9 ++++-
 src/backend/opencl/kernel/flood_fill.hpp |  4 +-
 src/backend/opencl/magma/getrs.cpp       |  3 +-
 src/backend/opencl/platform.cpp          | 51 +++++++++++++++++++-----
 src/backend/opencl/platform.hpp          |  4 +-
 src/backend/opencl/solve.cpp             |  4 +-
 10 files changed, 85 insertions(+), 32 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2fb83beed9..8985c797ff 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -50,7 +50,7 @@ set(MKL_THREAD_LAYER "Intel OpenMP" CACHE STRING "The thread layer to choose for
 
 find_package(CUDA 10.2)
 find_package(cuDNN 4.0)
-find_package(OpenCL 3.0)
+find_package(OpenCL 1.2)
 find_package(OpenGL)
 find_package(glad CONFIG QUIET)
 find_package(FreeImage)
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 225e9686ac..811f5551e3 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -308,7 +308,7 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     }
 
     bool isBufferLimit = getMemoryPressure() >= getMemoryPressureThreshold();
-    auto platform      = getActivePlatform();
+    auto platform      = getActivePlatformVendor();
 
     // The Apple platform can have the nvidia card or the AMD card
     bool isIntel = platform == AFCL_PLATFORM_INTEL;
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index d82e00d7d5..d79cc95705 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -139,7 +139,7 @@ file_to_string(
 set(opencl_compile_definitions
         CL_TARGET_OPENCL_VERSION=300
         CL_HPP_TARGET_OPENCL_VERSION=300
-        CL_HPP_MINIMUM_OPENCL_VERSION=300
+        CL_HPP_MINIMUM_OPENCL_VERSION=110
         CL_HPP_ENABLE_EXCEPTIONS)
 
 include(kernel/scan_by_key/CMakeLists.txt)
diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index 2befa70744..69a0da4f2c 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -256,21 +256,33 @@ DeviceManager::DeviceManager()
                 *mContexts.back(), *devices[i], cl::QueueProperties::None));
             mIsGLSharingOn.push_back(false);
             mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
-            mPlatforms.push_back(getPlatformEnum(*devices[i]));
+            mPlatforms.push_back(
+                std::make_pair<std::unique_ptr<cl::Platform>, afcl_platform>(
+                    make_unique<cl::Platform>(device_platform, true),
+                    getPlatformEnum(*devices[i])));
             mDevices.emplace_back(std::move(devices[i]));
 
-            auto device_versions =
-                mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
-            sort(begin(device_versions), end(device_versions),
-                 [](const auto& lhs, const auto& rhs) {
-                     return lhs.version < rhs.version;
-                 });
-            cl_name_version max_version = device_versions.back();
+            auto platform_version =
+                mPlatforms.back().first->getInfo<CL_PLATFORM_VERSION>();
             ostringstream options;
-            options << fmt::format(" -cl-std=CL{}.{}",
-                                   CL_VERSION_MAJOR(max_version.version),
-                                   CL_VERSION_MINOR(max_version.version))
-                    << fmt::format(" -D dim_t={}",
+            if (platform_version.substr(7).c_str()[0] >= '3') {
+                auto device_versions =
+                    mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
+                sort(begin(device_versions), end(device_versions),
+                     [](const auto& lhs, const auto& rhs) {
+                         return lhs.version < rhs.version;
+                     });
+                cl_name_version max_version = device_versions.back();
+                options << fmt::format(" -cl-std=CL{}.{}",
+                                       CL_VERSION_MAJOR(max_version.version),
+                                       CL_VERSION_MINOR(max_version.version));
+            } else {
+                auto device_version =
+                    mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_VERSION>();
+                options << fmt::format(" -cl-std=CL{}",
+                                       device_version.substr(9, 3));
+            }
+            options << fmt::format(" -D dim_t={}",
                                    dtype_traits<dim_t>::getName());
 #ifdef AF_WITH_FAST_MATH
             options << " -cl-fast-relaxed-math";
diff --git a/src/backend/opencl/device_manager.hpp b/src/backend/opencl/device_manager.hpp
index cce238533c..4e06582da3 100644
--- a/src/backend/opencl/device_manager.hpp
+++ b/src/backend/opencl/device_manager.hpp
@@ -9,6 +9,8 @@
 
 #pragma once
 
+#include <af/opencl.h>
+
 #include <memory>
 #include <mutex>
 #include <string>
@@ -131,7 +133,9 @@ class DeviceManager {
 
     friend int getActiveDeviceType();
 
-    friend int getActivePlatform();
+    friend cl::Platform& getActivePlatform();
+
+    friend afcl::platform getActivePlatformVendor();
 
    public:
     static const int MAX_DEVICES = 32;
@@ -165,7 +169,8 @@ class DeviceManager {
     std::vector<bool> mIsGLSharingOn;
     std::vector<std::string> mBaseBuildFlags;
     std::vector<int> mDeviceTypes;
-    std::vector<int> mPlatforms;
+    std::vector<std::pair<std::unique_ptr<cl::Platform>, afcl::platform>>
+        mPlatforms;
     unsigned mUserDeviceOffset;
 
     std::unique_ptr<arrayfire::common::ForgeManager> fgMngr;
diff --git a/src/backend/opencl/kernel/flood_fill.hpp b/src/backend/opencl/kernel/flood_fill.hpp
index 0b0b29fefe..793ae5adcd 100644
--- a/src/backend/opencl/kernel/flood_fill.hpp
+++ b/src/backend/opencl/kernel/flood_fill.hpp
@@ -84,8 +84,8 @@ void floodFill(Param out, const Param image, const Param seedsx,
         DefineKeyValue(LMEM_WIDTH, (THREADS_X + 2 * RADIUS)),
         DefineKeyValue(LMEM_HEIGHT, (THREADS_Y + 2 * RADIUS)),
         DefineKeyValue(GROUP_SIZE, (THREADS_Y * THREADS_X)),
-        DefineKeyValue(AF_IS_PLATFORM_NVIDIA,
-                       (int)(AFCL_PLATFORM_NVIDIA == getActivePlatform())),
+        DefineKeyValue(AF_IS_PLATFORM_NVIDIA, (int)(AFCL_PLATFORM_NVIDIA ==
+                                                    getActivePlatformVendor())),
         getTypeBuildDefinition<T>()};
 
     auto floodStep =
diff --git a/src/backend/opencl/magma/getrs.cpp b/src/backend/opencl/magma/getrs.cpp
index a689408a26..d945fa9def 100644
--- a/src/backend/opencl/magma/getrs.cpp
+++ b/src/backend/opencl/magma/getrs.cpp
@@ -165,7 +165,8 @@ magma_int_t magma_getrs_gpu(magma_trans_t trans, magma_int_t n,
             : (trans == MagmaTrans ? OPENCL_BLAS_TRANS
                                    : OPENCL_BLAS_CONJ_TRANS);
 
-    bool cond  = arrayfire::opencl::getActivePlatform() == AFCL_PLATFORM_NVIDIA;
+    bool cond =
+        arrayfire::opencl::getActivePlatformVendor() == AFCL_PLATFORM_NVIDIA;
     cl_mem dAT = 0;
     if (nrhs > 1 && cond) {
         magma_malloc<Ty>(&dAT, n * n);
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 26476b2057..ee2f1b83c6 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -255,15 +255,26 @@ int getActiveDeviceType() {
     return devMngr.mDeviceTypes[get<1>(devId)];
 }
 
-int getActivePlatform() {
+cl::Platform& getActivePlatform() {
     device_id_t& devId = tlocalActiveDeviceId();
 
     DeviceManager& devMngr = DeviceManager::getInstance();
 
     common::lock_guard_t lock(devMngr.deviceMutex);
 
-    return devMngr.mPlatforms[get<1>(devId)];
+    return *devMngr.mPlatforms[get<1>(devId)].first;
 }
+
+afcl::platform getActivePlatformVendor() {
+    device_id_t& devId = tlocalActiveDeviceId();
+
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mPlatforms[get<1>(devId)].second;
+}
+
 const Context& getContext() {
     device_id_t& devId = tlocalActiveDeviceId();
 
@@ -468,12 +479,17 @@ void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que) {
         auto tQueue =
             (que == NULL ? make_unique<cl::CommandQueue>(*tContext, *tDevice)
                          : make_unique<cl::CommandQueue>(que, true));
-        devMngr.mPlatforms.push_back(getPlatformEnum(*tDevice));
         // FIXME: add OpenGL Interop for user provided contexts later
         devMngr.mIsGLSharingOn.push_back(false);
         devMngr.mDeviceTypes.push_back(
             static_cast<int>(tDevice->getInfo<CL_DEVICE_TYPE>()));
 
+        auto device_platform = tDevice->getInfo<CL_DEVICE_PLATFORM>();
+        devMngr.mPlatforms.push_back(
+            std::make_pair<std::unique_ptr<cl::Platform>, afcl_platform>(
+                make_unique<cl::Platform>(device_platform, true),
+                getPlatformEnum(*tDevice)));
+
         devMngr.mDevices.push_back(move(tDevice));
         devMngr.mContexts.push_back(move(tContext));
         devMngr.mQueues.push_back(move(tQueue));
@@ -485,12 +501,29 @@ void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que) {
              [](const auto& lhs, const auto& rhs) {
                  return lhs.version < rhs.version;
              });
-        cl_name_version max_version = device_versions.back();
+
+        auto platform_version =
+            devMngr.mPlatforms.back().first->getInfo<CL_PLATFORM_VERSION>();
         ostringstream options;
-        options << fmt::format(" -cl-std=CL{}.{}",
-                               CL_VERSION_MAJOR(max_version.version),
-                               CL_VERSION_MINOR(max_version.version))
-                << fmt::format(" -D dim_t={}", dtype_traits<dim_t>::getName());
+        if (platform_version.substr(7).c_str()[0] >= '3') {
+            auto device_versions =
+                devMngr.mDevices.back()
+                    ->getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
+            sort(begin(device_versions), end(device_versions),
+                 [](const auto& lhs, const auto& rhs) {
+                     return lhs.version < rhs.version;
+                 });
+            cl_name_version max_version = device_versions.back();
+            options << fmt::format(" -cl-std=CL{}.{}",
+                                   CL_VERSION_MAJOR(max_version.version),
+                                   CL_VERSION_MINOR(max_version.version));
+        } else {
+            auto device_version =
+                devMngr.mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_VERSION>();
+            options << fmt::format(" -cl-std=CL{}",
+                                   device_version.substr(9, 3));
+        }
+        options << fmt::format(" -D dim_t={}", dtype_traits<dim_t>::getName());
 #ifdef AF_WITH_FAST_MATH
         options << " -cl-fast-relaxed-math";
 #endif
@@ -706,7 +739,7 @@ af_err afcl_get_device_type(afcl_device_type* res) {
 
 af_err afcl_get_platform(afcl_platform* res) {
     try {
-        *res = static_cast<afcl_platform>(getActivePlatform());
+        *res = static_cast<afcl_platform>(getActivePlatformVendor());
     }
     CATCHALL;
     return AF_SUCCESS;
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index dba60388f7..c7099bf818 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -147,7 +147,9 @@ bool synchronize_calls();
 
 int getActiveDeviceType();
 
-int getActivePlatform();
+cl::Platform& getActivePlatform();
+
+afcl::platform getActivePlatformVendor();
 
 bool& evalFlag();
 
diff --git a/src/backend/opencl/solve.cpp b/src/backend/opencl/solve.cpp
index 60d8f3a59b..e6e7aa99ea 100644
--- a/src/backend/opencl/solve.cpp
+++ b/src/backend/opencl/solve.cpp
@@ -230,7 +230,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
                               A.strides()[1], 1, (*dT)(),
                               tmp.getOffset() + NB * MN, NB, 0, queue);
 
-        if (getActivePlatform() == AFCL_PLATFORM_NVIDIA) {
+        if (getActivePlatformVendor() == AFCL_PLATFORM_NVIDIA) {
             Array<T> AT    = transpose<T>(A, true);
             Buffer *AT_buf = AT.get();
             OPENCL_BLAS_CHECK(gpu_blas_trsm(
@@ -269,7 +269,7 @@ Array<T> triangleSolve(const Array<T> &A, const Array<T> &b,
     cl_event event         = 0;
     cl_command_queue queue = getQueue()();
 
-    if (getActivePlatform() == AFCL_PLATFORM_NVIDIA &&
+    if (getActivePlatformVendor() == AFCL_PLATFORM_NVIDIA &&
         (options & AF_MAT_UPPER)) {
         Array<T> AT = transpose<T>(A, true);
 

From dcffa51a89377029816adf70ce86163adc1bf98d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 Jan 2023 15:05:43 -0500
Subject: [PATCH 178/473] Fix Version formatting function

---
 src/backend/common/ArrayFireTypesIO.hpp | 10 ++--------
 src/backend/cuda/device_manager.cpp     |  2 +-
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/backend/common/ArrayFireTypesIO.hpp b/src/backend/common/ArrayFireTypesIO.hpp
index 81b73f9988..bf2585c92d 100644
--- a/src/backend/common/ArrayFireTypesIO.hpp
+++ b/src/backend/common/ArrayFireTypesIO.hpp
@@ -14,13 +14,10 @@
 
 template<>
 struct fmt::formatter<af_seq> {
-    // Parses format specifications of the form ['f' | 'e'].
     constexpr auto parse(format_parse_context& ctx) -> decltype(ctx.begin()) {
         return ctx.begin();
     }
 
-    // Formats the point p using the parsed format specification (presentation)
-    // stored in this formatter.
     template<typename FormatContext>
     auto format(const af_seq& p, FormatContext& ctx) -> decltype(ctx.out()) {
         // ctx.out() is an output iterator to write to.
@@ -61,16 +58,13 @@ struct fmt::formatter<arrayfire::common::Version> {
             }
             ++it;
         } while (it != end && *it != '}');
-        return ctx.begin();
+        return it;
     }
 
-    // Formats the point p using the parsed format specification (presentation)
-    // stored in this formatter.
     template<typename FormatContext>
     auto format(const arrayfire::common::Version& ver, FormatContext& ctx)
         -> decltype(ctx.out()) {
-        // ctx.out() is an output iterator to write to.
-        // if (ver.major == -1) return format_to(ctx.out(), "N/A");
+        if (ver.major == -1) return format_to(ctx.out(), "N/A");
         if (ver.minor == -1) show_minor = false;
         if (ver.patch == -1) show_patch = false;
         if (show_major && !show_minor && !show_patch) {
diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 4f0d534b8d..00d2e68ee3 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -498,7 +498,7 @@ void DeviceManager::checkCudaVsDriverVersion() {
     if (runtime > driver) {
         string msg =
             "ArrayFire was built with CUDA {} which requires GPU driver "
-            "version {Mm} or later. Please download and install the latest "
+            "version {} or later. Please download and install the latest "
             "drivers from https://www.nvidia.com/drivers for your GPU. "
             "Alternatively, you could rebuild ArrayFire with CUDA Toolkit "
             "version {} to use the current drivers.";

From e5b1047f58f8524a9371e6aa24da2541ffb8c64f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 Jan 2023 15:06:09 -0500
Subject: [PATCH 179/473] Update convolve tests tolerances for floating point
 types

---
 test/convolve.cpp |  2 +-
 test/reduce.cpp   | 92 +++++++++++++++++++++++++----------------------
 2 files changed, 50 insertions(+), 44 deletions(-)

diff --git a/test/convolve.cpp b/test/convolve.cpp
index 5fb61e7ee0..8adeb40fd8 100644
--- a/test/convolve.cpp
+++ b/test/convolve.cpp
@@ -898,7 +898,7 @@ float tolerance();
 
 template<>
 float tolerance<float>() {
-    return 1e-4;
+    return 2e-3;
 }
 
 template<>
diff --git a/test/reduce.cpp b/test/reduce.cpp
index ef5b33bb1c..c6cc0d7d72 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -2012,15 +2012,14 @@ vector<ragged_params *> genRaggedRangeTests() {
           ragged_range_data<unsigned, Tv, To>("ragged_range", 1024 * 1025, 3),
     };
 }
+// clang-format on
 
 vector<ragged_params *> generateAllTypesRagged() {
     vector<ragged_params *> out;
-    vector<vector<ragged_params *> > tmp{
-        genRaggedRangeTests<int, int>(),
-        genRaggedRangeTests<float, float>(),
+    vector<vector<ragged_params *>> tmp{
+        genRaggedRangeTests<int, int>(), genRaggedRangeTests<float, float>(),
         genRaggedRangeTests<double, double>(),
-        genRaggedRangeTests<half_float::half, half_float::half>()
-    };
+        genRaggedRangeTests<half_float::half, half_float::half>()};
 
     for (auto &v : tmp) { copy(begin(v), end(v), back_inserter(out)); }
     return out;
@@ -2032,7 +2031,7 @@ string testNameGeneratorRagged(
     af_dtype lt = info.param->lType_;
     af_dtype vt = info.param->vType_;
     size_t size = info.param->reduceDimLen_;
-    int rdim = info.param->reduceDim_;
+    int rdim    = info.param->reduceDim_;
     std::stringstream s;
     s << info.param->testname_ << "_lenType_" << lt << "_valueType_" << vt
       << "_size_" << size << "_reduceDim_" << rdim;
@@ -2040,8 +2039,8 @@ string testNameGeneratorRagged(
 }
 
 INSTANTIATE_TEST_SUITE_P(RaggedReduceTests, RaggedReduceMaxRangeP,
-                        ::testing::ValuesIn(generateAllTypesRagged()),
-                        testNameGeneratorRagged<RaggedReduceMaxRangeP>);
+                         ::testing::ValuesIn(generateAllTypesRagged()),
+                         testNameGeneratorRagged<RaggedReduceMaxRangeP>);
 
 TEST_P(RaggedReduceMaxRangeP, rangeMaxTest) {
     if (noHalfTests(GetParam()->vType_)) { return; }
@@ -2052,13 +2051,12 @@ TEST_P(RaggedReduceMaxRangeP, rangeMaxTest) {
 
     ASSERT_ARRAYS_EQ(valsReducedGold, ragged_max);
     ASSERT_ARRAYS_EQ(idxsReducedGold, idx);
-
 }
 
 TEST(ReduceByKey, ISSUE_2955) {
-    int N = 256;
-    af::array val = af::randu(N);
-    af::array key = af::range(af::dim4(N), 0, af::dtype::s32);
+    int N                  = 256;
+    af::array val          = af::randu(N);
+    af::array key          = af::range(af::dim4(N), 0, af::dtype::s32);
     key(seq(127, af::end)) = 1;
 
     af::array ok, ov;
@@ -2068,9 +2066,9 @@ TEST(ReduceByKey, ISSUE_2955) {
 }
 
 TEST(ReduceByKey, ISSUE_2955_dim) {
-    int N = 256;
-    af::array val = af::randu(8, N);
-    af::array key = af::range(af::dim4(N), 0, af::dtype::s32);
+    int N                  = 256;
+    af::array val          = af::randu(8, N);
+    af::array key          = af::range(af::dim4(N), 0, af::dtype::s32);
     key(seq(127, af::end)) = 1;
 
     af::array ok, ov;
@@ -2082,7 +2080,7 @@ TEST(ReduceByKey, ISSUE_2955_dim) {
 TEST(ReduceByKey, ISSUE_3062) {
     size_t N = 129;
 
-    af::array ones = af::constant(1, N, u32);
+    af::array ones  = af::constant(1, N, u32);
     af::array zeros = af::constant(0, N, u32);
 
     af::array okeys;
@@ -2095,7 +2093,7 @@ TEST(ReduceByKey, ISSUE_3062) {
     ASSERT_EQ(ovalues.scalar<unsigned>(), 129);
 
     // test reduction on non-zero dimension as well
-    ones = af::constant(1, 2, N, u32);
+    ones  = af::constant(1, 2, N, u32);
     zeros = af::constant(0, N, u32);
 
     af::sumByKey(okeys, ovalues, zeros, ones, 1);
@@ -2109,15 +2107,16 @@ TEST(Reduce, Test_Sum_Global_Array) {
     const int num = 513;
     array a       = af::randn(num, 2, 33, 4);
 
-    float res          = af::sum<float>(a);
-    array full_reduce  = af::sum<af::array>(a);
+    float res         = af::sum<float>(a);
+    array full_reduce = af::sum<af::array>(a);
 
     float *h_a = a.host<float>();
     float gold = 0.f;
 
     for (int i = 0; i < a.elements(); i++) { gold += h_a[i]; }
 
-    float max_error = std::numeric_limits<float>::epsilon() * (float)a.elements();
+    float max_error =
+        std::numeric_limits<float>::epsilon() * (float)a.elements();
     ASSERT_NEAR(gold, res, max_error);
     ASSERT_NEAR(res, full_reduce.scalar<float>(), max_error);
     freeHost(h_a);
@@ -2127,15 +2126,16 @@ TEST(Reduce, Test_Product_Global_Array) {
     const int num = 512;
     array a       = 1 + (0.005 * af::randn(num, 2, 3, 4));
 
-    float res          = af::product<float>(a);
-    array full_reduce  = af::product<af::array>(a);
+    float res         = af::product<float>(a);
+    array full_reduce = af::product<af::array>(a);
 
     float *h_a = a.host<float>();
     float gold = 1.f;
 
     for (int i = 0; i < a.elements(); i++) { gold *= h_a[i]; }
 
-    float max_error = std::numeric_limits<float>::epsilon() * (float)a.elements();
+    float max_error =
+        std::numeric_limits<float>::epsilon() * (float)a.elements();
     ASSERT_NEAR(gold, res, max_error);
     ASSERT_NEAR(res, full_reduce.scalar<float>(), max_error);
     freeHost(h_a);
@@ -2149,7 +2149,7 @@ TEST(Reduce, Test_Count_Global_Array) {
     int res       = count<int>(b);
     array res_arr = count<af::array>(b);
     char *h_b     = b.host<char>();
-    unsigned gold      = 0;
+    unsigned gold = 0;
 
     for (int i = 0; i < a.elements(); i++) { gold += h_b[i]; }
 
@@ -2204,15 +2204,17 @@ TYPED_TEST(Reduce, Test_All_Global_Array) {
         TypeParam res = allTrue<TypeParam>(a);
         array res_arr = allTrue<array>(a);
         typed_assert_eq((TypeParam) true, res, false);
-        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(), false);
+        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(),
+                        false);
 
         h_vals[3] = false;
         a         = array(2, num / 2, &h_vals.front());
 
-        res = allTrue<TypeParam>(a);
+        res     = allTrue<TypeParam>(a);
         res_arr = allTrue<array>(a);
         typed_assert_eq((TypeParam) false, res, false);
-        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(), false);
+        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(),
+                        false);
     }
 
     // false value location test
@@ -2225,7 +2227,8 @@ TYPED_TEST(Reduce, Test_All_Global_Array) {
         TypeParam res = allTrue<TypeParam>(a);
         array res_arr = allTrue<array>(a);
         typed_assert_eq((TypeParam) false, res, false);
-        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(), false);
+        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(),
+                        false);
 
         h_vals[i] = true;
     }
@@ -2243,14 +2246,16 @@ TYPED_TEST(Reduce, Test_Any_Global_Array) {
         TypeParam res = anyTrue<TypeParam>(a);
         array res_arr = anyTrue<array>(a);
         typed_assert_eq((TypeParam) false, res, false);
-        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(), false);
+        typed_assert_eq((TypeParam) false, (TypeParam)res_arr.scalar<char>(),
+                        false);
 
         h_vals[3] = true;
         a         = array(2, num / 2, &h_vals.front());
 
-        res = anyTrue<TypeParam>(a);
+        res     = anyTrue<TypeParam>(a);
         res_arr = anyTrue<array>(a);
-        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(), false);
+        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(),
+                        false);
     }
 
     // true value location test
@@ -2263,25 +2268,25 @@ TYPED_TEST(Reduce, Test_Any_Global_Array) {
         TypeParam res = anyTrue<TypeParam>(a);
         array res_arr = anyTrue<array>(a);
         typed_assert_eq((TypeParam) true, res, false);
-        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(), false);
+        typed_assert_eq((TypeParam) true, (TypeParam)res_arr.scalar<char>(),
+                        false);
 
         h_vals[i] = false;
     }
 }
 
-
 TEST(Reduce, Test_Sum_Global_Array_nanval) {
     SKIP_IF_FAST_MATH_ENABLED();
     const int num = 100000;
-    array a = af::randn(num, 2, 34, 4);
+    array a       = af::randn(num, 2, 34, 4);
     a(1, 0, 0, 0) = NAN;
     a(0, 1, 0, 0) = NAN;
     a(0, 0, 1, 0) = NAN;
     a(0, 0, 0, 1) = NAN;
 
-    double nanval = 0.2;
-    float res          = af::sum<float>(a, nanval);
-    array full_reduce  = af::sum<af::array>(a, nanval);
+    double nanval     = 0.2;
+    float res         = af::sum<float>(a, nanval);
+    array full_reduce = af::sum<af::array>(a, nanval);
 
     float *h_a = a.host<float>();
     float gold = 0.f;
@@ -2289,7 +2294,8 @@ TEST(Reduce, Test_Sum_Global_Array_nanval) {
     for (int i = 0; i < a.elements(); i++) {
         gold += (isnan(h_a[i])) ? nanval : h_a[i];
     }
-    float max_error = std::numeric_limits<float>::epsilon() * (float)a.elements();
+    float max_error =
+        std::numeric_limits<float>::epsilon() * (float)a.elements();
     ASSERT_NEAR(gold, res, max_error);
     ASSERT_NEAR(res, full_reduce.scalar<float>(), max_error);
     freeHost(h_a);
@@ -2298,16 +2304,16 @@ TEST(Reduce, Test_Sum_Global_Array_nanval) {
 TEST(Reduce, nanval_issue_3255) {
     SKIP_IF_FAST_MATH_ENABLED();
     char *info_str;
-    af_array  ikeys, ivals, okeys, ovals;
+    af_array ikeys, ivals, okeys, ovals;
     dim_t dims[1] = {8};
 
-    int ikeys_src[8] = {0, 0,  1, 1, 1,  2, 2,  0};
+    int ikeys_src[8] = {0, 0, 1, 1, 1, 2, 2, 0};
     af_create_array(&ikeys, ikeys_src, 1, dims, u32);
 
     int i;
-    for (i=0; i<8; i++) {
-        double ivals_src[8] = {1, 2,  3, 4, 5,  6, 7,  8};
-        ivals_src[i] = NAN;
+    for (i = 0; i < 8; i++) {
+        double ivals_src[8] = {1, 2, 3, 4, 5, 6, 7, 8};
+        ivals_src[i]        = NAN;
         af_create_array(&ivals, ivals_src, 1, dims, f64);
 
         af_product_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0);

From dcbfb2dbe483edf7f318898454078b4ef6adccef Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 11 Jan 2023 15:24:54 -0500
Subject: [PATCH 180/473] Add support for building older OpenCL versions.

---
 src/backend/opencl/CMakeLists.txt     | 18 ++++--
 src/backend/opencl/device_manager.cpp | 34 ++++-------
 src/backend/opencl/platform.cpp       | 83 ++++++++++++++++-----------
 src/backend/opencl/platform.hpp       |  6 ++
 4 files changed, 80 insertions(+), 61 deletions(-)

diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index d79cc95705..5c694b632d 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -136,11 +136,19 @@ file_to_string(
     NAMESPACE "arrayfire opencl"
     )
 
-set(opencl_compile_definitions
-        CL_TARGET_OPENCL_VERSION=300
-        CL_HPP_TARGET_OPENCL_VERSION=300
-        CL_HPP_MINIMUM_OPENCL_VERSION=110
-        CL_HPP_ENABLE_EXCEPTIONS)
+if(OpenCL_VERSION_MAJOR LESS 3)
+  set(opencl_compile_definitions
+    CL_TARGET_OPENCL_VERSION=120
+    CL_HPP_TARGET_OPENCL_VERSION=120
+    CL_HPP_MINIMUM_OPENCL_VERSION=120
+    CL_HPP_ENABLE_EXCEPTIONS)
+else()
+  set(opencl_compile_definitions
+    CL_TARGET_OPENCL_VERSION=300
+    CL_HPP_TARGET_OPENCL_VERSION=300
+    CL_HPP_MINIMUM_OPENCL_VERSION=110
+    CL_HPP_ENABLE_EXCEPTIONS)
+endif()
 
 include(kernel/scan_by_key/CMakeLists.txt)
 include(kernel/sort_by_key/CMakeLists.txt)
diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index 69a0da4f2c..a8ca6e96c9 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -15,8 +15,10 @@
 #include <blas.hpp>
 #include <build_version.hpp>
 #include <clfft.hpp>
+#include <common/ArrayFireTypesIO.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
+#include <common/Version.hpp>
 #include <common/defines.hpp>
 #include <common/host_memory.hpp>
 #include <common/util.hpp>
@@ -264,30 +266,18 @@ DeviceManager::DeviceManager()
 
             auto platform_version =
                 mPlatforms.back().first->getInfo<CL_PLATFORM_VERSION>();
-            ostringstream options;
-            if (platform_version.substr(7).c_str()[0] >= '3') {
-                auto device_versions =
-                    mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
-                sort(begin(device_versions), end(device_versions),
-                     [](const auto& lhs, const auto& rhs) {
-                         return lhs.version < rhs.version;
-                     });
-                cl_name_version max_version = device_versions.back();
-                options << fmt::format(" -cl-std=CL{}.{}",
-                                       CL_VERSION_MAJOR(max_version.version),
-                                       CL_VERSION_MINOR(max_version.version));
-            } else {
-                auto device_version =
-                    mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_VERSION>();
-                options << fmt::format(" -cl-std=CL{}",
-                                       device_version.substr(9, 3));
-            }
-            options << fmt::format(" -D dim_t={}",
-                                   dtype_traits<dim_t>::getName());
+            string options;
+            common::Version version =
+                getOpenCLCDeviceVersion(*mDevices[i]).back();
 #ifdef AF_WITH_FAST_MATH
-            options << " -cl-fast-relaxed-math";
+            options = fmt::format(
+                " -cl-std=CL{:Mm} -D dim_t={} -cl-fast-relaxed-math", version,
+                dtype_traits<dim_t>::getName());
+#else
+            options = fmt::format(" -cl-std=CL{:Mm} -D dim_t={}", version,
+                                  dtype_traits<dim_t>::getName());
 #endif
-            mBaseBuildFlags.push_back(options.str());
+            mBaseBuildFlags.push_back(options);
         } catch (const cl::Error& err) {
             AF_TRACE("Error creating context for device {} with error {}\n",
                      devices[i]->getInfo<CL_DEVICE_NAME>(), err.what());
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index ee2f1b83c6..7e94cb0bde 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -15,8 +15,10 @@
 #include <blas.hpp>
 #include <build_version.hpp>
 #include <clfft.hpp>
+#include <common/ArrayFireTypesIO.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
+#include <common/Version.hpp>
 #include <common/host_memory.hpp>
 #include <common/util.hpp>
 #include <device_manager.hpp>
@@ -69,6 +71,7 @@ using std::vector;
 using arrayfire::common::getEnvVar;
 using arrayfire::common::ltrim;
 using arrayfire::common::MemoryManagerBase;
+using arrayfire::common::Version;
 using arrayfire::opencl::Allocator;
 using arrayfire::opencl::AllocatorPinned;
 
@@ -121,7 +124,7 @@ static string platformMap(string& platStr) {
     }
 }
 
-afcl::platform getPlatformEnum(cl::Device dev) {
+afcl::platform getPlatformEnum(Device dev) {
     string pname = getPlatformName(dev);
     if (verify_present(pname, "AMD"))
         return AFCL_PLATFORM_AMD;
@@ -188,7 +191,7 @@ string getDeviceInfo() noexcept {
     return info.str();
 }
 
-string getPlatformName(const cl::Device& device) {
+string getPlatformName(const Device& device) {
     const Platform platform(device.getInfo<CL_DEVICE_PLATFORM>());
     string platStr = platform.getInfo<CL_PLATFORM_NAME>();
     return platformMap(platStr);
@@ -295,7 +298,7 @@ CommandQueue& getQueue() {
     return *(devMngr.mQueues[get<1>(devId)]);
 }
 
-const cl::Device& getDevice(int id) {
+const Device& getDevice(int id) {
     device_id_t& devId = tlocalActiveDeviceId();
 
     if (id == -1) { id = get<1>(devId); }
@@ -314,6 +317,40 @@ const std::string& getActiveDeviceBaseBuildFlags() {
     return devMngr.mBaseBuildFlags[get<1>(devId)];
 }
 
+vector<Version> getOpenCLCDeviceVersion(const Device& device) {
+    Platform device_platform(device.getInfo<CL_DEVICE_PLATFORM>(), false);
+    auto platform_version = device_platform.getInfo<CL_PLATFORM_VERSION>();
+    vector<Version> out;
+
+    /// The ifdef allows us to support BUILDING ArrayFire with older versions of
+    /// OpenCL where as the if condition in the ifdef allows us to support older
+    /// versions of OpenCL at runtime
+#ifdef CL_DEVICE_OPENCL_C_ALL_VERSIONS
+    if (platform_version.substr(7).c_str()[0] >= '3') {
+        vector<cl_name_version> device_versions =
+            device.getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
+        sort(begin(device_versions), end(device_versions),
+             [](const auto& lhs, const auto& rhs) {
+                 return lhs.version < rhs.version;
+             });
+        transform(begin(device_versions), end(device_versions),
+                  std::back_inserter(out), [](const cl_name_version& version) {
+                      return Version(CL_VERSION_MAJOR(version.version),
+                                     CL_VERSION_MINOR(version.version),
+                                     CL_VERSION_PATCH(version.version));
+                  });
+    } else {
+#endif
+        auto device_version = device.getInfo<CL_DEVICE_OPENCL_C_VERSION>();
+        int major           = atoi(device_version.substr(9, 1).c_str());
+        int minor           = atoi(device_version.substr(11, 1).c_str());
+        out.emplace_back(major, minor);
+#ifdef CL_DEVICE_OPENCL_C_ALL_VERSIONS
+    }
+#endif
+    return out;
+}
+
 size_t getDeviceMemorySize(int device) {
     DeviceManager& devMngr = DeviceManager::getInstance();
 
@@ -495,39 +532,17 @@ void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que) {
         devMngr.mQueues.push_back(move(tQueue));
         nDevices = static_cast<int>(devMngr.mDevices.size()) - 1;
 
-        auto device_versions =
-            devMngr.mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
-        sort(begin(device_versions), end(device_versions),
-             [](const auto& lhs, const auto& rhs) {
-                 return lhs.version < rhs.version;
-             });
-
-        auto platform_version =
-            devMngr.mPlatforms.back().first->getInfo<CL_PLATFORM_VERSION>();
-        ostringstream options;
-        if (platform_version.substr(7).c_str()[0] >= '3') {
-            auto device_versions =
-                devMngr.mDevices.back()
-                    ->getInfo<CL_DEVICE_OPENCL_C_ALL_VERSIONS>();
-            sort(begin(device_versions), end(device_versions),
-                 [](const auto& lhs, const auto& rhs) {
-                     return lhs.version < rhs.version;
-                 });
-            cl_name_version max_version = device_versions.back();
-            options << fmt::format(" -cl-std=CL{}.{}",
-                                   CL_VERSION_MAJOR(max_version.version),
-                                   CL_VERSION_MINOR(max_version.version));
-        } else {
-            auto device_version =
-                devMngr.mDevices.back()->getInfo<CL_DEVICE_OPENCL_C_VERSION>();
-            options << fmt::format(" -cl-std=CL{}",
-                                   device_version.substr(9, 3));
-        }
-        options << fmt::format(" -D dim_t={}", dtype_traits<dim_t>::getName());
+        auto versions = getOpenCLCDeviceVersion(*(devMngr.mDevices.back()));
 #ifdef AF_WITH_FAST_MATH
-        options << " -cl-fast-relaxed-math";
+        std::string options =
+            fmt::format(" -cl-std=CL{:Mm} -D dim_t={} -cl-fast-relaxed-math",
+                        versions.back(), dtype_traits<dim_t>::getName());
+#else
+        std::string options =
+            fmt::format(" -cl-std=CL{:Mm} -D dim_t={}", versions.back(),
+                        dtype_traits<dim_t>::getName());
 #endif
-        devMngr.mBaseBuildFlags.push_back(options.str());
+        devMngr.mBaseBuildFlags.push_back(options);
 
         // cache the boost program_cache object, clean up done on program exit
         // not during removeDeviceContext
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index c7099bf818..050e44f8c3 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -35,6 +35,8 @@ namespace common {
 class ForgeManager;
 
 class MemoryManagerBase;
+
+class Version;
 }  // namespace common
 }  // namespace arrayfire
 
@@ -69,6 +71,10 @@ const cl::Device& getDevice(int id = -1);
 
 const std::string& getActiveDeviceBaseBuildFlags();
 
+/// Returns the set of all OpenCL C Versions the device supports. The values
+/// are sorted from oldest to latest.
+std::vector<common::Version> getOpenCLCDeviceVersion(const cl::Device& device);
+
 size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();

From 61e980582fdeb32734f33ba2f1cf52194d6e8f90 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 20 Jan 2023 00:41:14 -0500
Subject: [PATCH 181/473] Fix meanvar tests to avoid segfaults for unsupported
 types

---
 test/arrayfire_test.cpp | 13 +++++++++
 test/meanvar.cpp        | 65 ++++++++++++++++++++++++++++++-----------
 2 files changed, 61 insertions(+), 17 deletions(-)

diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index cf776b6e2b..2128f7fbd3 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -430,10 +430,23 @@ INSTANTIATE(unsigned char, unsigned char, float);
 INSTANTIATE(short, short, float);
 INSTANTIATE(unsigned short, unsigned short, float);
 INSTANTIATE(half_float::half, half_float::half, float);
+INSTANTIATE(half_float::half, half_float::half, double);
 
+INSTANTIATE(af_cdouble, af_cdouble, double);
 INSTANTIATE(double, af_cdouble, float);
 INSTANTIATE(float, af_cfloat, float);
 INSTANTIATE(half_float::half, uint, uint);
+INSTANTIATE(float, float, double);
+INSTANTIATE(int, float, double);
+INSTANTIATE(unsigned int, float, double);
+INSTANTIATE(short, float, double);
+INSTANTIATE(unsigned short, float, double);
+INSTANTIATE(char, float, double);
+INSTANTIATE(unsigned char, float, double);
+INSTANTIATE(long long, double, double);
+INSTANTIATE(unsigned long long, double, double);
+INSTANTIATE(af_cfloat, af_cfloat, double);
+INSTANTIATE(half_float::half, float, double);
 
 #undef INSTANTIATE
 
diff --git a/test/meanvar.cpp b/test/meanvar.cpp
index bd79c4015a..08e4702481 100644
--- a/test/meanvar.cpp
+++ b/test/meanvar.cpp
@@ -27,6 +27,7 @@ using std::string;
 using std::vector;
 
 af_err init_err = af_init();
+
 template<typename T>
 struct elseType {
     typedef typename cond_type<is_same_type<T, uintl>::value ||
@@ -59,8 +60,9 @@ struct meanvar_test {
     vector<outType<T>> variance_;
 
     meanvar_test(string description, af_array in, af_array weights,
-                 af_var_bias bias, int dim, vector<double> &&mean,
-                 vector<double> &&variance)
+                 af_var_bias bias, int dim,
+                 vector<typename varOutType<T>::type> &&mean,
+                 vector<typename varOutType<T>::type> &&variance)
         : test_description_(description)
         , in_(0)
         , weights_(0)
@@ -73,8 +75,21 @@ struct meanvar_test {
         for (auto &v : mean) mean_.push_back((outType<T>)v);
         for (auto &v : variance) variance_.push_back((outType<T>)v);
     }
-    meanvar_test()                                   = default;
-    meanvar_test(meanvar_test<T> &&other)            = default;
+
+    meanvar_test(std::string name)
+        : test_description_(name), in_(0), weights_(0) {}
+
+    meanvar_test(meanvar_test<T> &&other)
+        : test_description_(other.test_description_)
+        , in_(other.in_)
+        , weights_(other.weights_)
+        , bias_(other.bias_)
+        , dim_(other.dim_)
+        , mean_(other.mean_)
+        , variance_(other.variance_) {
+        other.in_      = 0;
+        other.weights_ = 0;
+    }
     meanvar_test &operator=(meanvar_test<T> &&other) = default;
     meanvar_test &operator=(meanvar_test<T> &other)  = delete;
 
@@ -86,7 +101,7 @@ struct meanvar_test {
         , dim_(other.dim_)
         , mean_(other.mean_)
         , variance_(other.variance_) {
-        af_retain_array(&in_, other.in_);
+        if (other.in_) af_retain_array(&in_, other.in_);
         if (other.weights_) { af_retain_array(&weights_, other.weights_); }
     }
 
@@ -109,6 +124,7 @@ class MeanVarTyped : public ::testing::TestWithParam<meanvar_test<T>> {
    public:
     void meanvar_test_function(const meanvar_test<T> &test) {
         SUPPORTED_TYPE_CHECK(T);
+        SUPPORTED_TYPE_CHECK(outType<T>);
         af_array mean, var;
 
         // Cast to the expected type
@@ -145,6 +161,7 @@ class MeanVarTyped : public ::testing::TestWithParam<meanvar_test<T>> {
 
     void meanvar_cpp_test_function(const meanvar_test<T> &test) {
         SUPPORTED_TYPE_CHECK(T);
+        SUPPORTED_TYPE_CHECK(outType<T>);
         array mean, var;
 
         // Cast to the expected type
@@ -188,19 +205,28 @@ template<typename T>
 meanvar_test<T> meanvar_test_gen(string name, int in_index, int weight_index,
                                  af_var_bias bias, int dim, int mean_index,
                                  int var_index, test_size size) {
+    if (noDoubleTests((af_dtype)af::dtype_traits<T>::af_type) ||
+        noDoubleTests((
+            af_dtype)af::dtype_traits<typename varOutType<T>::type>::af_type) ||
+        noHalfTests((af_dtype)af::dtype_traits<T>::af_type)) {
+        meanvar_test<T> out(name);
+        return out;
+    }
+
     vector<af_array> inputs;
-    vector<vector<double>> outputs;
+    vector<vector<typename varOutType<T>::type>> outputs;
     if (size == MEANVAR_SMALL) {
         vector<af::dim4> numDims_;
-        vector<vector<double>> in_;
-        vector<vector<double>> tests_;
-        readTests<double, typename varOutType<double>::type, double>(
+        vector<vector<T>> in_;
+        vector<vector<typename varOutType<T>::type>> tests_;
+        readTests<T, typename varOutType<T>::type, double>(
             TEST_DIR "/meanvar/meanvar.data", numDims_, in_, tests_);
 
         inputs.resize(in_.size());
         for (size_t i = 0; i < in_.size(); i++) {
             af_create_array(&inputs[i], &in_[i].front(), numDims_[i].ndims(),
-                            numDims_[i].get(), f64);
+                            numDims_[i].get(),
+                            (af_dtype)af::dtype_traits<T>::af_type);
         }
 
         outputs.resize(tests_.size());
@@ -219,21 +245,26 @@ meanvar_test<T> meanvar_test_gen(string name, int in_index, int weight_index,
             {50, 40, 1, 1}   // 5
         };
 
-        vector<double> large_(full_array_size);
+        vector<T> large_(full_array_size);
         for (size_t i = 0; i < large_.size(); i++) {
-            large_[i] = static_cast<double>(i);
+            large_[i] = static_cast<T>(i);
         }
 
         inputs.resize(dimensions.size());
         for (size_t i = 0; i < dimensions.size(); i++) {
             af_create_array(&inputs[i], &large_.front(), 4,
-                            dimensions[i].data(), f64);
+                            dimensions[i].data(),
+                            (af_dtype)af::dtype_traits<T>::af_type);
         }
 
-        outputs.push_back(vector<double>(1, 999.5));
-        outputs.push_back(vector<double>(1, 333500));
-        outputs.push_back({249.50, 749.50, 1249.50, 1749.50});
-        outputs.push_back(vector<double>(4, 20875));
+        outputs.push_back(
+            vector<typename varOutType<T>::type>(1, outType<T>(999.5)));
+        outputs.push_back(
+            vector<typename varOutType<T>::type>(1, outType<T>(333500)));
+        outputs.push_back({outType<T>(249.50), outType<T>(749.50),
+                           outType<T>(1249.50), outType<T>(1749.50)});
+        outputs.push_back(
+            vector<typename varOutType<T>::type>(4, outType<T>(20875)));
     }
     meanvar_test<T> out(name, inputs[in_index],
                         (weight_index == -1) ? empty : inputs[weight_index],

From d26d891633240e68b81fc9a75764345485591149 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 20 Jan 2023 00:56:28 -0500
Subject: [PATCH 182/473] Fix warnings related to Version class

---
 src/backend/common/ArrayFireTypesIO.hpp | 14 +++++++-------
 src/backend/common/DependencyModule.cpp |  9 +++++----
 src/backend/common/Version.hpp          | 25 +++++++++++++++----------
 src/backend/cuda/convolveNN.cpp         |  4 ++--
 src/backend/cuda/cudnn.cpp              |  4 ++--
 src/backend/cuda/cudnnModule.cpp        |  9 +++++----
 6 files changed, 36 insertions(+), 29 deletions(-)

diff --git a/src/backend/common/ArrayFireTypesIO.hpp b/src/backend/common/ArrayFireTypesIO.hpp
index bf2585c92d..8d36aa54c1 100644
--- a/src/backend/common/ArrayFireTypesIO.hpp
+++ b/src/backend/common/ArrayFireTypesIO.hpp
@@ -64,18 +64,18 @@ struct fmt::formatter<arrayfire::common::Version> {
     template<typename FormatContext>
     auto format(const arrayfire::common::Version& ver, FormatContext& ctx)
         -> decltype(ctx.out()) {
-        if (ver.major == -1) return format_to(ctx.out(), "N/A");
-        if (ver.minor == -1) show_minor = false;
-        if (ver.patch == -1) show_patch = false;
+        if (ver.major() == -1) return format_to(ctx.out(), "N/A");
+        if (ver.minor() == -1) show_minor = false;
+        if (ver.patch() == -1) show_patch = false;
         if (show_major && !show_minor && !show_patch) {
-            return format_to(ctx.out(), "{}", ver.major);
+            return format_to(ctx.out(), "{}", ver.major());
         }
         if (show_major && show_minor && !show_patch) {
-            return format_to(ctx.out(), "{}.{}", ver.major, ver.minor);
+            return format_to(ctx.out(), "{}.{}", ver.major(), ver.minor());
         }
         if (show_major && show_minor && show_patch) {
-            return format_to(ctx.out(), "{}.{}.{}", ver.major, ver.minor,
-                             ver.patch);
+            return format_to(ctx.out(), "{}.{}.{}", ver.major(), ver.minor(),
+                             ver.patch());
         }
         return ctx.out();
     }
diff --git a/src/backend/common/DependencyModule.cpp b/src/backend/common/DependencyModule.cpp
index d8552e450d..4ccb64bc9a 100644
--- a/src/backend/common/DependencyModule.cpp
+++ b/src/backend/common/DependencyModule.cpp
@@ -52,7 +52,7 @@ vector<string> libNames(const std::string& name, const string& suffix,
     UNUSED(suffix);
     const string noVerName = libraryPrefix + name + librarySuffix;
     if (ver != arrayfire::common::NullVersion) {
-        const string infix = "." + to_string(ver.major) + ".";
+        const string infix = "." + to_string(ver.major()) + ".";
         return {libraryPrefix + name + infix + librarySuffix, noVerName};
     } else {
         return {noVerName};
@@ -71,10 +71,11 @@ vector<string> libNames(const std::string& name, const string& suffix,
     UNUSED(suffix);
     const string noVerName = libraryPrefix + name + librarySuffix;
     if (ver != arrayfire::common::NullVersion) {
-        const string soname("." + to_string(ver.major));
+        const string soname("." + to_string(ver.major()));
 
-        const string vsfx = "." + to_string(ver.major) + "." +
-                            to_string(ver.minor) + "." + to_string(ver.patch);
+        const string vsfx = "." + to_string(ver.major()) + "." +
+                            to_string(ver.minor()) + "." +
+                            to_string(ver.patch());
         return {noVerName + vsfx, noVerName + soname, noVerName};
     } else {
         return {noVerName};
diff --git a/src/backend/common/Version.hpp b/src/backend/common/Version.hpp
index 0b88444222..55a6e79efb 100644
--- a/src/backend/common/Version.hpp
+++ b/src/backend/common/Version.hpp
@@ -21,11 +21,12 @@
 
 namespace arrayfire {
 namespace common {
-struct Version {
-    int major = -1;
-    int minor = -1;
-    int patch = -1;
+class Version {
+    int major_ = -1;
+    int minor_ = -1;
+    int patch_ = -1;
 
+   public:
     /// Checks if the major version is defined before minor and minor is defined
     /// before patch
     constexpr static bool validate(int major_, int minor_,
@@ -34,14 +35,18 @@ struct Version {
                !(minor_ < 0 && patch_ >= 0);
     }
 
+    constexpr int major() const { return major_; }
+    constexpr int minor() const { return minor_; }
+    constexpr int patch() const { return patch_; }
+
     constexpr Version(const int ver_major, const int ver_minor = -1,
                       const int ver_patch = -1) noexcept
-        : major(ver_major), minor(ver_minor), patch(ver_patch) {}
+        : major_(ver_major), minor_(ver_minor), patch_(ver_patch) {}
 };
 
 constexpr bool operator==(const Version& lhs, const Version& rhs) {
-    return lhs.major == rhs.major && lhs.minor == rhs.minor &&
-           lhs.patch == rhs.patch;
+    return lhs.major() == rhs.major() && lhs.minor() == rhs.minor() &&
+           lhs.patch() == rhs.patch();
 }
 
 constexpr bool operator!=(const Version& lhs, const Version& rhs) {
@@ -52,11 +57,11 @@ constexpr static Version NullVersion{-1, -1, -1};
 
 constexpr bool operator<(const Version& lhs, const Version& rhs) {
     if (lhs == NullVersion || rhs == NullVersion) return false;
-    if (lhs.major != -1 && rhs.major != -1 && lhs.major < rhs.major)
+    if (lhs.major() != -1 && rhs.major() != -1 && lhs.major() < rhs.major())
         return true;
-    if (lhs.minor != -1 && rhs.minor != -1 && lhs.minor < rhs.minor)
+    if (lhs.minor() != -1 && rhs.minor() != -1 && lhs.minor() < rhs.minor())
         return true;
-    if (lhs.patch != -1 && rhs.patch != -1 && lhs.patch < rhs.patch)
+    if (lhs.patch() != -1 && rhs.patch() != -1 && lhs.patch() < rhs.patch())
         return true;
     return false;
 }
diff --git a/src/backend/cuda/convolveNN.cpp b/src/backend/cuda/convolveNN.cpp
index 4988d807f3..1110d81506 100644
--- a/src/backend/cuda/convolveNN.cpp
+++ b/src/backend/cuda/convolveNN.cpp
@@ -70,7 +70,7 @@ pair<cudnnConvolutionFwdAlgo_t, size_t> getForwardAlgorithm(
     size_t workspace_bytes = 0;
 
     auto version = getCudnnPlugin().getVersion();
-    if (version.major >= 8) {
+    if (version.major() >= 8) {
         int maxAlgoCount = 0;
         CUDNN_CHECK(cuda::cudnnGetConvolutionForwardAlgorithmMaxCount(
             cudnn, &maxAlgoCount));
@@ -419,7 +419,7 @@ pair<cudnnConvolutionBwdFilterAlgo_t, size_t> getBackwardFilterAlgorithm(
     size_t workspace_bytes = 0;
 
     auto version = getCudnnPlugin().getVersion();
-    if (version.major >= 8) {
+    if (version.major() >= 8) {
         int maxAlgoCount = 0;
         CUDNN_CHECK(cuda::cudnnGetConvolutionBackwardFilterAlgorithmMaxCount(
             cudnn, &maxAlgoCount));
diff --git a/src/backend/cuda/cudnn.cpp b/src/backend/cuda/cudnn.cpp
index b6fd903729..39ee3305e6 100644
--- a/src/backend/cuda/cudnn.cpp
+++ b/src/backend/cuda/cudnn.cpp
@@ -238,7 +238,7 @@ cudnnStatus_t cudnnGetConvolutionForwardAlgorithm(
     cudnnConvolutionFwdPreference_t preference, size_t memoryLimitInBytes,
     cudnnConvolutionFwdAlgo_t *algo) {
     auto version = getCudnnPlugin().getVersion();
-    if (version.major < 8) {
+    if (version.major() < 8) {
         return getCudnnPlugin().cudnnGetConvolutionForwardAlgorithm(
             handle, xDesc, wDesc, convDesc, yDesc, preference,
             memoryLimitInBytes, algo);
@@ -259,7 +259,7 @@ cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm(
     cudnnConvolutionBwdFilterPreference_t preference, size_t memoryLimitInBytes,
     cudnnConvolutionBwdFilterAlgo_t *algo) {
     auto version = getCudnnPlugin().getVersion();
-    if (version.major < 8) {
+    if (version.major() < 8) {
         return getCudnnPlugin().cudnnGetConvolutionBackwardFilterAlgorithm(
             handle, xDesc, dyDesc, convDesc, dwDesc, preference,
             memoryLimitInBytes, algo);
diff --git a/src/backend/cuda/cudnnModule.cpp b/src/backend/cuda/cudnnModule.cpp
index 657c867156..66c4b4ab06 100644
--- a/src/backend/cuda/cudnnModule.cpp
+++ b/src/backend/cuda/cudnnModule.cpp
@@ -111,12 +111,13 @@ cudnnModule::cudnnModule()
 
     // Check to see if the version of cuDNN ArrayFire was compiled against
     // is compatible with the version loaded at runtime
-    if (compiled_cudnn_version.major <= 6 &&
+    if (compiled_cudnn_version.major() <= 6 &&
         compiled_cudnn_version < cudnn_version) {
         string error_msg = fmt::format(
             "ArrayFire was compiled with an older version of cuDNN({}.{}) that "
             "does not support the version that was loaded at runtime({}.{}).",
-            CUDNN_MAJOR, CUDNN_MINOR, cudnn_version.major, cudnn_version.minor);
+            CUDNN_MAJOR, CUDNN_MINOR, cudnn_version.major(),
+            cudnn_version.minor());
         AF_ERROR(error_msg, AF_ERR_NOT_SUPPORTED);
     }
 
@@ -152,14 +153,14 @@ cudnnModule::cudnnModule()
     MODULE_FUNCTION_INIT(cudnnGetConvolutionBackwardFilterWorkspaceSize);
     MODULE_FUNCTION_INIT(cudnnFindConvolutionForwardAlgorithm);
     MODULE_FUNCTION_INIT(cudnnFindConvolutionBackwardFilterAlgorithm);
-    if (cudnn_version.major < 8) {
+    if (cudnn_version.major() < 8) {
         MODULE_FUNCTION_INIT(cudnnGetConvolutionForwardAlgorithm);
         MODULE_FUNCTION_INIT(cudnnGetConvolutionBackwardFilterAlgorithm);
     }
     MODULE_FUNCTION_INIT(cudnnGetConvolutionNdForwardOutputDim);
     MODULE_FUNCTION_INIT(cudnnSetConvolution2dDescriptor);
     MODULE_FUNCTION_INIT(cudnnSetFilter4dDescriptor);
-    if (cudnn_version.major == 4) {
+    if (cudnn_version.major() == 4) {
         MODULE_FUNCTION_INIT(cudnnSetFilter4dDescriptor_v4);
     }
     MODULE_FUNCTION_INIT(cudnnSetStream);

From 4e8e9389b9338bae7a07b35fe43680cb864cf76d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 20 Jan 2023 14:36:14 -0500
Subject: [PATCH 183/473] Update vcpkg baseline to update OpenCL version

---
 .github/workflows/win_cpu_build.yml                      | 2 +-
 CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake | 2 +-
 vcpkg.json                                               | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/win_cpu_build.yml b/.github/workflows/win_cpu_build.yml
index dc73cf7c28..8564bd03b8 100644
--- a/.github/workflows/win_cpu_build.yml
+++ b/.github/workflows/win_cpu_build.yml
@@ -13,7 +13,7 @@ jobs:
         name: CPU (fftw, OpenBLAS, windows-latest)
         runs-on: windows-latest
         env:
-          VCPKG_HASH: 6ca56aeb457f033d344a7106cb3f9f1abf8f4e98
+          VCPKG_HASH: f14984af3738e69f197bf0e647a8dca12de92996
           VCPKG_DEFAULT_TRIPLET: x64-windows
         steps:
             - name: Checkout Repository
diff --git a/CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake b/CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake
index ba8999d36e..f1a180065a 100644
--- a/CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake
+++ b/CMakeModules/vcpkg/ports/lapack-reference/portfile.cmake
@@ -68,7 +68,7 @@ vcpkg_cmake_configure(
     OPTIONS
         "-DUSE_OPTIMIZED_BLAS=${USE_OPTIMIZED_BLAS}"
         "-DCBLAS=${CBLAS}"
-	"-DLAPACKE=ON"
+        "-DLAPACKE=ON"
         ${FORTRAN_CMAKE}
 )
 
diff --git a/vcpkg.json b/vcpkg.json
index 4562e14f80..72625d8fa9 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -77,5 +77,5 @@
             ]
         }
     },
-    "builtin-baseline": "6ca56aeb457f033d344a7106cb3f9f1abf8f4e98"
+    "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996"
 }

From 3bd7991fccab353e639ed5f2a1b91ffe2c9c3691 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 20 Jan 2023 14:38:57 -0500
Subject: [PATCH 184/473] Add group flags around LAPACKE libraries to avoid
 missing symbol errs

---
 src/backend/opencl/CMakeLists.txt | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 5c694b632d..8a0e55d2e4 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -595,11 +595,19 @@ if(LAPACK_FOUND OR BUILD_WITH_MKL)
       SYSTEM PRIVATE
         ${CBLAS_INCLUDE_DIR})
 
+    check_cxx_compiler_flag("-Wl,--start-group -Werror" group_flags)
+    if(group_flags)
+      set(START_GROUP -Wl,--start-group)
+      set(END_GROUP -Wl,--end-group)
+    endif()
     target_link_libraries(afopencl
       PRIVATE
-        ${CBLAS_LIBRARIES}
+        ${START_GROUP}
         ${LAPACK_LIBRARIES}
-        LAPACKE::LAPACKE)
+        LAPACKE::LAPACKE
+        ${CBLAS_LIBRARIES}
+        ${END_GROUP}
+      )
   endif()
 
   target_compile_definitions(afopencl PRIVATE WITH_LINEAR_ALGEBRA)

From 727a7960e28275bee1bbd3ce95c546c4725921c2 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 20 Jan 2023 14:39:34 -0500
Subject: [PATCH 185/473] Fix extern half include directories command in cmake

---
 test/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 16ba6f71ec..dbd81ea6e7 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -177,7 +177,8 @@ function(make_test)
     target_include_directories(${target}
       PRIVATE
         ${CMAKE_SOURCE_DIR}
-        ${CMAKE_CURRENT_SOURCE_DIR}
+        ${CMAKE_CURRENT_SOURCE_DIR})
+    target_include_directories(${target}
       SYSTEM PRIVATE
         ${ArrayFire_SOURCE_DIR}/extern/half/include
       )

From 225a828fcd4e34076d61b1d20495d3acc9b9da8a Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 20 Jan 2023 16:08:23 -0500
Subject: [PATCH 186/473] Fix error due to an extra brace during the namespace
 refactor

---
 src/api/c/imageio.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/api/c/imageio.cpp b/src/api/c/imageio.cpp
index 41e713e631..be5f528922 100644
--- a/src/api/c/imageio.cpp
+++ b/src/api/c/imageio.cpp
@@ -1091,5 +1091,4 @@ af_err af_delete_image_memory(void *ptr) {
     AF_RETURN_ERROR("ArrayFire compiled without Image IO (FreeImage) support",
                     AF_ERR_NOT_CONFIGURED);
 }
-}  // namespace arrayfire
 #endif  // WITH_FREEIMAGE

From f9259985a144ec2b85af820ce12971f66da8a464 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 19 Dec 2022 21:40:04 -0500
Subject: [PATCH 187/473] use doxygen-awesome css theme

---
 docs/CMakeLists.txt                          |    3 +-
 docs/arrayfire.css                           |  196 --
 docs/doxygen-awesome-darkmode-toggle.js      |  157 ++
 docs/doxygen-awesome-fragment-copy-button.js |   85 +
 docs/doxygen-awesome-interactive-toc.js      |   81 +
 docs/doxygen-awesome-sidebar-only.css        |  115 +
 docs/doxygen-awesome.css                     | 2405 ++++++++++++++++++
 docs/doxygen.mk                              |  226 +-
 docs/header.htm                              |   74 +-
 9 files changed, 3063 insertions(+), 279 deletions(-)
 delete mode 100644 docs/arrayfire.css
 create mode 100644 docs/doxygen-awesome-darkmode-toggle.js
 create mode 100644 docs/doxygen-awesome-fragment-copy-button.js
 create mode 100644 docs/doxygen-awesome-interactive-toc.js
 create mode 100644 docs/doxygen-awesome-sidebar-only.css
 create mode 100644 docs/doxygen-awesome.css

diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt
index 1310b3c87b..93ba6615e8 100644
--- a/docs/CMakeLists.txt
+++ b/docs/CMakeLists.txt
@@ -39,10 +39,9 @@ configure_file(
     ${DOCS_DIR}/details/examples.dox
 )
 ###########################################################
-
 add_custom_target(docs
     ALL
-    COMMAND ${DOXYGEN_EXECUTABLE} ${AF_DOCS_CONFIG_OUT}
+    COMMAND Doxygen::doxygen ${AF_DOCS_CONFIG_OUT}
     COMMAND cmake -E copy_directory ${ASSETS_DIR} ${CMAKE_CURRENT_BINARY_DIR}/html
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     COMMENT "Generating Documentation"
diff --git a/docs/arrayfire.css b/docs/arrayfire.css
deleted file mode 100644
index 397e8089d5..0000000000
--- a/docs/arrayfire.css
+++ /dev/null
@@ -1,196 +0,0 @@
-/* The standard CSS for doxygen 1.8.5 */
-
-body, table, div, p, dl
-{
-    font            :   400 12px/22px Lucida Grande, Verdana, Geneva, Arial, sans-serif;
-}
-
-p
-{
-    padding-left    :   10px;
-}
-
-p code
-{
-    font-weight     :   bold;
-    background-color:   #F7F7F7;
-}
-
-/* @group Heading Levels */
-/* Increase the size of the page title */
-.title
-{
-    font-size       :   250%;
-}
-
-/* Remove space above line items */
-ul
-{
-    margin-top      :   0em;
-}
-
-/* Slightly pad subsections */
-h2, h3, h4, h5
-{
-    padding-left    :   10px;
-    margin-bottom   :   0px;
-}
-
-/* Margins on the left of the code */
-div.line
-{
-    margin-left :   15px;
-}
-
-a.code, a.code:visited, a.line, a.line:visited
-{
-    color       :   #4665A2;
-}
-
-a.codeRef, a.codeRef:visited, a.lineRef, a.lineRef:visited
-{
-    color       :   #4665A2;
-}
-
-/*image and image groups*/
-div.image_group
-{
-    text-align  :   center;
-}
-
-div.image_group > div
-{
-    display     :   inline-block;
-}
-
-div.scaled > img
-{
-    max-width   :   250px;
-}
-
-div.scaled > img:hover
-{
-    z-index             :   255; /* Hovered image to be shown on top of all */
-    background          :   #ffffff;
-    border              :   1px solid #000000;
-    -ms-transform       :   scale(2, 2);
-    -webkit-transform   :   scale(2, 2);
-    -moz-transform      :   scale(2, 2);
-    transform           :   scale(2, 2);
-}
-
-/*ArrayFire Feature Support Settings*/
-div.support
-{
-    text-align  :   right;
-}
-
-div.support *
-{
-    display     :   inline-block;
-    max-width   :   50px;
-}
-
-#under_logo
-{
-    font-size   :   2em;
-    max-width   :   25px;
-    color       :   #000000;
-}
-
-#projectbrief
-{
-    color       :   #555555
-}
-
-#projectlogo
-{
-    width       :   300px;
-    text-align  :   left;
-}
-
-#projectnumber
-{
-    max-width   :   25px;
-}
-
-#projectname
-{
-    font-size       :   3em;
-    max-width       :   25px;
-    color           :   #555555
-}
-
-#gsearch
-{
-    width       :   20%;
-}
-
-.tablist span
-{
-    font-weight     :   normal;
-    font-family     :   "Raleway","Helvetica Neue",Helvetica,sans-serif;
-    color           :   #FFFFFF;
-    text-shadow     :   none;
-}
-
-#side-nav {
-    height: 100%
-}
-
-#nav-tree
-{
-    background-color    : #F7F7F7;
-}
-
-div.toc
-{
-    background-color    : #F7F7F7;
-    border              : 1px solid #DFDFDF;
-}
-
-#nav-tree
-{
-    background-color    : #F7F7F7;
-}
-
-div.toc
-{
-    background-color    : #F7F7F7;
-    border              : 1px solid #DFDFDF;
-}
-
-.tablist a
-{
-    background-image:url('tab_b.png');
-}
-
-div.header
-{
-    background-image    :   none;
-    background-color    :   #F7F7F7;
-    border-bottom       :   1px solid #DFDFDF;
-}
-
-#nav-tree
-{
-    background-image    :   none;
-}
-
-.ui-resizable-e
-{
-    background  :   url("ftv2splitbar1.png") repeat scroll right center transparent;
-}
-
-div.fragment
-{
-    background-color    :   #F7F7F7;
-    border              :   1px solid #DFDFDF;
-}
-
-pre
-{
-    overflow            : hidden;
-}
-
-/* @end */
diff --git a/docs/doxygen-awesome-darkmode-toggle.js b/docs/doxygen-awesome-darkmode-toggle.js
new file mode 100644
index 0000000000..2032f02c0b
--- /dev/null
+++ b/docs/doxygen-awesome-darkmode-toggle.js
@@ -0,0 +1,157 @@
+/**
+
+Doxygen Awesome
+https://github.com/jothepro/doxygen-awesome-css
+
+MIT License
+
+Copyright (c) 2021 - 2022 jothepro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+class DoxygenAwesomeDarkModeToggle extends HTMLElement {
+    // SVG icons from https://fonts.google.com/icons
+    // Licensed under the Apache 2.0 license:
+    // https://www.apache.org/licenses/LICENSE-2.0.html
+    static lightModeIcon = `<svg xmlns="http://www.w3.org/2000/svg" enable-background="new 0 0 24 24" height="24px" viewBox="0 0 24 24" width="24px" fill="#FCBF00"><rect fill="none" height="24" width="24"/><circle cx="12" cy="12" opacity=".3" r="3"/><path d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"/></svg>`
+    static darkModeIcon = `<svg xmlns="http://www.w3.org/2000/svg" enable-background="new 0 0 24 24" height="24px" viewBox="0 0 24 24" width="24px" fill="#FE9700"><rect fill="none" height="24" width="24"/><path d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27 C17.45,17.19,14.93,19,12,19c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z" opacity=".3"/><path d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"/></svg>`
+    static title = "Toggle Light/Dark Mode"
+
+    static prefersLightModeInDarkModeKey = "prefers-light-mode-in-dark-mode"
+    static prefersDarkModeInLightModeKey = "prefers-dark-mode-in-light-mode"
+
+    static _staticConstructor = function() {
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.userPreference)
+        // Update the color scheme when the browsers preference changes
+        // without user interaction on the website.
+        window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', event => {
+            DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
+        })
+        // Update the color scheme when the tab is made visible again.
+        // It is possible that the appearance was changed in another tab 
+        // while this tab was in the background.
+        document.addEventListener("visibilitychange", visibilityState => {
+            if (document.visibilityState === 'visible') {
+                DoxygenAwesomeDarkModeToggle.onSystemPreferenceChanged()
+            }
+        });
+    }()
+
+    static init() {
+        $(function() {
+            $(document).ready(function() {
+                const toggleButton = document.createElement('doxygen-awesome-dark-mode-toggle')
+                toggleButton.title = DoxygenAwesomeDarkModeToggle.title
+                toggleButton.updateIcon()
+
+                window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', event => {
+                    toggleButton.updateIcon()
+                })
+                document.addEventListener("visibilitychange", visibilityState => {
+                    if (document.visibilityState === 'visible') {
+                        toggleButton.updateIcon()
+                    }
+                });
+
+                $(document).ready(function(){
+                    document.getElementById("togglediv").parentNode.appendChild(toggleButton)
+                })
+                $(window).resize(function(){
+                    document.getElementById("togglediv").parentNode.appendChild(toggleButton)
+                })
+            })
+        })
+    }
+
+    constructor() {
+        super();
+        this.onclick=this.toggleDarkMode
+    }
+
+    /**
+     * @returns `true` for dark-mode, `false` for light-mode system preference
+     */
+    static get systemPreference() {
+        return window.matchMedia('(prefers-color-scheme: dark)').matches
+    }
+
+    /**
+     * @returns `true` for dark-mode, `false` for light-mode user preference
+     */
+    static get userPreference() {
+        return (!DoxygenAwesomeDarkModeToggle.systemPreference && localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)) || 
+        (DoxygenAwesomeDarkModeToggle.systemPreference && !localStorage.getItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey))
+    }
+
+    static set userPreference(userPreference) {
+        DoxygenAwesomeDarkModeToggle.darkModeEnabled = userPreference
+        if(!userPreference) {
+            if(DoxygenAwesomeDarkModeToggle.systemPreference) {
+                localStorage.setItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey, true)
+            } else {
+                localStorage.removeItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey)
+            }
+        } else {
+            if(!DoxygenAwesomeDarkModeToggle.systemPreference) {
+                localStorage.setItem(DoxygenAwesomeDarkModeToggle.prefersDarkModeInLightModeKey, true)
+            } else {
+                localStorage.removeItem(DoxygenAwesomeDarkModeToggle.prefersLightModeInDarkModeKey)
+            }
+        }
+        DoxygenAwesomeDarkModeToggle.onUserPreferenceChanged()
+    }
+
+    static enableDarkMode(enable) {
+        if(enable) {
+            DoxygenAwesomeDarkModeToggle.darkModeEnabled = true
+            document.documentElement.classList.add("dark-mode")
+            document.documentElement.classList.remove("light-mode")
+        } else {
+            DoxygenAwesomeDarkModeToggle.darkModeEnabled = false
+            document.documentElement.classList.remove("dark-mode")
+            document.documentElement.classList.add("light-mode")
+        }
+    }
+
+    static onSystemPreferenceChanged() {
+        DoxygenAwesomeDarkModeToggle.darkModeEnabled = DoxygenAwesomeDarkModeToggle.userPreference
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.darkModeEnabled)
+    }
+
+    static onUserPreferenceChanged() {
+        DoxygenAwesomeDarkModeToggle.enableDarkMode(DoxygenAwesomeDarkModeToggle.darkModeEnabled)
+    }
+
+    toggleDarkMode() {
+        DoxygenAwesomeDarkModeToggle.userPreference = !DoxygenAwesomeDarkModeToggle.userPreference
+        this.updateIcon()
+    }
+
+    updateIcon() {
+        if(DoxygenAwesomeDarkModeToggle.darkModeEnabled) {
+            this.innerHTML = DoxygenAwesomeDarkModeToggle.darkModeIcon
+        } else {
+            this.innerHTML = DoxygenAwesomeDarkModeToggle.lightModeIcon
+        }
+    }
+}
+
+customElements.define("doxygen-awesome-dark-mode-toggle", DoxygenAwesomeDarkModeToggle);
diff --git a/docs/doxygen-awesome-fragment-copy-button.js b/docs/doxygen-awesome-fragment-copy-button.js
new file mode 100644
index 0000000000..7d06b348d6
--- /dev/null
+++ b/docs/doxygen-awesome-fragment-copy-button.js
@@ -0,0 +1,85 @@
+/**
+
+Doxygen Awesome
+https://github.com/jothepro/doxygen-awesome-css
+
+MIT License
+
+Copyright (c) 2022 jothepro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+class DoxygenAwesomeFragmentCopyButton extends HTMLElement {
+    constructor() {
+        super();
+        this.onclick=this.copyContent
+    }
+    static title = "Copy to clipboard"
+    static copyIcon = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24"><path d="M0 0h24v24H0V0z" fill="none"/><path d="M16 1H4c-1.1 0-2 .9-2 2v14h2V3h12V1zm3 4H8c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z"/></svg>`
+    static successIcon = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24"><path d="M0 0h24v24H0V0z" fill="none"/><path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41L9 16.17z"/></svg>`
+    static successDuration = 980
+    static init() {
+        $(function() {
+            $(document).ready(function() {
+                if(navigator.clipboard) {
+                    const fragments = document.getElementsByClassName("fragment")
+                    for(const fragment of fragments) {
+                        const fragmentWrapper = document.createElement("div")
+                        fragmentWrapper.className = "doxygen-awesome-fragment-wrapper"
+                        const fragmentCopyButton = document.createElement("doxygen-awesome-fragment-copy-button")
+                        fragmentCopyButton.innerHTML = DoxygenAwesomeFragmentCopyButton.copyIcon
+                        fragmentCopyButton.title = DoxygenAwesomeFragmentCopyButton.title
+                
+                        fragment.parentNode.replaceChild(fragmentWrapper, fragment)
+                        fragmentWrapper.appendChild(fragment)
+                        fragmentWrapper.appendChild(fragmentCopyButton)
+            
+                    }
+                }
+            })
+        })
+    }
+
+
+    copyContent() {
+        const content = this.previousSibling.cloneNode(true)
+        // filter out line number from file listings
+        content.querySelectorAll(".lineno, .ttc").forEach((node) => {
+            node.remove()
+        })
+        let textContent = content.textContent
+        // remove trailing newlines that appear in file listings
+        let numberOfTrailingNewlines = 0
+        while(textContent.charAt(textContent.length - (numberOfTrailingNewlines + 1)) == '\n') {
+            numberOfTrailingNewlines++;
+        }
+        textContent = textContent.substring(0, textContent.length - numberOfTrailingNewlines)
+        navigator.clipboard.writeText(textContent);
+        this.classList.add("success")
+        this.innerHTML = DoxygenAwesomeFragmentCopyButton.successIcon
+        window.setTimeout(() => {
+            this.classList.remove("success")
+            this.innerHTML = DoxygenAwesomeFragmentCopyButton.copyIcon
+        }, DoxygenAwesomeFragmentCopyButton.successDuration);
+    }
+}
+
+customElements.define("doxygen-awesome-fragment-copy-button", DoxygenAwesomeFragmentCopyButton)
diff --git a/docs/doxygen-awesome-interactive-toc.js b/docs/doxygen-awesome-interactive-toc.js
new file mode 100644
index 0000000000..b049f57331
--- /dev/null
+++ b/docs/doxygen-awesome-interactive-toc.js
@@ -0,0 +1,81 @@
+/**
+
+Doxygen Awesome
+https://github.com/jothepro/doxygen-awesome-css
+
+MIT License
+
+Copyright (c) 2022 jothepro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+class DoxygenAwesomeInteractiveToc {
+    static topOffset = 38
+    static hideMobileMenu = true
+    static headers = []
+
+    static init() {
+        window.addEventListener("load", () => {
+            let toc = document.querySelector(".contents > .toc")
+            if(toc) {
+                toc.classList.add("interactive")
+                if(!DoxygenAwesomeInteractiveToc.hideMobileMenu) {
+                    toc.classList.add("open")
+                }
+                document.querySelector(".contents > .toc > h3")?.addEventListener("click", () => {
+                    if(toc.classList.contains("open")) {
+                        toc.classList.remove("open")
+                    } else {
+                        toc.classList.add("open")
+                    }
+                })
+
+                document.querySelectorAll(".contents > .toc > ul a").forEach((node) => {
+                    let id = node.getAttribute("href").substring(1)
+                    DoxygenAwesomeInteractiveToc.headers.push({
+                        node: node,
+                        headerNode: document.getElementById(id)
+                    })
+
+                    document.getElementById("doc-content")?.addEventListener("scroll", () => {
+                        DoxygenAwesomeInteractiveToc.update()
+                    })
+                })
+                DoxygenAwesomeInteractiveToc.update()
+            }
+        })
+    }
+
+    static update() {
+        let active = DoxygenAwesomeInteractiveToc.headers[0]?.node
+        DoxygenAwesomeInteractiveToc.headers.forEach((header) => {
+            let position = header.headerNode.getBoundingClientRect().top
+            header.node.classList.remove("active")
+            header.node.classList.remove("aboveActive")
+            if(position < DoxygenAwesomeInteractiveToc.topOffset) {
+                active = header.node
+                active?.classList.add("aboveActive")
+            }
+        })
+        active?.classList.add("active")
+        active?.classList.remove("aboveActive")
+    }
+}
\ No newline at end of file
diff --git a/docs/doxygen-awesome-sidebar-only.css b/docs/doxygen-awesome-sidebar-only.css
new file mode 100644
index 0000000000..65e1a71fd2
--- /dev/null
+++ b/docs/doxygen-awesome-sidebar-only.css
@@ -0,0 +1,115 @@
+/**
+
+Doxygen Awesome
+https://github.com/jothepro/doxygen-awesome-css
+
+MIT License
+
+Copyright (c) 2021 jothepro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+ */
+
+html {
+    /* side nav width. MUST be = `TREEVIEW_WIDTH`.
+     * Make sure it is wide enough to contain the page title (logo + title + version)
+     */
+    --side-nav-fixed-width: 335px;
+    --menu-display: none;
+
+    --top-height: 170px;
+    --toc-sticky-top: -25px;
+    --toc-max-height: calc(100vh - 2 * var(--spacing-medium) - 25px);
+}
+
+#projectname {
+    white-space: nowrap;
+}
+
+
+@media screen and (min-width: 768px) {
+    html {
+        --searchbar-background: var(--page-background-color);
+    }
+
+    #side-nav {
+        min-width: var(--side-nav-fixed-width);
+        max-width: var(--side-nav-fixed-width);
+        top: var(--top-height);
+        overflow: visible;
+    }
+
+    #nav-tree, #side-nav {
+        height: calc(100vh - var(--top-height)) !important;
+    }
+
+    #nav-tree {
+        padding: 0;
+    }
+
+    #top {
+        display: block;
+        border-bottom: none;
+        height: var(--top-height);
+        margin-bottom: calc(0px - var(--top-height));
+        max-width: var(--side-nav-fixed-width);
+        overflow: hidden;
+        background: var(--side-nav-background);
+    }
+    #main-nav {
+        float: left;
+        padding-right: 0;
+    }
+
+    .ui-resizable-handle {
+        cursor: default;
+        width: 1px !important;
+        box-shadow: 0 calc(-2 * var(--top-height)) 0 0 var(--separator-color);
+    }
+
+    #nav-path {
+        position: fixed;
+        right: 0;
+        left: var(--side-nav-fixed-width);
+        bottom: 0;
+        width: auto;
+    }
+
+    #doc-content {
+        height: calc(100vh - 31px) !important;
+        padding-bottom: calc(3 * var(--spacing-large));
+        padding-top: calc(var(--top-height) - 80px);
+        box-sizing: border-box;
+        margin-left: var(--side-nav-fixed-width) !important;
+    }
+
+    #MSearchBox {
+        width: calc(var(--side-nav-fixed-width) - calc(2 * var(--spacing-medium)));
+    }
+
+    #MSearchField {
+        width: calc(var(--side-nav-fixed-width) - calc(2 * var(--spacing-medium)) - 65px);
+    }
+
+    #MSearchResultsWindow {
+        left: var(--spacing-medium) !important;
+        right: auto;
+    }
+}
diff --git a/docs/doxygen-awesome.css b/docs/doxygen-awesome.css
new file mode 100644
index 0000000000..e9a1553123
--- /dev/null
+++ b/docs/doxygen-awesome.css
@@ -0,0 +1,2405 @@
+/**
+
+Doxygen Awesome
+https://github.com/jothepro/doxygen-awesome-css
+
+MIT License
+
+Copyright (c) 2021 - 2022 jothepro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+html {
+    /* primary theme color. This will affect the entire websites color scheme: links, arrows, labels, ... */
+    --primary-color: #1779c4;
+    --primary-dark-color: #335c80;
+    --primary-light-color: #70b1e9;
+
+    /* page base colors */
+    --page-background-color: #ffffff;
+    --page-foreground-color: #2f4153;
+    --page-secondary-foreground-color: #6f7e8e;
+
+    /* color for all separators on the website: hr, borders, ... */
+    --separator-color: #dedede;
+
+    /* border radius for all rounded components. Will affect many components, like dropdowns, memitems, codeblocks, ... */
+    --border-radius-large: 6px;
+    --border-radius-small: 3px;
+    --border-radius-medium: 5px;
+
+    /* default spacings. Most components reference these values for spacing, to provide uniform spacing on the page. */
+    --spacing-small: 5px;
+    --spacing-medium: 8px;
+    --spacing-large: 10px;
+
+    /* default box shadow used for raising an element above the normal content. Used in dropdowns, search result, ... */
+    --box-shadow: 0 2px 8px 0 rgba(0,0,0,.075);
+
+    --odd-color: rgba(0,0,0,.028);
+
+    /* font-families. will affect all text on the website
+     * font-family: the normal font for text, headlines, menus
+     * font-family-monospace: used for preformatted text in memtitle, code, fragments
+     */
+    --font-family: -apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;
+    --font-family-monospace: ui-monospace,SFMono-Regular,SF Mono,Menlo,Consolas,Liberation Mono,monospace;
+
+    /* font sizes */
+    --page-font-size: 15.6px;
+    --navigation-font-size: 14.4px;
+    --toc-font-size: 13.4px;
+    --code-font-size: 14px; /* affects code, fragment */
+    --title-font-size: 22px;
+
+    /* content text properties. These only affect the page content, not the navigation or any other ui elements */
+    --content-line-height: 25px;
+    /* The content is centered and constraint in it's width. To make the content fill the whole page, set the variable to auto.*/
+    --content-maxwidth: 1050px;
+    --table-line-height: 24px;
+    --toc-sticky-top: var(--spacing-medium);
+    --toc-width: 200px;
+    --toc-max-height: calc(100vh - 2 * var(--spacing-medium) - 85px);
+
+    /* colors for various content boxes: @warning, @note, @deprecated @bug */
+    --warning-color: #f8d1cc;
+    --warning-color-dark: #b61825;
+    --warning-color-darker: #75070f;
+    --note-color: #faf3d8;
+    --note-color-dark: #f3a600;
+    --note-color-darker: #5f4204;
+    --todo-color: #e4f3ff;
+    --todo-color-dark: #1879C4;
+    --todo-color-darker: #274a5c;
+    --deprecated-color: #ecf0f3;
+    --deprecated-color-dark: #5b6269;
+    --deprecated-color-darker: #43454a;
+    --bug-color: #e4dafd;
+    --bug-color-dark: #5b2bdd;
+    --bug-color-darker: #2a0d72;
+    --invariant-color: #d8f1e3;
+    --invariant-color-dark: #44b86f;
+    --invariant-color-darker: #265532;
+
+    /* blockquote colors */
+    --blockquote-background: #f8f9fa;
+    --blockquote-foreground: #636568;
+
+    /* table colors */
+    --tablehead-background: #f1f1f1;
+    --tablehead-foreground: var(--page-foreground-color);
+
+    /* menu-display: block | none
+     * Visibility of the top navigation on screens >= 768px. On smaller screen the menu is always visible.
+     * `GENERATE_TREEVIEW` MUST be enabled!
+     */
+    --menu-display: block;
+
+    --menu-focus-foreground: var(--page-background-color);
+    --menu-focus-background: var(--primary-color);
+    --menu-selected-background: rgba(0,0,0,.05);
+
+
+    --header-background: var(--page-background-color);
+    --header-foreground: var(--page-foreground-color);
+
+    /* searchbar colors */
+    --searchbar-background: var(--side-nav-background);
+    --searchbar-foreground: var(--page-foreground-color);
+
+    /* searchbar size
+     * (`searchbar-width` is only applied on screens >= 768px.
+     * on smaller screens the searchbar will always fill the entire screen width) */
+    --searchbar-height: 33px;
+    --searchbar-width: 210px;
+    --searchbar-border-radius: var(--searchbar-height);
+
+    /* code block colors */
+    --code-background: #f5f5f5;
+    --code-foreground: var(--page-foreground-color);
+
+    /* fragment colors */
+    --fragment-background: #F8F9FA;
+    --fragment-foreground: #37474F;
+    --fragment-keyword: #bb6bb2;
+    --fragment-keywordtype: #8258b3;
+    --fragment-keywordflow: #d67c3b;
+    --fragment-token: #438a59;
+    --fragment-comment: #969696;
+    --fragment-link: #5383d6;
+    --fragment-preprocessor: #46aaa5;
+    --fragment-linenumber-color: #797979;
+    --fragment-linenumber-background: #f4f4f5;
+    --fragment-linenumber-border: #e3e5e7;
+    --fragment-lineheight: 19px;
+
+    /* sidebar navigation (treeview) colors */
+    --side-nav-background: #fbfbfb;
+    --side-nav-foreground: var(--page-foreground-color);
+    --side-nav-arrow-opacity: 0;
+    --side-nav-arrow-hover-opacity: 0.9;
+
+    --toc-background: var(--side-nav-background);
+    --toc-foreground: var(--side-nav-foreground);
+
+    /* height of an item in any tree / collapsable table */
+    --tree-item-height: 27px;
+
+    --memname-font-size: var(--code-font-size);
+    --memtitle-font-size: 18px;
+
+    --webkit-scrollbar-size: 7px;
+    --webkit-scrollbar-padding: 4px;
+    --webkit-scrollbar-color: var(--separator-color);
+}
+
+@media screen and (max-width: 767px) {
+    html {
+        --page-font-size: 16px;
+        --navigation-font-size: 16px;
+        --toc-font-size: 15px;
+        --code-font-size: 15px; /* affects code, fragment */
+        --title-font-size: 22px;
+    }
+}
+
+@media (prefers-color-scheme: dark) {
+    html:not(.light-mode) {
+        color-scheme: dark;
+
+        --primary-color: #1982d2;
+        --primary-dark-color: #86a9c4;
+        --primary-light-color: #4779ac;
+
+        --box-shadow: 0 2px 8px 0 rgba(0,0,0,.35);
+
+        --odd-color: rgba(100,100,100,.06);
+
+        --menu-selected-background: rgba(0,0,0,.4);
+
+        --page-background-color: #1C1D1F;
+        --page-foreground-color: #d2dbde;
+        --page-secondary-foreground-color: #859399;
+        --separator-color: #38393b;
+        --side-nav-background: #252628;
+
+        --code-background: #2a2c2f;
+
+        --tablehead-background: #2a2c2f;
+    
+        --blockquote-background: #222325;
+        --blockquote-foreground: #7e8c92;
+
+        --warning-color: #2e1917;
+        --warning-color-dark: #ad2617;
+        --warning-color-darker: #f5b1aa;
+        --note-color: #3b2e04;
+        --note-color-dark: #f1b602;
+        --note-color-darker: #ceb670;
+        --todo-color: #163750;
+        --todo-color-dark: #1982D2;
+        --todo-color-darker: #dcf0fa;
+        --deprecated-color: #2e323b;
+        --deprecated-color-dark: #738396;
+        --deprecated-color-darker: #abb0bd;
+        --bug-color: #2a2536;
+        --bug-color-dark: #7661b3;
+        --bug-color-darker: #ae9ed6;
+        --invariant-color: #303a35;
+        --invariant-color-dark: #76ce96;
+        --invariant-color-darker: #cceed5;
+
+        --fragment-background: #282c34;
+        --fragment-foreground: #dbe4eb;
+        --fragment-keyword: #cc99cd;
+        --fragment-keywordtype: #ab99cd;
+        --fragment-keywordflow: #e08000;
+        --fragment-token: #7ec699;
+        --fragment-comment: #999999;
+        --fragment-link: #98c0e3;
+        --fragment-preprocessor: #65cabe;
+        --fragment-linenumber-color: #cccccc;
+        --fragment-linenumber-background: #35393c;
+        --fragment-linenumber-border: #1f1f1f;
+    }
+}
+
+/* dark mode variables are defined twice, to support both the dark-mode without and with doxygen-awesome-darkmode-toggle.js */
+html.dark-mode {
+    color-scheme: dark;
+
+    --primary-color: #1982d2;
+    --primary-dark-color: #86a9c4;
+    --primary-light-color: #4779ac;
+
+    --box-shadow: 0 2px 8px 0 rgba(0,0,0,.30);
+
+    --odd-color: rgba(100,100,100,.06);
+
+    --menu-selected-background: rgba(0,0,0,.4);
+
+    --page-background-color: #1C1D1F;
+    --page-foreground-color: #d2dbde;
+    --page-secondary-foreground-color: #859399;
+    --separator-color: #38393b;
+    --side-nav-background: #252628;
+
+    --code-background: #2a2c2f;
+
+    --tablehead-background: #2a2c2f;
+
+    --blockquote-background: #222325;
+    --blockquote-foreground: #7e8c92;
+
+    --warning-color: #2e1917;
+    --warning-color-dark: #ad2617;
+    --warning-color-darker: #f5b1aa;
+    --note-color: #3b2e04;
+    --note-color-dark: #f1b602;
+    --note-color-darker: #ceb670;
+    --todo-color: #163750;
+    --todo-color-dark: #1982D2;
+    --todo-color-darker: #dcf0fa;
+    --deprecated-color: #2e323b;
+    --deprecated-color-dark: #738396;
+    --deprecated-color-darker: #abb0bd;
+    --bug-color: #2a2536;
+    --bug-color-dark: #7661b3;
+    --bug-color-darker: #ae9ed6;
+    --invariant-color: #303a35;
+    --invariant-color-dark: #76ce96;
+    --invariant-color-darker: #cceed5;
+
+    --fragment-background: #282c34;
+    --fragment-foreground: #dbe4eb;
+    --fragment-keyword: #cc99cd;
+    --fragment-keywordtype: #ab99cd;
+    --fragment-keywordflow: #e08000;
+    --fragment-token: #7ec699;
+    --fragment-comment: #999999;
+    --fragment-link: #98c0e3;
+    --fragment-preprocessor: #65cabe;
+    --fragment-linenumber-color: #cccccc;
+    --fragment-linenumber-background: #35393c;
+    --fragment-linenumber-border: #1f1f1f;
+}
+
+body {
+    color: var(--page-foreground-color);
+    background-color: var(--page-background-color);
+    font-size: var(--page-font-size);
+}
+
+body, table, div, p, dl, #nav-tree .label, .title,
+.sm-dox a, .sm-dox a:hover, .sm-dox a:focus, #projectname,
+.SelectItem, #MSearchField, .navpath li.navelem a,
+.navpath li.navelem a:hover, p.reference, p.definition {
+    font-family: var(--font-family);
+}
+
+h1, h2, h3, h4, h5 {
+    margin-top: .9em;
+    font-weight: 600;
+    line-height: initial;
+}
+
+p, div, table, dl, p.reference, p.definition {
+    font-size: var(--page-font-size);
+}
+
+p.reference, p.definition {
+    color: var(--page-secondary-foreground-color);
+}
+
+a:link, a:visited, a:hover, a:focus, a:active {
+    color: var(--primary-color) !important;
+    font-weight: 500;
+}
+
+a.anchor {
+    scroll-margin-top: var(--spacing-large);
+    display: block;
+}
+
+/*
+ Title and top navigation
+ */
+
+#top {
+    background: var(--header-background);
+    border-bottom: 1px solid var(--separator-color);
+}
+
+@media screen and (min-width: 768px) {
+    #top {
+        display: flex;
+        flex-wrap: wrap;
+        justify-content: space-between;
+        align-items: center;
+    }
+}
+
+#main-nav {
+    flex-grow: 5;
+    padding: var(--spacing-small) var(--spacing-medium);
+}
+
+#titlearea {
+    width: auto;
+    padding: var(--spacing-medium) var(--spacing-large);
+    background: none;
+    color: var(--header-foreground);
+    border-bottom: none;
+}
+
+@media screen and (max-width: 767px) {
+    #titlearea {
+        padding-bottom: var(--spacing-small);
+    }
+}
+
+#titlearea table tbody tr {
+    height: auto !important;
+}
+
+#projectname {
+    font-size: var(--title-font-size);
+    font-weight: 600;
+}
+
+#projectnumber {
+    font-family: inherit;
+    font-size: 60%;
+}
+
+#projectbrief {
+    font-family: inherit;
+    font-size: 80%;
+}
+
+#projectlogo {
+    vertical-align: middle;
+}
+
+#projectlogo img {
+    max-height: calc(var(--title-font-size) * 2);
+    margin-right: var(--spacing-small);
+}
+
+.sm-dox, .tabs, .tabs2, .tabs3 {
+    background: none;
+    padding: 0;
+}
+
+.tabs, .tabs2, .tabs3 {
+    border-bottom: 1px solid var(--separator-color);
+    margin-bottom: -1px;
+}
+
+.main-menu-btn-icon, .main-menu-btn-icon:before, .main-menu-btn-icon:after {
+    background: var(--page-secondary-foreground-color);
+}
+
+@media screen and (max-width: 767px) {
+    .sm-dox a span.sub-arrow {
+        background: var(--code-background);
+    }
+
+    #main-menu a.has-submenu span.sub-arrow {
+        color: var(--page-secondary-foreground-color);
+        border-radius: var(--border-radius-medium);
+    }
+
+    #main-menu a.has-submenu:hover span.sub-arrow {
+        color: var(--page-foreground-color);
+    }
+}
+
+@media screen and (min-width: 768px) {
+    .sm-dox li, .tablist li {
+        display: var(--menu-display);
+    }
+
+    .sm-dox a span.sub-arrow {
+        border-color: var(--header-foreground) transparent transparent transparent;
+    }
+
+    .sm-dox a:hover span.sub-arrow {
+        border-color: var(--menu-focus-foreground) transparent transparent transparent;
+    }
+
+    .sm-dox ul a span.sub-arrow {
+        border-color: transparent transparent transparent var(--page-foreground-color);
+    }
+
+    .sm-dox ul a:hover span.sub-arrow {
+        border-color: transparent transparent transparent var(--menu-focus-foreground);
+    }
+}
+
+.sm-dox ul {
+    background: var(--page-background-color);
+    box-shadow: var(--box-shadow);
+    border: 1px solid var(--separator-color);
+    border-radius: var(--border-radius-medium) !important;
+    padding: var(--spacing-small);
+    animation: ease-out 150ms slideInMenu;
+}
+
+@keyframes slideInMenu {
+    from {
+        opacity: 0;
+        transform: translate(0px, -2px);
+    }
+
+    to {
+        opacity: 1;
+        transform: translate(0px, 0px);
+    }
+}
+
+.sm-dox ul a {
+    color: var(--page-foreground-color) !important;
+    background: var(--page-background-color);
+    font-size: var(--navigation-font-size);
+}
+
+.sm-dox>li>ul:after {
+    border-bottom-color: var(--page-background-color) !important;
+}
+
+.sm-dox>li>ul:before {
+    border-bottom-color: var(--separator-color) !important;
+}
+
+.sm-dox ul a:hover, .sm-dox ul a:active, .sm-dox ul a:focus {
+    font-size: var(--navigation-font-size) !important;
+    color: var(--menu-focus-foreground) !important;
+    text-shadow: none;
+    background-color: var(--menu-focus-background);
+    border-radius: var(--border-radius-small) !important;
+}
+
+.sm-dox a, .sm-dox a:focus, .tablist li, .tablist li a, .tablist li.current a {
+    text-shadow: none;
+    background: transparent;
+    background-image: none !important;
+    color: var(--header-foreground) !important;
+    font-weight: normal;
+    font-size: var(--navigation-font-size);
+    border-radius: var(--border-radius-small) !important;
+}
+
+.sm-dox a:focus {
+    outline: auto;
+}
+
+.sm-dox a:hover, .sm-dox a:active, .tablist li a:hover {
+    text-shadow: none;
+    font-weight: normal;
+    background: var(--menu-focus-background);
+    color: var(--menu-focus-foreground) !important;
+    border-radius: var(--border-radius-small) !important;
+    font-size: var(--navigation-font-size);
+}
+
+.tablist li.current {
+    border-radius: var(--border-radius-small);
+    background: var(--menu-selected-background);
+}
+
+.tablist li {
+    margin: var(--spacing-small) 0 var(--spacing-small) var(--spacing-small);
+}
+
+.tablist a {
+    padding: 0 var(--spacing-large);
+}
+
+
+/*
+ Search box
+ */
+
+#MSearchBox {
+    height: var(--searchbar-height);
+    background: var(--searchbar-background);
+    border-radius: var(--searchbar-border-radius);
+    border: 1px solid var(--separator-color);
+    overflow: hidden;
+    width: var(--searchbar-width);
+    position: relative;
+    box-shadow: none;
+    display: block;
+    margin-top: 0;
+}
+
+/* until Doxygen 1.9.4 */
+.left img#MSearchSelect {
+    left: 0;
+    user-select: none;
+    padding-left: 8px;
+}
+
+/* Doxygen 1.9.5 */
+.left span#MSearchSelect {
+    left: 0;
+    user-select: none;
+    margin-left: 8px;
+    padding: 0;
+}
+
+.left #MSearchSelect[src$=".png"] {
+    padding-left: 0
+}
+
+.SelectionMark {
+    user-select: none;
+}
+
+.tabs .left #MSearchSelect {
+    padding-left: 0;
+}
+
+.tabs #MSearchBox {
+    position: absolute;
+    right: var(--spacing-medium);
+}
+
+@media screen and (max-width: 767px) {
+    .tabs #MSearchBox {
+        position: relative;
+        right: 0;
+        margin-left: var(--spacing-medium);
+        margin-top: 0;
+    }
+}
+
+#MSearchSelectWindow, #MSearchResultsWindow {
+    z-index: 9999;
+}
+
+#MSearchBox.MSearchBoxActive {
+    border-color: var(--primary-color);
+    box-shadow: inset 0 0 0 1px var(--primary-color);
+}
+
+#main-menu > li:last-child {
+    margin-right: 0;
+}
+
+@media screen and (max-width: 767px) {
+    #main-menu > li:last-child {
+        height: 50px;
+    }
+}
+
+#MSearchField {
+    font-size: var(--navigation-font-size);
+    height: calc(var(--searchbar-height) - 2px);
+    background: transparent;
+    width: calc(var(--searchbar-width) - 64px);
+}
+
+.MSearchBoxActive #MSearchField {
+    color: var(--searchbar-foreground);
+}
+
+#MSearchSelect {
+    top: calc(calc(var(--searchbar-height) / 2) - 11px);
+}
+
+#MSearchBox span.left, #MSearchBox span.right {
+    background: none;
+    background-image: none;
+}
+
+#MSearchBox span.right {
+    padding-top: calc(calc(var(--searchbar-height) / 2) - 12px);
+    position: absolute;
+    right: var(--spacing-small);
+}
+
+.tabs #MSearchBox span.right {
+    top: calc(calc(var(--searchbar-height) / 2) - 12px);
+}
+
+@keyframes slideInSearchResults {
+    from {
+        opacity: 0;
+        transform: translate(0, 15px);
+    }
+
+    to {
+        opacity: 1;
+        transform: translate(0, 20px);
+    }
+}
+
+#MSearchResultsWindow {
+    left: auto !important;
+    right: var(--spacing-medium);
+    border-radius: var(--border-radius-large);
+    border: 1px solid var(--separator-color);
+    transform: translate(0, 20px);
+    box-shadow: var(--box-shadow);
+    animation: ease-out 280ms slideInSearchResults;
+    background: var(--page-background-color);
+}
+
+iframe#MSearchResults {
+    margin: 4px;
+}
+
+iframe {
+    color-scheme: normal;
+}
+
+@media (prefers-color-scheme: dark) {
+    html:not(.light-mode) iframe#MSearchResults {
+        filter: invert() hue-rotate(180deg);
+    }
+}
+
+html.dark-mode iframe#MSearchResults {
+    filter: invert() hue-rotate(180deg);
+}
+
+#MSearchResults .SRPage {
+    background-color: transparent;
+}
+
+#MSearchResults .SRPage .SREntry {
+    font-size: 10pt;
+    padding: var(--spacing-small) var(--spacing-medium);
+}
+
+#MSearchSelectWindow {
+    border: 1px solid var(--separator-color);
+    border-radius: var(--border-radius-medium);
+    box-shadow: var(--box-shadow);
+    background: var(--page-background-color);
+    padding-top: var(--spacing-small);
+    padding-bottom: var(--spacing-small);
+}
+
+#MSearchSelectWindow a.SelectItem {
+    font-size: var(--navigation-font-size);
+    line-height: var(--content-line-height);
+    margin: 0 var(--spacing-small);
+    border-radius: var(--border-radius-small);
+    color: var(--page-foreground-color) !important;
+    font-weight: normal;
+}
+
+#MSearchSelectWindow a.SelectItem:hover {
+    background: var(--menu-focus-background);
+    color: var(--menu-focus-foreground) !important;
+}
+
+@media screen and (max-width: 767px) {
+    #MSearchBox {
+        margin-top: var(--spacing-medium);
+        margin-bottom: var(--spacing-medium);
+        width: calc(100vw - 30px);
+    }
+
+    #main-menu > li:last-child {
+        float: none !important;
+    }
+
+    #MSearchField {
+        width: calc(100vw - 110px);
+    }
+
+    @keyframes slideInSearchResultsMobile {
+        from {
+            opacity: 0;
+            transform: translate(0, 15px);
+        }
+
+        to {
+            opacity: 1;
+            transform: translate(0, 20px);
+        }
+    }
+
+    #MSearchResultsWindow {
+        left: var(--spacing-medium) !important;
+        right: var(--spacing-medium);
+        overflow: auto;
+        transform: translate(0, 20px);
+        animation: ease-out 280ms slideInSearchResultsMobile;
+        width: auto !important;
+    }
+
+    /*
+     * Overwrites for fixing the searchbox on mobile in doxygen 1.9.2
+     */
+    label.main-menu-btn ~ #searchBoxPos1 {
+        top: 3px !important;
+        right: 6px !important;
+        left: 45px;
+        display: flex;
+    }
+
+    label.main-menu-btn ~ #searchBoxPos1 > #MSearchBox {
+        margin-top: 0;
+        margin-bottom: 0;
+        flex-grow: 2;
+        float: left;
+    }
+}
+
+/*
+ Tree view
+ */
+
+#side-nav {
+    padding: 0 !important;
+    background: var(--side-nav-background);
+}
+
+@media screen and (max-width: 767px) {
+    #side-nav {
+        display: none;
+    }
+
+    #doc-content {
+        margin-left: 0 !important;
+    }
+}
+
+#nav-tree {
+    background: transparent;
+}
+
+#nav-tree .label {
+    font-size: var(--navigation-font-size);
+}
+
+#nav-tree .item {
+    height: var(--tree-item-height);
+    line-height: var(--tree-item-height);
+}
+
+#nav-sync {
+    bottom: 12px;
+    right: 12px;
+    top: auto !important;
+    user-select: none;
+}
+
+#nav-tree .selected {
+    text-shadow: none;
+    background-image: none;
+    background-color: transparent;
+    position: relative;
+}
+
+#nav-tree .selected::after {
+    content: "";
+    position: absolute;
+    top: 1px;
+    bottom: 1px;
+    left: 0;
+    width: 4px;
+    border-radius: 0 var(--border-radius-small) var(--border-radius-small) 0;
+    background: var(--primary-color);
+}
+
+
+#nav-tree a {
+    color: var(--side-nav-foreground) !important;
+    font-weight: normal;
+}
+
+#nav-tree a:focus {
+    outline-style: auto;
+}
+
+#nav-tree .arrow {
+    opacity: var(--side-nav-arrow-opacity);
+}
+
+.arrow {
+    color: inherit;
+    cursor: pointer;
+    font-size: 45%;
+    vertical-align: middle;
+    margin-right: 2px;
+    font-family: serif;
+    height: auto;
+    text-align: right;
+}
+
+#nav-tree div.item:hover .arrow, #nav-tree a:focus .arrow {
+    opacity: var(--side-nav-arrow-hover-opacity);
+}
+
+#nav-tree .selected a {
+    color: var(--primary-color) !important;
+    font-weight: bolder;
+    font-weight: 600;
+}
+
+.ui-resizable-e {
+    background: var(--separator-color);
+    width: 1px;
+}
+
+/*
+ Contents
+ */
+
+div.header {
+    border-bottom: 1px solid var(--separator-color);
+    background-color: var(--page-background-color);
+    background-image: none;
+}
+
+@media screen and (min-width: 1000px) {
+    #doc-content > div > div.contents,
+    .PageDoc > div.contents {
+        display: flex;
+        flex-direction: row-reverse;
+        flex-wrap: nowrap;
+        align-items: flex-start;
+    }
+    
+    div.contents .textblock {
+        min-width: 200px;
+        flex-grow: 1;
+    }
+}
+
+div.contents, div.header .title, div.header .summary {
+    max-width: var(--content-maxwidth);
+}
+
+div.contents, div.header .title  {
+    line-height: initial;
+    margin: calc(var(--spacing-medium) + .2em) auto var(--spacing-medium) auto;
+}
+
+div.header .summary {
+    margin: var(--spacing-medium) auto 0 auto;
+}
+
+div.headertitle {
+    padding: 0;
+}
+
+div.header .title {
+    font-weight: 600;
+    font-size: 225%;
+    padding: var(--spacing-medium) var(--spacing-large);
+    word-break: break-word;
+}
+
+div.header .summary {
+    width: auto;
+    display: block;
+    float: none;
+    padding: 0 var(--spacing-large);
+}
+
+td.memSeparator {
+    border-color: var(--separator-color);
+}
+
+span.mlabel {
+    background: var(--primary-color);
+    border: none;
+    padding: 4px 9px;
+    border-radius: 12px;
+    margin-right: var(--spacing-medium);
+}
+
+span.mlabel:last-of-type {
+    margin-right: 2px;
+}
+
+div.contents {
+    padding: 0 var(--spacing-large);
+}
+
+div.contents p, div.contents li {
+    line-height: var(--content-line-height);
+}
+
+div.contents div.dyncontent {
+    margin: var(--spacing-medium) 0;
+}
+
+@media (prefers-color-scheme: dark) {
+    html:not(.light-mode) div.contents div.dyncontent img,
+    html:not(.light-mode) div.contents center img,
+    html:not(.light-mode) div.contents > table img,
+    html:not(.light-mode) div.contents div.dyncontent iframe,
+    html:not(.light-mode) div.contents center iframe,
+    html:not(.light-mode) div.contents table iframe {
+        filter: hue-rotate(180deg) invert();
+    }
+}
+
+html.dark-mode div.contents div.dyncontent img,
+html.dark-mode div.contents center img,
+html.dark-mode div.contents > table img,
+html.dark-mode div.contents div.dyncontent iframe,
+html.dark-mode div.contents center iframe,
+html.dark-mode div.contents table iframe {
+    filter: hue-rotate(180deg) invert();
+}
+
+h2.groupheader {
+    border-bottom: 0px;
+    color: var(--page-foreground-color);
+    box-shadow: 
+        100px 0 var(--page-background-color), 
+        -100px 0 var(--page-background-color),
+        100px 0.75px var(--separator-color),
+        -100px 0.75px var(--separator-color),
+        500px 0 var(--page-background-color), 
+        -500px 0 var(--page-background-color),
+        500px 0.75px var(--separator-color),
+        -500px 0.75px var(--separator-color),
+        900px 0 var(--page-background-color), 
+        -900px 0 var(--page-background-color),
+        900px 0.75px var(--separator-color),
+        -900px 0.75px var(--separator-color),
+        1400px 0 var(--page-background-color),
+        -1400px 0 var(--page-background-color), 
+        1400px 0.75px var(--separator-color),
+        -1400px 0.75px var(--separator-color),
+        1900px 0 var(--page-background-color),
+        -1900px 0 var(--page-background-color),
+        1900px 0.75px var(--separator-color),
+        -1900px 0.75px var(--separator-color);
+}
+
+blockquote {
+    margin: 0 var(--spacing-medium) 0 var(--spacing-medium);
+    padding: var(--spacing-small) var(--spacing-large);
+    background: var(--blockquote-background);
+    color: var(--blockquote-foreground);
+    border-left: 0;
+    overflow: visible;
+    border-radius: var(--border-radius-medium);
+    overflow: visible;
+    position: relative;
+}
+
+blockquote::before, blockquote::after {
+    font-weight: bold;
+    font-family: serif;
+    font-size: 360%;
+    opacity: .15;
+    position: absolute;
+}
+
+blockquote::before {
+    content: "“";
+    left: -10px;
+    top: 4px;
+}
+
+blockquote::after {
+    content: "”";
+    right: -8px;
+    bottom: -25px;
+}
+
+blockquote p {
+    margin: var(--spacing-small) 0 var(--spacing-medium) 0;
+}
+.paramname {
+    font-weight: 600;
+    color: var(--primary-dark-color);
+}
+
+.paramname > code {
+    border: 0;
+}
+
+table.params .paramname {
+    font-weight: 600;
+    font-family: var(--font-family-monospace);
+    font-size: var(--code-font-size);
+    padding-right: var(--spacing-small);
+    line-height: var(--table-line-height);
+}
+
+h1.glow, h2.glow, h3.glow, h4.glow, h5.glow, h6.glow {
+    text-shadow: 0 0 15px var(--primary-light-color);
+}
+
+.alphachar a {
+    color: var(--page-foreground-color);
+}
+
+/*
+ Table of Contents
+ */
+
+div.contents .toc {
+    max-height: var(--toc-max-height);
+    min-width: var(--toc-width);
+    border: 0;
+    border-left: 1px solid var(--separator-color);
+    border-radius: 0;
+    background-color: transparent;
+    box-shadow: none;
+    position: sticky;
+    top: var(--toc-sticky-top);
+    padding: 0 var(--spacing-large);
+    margin: var(--spacing-small) 0 var(--spacing-large) var(--spacing-large);
+}
+
+div.toc h3 {
+    color: var(--toc-foreground);
+    font-size: var(--navigation-font-size);
+    margin: var(--spacing-large) 0 var(--spacing-medium) 0;
+}
+
+div.toc li {
+    padding: 0;
+    background: none;
+    line-height: var(--toc-font-size);
+    margin: var(--toc-font-size) 0 0 0;
+}
+
+div.toc li::before {
+    display: none;
+}
+
+div.toc ul {
+    margin-top: 0
+}
+
+div.toc li a {
+    font-size: var(--toc-font-size);
+    color: var(--page-foreground-color) !important;
+    text-decoration: none;
+}
+
+div.toc li a:hover, div.toc li a.active {
+    color: var(--primary-color) !important;
+}
+
+div.toc li a.aboveActive {
+    color: var(--page-secondary-foreground-color) !important;
+}
+
+
+@media screen and (max-width: 999px) {
+    div.contents .toc {
+        max-height: 45vh;
+        float: none;
+        width: auto;
+        margin: 0 0 var(--spacing-medium) 0;
+        position: relative;
+        top: 0;
+        position: relative;
+        border: 1px solid var(--separator-color);
+        border-radius: var(--border-radius-medium);
+        background-color: var(--toc-background);
+        box-shadow: var(--box-shadow);
+    }
+
+    div.contents .toc.interactive {
+        max-height: calc(var(--navigation-font-size) + 2 * var(--spacing-large));
+        overflow: hidden;
+    }
+
+    div.contents .toc > h3 {
+        -webkit-tap-highlight-color: transparent;
+        cursor: pointer;
+        position: sticky;
+        top: 0;
+        background-color: var(--toc-background);
+        margin: 0;
+        padding: var(--spacing-large) 0;
+        display: block;
+    }
+
+    div.contents .toc.interactive > h3::before {
+        content: "";
+        width: 0; 
+        height: 0; 
+        border-left: 4px solid transparent;
+        border-right: 4px solid transparent;
+        border-top: 5px solid var(--primary-color);
+        display: inline-block;
+        margin-right: var(--spacing-small);
+        margin-bottom: calc(var(--navigation-font-size) / 4);
+        transform: rotate(-90deg);
+        transition: transform 0.25s ease-out;
+    }
+
+    div.contents .toc.interactive.open > h3::before {
+        transform: rotate(0deg);
+    }
+
+    div.contents .toc.interactive.open {
+        max-height: 45vh;
+        overflow: auto;
+        transition: max-height 0.2s ease-in-out;
+    }
+
+    div.contents .toc a, div.contents .toc a.active {
+        color: var(--primary-color) !important;
+    }
+
+    div.contents .toc a:hover {
+        text-decoration: underline;
+    }
+}
+
+/*
+ Code & Fragments
+ */
+
+code, div.fragment, pre.fragment {
+    border-radius: var(--border-radius-small);
+    border: 1px solid var(--separator-color);
+    overflow: hidden;
+}
+
+code {
+    display: inline;
+    background: var(--code-background);
+    color: var(--code-foreground);
+    padding: 2px 6px;
+}
+
+div.fragment, pre.fragment {
+    margin: var(--spacing-medium) 0;
+    padding: calc(var(--spacing-large) - (var(--spacing-large) / 6)) var(--spacing-large);
+    background: var(--fragment-background);
+    color: var(--fragment-foreground);
+    overflow-x: auto;
+}
+
+@media screen and (max-width: 767px) {
+    div.fragment, pre.fragment {
+        border-top-right-radius: 0;
+        border-bottom-right-radius: 0;
+        border-right: 0;
+    }
+
+    .contents > div.fragment,
+    .textblock > div.fragment,
+    .textblock > pre.fragment,
+    .contents > .doxygen-awesome-fragment-wrapper > div.fragment,
+    .textblock > .doxygen-awesome-fragment-wrapper > div.fragment,
+    .textblock > .doxygen-awesome-fragment-wrapper > pre.fragment {
+        margin: var(--spacing-medium) calc(0px - var(--spacing-large));
+        border-radius: 0;
+        border-left: 0;
+    }
+
+    .textblock li > .fragment,
+    .textblock li > .doxygen-awesome-fragment-wrapper > .fragment {
+        margin: var(--spacing-medium) calc(0px - var(--spacing-large));
+    }
+
+    .memdoc li > .fragment,
+    .memdoc li > .doxygen-awesome-fragment-wrapper > .fragment {
+        margin: var(--spacing-medium) calc(0px - var(--spacing-medium));
+    }
+
+    .textblock ul, .memdoc ul {
+        overflow: initial;
+    }
+
+    .memdoc > div.fragment,
+    .memdoc > pre.fragment,
+    dl dd > div.fragment,
+    dl dd pre.fragment,
+    .memdoc > .doxygen-awesome-fragment-wrapper > div.fragment,
+    .memdoc > .doxygen-awesome-fragment-wrapper > pre.fragment,
+    dl dd > .doxygen-awesome-fragment-wrapper > div.fragment,
+    dl dd .doxygen-awesome-fragment-wrapper > pre.fragment {
+        margin: var(--spacing-medium) calc(0px - var(--spacing-medium));
+        border-radius: 0;
+        border-left: 0;
+    }
+}
+
+code, code a, pre.fragment, div.fragment, div.fragment .line, div.fragment span, div.fragment .line a, div.fragment .line span {
+    font-family: var(--font-family-monospace);
+    font-size: var(--code-font-size) !important;
+}
+
+div.line:after {
+    margin-right: var(--spacing-medium);
+}
+
+div.fragment .line, pre.fragment {
+    white-space: pre;
+    word-wrap: initial;
+    line-height: var(--fragment-lineheight);
+}
+
+div.fragment span.keyword {
+    color: var(--fragment-keyword);
+}
+
+div.fragment span.keywordtype {
+    color: var(--fragment-keywordtype);
+}
+
+div.fragment span.keywordflow {
+    color: var(--fragment-keywordflow);
+}
+
+div.fragment span.stringliteral {
+    color: var(--fragment-token)
+}
+
+div.fragment span.comment {
+    color: var(--fragment-comment);
+}
+
+div.fragment a.code {
+    color: var(--fragment-link) !important;
+}
+
+div.fragment span.preprocessor {
+    color: var(--fragment-preprocessor);
+}
+
+div.fragment span.lineno {
+    display: inline-block;
+    width: 27px;
+    border-right: none;
+    background: var(--fragment-linenumber-background);
+    color: var(--fragment-linenumber-color);
+}
+
+div.fragment span.lineno a {
+    background: none;
+    color: var(--fragment-link) !important;
+}
+
+div.fragment .line:first-child .lineno {
+    box-shadow: -999999px 0px 0 999999px var(--fragment-linenumber-background), -999998px 0px 0 999999px var(--fragment-linenumber-border);
+}
+
+div.line {
+    border-radius: var(--border-radius-small);
+}
+
+div.line.glow {
+    background-color: var(--primary-light-color);
+    box-shadow: none;
+}
+
+/*
+ dl warning, attention, note, deprecated, bug, ...
+ */
+
+dl.bug dt a, dl.deprecated dt a, dl.todo dt a {
+    font-weight: bold !important;
+}
+
+dl.warning, dl.attention, dl.note, dl.deprecated, dl.bug, dl.invariant, dl.pre, dl.post, dl.todo, dl.remark {
+    padding: var(--spacing-medium);
+    margin: var(--spacing-medium) 0;
+    color: var(--page-background-color);
+    overflow: hidden;
+    margin-left: 0;
+    border-radius: var(--border-radius-small);
+}
+
+dl.section dd {
+    margin-bottom: 2px;
+}
+
+dl.warning, dl.attention {
+    background: var(--warning-color);
+    border-left: 8px solid var(--warning-color-dark);
+    color: var(--warning-color-darker);
+}
+
+dl.warning dt, dl.attention dt {
+    color: var(--warning-color-dark);
+}
+
+dl.note, dl.remark {
+    background: var(--note-color);
+    border-left: 8px solid var(--note-color-dark);
+    color: var(--note-color-darker);
+}
+
+dl.note dt, dl.remark dt {
+    color: var(--note-color-dark);
+}
+
+dl.todo {
+    background: var(--todo-color);
+    border-left: 8px solid var(--todo-color-dark);
+    color: var(--todo-color-darker);
+}
+
+dl.todo dt {
+    color: var(--todo-color-dark);
+}
+
+dl.bug dt a {
+    color: var(--todo-color-dark) !important;
+}
+
+dl.bug {
+    background: var(--bug-color);
+    border-left: 8px solid var(--bug-color-dark);
+    color: var(--bug-color-darker);
+}
+
+dl.bug dt a {
+    color: var(--bug-color-dark) !important;
+}
+
+dl.deprecated {
+    background: var(--deprecated-color);
+    border-left: 8px solid var(--deprecated-color-dark);
+    color: var(--deprecated-color-darker);
+}
+
+dl.deprecated dt a {
+    color: var(--deprecated-color-dark) !important;
+}
+
+dl.section dd, dl.bug dd, dl.deprecated dd, dl.todo dd {
+    margin-inline-start: 0px;
+}
+
+dl.invariant, dl.pre, dl.post {
+    background: var(--invariant-color);
+    border-left: 8px solid var(--invariant-color-dark);
+    color: var(--invariant-color-darker);
+}
+
+dl.invariant dt, dl.pre dt, dl.post dt {
+    color: var(--invariant-color-dark);
+}
+
+/*
+ memitem
+ */
+
+div.memdoc, div.memproto, h2.memtitle {
+    box-shadow: none;
+    background-image: none;
+    border: none;
+}
+
+div.memdoc {
+    padding: 0 var(--spacing-medium);
+    background: var(--page-background-color);
+}
+
+h2.memtitle, div.memitem {
+    border: 1px solid var(--separator-color);
+    box-shadow: var(--box-shadow);
+}
+
+h2.memtitle {
+    box-shadow: 0px var(--spacing-medium) 0 -1px var(--fragment-background), var(--box-shadow);
+}
+
+div.memitem {
+    transition: none;
+}
+
+div.memproto, h2.memtitle {
+    background: var(--fragment-background);
+}
+
+h2.memtitle {
+    font-weight: 500;
+    font-size: var(--memtitle-font-size);
+    font-family: var(--font-family-monospace);
+    border-bottom: none;
+    border-top-left-radius: var(--border-radius-medium);
+    border-top-right-radius: var(--border-radius-medium);
+    word-break: break-all;
+    position: relative;
+}
+
+h2.memtitle:after {
+    content: "";
+    display: block;
+    background: var(--fragment-background);
+    height: var(--spacing-medium);
+    bottom: calc(0px - var(--spacing-medium));
+    left: 0;
+    right: -14px;
+    position: absolute;
+    border-top-right-radius: var(--border-radius-medium);
+}
+
+h2.memtitle > span.permalink {
+    font-size: inherit;
+}
+
+h2.memtitle > span.permalink > a {
+    text-decoration: none;
+    padding-left: 3px;
+    margin-right: -4px;
+    user-select: none;
+    display: inline-block;
+    margin-top: -6px;
+}
+
+h2.memtitle > span.permalink > a:hover {
+    color: var(--primary-dark-color) !important;
+}
+
+a:target + h2.memtitle, a:target + h2.memtitle + div.memitem {
+    border-color: var(--primary-light-color);
+}
+
+div.memitem {
+    border-top-right-radius: var(--border-radius-medium);
+    border-bottom-right-radius: var(--border-radius-medium);
+    border-bottom-left-radius: var(--border-radius-medium);
+    overflow: hidden;
+    display: block !important;
+}
+
+div.memdoc {
+    border-radius: 0;
+}
+
+div.memproto {
+    border-radius: 0 var(--border-radius-small) 0 0;
+    overflow: auto;
+    border-bottom: 1px solid var(--separator-color);
+    padding: var(--spacing-medium);
+    margin-bottom: -1px;
+}
+
+div.memtitle {
+    border-top-right-radius: var(--border-radius-medium);
+    border-top-left-radius: var(--border-radius-medium);
+}
+
+div.memproto table.memname {
+    font-family: var(--font-family-monospace);
+    color: var(--page-foreground-color);
+    font-size: var(--memname-font-size);
+    text-shadow: none;
+}
+
+div.memproto div.memtemplate {
+    font-family: var(--font-family-monospace);
+    color: var(--primary-dark-color);
+    font-size: var(--memname-font-size);
+    margin-left: 2px;
+    text-shadow: none;
+}
+
+table.mlabels, table.mlabels > tbody {
+    display: block;
+}
+
+td.mlabels-left {
+    width: auto;
+}
+
+td.mlabels-right {
+    margin-top: 3px;
+    position: sticky;
+    left: 0;
+}
+
+table.mlabels > tbody > tr:first-child {
+    display: flex;
+    justify-content: space-between;
+    flex-wrap: wrap;
+}
+
+.memname, .memitem span.mlabels {
+    margin: 0
+}
+
+/*
+ reflist
+ */
+
+dl.reflist {
+    box-shadow: var(--box-shadow);
+    border-radius: var(--border-radius-medium);
+    border: 1px solid var(--separator-color);
+    overflow: hidden;
+    padding: 0;
+}
+
+
+dl.reflist dt, dl.reflist dd {
+    box-shadow: none;
+    text-shadow: none;
+    background-image: none;
+    border: none;
+    padding: 12px;
+}
+
+
+dl.reflist dt {
+    font-weight: 500;
+    border-radius: 0;
+    background: var(--code-background);
+    border-bottom: 1px solid var(--separator-color);
+    color: var(--page-foreground-color)
+}
+
+
+dl.reflist dd {
+    background: none;
+}
+
+/*
+ Table
+ */
+
+.contents table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname),
+.contents table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname) tbody {
+    display: inline-block;
+    max-width: 100%;
+}
+
+.contents > table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname):not(.classindex) {
+    margin-left: calc(0px - var(--spacing-large));
+    margin-right: calc(0px - var(--spacing-large));
+    max-width: calc(100% + 2 * var(--spacing-large));
+}
+
+table.fieldtable,
+table.markdownTable tbody,
+table.doxtable tbody {
+    border: none;
+    margin: var(--spacing-medium) 0;
+    box-shadow: 0 0 0 1px var(--separator-color);
+    border-radius: var(--border-radius-small);
+}
+
+table.doxtable caption {
+    display: block;
+}
+
+table.fieldtable {
+    border-collapse: collapse;
+    width: 100%;
+}
+
+th.markdownTableHeadLeft,
+th.markdownTableHeadRight,
+th.markdownTableHeadCenter,
+th.markdownTableHeadNone,
+table.doxtable th {
+    background: var(--tablehead-background);
+    color: var(--tablehead-foreground);
+    font-weight: 600;
+    font-size: var(--page-font-size);
+}
+
+th.markdownTableHeadLeft:first-child,
+th.markdownTableHeadRight:first-child,
+th.markdownTableHeadCenter:first-child,
+th.markdownTableHeadNone:first-child,
+table.doxtable tr th:first-child {
+    border-top-left-radius: var(--border-radius-small);
+}
+
+th.markdownTableHeadLeft:last-child,
+th.markdownTableHeadRight:last-child,
+th.markdownTableHeadCenter:last-child,
+th.markdownTableHeadNone:last-child,
+table.doxtable tr th:last-child {
+    border-top-right-radius: var(--border-radius-small);
+}
+
+table.markdownTable td,
+table.markdownTable th,
+table.fieldtable td,
+table.fieldtable th,
+table.doxtable td,
+table.doxtable th {
+    border: 1px solid var(--separator-color);
+    padding: var(--spacing-small) var(--spacing-medium);
+}
+
+table.markdownTable td:last-child,
+table.markdownTable th:last-child,
+table.fieldtable td:last-child,
+table.fieldtable th:last-child,
+table.doxtable td:last-child,
+table.doxtable th:last-child {
+    border-right: none;
+}
+
+table.markdownTable td:first-child,
+table.markdownTable th:first-child,
+table.fieldtable td:first-child,
+table.fieldtable th:first-child,
+table.doxtable td:first-child,
+table.doxtable th:first-child {
+    border-left: none;
+}
+
+table.markdownTable tr:first-child td,
+table.markdownTable tr:first-child th,
+table.fieldtable tr:first-child td,
+table.fieldtable tr:first-child th,
+table.doxtable tr:first-child td,
+table.doxtable tr:first-child th {
+    border-top: none;
+}
+
+table.markdownTable tr:last-child td,
+table.markdownTable tr:last-child th,
+table.fieldtable tr:last-child td,
+table.fieldtable tr:last-child th,
+table.doxtable tr:last-child td,
+table.doxtable tr:last-child th {
+    border-bottom: none;
+}
+
+table.markdownTable tr, table.doxtable tr {
+    border-bottom: 1px solid var(--separator-color);
+}
+
+table.markdownTable tr:last-child, table.doxtable tr:last-child {
+    border-bottom: none;
+}
+
+table.fieldtable th {
+    font-size: var(--page-font-size);
+    font-weight: 600;
+    background-image: none;
+    background-color: var(--tablehead-background);
+    color: var(--tablehead-foreground);
+}
+
+table.fieldtable td.fieldtype, .fieldtable td.fieldname, .fieldtable td.fielddoc, .fieldtable th {
+    border-bottom: 1px solid var(--separator-color);
+    border-right: 1px solid var(--separator-color);
+}
+
+table.fieldtable tr:last-child td:first-child {
+    border-bottom-left-radius: var(--border-radius-small);
+}
+
+table.fieldtable tr:last-child td:last-child {
+    border-bottom-right-radius: var(--border-radius-small);
+}
+
+.memberdecls td.glow, .fieldtable tr.glow {
+    background-color: var(--primary-light-color);
+    box-shadow: none;
+}
+
+table.memberdecls {
+    display: block;
+    -webkit-tap-highlight-color: transparent;
+}
+
+table.memberdecls tr[class^='memitem'] {
+    font-family: var(--font-family-monospace);
+    font-size: var(--code-font-size);
+}
+
+table.memberdecls tr[class^='memitem'] .memTemplParams {
+    font-family: var(--font-family-monospace);
+    font-size: var(--code-font-size);
+    color: var(--primary-dark-color);
+    white-space: normal;
+}
+
+table.memberdecls .memItemLeft,
+table.memberdecls .memItemRight,
+table.memberdecls .memTemplItemLeft,
+table.memberdecls .memTemplItemRight,
+table.memberdecls .memTemplParams {
+    transition: none;
+    padding-top: var(--spacing-small);
+    padding-bottom: var(--spacing-small);
+    border-top: 1px solid var(--separator-color);
+    border-bottom: 1px solid var(--separator-color);
+    background-color: var(--fragment-background);
+}
+
+table.memberdecls .memTemplItemLeft,
+table.memberdecls .memTemplItemRight {
+    padding-top: 2px;
+}
+
+table.memberdecls .memTemplParams {
+    border-bottom: 0;
+    border-left: 1px solid var(--separator-color);
+    border-right: 1px solid var(--separator-color);
+    border-radius: var(--border-radius-small) var(--border-radius-small) 0 0;
+    padding-bottom: var(--spacing-small);
+}
+
+table.memberdecls .memTemplItemLeft {
+    border-radius: 0 0 0 var(--border-radius-small);
+    border-left: 1px solid var(--separator-color);
+    border-top: 0;
+}
+
+table.memberdecls .memTemplItemRight {
+    border-radius: 0 0 var(--border-radius-small) 0;
+    border-right: 1px solid var(--separator-color);
+    padding-left: 0;
+    border-top: 0;
+}
+
+table.memberdecls .memItemLeft {
+    border-radius: var(--border-radius-small) 0 0 var(--border-radius-small);
+    border-left: 1px solid var(--separator-color);
+    padding-left: var(--spacing-medium);
+    padding-right: 0;
+}
+
+table.memberdecls .memItemRight  {
+    border-radius: 0 var(--border-radius-small) var(--border-radius-small) 0;
+    border-right: 1px solid var(--separator-color);
+    padding-right: var(--spacing-medium);
+    padding-left: 0;
+
+}
+
+table.memberdecls .mdescLeft, table.memberdecls .mdescRight {
+    background: none;
+    color: var(--page-foreground-color);
+    padding: var(--spacing-small) 0;
+}
+
+table.memberdecls .memItemLeft,
+table.memberdecls .memTemplItemLeft {
+    padding-right: var(--spacing-medium);
+}
+
+table.memberdecls .memSeparator {
+    background: var(--page-background-color);
+    height: var(--spacing-large);
+    border: 0;
+    transition: none;
+}
+
+table.memberdecls .groupheader {
+    margin-bottom: var(--spacing-large);
+}
+
+table.memberdecls .inherit_header td {
+    padding: 0 0 var(--spacing-medium) 0;
+    text-indent: -12px;
+    color: var(--page-secondary-foreground-color);
+}
+
+table.memberdecls img[src="closed.png"],
+table.memberdecls img[src="open.png"],
+div.dynheader img[src="open.png"],
+div.dynheader img[src="closed.png"] {
+    width: 0; 
+    height: 0; 
+    border-left: 4px solid transparent;
+    border-right: 4px solid transparent;
+    border-top: 5px solid var(--primary-color);
+    margin-top: 8px;
+    display: block;
+    float: left;
+    margin-left: -10px;
+    transition: transform 0.25s ease-out;
+}
+
+table.memberdecls img {
+    margin-right: 10px;
+}
+
+table.memberdecls img[src="closed.png"],
+div.dynheader img[src="closed.png"] {
+    transform: rotate(-90deg);
+    
+}
+
+.compoundTemplParams {
+    font-family: var(--font-family-monospace);
+    color: var(--primary-dark-color);
+    font-size: var(--code-font-size);
+}
+
+@media screen and (max-width: 767px) {
+
+    table.memberdecls .memItemLeft,
+    table.memberdecls .memItemRight,
+    table.memberdecls .mdescLeft,
+    table.memberdecls .mdescRight,
+    table.memberdecls .memTemplItemLeft,
+    table.memberdecls .memTemplItemRight,
+    table.memberdecls .memTemplParams {
+        display: block;
+        text-align: left;
+        padding-left: var(--spacing-large);
+        margin: 0 calc(0px - var(--spacing-large)) 0 calc(0px - var(--spacing-large));
+        border-right: none;
+        border-left: none;
+        border-radius: 0;
+        white-space: normal;
+    }
+
+    table.memberdecls .memItemLeft,
+    table.memberdecls .mdescLeft,
+    table.memberdecls .memTemplItemLeft {
+        border-bottom: 0;
+        padding-bottom: 0;
+    }
+
+    table.memberdecls .memTemplItemLeft {
+        padding-top: 0;
+    }
+
+    table.memberdecls .mdescLeft {
+        margin-bottom: calc(0px - var(--page-font-size));
+    }
+
+    table.memberdecls .memItemRight, 
+    table.memberdecls .mdescRight,
+    table.memberdecls .memTemplItemRight {
+        border-top: 0;
+        padding-top: 0;
+        padding-right: var(--spacing-large);
+        overflow-x: auto;
+    }
+
+    table.memberdecls tr[class^='memitem']:not(.inherit) {
+        display: block;
+        width: calc(100vw - 2 * var(--spacing-large));
+    }
+
+    table.memberdecls .mdescRight {
+        color: var(--page-foreground-color);
+    }
+
+    table.memberdecls tr.inherit {
+        visibility: hidden;
+    }
+
+    table.memberdecls tr[style="display: table-row;"] {
+        display: block !important;
+        visibility: visible;
+        width: calc(100vw - 2 * var(--spacing-large));
+        animation: fade .5s;
+    }
+
+    @keyframes fade {
+        0% {
+            opacity: 0;
+            max-height: 0;
+        }
+
+        100% {
+            opacity: 1;
+            max-height: 200px;
+        }
+    }
+}
+
+
+/*
+ Horizontal Rule
+ */
+
+hr {
+    margin-top: var(--spacing-large);
+    margin-bottom: var(--spacing-large);
+    height: 1px;
+    background-color: var(--separator-color);
+    border: 0;
+}
+
+.contents hr {
+    box-shadow: 100px 0 0 var(--separator-color),
+                -100px 0 0 var(--separator-color),
+                500px 0 0 var(--separator-color),
+                -500px 0 0 var(--separator-color),
+                1500px 0 0 var(--separator-color),
+                -1500px 0 0 var(--separator-color),
+                2000px 0 0 var(--separator-color),
+                -2000px 0 0 var(--separator-color);
+}
+
+.contents img, .contents .center, .contents center, .contents div.image object {
+    max-width: 100%;
+    overflow: auto;
+}
+
+@media screen and (max-width: 767px) {
+    .contents .dyncontent > .center, .contents > center {
+        margin-left: calc(0px - var(--spacing-large));
+        margin-right: calc(0px - var(--spacing-large));
+        max-width: calc(100% + 2 * var(--spacing-large));
+    }
+}
+
+/*
+ Directories
+ */
+div.directory {
+    border-top: 1px solid var(--separator-color);
+    border-bottom: 1px solid var(--separator-color);
+    width: auto;
+}
+
+table.directory {
+    font-family: var(--font-family);
+    font-size: var(--page-font-size);
+    font-weight: normal;
+    width: 100%;
+}
+
+table.directory td.entry, table.directory td.desc {
+    padding: calc(var(--spacing-small) / 2) var(--spacing-small);
+    line-height: var(--table-line-height);
+}
+
+table.directory tr.even td:last-child {
+    border-radius: 0 var(--border-radius-small) var(--border-radius-small) 0;
+}
+
+table.directory tr.even td:first-child {
+    border-radius: var(--border-radius-small) 0 0 var(--border-radius-small);
+}
+
+table.directory tr.even:last-child td:last-child {
+    border-radius: 0 var(--border-radius-small) 0 0;
+}
+
+table.directory tr.even:last-child td:first-child {
+    border-radius: var(--border-radius-small) 0 0 0;
+}
+
+table.directory td.desc {
+    min-width: 250px;
+}
+
+table.directory tr.even {
+    background-color: var(--odd-color);
+}
+
+table.directory tr.odd {
+    background-color: transparent;
+}
+
+.icona {
+    width: auto;
+    height: auto;
+    margin: 0 var(--spacing-small);
+}
+
+.icon {
+    background: var(--primary-color);
+    border-radius: var(--border-radius-small);
+    font-size: var(--page-font-size);
+    padding: calc(var(--page-font-size) / 5);
+    line-height: var(--page-font-size);
+    transform: scale(0.8);
+    height: auto;
+    width: var(--page-font-size);
+    user-select: none;
+}
+
+.iconfopen, .icondoc, .iconfclosed {
+    background-position: center;
+    margin-bottom: 0;
+    height: var(--table-line-height);
+}
+
+.icondoc {
+    filter: saturate(0.2);
+}
+
+@media screen and (max-width: 767px) {
+    div.directory {
+        margin-left: calc(0px - var(--spacing-large));
+        margin-right: calc(0px - var(--spacing-large));
+    }
+}
+
+@media (prefers-color-scheme: dark) {
+    html:not(.light-mode) .iconfopen, html:not(.light-mode) .iconfclosed {
+        filter: hue-rotate(180deg) invert();
+    }
+}
+
+html.dark-mode .iconfopen, html.dark-mode .iconfclosed {
+    filter: hue-rotate(180deg) invert();
+}
+
+/*
+ Class list
+ */
+
+.classindex dl.odd {
+    background: var(--odd-color);
+    border-radius: var(--border-radius-small);
+}
+
+.classindex dl.even {
+    background-color: transparent;
+}
+
+/* 
+ Class Index Doxygen 1.8 
+*/
+
+table.classindex {
+    margin-left: 0;
+    margin-right: 0;
+    width: 100%;
+}
+
+table.classindex table div.ah {
+    background-image: none;
+    background-color: initial;
+    border-color: var(--separator-color);
+    color: var(--page-foreground-color);
+    box-shadow: var(--box-shadow);
+    border-radius: var(--border-radius-large);
+    padding: var(--spacing-small);
+}
+
+div.qindex {
+    background-color: var(--odd-color);
+    border-radius: var(--border-radius-small);
+    border: 1px solid var(--separator-color);
+    padding: var(--spacing-small) 0;
+}
+
+/*
+  Footer and nav-path
+ */
+
+#nav-path {
+    width: 100%;
+}
+
+#nav-path ul {
+    background-image: none;
+    background: var(--page-background-color);
+    border: none;
+    border-top: 1px solid var(--separator-color);
+    border-bottom: 1px solid var(--separator-color);
+    border-bottom: 0;
+    box-shadow: 0 0.75px 0 var(--separator-color);
+    font-size: var(--navigation-font-size);
+}
+
+img.footer {
+    width: 60px;
+}
+
+.navpath li.footer {
+    color: var(--page-secondary-foreground-color);
+}
+
+address.footer {
+    color: var(--page-secondary-foreground-color);
+    margin-bottom: var(--spacing-large);
+}
+
+#nav-path li.navelem {
+    background-image: none;
+    display: flex;
+    align-items: center;
+}
+
+.navpath li.navelem a {
+    text-shadow: none;
+    display: inline-block;
+    color: var(--primary-color) !important;
+}
+
+.navpath li.navelem b {
+    color: var(--primary-dark-color);
+    font-weight: 500;
+}
+
+li.navelem {
+    padding: 0;
+    margin-left: -8px;
+}
+
+li.navelem:first-child {
+    margin-left: var(--spacing-large);
+}
+
+li.navelem:first-child:before {
+    display: none;
+}
+
+#nav-path li.navelem:after {
+    content: '';
+    border: 5px solid var(--page-background-color);
+    border-bottom-color: transparent;
+    border-right-color: transparent;
+    border-top-color: transparent;
+    transform: translateY(-1px) scaleY(4.2);
+    z-index: 10;
+    margin-left: 6px;
+}
+
+#nav-path li.navelem:before {
+    content: '';
+    border: 5px solid var(--separator-color);
+    border-bottom-color: transparent;
+    border-right-color: transparent;
+    border-top-color: transparent;
+    transform: translateY(-1px) scaleY(3.2);
+    margin-right: var(--spacing-small);
+}
+
+.navpath li.navelem a:hover {
+    color: var(--primary-color);
+}
+
+/*
+ Scrollbars for Webkit
+*/
+
+#nav-tree::-webkit-scrollbar,
+div.fragment::-webkit-scrollbar,
+pre.fragment::-webkit-scrollbar,
+div.memproto::-webkit-scrollbar,
+.contents center::-webkit-scrollbar,
+.contents .center::-webkit-scrollbar,
+.contents table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname) tbody::-webkit-scrollbar,
+div.contents .toc::-webkit-scrollbar {
+    background: transparent;
+    width: calc(var(--webkit-scrollbar-size) + var(--webkit-scrollbar-padding) + var(--webkit-scrollbar-padding));
+    height: calc(var(--webkit-scrollbar-size) + var(--webkit-scrollbar-padding) + var(--webkit-scrollbar-padding));
+}
+
+#nav-tree::-webkit-scrollbar-thumb,
+div.fragment::-webkit-scrollbar-thumb,
+pre.fragment::-webkit-scrollbar-thumb,
+div.memproto::-webkit-scrollbar-thumb,
+.contents center::-webkit-scrollbar-thumb,
+.contents .center::-webkit-scrollbar-thumb,
+.contents table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname) tbody::-webkit-scrollbar-thumb,
+div.contents .toc::-webkit-scrollbar-thumb {
+    background-color: transparent;
+    border: var(--webkit-scrollbar-padding) solid transparent;
+    border-radius: calc(var(--webkit-scrollbar-padding) + var(--webkit-scrollbar-padding));
+    background-clip: padding-box;  
+}
+
+#nav-tree:hover::-webkit-scrollbar-thumb,
+div.fragment:hover::-webkit-scrollbar-thumb,
+pre.fragment:hover::-webkit-scrollbar-thumb,
+div.memproto:hover::-webkit-scrollbar-thumb,
+.contents center:hover::-webkit-scrollbar-thumb,
+.contents .center:hover::-webkit-scrollbar-thumb,
+.contents table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname) tbody:hover::-webkit-scrollbar-thumb,
+div.contents .toc:hover::-webkit-scrollbar-thumb {
+    background-color: var(--webkit-scrollbar-color);
+}
+
+#nav-tree::-webkit-scrollbar-track,
+div.fragment::-webkit-scrollbar-track,
+pre.fragment::-webkit-scrollbar-track,
+div.memproto::-webkit-scrollbar-track,
+.contents center::-webkit-scrollbar-track,
+.contents .center::-webkit-scrollbar-track,
+.contents table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname) tbody::-webkit-scrollbar-track,
+div.contents .toc::-webkit-scrollbar-track {
+    background: transparent;
+}
+
+#nav-tree::-webkit-scrollbar-corner {
+    background-color: var(--side-nav-background);
+}
+
+#nav-tree,
+div.fragment,
+pre.fragment,
+div.memproto,
+.contents center,
+.contents .center,
+.contents table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname) tbody,
+div.contents .toc {
+    overflow-x: auto;
+    overflow-x: overlay;
+}
+
+#nav-tree {
+    overflow-x: auto;
+    overflow-y: auto;
+    overflow-y: overlay;
+}
+
+/*
+ Scrollbars for Firefox
+*/
+
+#nav-tree,
+div.fragment,
+pre.fragment,
+div.memproto,
+.contents center,
+.contents .center,
+.contents table:not(.memberdecls):not(.mlabels):not(.fieldtable):not(.memname) tbody,
+div.contents .toc {
+    scrollbar-width: thin;
+}
+
+/*
+  Optional Dark mode toggle button
+*/
+
+doxygen-awesome-dark-mode-toggle {
+    display: inline-block;
+    margin: 0 0 0 var(--spacing-small);
+    padding: 0;
+    width: var(--searchbar-height);
+    height: var(--searchbar-height);
+    background: none;
+    border: none;
+    border-radius: var(--searchbar-height);
+    vertical-align: middle;
+    text-align: center;
+    line-height: var(--searchbar-height);
+    font-size: 22px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    user-select: none;
+    cursor: pointer;
+}
+
+doxygen-awesome-dark-mode-toggle > svg {
+    transition: transform .1s ease-in-out;
+}
+
+doxygen-awesome-dark-mode-toggle:active > svg {
+    transform: scale(.5);
+}
+
+doxygen-awesome-dark-mode-toggle:hover {
+    background-color: rgba(0,0,0,.03);
+}
+
+html.dark-mode doxygen-awesome-dark-mode-toggle:hover {
+    background-color: rgba(0,0,0,.18);
+}
+
+/*
+ Optional fragment copy button
+*/
+.doxygen-awesome-fragment-wrapper {
+    position: relative;
+}
+
+doxygen-awesome-fragment-copy-button {
+    opacity: 0;
+    background: var(--fragment-background);
+    width: 28px;
+    height: 28px;
+    position: absolute;
+    right: calc(var(--spacing-large) - (var(--spacing-large) / 2.5));
+    top: calc(var(--spacing-large) - (var(--spacing-large) / 2.5));
+    border: 1px solid var(--fragment-foreground);
+    cursor: pointer;
+    border-radius: var(--border-radius-small);
+    display: flex;
+    justify-content: center;
+    align-items: center;
+}
+
+.doxygen-awesome-fragment-wrapper:hover doxygen-awesome-fragment-copy-button, doxygen-awesome-fragment-copy-button.success {
+    opacity: .28;
+}
+
+doxygen-awesome-fragment-copy-button:hover, doxygen-awesome-fragment-copy-button.success {
+    opacity: 1 !important;
+}
+
+doxygen-awesome-fragment-copy-button:active:not([class~=success]) svg {
+    transform: scale(.91);
+}
+
+doxygen-awesome-fragment-copy-button svg {
+    fill: var(--fragment-foreground);
+    width: 18px;
+    height: 18px;
+}
+
+doxygen-awesome-fragment-copy-button.success svg {
+    fill: rgb(14, 168, 14);
+}
+
+doxygen-awesome-fragment-copy-button.success {
+    border-color: rgb(14, 168, 14);
+}
+
+@media screen and (max-width: 767px) {
+    .textblock > .doxygen-awesome-fragment-wrapper > doxygen-awesome-fragment-copy-button,
+    .textblock li > .doxygen-awesome-fragment-wrapper > doxygen-awesome-fragment-copy-button,
+    .memdoc li > .doxygen-awesome-fragment-wrapper > doxygen-awesome-fragment-copy-button,
+    .memdoc > .doxygen-awesome-fragment-wrapper > doxygen-awesome-fragment-copy-button,
+    dl dd > .doxygen-awesome-fragment-wrapper > doxygen-awesome-fragment-copy-button {
+        right: 0;
+    }
+}
+
+/*
+ Optional paragraph link button
+*/
+
+a.anchorlink {
+    font-size: 90%;
+    margin-left: var(--spacing-small);
+    color: var(--page-foreground-color) !important;
+    text-decoration: none;
+    opacity: .15;
+    display: none;
+    transition: opacity .1s ease-in-out, color .1s ease-in-out;
+}
+
+a.anchorlink svg {
+    fill: var(--page-foreground-color);
+}
+
+h3 a.anchorlink svg, h4 a.anchorlink svg {
+    margin-bottom: -3px;
+    margin-top: -4px;
+}
+
+a.anchorlink:hover {
+    opacity: .45;
+}
+
+h2:hover a.anchorlink, h1:hover a.anchorlink, h3:hover a.anchorlink, h4:hover a.anchorlink  {
+    display: inline-block;
+}
diff --git a/docs/doxygen.mk b/docs/doxygen.mk
index b7eded0238..4ec7155d51 100644
--- a/docs/doxygen.mk
+++ b/docs/doxygen.mk
@@ -1,4 +1,4 @@
-# Doxyfile 1.9.3
+# Doxyfile 1.9.5
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -12,6 +12,16 @@
 # For lists, items can also be appended using:
 # TAG += value [value, ...]
 # Values that contain spaces should be placed between quotes (\" \").
+#
+# Note:
+#
+# Use doxygen to compare the used configuration file with the template
+# configuration file:
+# doxygen -x [configFile]
+# Use doxygen to compare the used configuration file with the template
+# configuration file without replacing the environment variables or CMake type
+# replacement variables:
+# doxygen -x_noenv [configFile]
 
 #---------------------------------------------------------------------------
 # Project related configuration options
@@ -60,16 +70,28 @@ PROJECT_LOGO           = ${ASSETS_DIR}/arrayfire_logo.png
 
 OUTPUT_DIRECTORY       = ${CMAKE_CURRENT_BINARY_DIR}
 
-# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
-# directories (in 2 levels) under the output directory of each output format and
-# will distribute the generated files over these directories. Enabling this
+# If the CREATE_SUBDIRS tag is set to YES then doxygen will create up to 4096
+# sub-directories (in 2 levels) under the output directory of each output format
+# and will distribute the generated files over these directories. Enabling this
 # option can be useful when feeding doxygen a huge amount of source files, where
 # putting all generated files in the same directory would otherwise causes
-# performance problems for the file system.
+# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to
+# control the number of sub-directories.
 # The default value is: NO.
 
 CREATE_SUBDIRS         = NO
 
+# Controls the number of sub-directories that will be created when
+# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every
+# level increment doubles the number of directories, resulting in 4096
+# directories at level 8 which is the default and also the maximum value. The
+# sub-directories are organized in 2 levels, the first level always has a fixed
+# numer of 16 directories.
+# Minimum value: 0, maximum value: 8, default value: 8.
+# This tag requires that the tag CREATE_SUBDIRS is set to YES.
+
+CREATE_SUBDIRS_LEVEL   = 8
+
 # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
 # characters to appear in the names of generated files. If set to NO, non-ASCII
 # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
@@ -81,14 +103,14 @@ ALLOW_UNICODE_NAMES    = NO
 # The OUTPUT_LANGUAGE tag is used to specify the language in which all
 # documentation generated by doxygen is written. Doxygen will use this
 # information to generate all constant output in the proper language.
-# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
-# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
-# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
-# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
-# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
-# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
-# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
-# Ukrainian and Vietnamese.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian,
+# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English
+# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek,
+# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with
+# English messages), Korean, Korean-en (Korean with English messages), Latvian,
+# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese,
+# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish,
+# Swedish, Turkish, Ukrainian and Vietnamese.
 # The default value is: English.
 
 OUTPUT_LANGUAGE        = English
@@ -466,7 +488,7 @@ TYPEDEF_HIDES_STRUCT   = NO
 
 LOOKUP_CACHE_SIZE      = 0
 
-# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use
+# The NUM_PROC_THREADS specifies the number of threads doxygen is allowed to use
 # during processing. When set to 0 doxygen will based this on the number of
 # cores available in the system. You can set it explicitly to a value larger
 # than 0 to get more control over the balance between CPU load and processing
@@ -591,14 +613,15 @@ INTERNAL_DOCS          = NO
 # filesystem is case sensitive (i.e. it supports files in the same directory
 # whose names only differ in casing), the option must be set to YES to properly
 # deal with such files in case they appear in the input. For filesystems that
-# are not case sensitive the option should be be set to NO to properly deal with
+# are not case sensitive the option should be set to NO to properly deal with
 # output files written for symbols that only differ in casing, such as for two
 # classes, one named CLASS and the other named Class, and to also support
 # references to files without having to specify the exact matching casing. On
 # Windows (including Cygwin) and MacOS, users should typically set this option
 # to NO, whereas on Linux or other Unix flavors it should typically be set to
 # YES.
-# The default value is: system dependent.
+# Possible values are: SYSTEM, NO and YES.
+# The default value is: SYSTEM.
 
 CASE_SENSE_NAMES       = YES
 
@@ -865,10 +888,21 @@ WARN_AS_ERROR          = NO
 # and the warning text. Optionally the format may contain $version, which will
 # be replaced by the version of the file (if it could be obtained via
 # FILE_VERSION_FILTER)
+# See also: WARN_LINE_FORMAT
 # The default value is: $file:$line: $text.
 
 WARN_FORMAT            = "$file:$line: $text"
 
+# In the $text part of the WARN_FORMAT command it is possible that a reference
+# to a more specific place is given. To make it easier to jump to this place
+# (outside of doxygen) the user can define a custom "cut" / "paste" string.
+# Example:
+# WARN_LINE_FORMAT = "'vi $file +$line'"
+# See also: WARN_FORMAT
+# The default value is: at line $line of file $file.
+
+WARN_LINE_FORMAT       = "at line $line of file $file"
+
 # The WARN_LOGFILE tag can be used to specify a file to which warning and error
 # messages should be written. If left blank the output is written to standard
 # error (stderr). In case the file specified cannot be opened for writing the
@@ -898,10 +932,21 @@ INPUT                  = ${DOCS_DIR}/pages \
 # libiconv (or the iconv built into libc) for the transcoding. See the libiconv
 # documentation (see:
 # https://www.gnu.org/software/libiconv/) for the list of possible encodings.
+# See also: INPUT_FILE_ENCODING
 # The default value is: UTF-8.
 
 INPUT_ENCODING         = UTF-8
 
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses The INPUT_FILE_ENCODING tag can be used to specify
+# character encoding on a per file pattern basis. Doxygen will compare the file
+# name with each pattern and apply the encoding instead of the default
+# INPUT_ENCODING) if there is a match. The character encodings are a list of the
+# form: pattern=encoding (like *.php=ISO-8859-1). See cfg_input_encoding
+# "INPUT_ENCODING" for further information on supported encodings.
+
+INPUT_FILE_ENCODING    =
+
 # If the value of the INPUT tag contains directories, you can use the
 # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
 # *.h) to filter out the source-files in the directories.
@@ -1009,6 +1054,11 @@ IMAGE_PATH             = ${ASSETS_DIR} \
 # code is scanned, but not when the output code is generated. If lines are added
 # or removed, the anchors will not be placed correctly.
 #
+# Note that doxygen will use the data processed and written to standard output
+# for further processing, therefore nothing else, like debug statements or used
+# commands (so in case of a Windows batch file always use @echo OFF), should be
+# written to standard output.
+#
 # Note that for custom extensions or not directly supported extensions you also
 # need to set EXTENSION_MAPPING for the extension otherwise the files are not
 # properly processed by doxygen.
@@ -1050,6 +1100,15 @@ FILTER_SOURCE_PATTERNS =
 
 USE_MDFILE_AS_MAINPAGE = ${DOCS_DIR}/pages/README.md
 
+# The Fortran standard specifies that for fixed formatted Fortran code all
+# characters from position 72 are to be considered as comment. A common
+# extension is to allow longer lines before the automatic comment starts. The
+# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can
+# be processed before the automatic comment starts.
+# Minimum value: 7, maximum value: 10000, default value: 72.
+
+FORTRAN_COMMENT_AFTER  = 72
+
 #---------------------------------------------------------------------------
 # Configuration options related to source browsing
 #---------------------------------------------------------------------------
@@ -1136,6 +1195,46 @@ USE_HTAGS              = NO
 
 VERBATIM_HEADERS       = YES
 
+# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the
+# clang parser (see:
+# http://clang.llvm.org/) for more accurate parsing at the cost of reduced
+# performance. This can be particularly helpful with template rich C++ code for
+# which doxygen's built-in parser lacks the necessary type information.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+# The default value is: NO.
+
+CLANG_ASSISTED_PARSING = NO
+
+# If the CLANG_ASSISTED_PARSING tag is set to YES and the CLANG_ADD_INC_PATHS
+# tag is set to YES then doxygen will add the directory of each input to the
+# include path.
+# The default value is: YES.
+# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
+
+CLANG_ADD_INC_PATHS    = YES
+
+# If clang assisted parsing is enabled you can provide the compiler with command
+# line options that you would normally use when invoking the compiler. Note that
+# the include paths will already be set by doxygen for the files and directories
+# specified with INPUT and INCLUDE_PATH.
+# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
+
+CLANG_OPTIONS          =
+
+# If clang assisted parsing is enabled you can provide the clang parser with the
+# path to the directory containing a file called compile_commands.json. This
+# file is the compilation database (see:
+# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) containing the
+# options used when the source files were built. This is equivalent to
+# specifying the -p option to a clang tool, such as clang-check. These options
+# will then be passed to the parser. Any options specified with CLANG_OPTIONS
+# will be added as well.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+
+CLANG_DATABASE_PATH    =
+
 #---------------------------------------------------------------------------
 # Configuration options related to the alphabetical class index
 #---------------------------------------------------------------------------
@@ -1232,7 +1331,8 @@ HTML_STYLESHEET        =
 # list). For an example see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_STYLESHEET  = ${DOCS_DIR}/arrayfire.css
+HTML_EXTRA_STYLESHEET  = ${DOCS_DIR}/doxygen-awesome.css \
+                         ${DOCS_DIR}/doxygen-awesome-sidebar-only.css
 
 # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
 # other source files which should be copied to the HTML output directory. Note
@@ -1242,7 +1342,26 @@ HTML_EXTRA_STYLESHEET  = ${DOCS_DIR}/arrayfire.css
 # files will be copied as-is; there are no commands or markers available.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_FILES       =
+HTML_EXTRA_FILES       = ${DOCS_DIR}/doxygen-awesome-darkmode-toggle.js \
+                         ${DOCS_DIR}/doxygen-awesome-fragment-copy-button.js \
+                         ${DOCS_DIR}/doxygen-awesome-interactive-toc.js
+
+# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output
+# should be rendered with a dark or light theme. Default setting AUTO_LIGHT
+# enables light output unless the user preference is dark output. Other options
+# are DARK to always use dark mode, LIGHT to always use light mode, AUTO_DARK to
+# default to dark mode unless the user prefers light mode, and TOGGLE to let the
+# user toggle between dark and light mode via a button.
+# Possible values are: LIGHT Always generate light output., DARK Always generate
+# dark output., AUTO_LIGHT Automatically set the mode according to the user
+# preference, use light mode if no preference is set (the default)., AUTO_DARK
+# Automatically set the mode according to the user preference, use dark mode if
+# no preference is set. and TOGGLE Allow to user to switch between light and
+# dark mode via a button..
+# The default value is: AUTO_LIGHT.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE        = LIGHT
 
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
@@ -1571,7 +1690,7 @@ ENUM_VALUES_PER_LINE   = 4
 # Minimum value: 0, maximum value: 1500, default value: 250.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-TREEVIEW_WIDTH         = 250
+TREEVIEW_WIDTH         = 335
 
 # If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
 # external symbols imported via tag files in a separate window.
@@ -1607,17 +1726,6 @@ HTML_FORMULA_FORMAT    = png
 
 FORMULA_FONTSIZE       = 12
 
-# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
-# generated for formulas are transparent PNGs. Transparent PNGs are not
-# supported properly for IE 6.0, but are supported on all modern browsers.
-#
-# Note that when changing this option you need to delete any form_*.png files in
-# the HTML output directory before the changes have effect.
-# The default value is: YES.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-FORMULA_TRANSPARENT    = YES
-
 # The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands
 # to create new LaTeX commands to be used in formulas as building blocks. See
 # the section "Including formulas" for details.
@@ -2208,7 +2316,8 @@ SEARCH_INCLUDES        = NO
 
 # The INCLUDE_PATH tag can be used to specify one or more directories that
 # contain include files that are not input files but should be processed by the
-# preprocessor.
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
 # This tag requires that the tag SEARCH_INCLUDES is set to YES.
 
 INCLUDE_PATH           =
@@ -2336,26 +2445,38 @@ HAVE_DOT               = NO
 
 DOT_NUM_THREADS        = 0
 
-# When you want a differently looking font in the dot files that doxygen
-# generates you can specify the font name using DOT_FONTNAME. You need to make
-# sure dot is able to find the font, which can be done by putting it in a
-# standard location or by setting the DOTFONTPATH environment variable or by
-# setting DOT_FONTPATH to the directory containing the font.
-# The default value is: Helvetica.
+# DOT_COMMON_ATTR is common attributes for nodes, edges and labels of
+# subgraphs. When you want a differently looking font in the dot files that
+# doxygen generates you can specify fontname, fontcolor and fontsize attributes.
+# For details please see <a href=https://graphviz.org/doc/info/attrs.html>Node,
+# Edge and Graph Attributes specification</a> You need to make sure dot is able
+# to find the font, which can be done by putting it in a standard location or by
+# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the
+# directory containing the font. Default graphviz fontsize is 14.
+# The default value is: fontname=Helvetica,fontsize=10.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_FONTNAME           = Helvetica
+DOT_COMMON_ATTR        = "fontname=Helvetica,fontsize=10"
 
-# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
-# dot graphs.
-# Minimum value: 4, maximum value: 24, default value: 10.
+# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can
+# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. <a
+# href=https://graphviz.org/doc/info/arrows.html>Complete documentation about
+# arrows shapes.</a>
+# The default value is: labelfontname=Helvetica,labelfontsize=10.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_FONTSIZE           = 10
+DOT_EDGE_ATTR          = "labelfontname=Helvetica,labelfontsize=10"
 
-# By default doxygen will tell dot to use the default font as specified with
-# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
-# the path where dot can find it using this tag.
+# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes
+# around nodes set 'shape=plain' or 'shape=plaintext' <a
+# href=https://www.graphviz.org/doc/info/shapes.html>Shapes specification</a>
+# The default value is: shape=box,height=0.2,width=0.4.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NODE_ATTR          = "shape=box,height=0.2,width=0.4"
+
+# You can set the path where dot can find font specified with fontname in
+# DOT_COMMON_ATTR and others dot attributes.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
 DOT_FONTPATH           =
@@ -2381,7 +2502,8 @@ CLASS_GRAPH            = YES
 COLLABORATION_GRAPH    = YES
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
-# groups, showing the direct groups dependencies.
+# groups, showing the direct groups dependencies. See also the chapter Grouping
+# in the manual.
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
@@ -2597,18 +2719,6 @@ DOT_GRAPH_MAX_NODES    = 50
 
 MAX_DOT_GRAPH_DEPTH    = 0
 
-# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
-# background. This is disabled by default, because dot on Windows does not seem
-# to support this out of the box.
-#
-# Warning: Depending on the platform used, enabling this option may lead to
-# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
-# read).
-# The default value is: NO.
-# This tag requires that the tag HAVE_DOT is set to YES.
-
-DOT_TRANSPARENT        = NO
-
 # Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
 # files in one run (i.e. multiple -o and -T options on the command line). This
 # makes dot run faster, but since only newer versions of dot (>1.8.10) support
diff --git a/docs/header.htm b/docs/header.htm
index 5704d89dfb..7709ca014c 100644
--- a/docs/header.htm
+++ b/docs/header.htm
@@ -1,6 +1,6 @@
-<!-- HTML header for doxygen 1.9.3-->
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml">
+<!-- HTML header for doxygen 1.9.5-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="$langISO">
 <head>
 <!-- Global site tag (gtag.js) - Google Analytics -->
 <script async src="https://www.googletagmanager.com/gtag/js?id=UA-130950618-1"></script>
@@ -28,8 +28,17 @@
 $treeview
 $search
 $mathjax
+$darkmode
 <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
 $extrastylesheet
+<script type="text/javascript" src="$relpath^doxygen-awesome-darkmode-toggle.js"></script>
+<script type="text/javascript" src="$relpath^doxygen-awesome-fragment-copy-button.js"></script>
+<script type="text/javascript" src="$relpath^doxygen-awesome-interactive-toc.js"></script>
+<script type="text/javascript">
+    DoxygenAwesomeDarkModeToggle.init()
+    DoxygenAwesomeInteractiveToc.init()
+	DoxygenAwesomeFragmentCopyButton.init()
+</script>
 </head>
 <body>
 <!--BEGIN DISABLE_INDEX-->
@@ -42,45 +51,64 @@
 
 <!--BEGIN TITLEAREA-->
 <div id="titlearea">
-<table cellspacing="0" cellpadding="0" width="100%">
+<table cellspacing="2" cellpadding="2" width="100%">
  <tbody>
- <tr id="projectrow">
+  <tr id="projectrow">
   <!--BEGIN PROJECT_LOGO-->
-  <td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td>
+  <td id="projectlogo"><a  href="index.html"><img alt="Logo" src="$relpath^$projectlogo"/></a></td>
   <!--END PROJECT_LOGO-->
-  <!--BEGIN PROJECT_NAME-->
-  <!--<td id="projectalign">
-   <div id="projectname">$projectname<!--BEGIN PROJECT_NUMBER--><span id="projectnumber">&#160;$projectnumber</span><!--END PROJECT_NUMBER-->
-   </div>
-   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF-->
-  </td>-->
-  <!--END PROJECT_NAME-->
-  <!--BEGIN !PROJECT_NAME-->
-   <!--BEGIN PROJECT_BRIEF-->
-    <td>
-    <div id="projectbrief">$projectbrief</div>
-    </td>
-   <!--END PROJECT_BRIEF-->
+  </tr>
+  <!--BEGIN PROJECT_BRIEF-->
+  <tr id="projectrow">
+  <td>
+  <div id="projectbrief">$projectbrief</div>
+  </td>
+  </tr>
+  <!--END PROJECT_BRIEF-->
   <!--END !PROJECT_NAME-->
   <!--BEGIN DISABLE_INDEX-->
    <!--BEGIN SEARCHENGINE-->
      <!--BEGIN !FULL_SIDEBAR-->
-    <td>$searchbox</td>
+  <tr>   
      <!--END !FULL_SIDEBAR-->
    <!--END SEARCHENGINE-->
   <!--END DISABLE_INDEX-->
+  <div>
+    <style>
+	.cse input.gsc-input,input.gsc-input,.gsc-input-box-focus{
+		border-radius: var(--searchbar-border-radius) !important;
+		background-image:none !important;
+		color-scheme: light !important;
+		-webkit-box-sizing: border-box !important;
+		-moz-box-sizing: content-box !important;
+		box-sizing: content-box !important;
+		text-align: center !important;
+		border: none !important;
+		outline: none !important;
+	}
+	.gsc-clear-button {
+	    display:none !important;
+	}
+    </style>
     <td id="gsearch">
         <script async src="https://cse.google.com/cse.js?cx=004356362924927882526:zup3ehe-7bs"></script>
         <div class="gcse-search"></div>
     </td>
+  </div>
+ </tr>
+ <tr>
+  <td>
+    <div id="togglediv"></div>
+  </td>
  </tr>
   <!--BEGIN SEARCHENGINE-->
-   <!--BEGIN FULL_SIDEBAR-->
- <tr><td colspan="2">$searchbox</td></tr>
+  <!--BEGIN FULL_SIDEBAR-->
+   
+ 
    <!--END FULL_SIDEBAR-->
   <!--END SEARCHENGINE-->
  </tbody>
 </table>
 </div>
 <!--END TITLEAREA-->
-<!-- end header part -->
+<!-- end header part -->
\ No newline at end of file

From 8c7eff36460a2e943525e05ac875306157f14529 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 21 Dec 2022 17:50:52 -0500
Subject: [PATCH 188/473] fix exccessive padding w/gsearch on firefox

---
 docs/arrayfire.css | 22 ++++++++++++++++++++++
 docs/doxygen.mk    |  3 ++-
 docs/header.htm    | 18 +-----------------
 3 files changed, 25 insertions(+), 18 deletions(-)
 create mode 100644 docs/arrayfire.css

diff --git a/docs/arrayfire.css b/docs/arrayfire.css
new file mode 100644
index 0000000000..c9a0417fb0
--- /dev/null
+++ b/docs/arrayfire.css
@@ -0,0 +1,22 @@
+/*
+Overwrite google search bar .css to better match doxygen-awesome dark theme
+*/
+.cse input.gsc-input,input.gsc-input,.gsc_input-box,.gsc-input-box-focus{
+	border-radius: 4px !important;
+	background-image:none !important;
+	color-scheme: light !important;
+	-webkit-box-sizing: border-box !important;
+	-moz-box-sizing: content-box !important;
+	box-sizing: content-box !important;
+	border: none !important;
+	outline: none !important;
+}
+.gsc-control-cse {
+	padding: 0px !important;
+	border: none !important;
+	outline: none !important;
+	background-color: transparent !important;
+}
+.gsc-clear-button {
+	display:none !important;
+}
\ No newline at end of file
diff --git a/docs/doxygen.mk b/docs/doxygen.mk
index 4ec7155d51..2e4da59f66 100644
--- a/docs/doxygen.mk
+++ b/docs/doxygen.mk
@@ -1331,7 +1331,8 @@ HTML_STYLESHEET        =
 # list). For an example see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_STYLESHEET  = ${DOCS_DIR}/doxygen-awesome.css \
+HTML_EXTRA_STYLESHEET  = ${DOCS_DIR}/arrayfire.css \
+                         ${DOCS_DIR}/doxygen-awesome.css \
                          ${DOCS_DIR}/doxygen-awesome-sidebar-only.css
 
 # The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
diff --git a/docs/header.htm b/docs/header.htm
index 7709ca014c..9d7542fe1b 100644
--- a/docs/header.htm
+++ b/docs/header.htm
@@ -55,7 +55,7 @@
  <tbody>
   <tr id="projectrow">
   <!--BEGIN PROJECT_LOGO-->
-  <td id="projectlogo"><a  href="index.html"><img alt="Logo" src="$relpath^$projectlogo"/></a></td>
+  <td id="projectlogo"><a  href="index.htm"><img alt="Logo" src="$relpath^$projectlogo"/></a></td>
   <!--END PROJECT_LOGO-->
   </tr>
   <!--BEGIN PROJECT_BRIEF-->
@@ -74,22 +74,6 @@
    <!--END SEARCHENGINE-->
   <!--END DISABLE_INDEX-->
   <div>
-    <style>
-	.cse input.gsc-input,input.gsc-input,.gsc-input-box-focus{
-		border-radius: var(--searchbar-border-radius) !important;
-		background-image:none !important;
-		color-scheme: light !important;
-		-webkit-box-sizing: border-box !important;
-		-moz-box-sizing: content-box !important;
-		box-sizing: content-box !important;
-		text-align: center !important;
-		border: none !important;
-		outline: none !important;
-	}
-	.gsc-clear-button {
-	    display:none !important;
-	}
-    </style>
     <td id="gsearch">
         <script async src="https://cse.google.com/cse.js?cx=004356362924927882526:zup3ehe-7bs"></script>
         <div class="gcse-search"></div>

From eb23625f43479b26661face4eb8f23af5bd52b7e Mon Sep 17 00:00:00 2001
From: John Melonakos <john@arrayfire.com>
Date: Thu, 29 Dec 2022 12:35:00 -0500
Subject: [PATCH 189/473] docs updates to arith, blas, data.. new examples

---
 docs/details/arith.dox    | 20 ++++++----
 docs/details/blas.dox     | 12 +++++-
 docs/details/data.dox     | 35 +++++------------
 docs/details/examples.dox | 58 +++++++++++++++++++++++++++
 include/af/arith.h        | 72 +++++++++++++++++-----------------
 include/af/blas.h         | 34 +++++++---------
 include/af/data.h         | 82 +++++++++++++++++++++++----------------
 test/complex.cpp          | 59 ++++++++++++++++++++++++++++
 test/getting_started.cpp  | 14 +++++++
 test/moddims.cpp          | 34 ++++++++++++++++
 test/range.cpp            | 38 ++++++++++++++++++
 test/reduce.cpp           | 38 ++++++++++++++++++
 test/transpose.cpp        | 22 +++++++++++
 13 files changed, 392 insertions(+), 126 deletions(-)
 create mode 100644 docs/details/examples.dox

diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index 8461ecd100..ca3968db68 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -190,6 +190,7 @@ Bitwise xor operation of two inputs
 Minimum of two inputs.
 
 
+
 \defgroup arith_func_max max
 
 \ingroup numeric_mat
@@ -197,12 +198,6 @@ Minimum of two inputs.
 Maximum of two inputs.
 
 
-\defgroup arith_func_clamp clamp
-
-\ingroup numeric_mat
-
-Limits the range of the in array to the values between lo and hi
-
 
 \defgroup arith_func_rem rem
 
@@ -385,7 +380,18 @@ atanh of input
 
 \ingroup complex_mat
 
-create complex arrays
+Create complex arrays.
+
+Complex arrays are created from any of the following four inputs:
+
+1. a single real array, returning zeros for the imaginary component. See `array b` in the example.
+2. two real arrays, one for the real component and one for the imaginary component. See `array c` in the example.
+3. a single real array for the real component and a single scalar for each imaginary component. See `array d` in the example.
+4. a single scalar for each real component and a single real array for the imaginary component. See `array e` in the example.
+
+__Examples:__
+
+\snippet test/complex.cpp ex_arith_func_complex
 
 
 
diff --git a/docs/details/blas.dox b/docs/details/blas.dox
index 7ec09af9c3..3765ed446c 100644
--- a/docs/details/blas.dox
+++ b/docs/details/blas.dox
@@ -50,9 +50,17 @@ and restrictions.
 \ingroup blas_mat
 \ingroup manip_mat
 
-\brief Matrix Transpose
+\brief Transpose a matrix.
 
-Transposes a matrix
+Reverse or permute the dimensions of an array; returns the modified array. For an array a with two dimensions, `transpose(a)` gives the matrix transpose. For an array with more than two dimensions, the first two dimensions are transposed across higher dimensions.
+
+Set `conjugate=true` to perform the complex conjugate transpose of a matrix which interchanges the row and column index for each element, reflecting the elements across the main diagonal and negating the imaginary part of any complex numbers. For example, if `b = transpose(a, true)` and element `a(2, 1)` is `(1, 2)`, then element `b(1, 2)` is `(1, -2)`.
+
+In-place versions perform matrix transposition by reordering the input, reducing memory footprint.
+
+__Examples:__
+
+\snippet test/transpose.cpp ex_blas_func_transpose
 
 =======================================================================
 
diff --git a/docs/details/data.dox b/docs/details/data.dox
index f8db9586f0..99a94f1202 100644
--- a/docs/details/data.dox
+++ b/docs/details/data.dox
@@ -45,30 +45,11 @@ array a = identity(5, 3);
 
 \defgroup data_func_range range
 
-\brief Creates an array with [0, n] values along the seq_dim which is tiled across other dimensions
+\brief Create an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions.
 
-\code
-// Generates an array of [0, 4] along first dimension
-array a = range(dim4(5));        // a = [0,
-                                 //      1,
-                                 //      2,
-                                 //      3,
-                                 //      4]
-
-// Generates an array of [0, 4] along first dimension, tiled along second dimension
-array b = range(dim4(5, 2));     // a = [0, 0,
-                                 //      1, 1,
-                                 //      2, 2,
-                                 //      3, 3,
-                                 //      4, 4]
-
-// Generates an array of [0, 2] along second dimension, tiled along first dimension
-array c = range(dim4(5, 3), 1);  // c = [0, 1, 2,
-                                 //      0, 1, 2,
-                                 //      0, 1, 2,
-                                 //      0, 1, 2,
-                                 //      0, 1, 2]
-\endcode
+__Examples:__
+
+\snippet test/range.cpp ex_data_func_range
 
 \ingroup data_mat
 \ingroup arrayfire_func
@@ -259,9 +240,13 @@ Shifts the values in a circular fashion along the specified dimesion.
 
 \defgroup manip_func_moddims moddims
 
-\brief Modify the input dimensions without changing the data order
+\brief Modify the dimensions of an array without changing the order of its elements.
+
+This function only modifies array metadata and requires no computation. It is a NOOP.
+
+__Examples:__
 
-Simply modifies the metadata. This is a noop.
+\snippet test/moddims.cpp ex_data_func_moddims
 
 \ingroup manip_mat
 \ingroup arrayfire_func
diff --git a/docs/details/examples.dox b/docs/details/examples.dox
new file mode 100644
index 0000000000..a61ffbc271
--- /dev/null
+++ b/docs/details/examples.dox
@@ -0,0 +1,58 @@
+/**
+\example benchmarks/blas.cpp
+\example benchmarks/cg.cpp
+\example benchmarks/fft.cpp
+\example benchmarks/pi.cpp
+\example computer_vision/fast.cpp
+\example computer_vision/harris.cpp
+\example computer_vision/matching.cpp
+\example computer_vision/susan.cpp
+\example financial/black_scholes_options.cpp
+\example financial/heston_model.cpp
+\example financial/monte_carlo_options.cpp
+\example getting_started/convolve.cpp
+\example getting_started/integer.cpp
+\example getting_started/rainfall.cpp
+\example getting_started/vectorize.cpp
+\example graphics/conway.cpp
+\example graphics/conway_pretty.cpp
+\example graphics/field.cpp
+\example graphics/fractal.cpp
+\example graphics/gravity_sim.cpp
+\example graphics/histogram.cpp
+\example graphics/plot2d.cpp
+\example graphics/plot3.cpp
+\example graphics/surface.cpp
+\example helloworld/helloworld.cpp
+\example image_processing/adaptive_thresholding.cpp
+\example image_processing/binary_thresholding.cpp
+\example image_processing/brain_segmentation.cpp
+\example image_processing/confidence_connected_components.cpp
+\example image_processing/deconvolution.cpp
+\example image_processing/edge.cpp
+\example image_processing/filters.cpp
+\example image_processing/gradient_diffusion.cpp
+\example image_processing/image_demo.cpp
+\example image_processing/image_editing.cpp
+\example image_processing/morphing.cpp
+\example image_processing/optical_flow.cpp
+\example image_processing/pyramids.cpp
+\example lin_algebra/cholesky.cpp
+\example lin_algebra/lu.cpp
+\example lin_algebra/qr.cpp
+\example lin_algebra/svd.cpp
+\example machine_learning/bagging.cpp
+\example machine_learning/deep_belief_net.cpp
+\example machine_learning/geneticalgorithm.cpp
+\example machine_learning/kmeans.cpp
+\example machine_learning/knn.cpp
+\example machine_learning/logistic_regression.cpp
+\example machine_learning/naive_bayes.cpp
+\example machine_learning/neural_network.cpp
+\example machine_learning/perceptron.cpp
+\example machine_learning/rbm.cpp
+\example machine_learning/softmax_regression.cpp
+\example pde/swe.cpp
+\example unified/basic.cpp
+
+*/
diff --git a/include/af/arith.h b/include/af/arith.h
index 89bd39bd64..e2f695601d 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -259,36 +259,34 @@ namespace af
     AFAPI array atan2  (const double lhs, const array &rhs);
     /// @}
 
-    /// \ingroup trig_func_cplx2
+    /// \ingroup arith_func_cplx
     /// @{
-    /// C++ Interface for creating complex array from two inputs
+    /// C++ Interface for creating a complex array from a single real array.
     ///
-    /// Creates a complex number from two sets of inputs. The left hand side is
-    /// the real part and the right hand side is the imaginary part. This
-    /// function accepts two \ref af::array or one \ref af::array and a scalar
-    /// as nputs.
+    /// \param[in] in a real array
+    /// \return the returned complex array
+    AFAPI array complex(const array& in);
+ 
+    /// C++ Interface for creating a complex array from two real arrays.
     ///
-    /// \param[in] real is real value(s)
-    /// \param[in] imaginary is imaginary value(s)
-    /// \return complex array from inputs
-    /// \ingroup arith_func_cplx
-    AFAPI array complex(const array &real, const array &imaginary);
-
-    /// \copydoc complex(const array&, const array&)
-    /// \ingroup arith_func_cplx
-    AFAPI array complex(const array &real, const double imaginary);
-
-    /// \copydoc complex(const array&, const array&)
-    /// \ingroup arith_func_cplx
-    AFAPI array complex(const double real, const array &imaginary);
+    /// \param[in] real_ a real array to be assigned as the real component of the returned complex array
+    /// \param[in] imag_ a real array to be assigned as the imaginary component of the returned complex array
+    /// \return the returned complex array
+    AFAPI array complex(const array &real_, const array &imag_);
 
-    /// C++ Interface for creating complex array from real array
+    /// C++ Interface for creating a complex array from a single real array for the real component and a single scalar for each imaginary component.
     ///
-    /// \param[in] in is real array
-    /// \return complex array from \p in
+    /// \param[in] real_ a real array to be assigned as the real component of the returned complex array
+    /// \param[in] imag_ a single scalar to be assigned as the imaginary component of each value of the returned complex array
+    /// \return the returned complex array
+    AFAPI array complex(const array &real_, const double imag_);
+
+    /// C++ Interface for creating a complex array from a single scalar for each real component and a single real array for the imaginary component.
     ///
-    /// \ingroup arith_func_cplx
-    AFAPI array complex(const array &in);
+    /// \param[in] real_ a single scalar to be assigned as the real component of each value of the returned complex array
+    /// \param[in] imag_ a real array to be assigned as the imaginary component of the returned complex array
+    /// \return the returned complex array
+    AFAPI array complex(const double real_, const array &imag_);
     /// @}
 
     /// C++ Interface for getting real part from complex array
@@ -888,16 +886,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for clamp
+       C Interface for max of two arrays
 
-       \param[out] out will contain the values from \p in clamped between \p lo and \p hi
+       \param[out] out will contain the values from \p clamped between \p lo and \p hi
        \param[in] in Input array
        \param[in] lo Value for lower limit
        \param[in] hi Value for upper limit
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_clamp
+       \ingroup arith_func_max
     */
     AFAPI af_err af_clamp(af_array *out, const af_array in,
                           const af_array lo, const af_array hi, const bool batch);
@@ -1103,28 +1101,28 @@ extern "C" {
     AFAPI af_err af_atan2 (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for creating complex array from two input arrays
+       C Interface for creating a complex array from a single real array.
 
-       \param[out] out will contain the complex array generated from inputs
-       \param[in] real is real array
-       \param[in] imaginary is imaginary array
-       \param[in] batch specifies if operations need to be performed in batch mode
+       \param[out] out the returned complex array
+       \param[in] in a real array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_cplx
     */
-    AFAPI af_err af_cplx2 (af_array *out, const af_array real, const af_array imaginary, const bool batch);
+    AFAPI af_err af_cplx(af_array* out, const af_array in);
 
     /**
-       C Interface for creating complex array from real array
+       C Interface for creating a complex array from two real arrays.
 
-       \param[out] out will contain complex array created from real input \p in
-       \param[in] in is real array
+       \param[out] out the returned complex array
+       \param[in] real a real array to be assigned as the real component of the returned complex array
+       \param[in] imag a real array to be assigned as the imaginary component of the returned complex array
+       \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_cplx
     */
-    AFAPI af_err af_cplx    (af_array *out, const af_array in);
+    AFAPI af_err af_cplx2 (af_array *out, const af_array real, const af_array imag, const bool batch);
 
     /**
        C Interface for getting real part from complex array
diff --git a/include/af/blas.h b/include/af/blas.h
index 6023717d0e..d20986b215 100644
--- a/include/af/blas.h
+++ b/include/af/blas.h
@@ -181,24 +181,20 @@ namespace af
                     const matProp optRhs = AF_MAT_NONE);
 
     /**
-        \brief Transposes a matrix
+        \brief C++ Interface for transposing a matrix
 
-        \copydetails blas_func_transpose
-
-        \param[in] in Input Matrix
-        \param[in] conjugate If true a congugate transposition is performed
-        \return Transposed matrix
+        \param[in] in an input matrix
+        \param[in] conjugate if true, a conjugate transposition is performed
+        \return the transposed matrix
         \ingroup blas_func_transpose
     */
     AFAPI array transpose(const array &in, const bool conjugate = false);
 
     /**
-        \brief Transposes a matrix in-place
-
-        \copydetails blas_func_transpose
+        \brief C++ Interface for transposing a matrix in-place
 
-        \param[in,out] in is the matrix to be transposed in place
-        \param[in] conjugate If true a congugate transposition is performed
+        \param[in,out] in the matrix to be transposed in-place
+        \param[in] conjugate if true, a conjugate transposition is performed
 
         \ingroup blas_func_transpose
     */
@@ -356,13 +352,11 @@ extern "C" {
 #endif
 
     /**
-        \brief Transposes a matrix
+        \brief C Interface for transposing a matrix
 
-        This funciton will tranpose the matrix in.
-
-        \param[out] out The transposed matrix
-        \param[in] in Input matrix which will be transposed
-        \param[in] conjugate Perform a congugate transposition
+        \param[out] out the transposed matrix
+        \param[in] in an input matrix
+        \param[in] conjugate if true, a conjugate transposition is performed
 
         \return AF_SUCCESS if the process is successful.
         \ingroup blas_func_transpose
@@ -370,12 +364,10 @@ extern "C" {
     AFAPI af_err af_transpose(af_array *out, af_array in, const bool conjugate);
 
     /**
-        \brief Transposes a matrix in-place
-
-        \copydetails blas_func_transpose
+        \brief C Interface for transposing a matrix in-place
 
         \param[in,out] in is the matrix to be transposed in place
-        \param[in] conjugate If true a congugate transposition is performed
+        \param[in] conjugate if true, a conjugate transposition is performed
 
         \ingroup blas_func_transpose
     */
diff --git a/include/af/data.h b/include/af/data.h
index 6da90fe801..1559ea204f 100644
--- a/include/af/data.h
+++ b/include/af/data.h
@@ -144,25 +144,29 @@ namespace af
                          const dim_t d2, const dim_t d3, const dtype ty=f32);
 
     /**
-        \param[in] dims is dim4 for size of all dimensions
-        \param[in] seq_dim is dimesion along which [0, dim[seq_dim] - 1] is generated
-        \param[in] ty is the type of array to generate
+    *  C++ Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions of shape `dim4`.
+    *
+        \param[in] dims the `dim4` object describing the shape of the generated array
+        \param[in] seq_dim the dimesion along which `[0, dim[seq_dim] - 1]` is created
+        \param[in] ty the type of the generated array
 
-        \returns an array of integral range specified dimension and type
+        \returns the generated array
 
         \ingroup data_func_range
     */
     AFAPI array range(const dim4 &dims, const int seq_dim = -1, const dtype ty=f32);
 
     /**
-        \param[in] d0 is size of first dimension
-        \param[in] d1 is size of second dimension
-        \param[in] d2 is size of third dimension
-        \param[in] d3 is size of fourth dimension
-        \param[in] seq_dim is dimesion along which [0, dim[seq_dim] - 1] is generated
-        \param[in] ty is the type of array to generate
+    *  C++ Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions described by dimension parameters.
+    *
+        \param[in] d0 the size of first dimension
+        \param[in] d1 the size of second dimension
+        \param[in] d2 the size of third dimension
+        \param[in] d3 the size of fourth dimension
+        \param[in] seq_dim the dimesion along which `[0, dim[seq_dim] - 1]` is created
+        \param[in] ty the type of the generated array
 
-        \returns an array of integral range specified dimension and type
+        \returns the generated array
 
         \ingroup data_func_range
     */
@@ -295,35 +299,41 @@ namespace af
     AFAPI array shift(const array& in, const int x, const int y=0, const int z=0, const int w=0);
 
     /**
-        \param[in] in is the input array
-        \param[in] ndims is the number of dimensions
-        \param[in] dims is the array containing the new dimensions
+    * C++ Interface for modifying the dimensions of an input array to the shape specified by a `dim4` object
+    *
+        \param[in] in the input array
+        \param[in] dims the array of new dimension sizes
         \return the modded output
 
         \ingroup manip_func_moddims
     */
-    AFAPI array moddims(const array& in, const unsigned ndims, const dim_t * const dims);
+    AFAPI array moddims(const array& in, const dim4& dims);
 
     /**
-        \param[in] in is the input array
-        \param[in] dims is the new dimensions
+    * C++ Interface for modifying the dimensions of an input array to the shape specified by dimension length parameters
+    *
+        \param[in] in the input array
+        \param[in] d0 the new size of the first dimension
+        \param[in] d1 the new size of the second dimension (optional)
+        \param[in] d2 the new size of the third dimension (optional)
+        \param[in] d3 the new size of the fourth dimension (optional)
         \return the modded output
 
         \ingroup manip_func_moddims
     */
-    AFAPI array moddims(const array& in, const dim4& dims);
+    AFAPI array moddims(const array& in, const dim_t d0, const dim_t d1=1, const dim_t d2=1, const dim_t d3=1);
 
     /**
-        \param[in] in is the input array
-        \param[in] d0 specifies the new size of the first dimension
-        \param[in] d1 specifies the new size of the second dimension
-        \param[in] d2 specifies the new size of the third dimension
-        \param[in] d3 specifies the new size of the fourth dimension
-        \return the modded array
+    * C++ Interface for modifying the dimensions of an input array to the shape specified by an array of `ndims` dimensions
+    *
+        \param[in] in the input array
+        \param[in] ndims the number of dimensions
+        \param[in] dims the array of new dimension sizes
+        \return the modded output
 
         \ingroup manip_func_moddims
     */
-    AFAPI array moddims(const array& in, const dim_t d0, const dim_t d1=1, const dim_t d2=1, const dim_t d3=1);
+    AFAPI array moddims(const array& in, const unsigned ndims, const dim_t* const dims);
 
     /**
         \param[in] in is the input array
@@ -567,11 +577,13 @@ extern "C" {
     AFAPI af_err af_constant_ulong(af_array *arr, const unsigned long long val, const unsigned ndims, const dim_t * const dims);
 
     /**
-        \param[out] out is the generated array
-        \param[in] ndims is size of dimension array \p dims
-        \param[in] dims is the array containing sizes of the dimension
-        \param[in] seq_dim is dimension along which [0, dim[seq_dim] - 1] is generated
-        \param[in] type is the type of array to generate
+    * C Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions specified by an array of `ndims` dimensions.
+    *
+        \param[out] out the generated array
+        \param[in] ndims the size of dimension array `dims`
+        \param[in] dims the array containing the dimension sizes
+        \param[in] seq_dim the dimension along which `[0, dim[seq_dim] - 1]` is created
+        \param[in] type the type of the generated array
 
         \ingroup data_func_range
     */
@@ -693,10 +705,12 @@ extern "C" {
     AFAPI af_err af_shift(af_array *out, const af_array in, const int x, const int y, const int z, const int w);
 
     /**
-        \param[out] out is the modded array
-        \param[in] in is the input array
-        \param[in] ndims is the number of dimensions
-        \param[in] dims is the array containing the new dimensions
+    * C Interface for modifying the dimensions of an input array to the shape specified by an array of `ndims` dimensions
+    *
+        \param[out] out the modded output
+        \param[in] in the input array
+        \param[in] ndims the number of dimensions
+        \param[in] dims the array of new dimension sizes
 
         \ingroup manip_func_moddims
     */
diff --git a/test/complex.cpp b/test/complex.cpp
index 93a5d47b18..b63fd63bba 100644
--- a/test/complex.cpp
+++ b/test/complex.cpp
@@ -134,3 +134,62 @@ const int num = 10;
 COMPLEX_TESTS(float, float, float)
 COMPLEX_TESTS(double, double, double)
 COMPLEX_TESTS(float, double, double)
+
+TEST(Complex, SNIPPET_arith_func_complex) {
+    //! [ex_arith_func_complex]
+    //!
+    // Create a, a 2x3 array
+    array a = iota(dim4(2, 3));    // a = [0, 2, 4,
+                                   //      1, 3, 5]
+
+    // Create b from a single real array, returning zeros for the imaginary component
+    array b = complex(a);          // b = [(0, 0), (2, 0), (4, 0),
+                                   //      (1, 0), (3, 0), (5, 0)]
+
+    // Create c from two real arrays, one for the real component and one for the imaginary component
+    array c = complex(a, a);       // c = [(0, 0), (2, 2), (4, 4),
+                                   //      (1, 1), (3, 3), (5, 5)]
+
+    // Create d from a single real array for the real component and a single scalar for each imaginary component
+    array d = complex(a, 2);       // d = [(0, 2), (2, 2), (4, 2),
+                                   //      (1, 2), (3, 2), (5, 2)]
+
+    // Create e from a single scalar for each real component and a single real array for the imaginary component
+    array e = complex(2, a);       // e = [(2, 0), (2, 2), (2, 4),
+                                   //      (2, 1), (2, 3), (2, 5)]
+
+    //! [ex_arith_func_complex]
+
+    using std::complex;
+    using std::vector;
+    vector<float> ha(a.elements());
+    a.host(ha.data());
+
+    vector<cfloat> gold_b(a.elements());
+    for (int i = 0; i < a.elements(); i++) {
+        gold_b[i].real = ha[i];
+        gold_b[i].imag = 0;
+    }
+    ASSERT_VEC_ARRAY_EQ(gold_b, a.dims(), b);
+
+    vector<cfloat> gold_c(a.elements());
+    for (int i = 0; i < a.elements(); i++) {
+        gold_c[i].real = ha[i];
+        gold_c[i].imag = ha[i];
+    }
+    ASSERT_VEC_ARRAY_EQ(gold_c, a.dims(), c);
+
+    vector<cfloat> gold_d(a.elements());
+    for (int i = 0; i < a.elements(); i++) {
+        gold_d[i].real = ha[i];
+        gold_d[i].imag = 2;
+    }
+    ASSERT_VEC_ARRAY_EQ(gold_d, a.dims(), d);
+
+    vector<cfloat> gold_e(a.elements());
+    for (int i = 0; i < a.elements(); i++) {
+        gold_e[i].real = 2;
+        gold_e[i].imag = ha[i];
+    }
+    ASSERT_VEC_ARRAY_EQ(gold_e, a.dims(), e);
+}
\ No newline at end of file
diff --git a/test/getting_started.cpp b/test/getting_started.cpp
index ac77f58cf5..c9e73ef6b5 100644
--- a/test/getting_started.cpp
+++ b/test/getting_started.cpp
@@ -307,3 +307,17 @@ TEST(GettingStarted, SNIPPET_getting_started_constants) {
 
     ASSERT_LE(fabs(Pi - pi_est), 0.005);
 }
+
+TEST(GettingStarted, SNIPPET_JohnTest) {
+    array a = iota(dim4(2, 3));
+    array b = sum(a);     // sum across the first axis, same as sum(a, 0)
+    array c = sum(a, 1);  // sum across the second axis
+    array d = sum(a, 2);  // sum across the third axis
+    array e = sum(a, 3);  // sum acorss the fourth axis
+    // array f = sum(a, 4); fails due to stepping out of bounds
+    af_print(a);
+    af_print(b);
+    af_print(c);
+    af_print(d);
+    af_print(e);
+}
\ No newline at end of file
diff --git a/test/moddims.cpp b/test/moddims.cpp
index 9674c5a4f1..a7dea52a00 100644
--- a/test/moddims.cpp
+++ b/test/moddims.cpp
@@ -346,3 +346,37 @@ TEST(Moddims, JitMultipleModdimsThenTiled) {
     gold.eval();
     ASSERT_ARRAYS_EQ(gold, c);
 }
+
+TEST(Moddims, SNIPPET_data_func_moddims) {
+    // clang-format off
+    //! [ex_data_func_moddims]
+    //!
+    // Create a, a 2x3 array
+    array a = iota(dim4(2, 3));           // a = [0, 2, 4,
+                                          //      1, 3, 5]
+
+    // Create b by modifying the dimensions of a to the shape described by a dim4 object
+    array b = moddims(a, dim4(3, 2));     // b = [0, 3,
+                                          //      1, 4,
+                                          //      2, 5]
+
+    // Create c by modifying the dimensions of a to the shape described by dimension length parameters
+    array c = moddims(a, 3, 2);           // c = [0, 3,
+                                          //      1, 4,
+                                          //      2, 5]
+
+    // Create d by modifying the dimensions of a to the shape described by an array of ndims dimensions
+    vector<dim_t> x{3, 2};
+    array d = moddims(a, 2, x.data());    // d = [0, 3,
+                                          //      1, 4,
+                                          //      2, 5]
+
+    //! [ex_data_func_moddims]
+    // clang-format on
+
+    vector<float> gold_a{0, 1, 2, 3, 4, 5};
+
+    ASSERT_VEC_ARRAY_EQ(gold_a, dim4(3, 2), b);
+    ASSERT_VEC_ARRAY_EQ(gold_a, dim4(3, 2), c);
+    ASSERT_VEC_ARRAY_EQ(gold_a, dim4(3, 2), d);
+}
\ No newline at end of file
diff --git a/test/range.cpp b/test/range.cpp
index 4d90b8a42f..35708bde09 100644
--- a/test/range.cpp
+++ b/test/range.cpp
@@ -171,3 +171,41 @@ TEST(Range, CPP) {
     // Delete
     delete[] outData;
 }
+
+TEST(Range, SNIPPET_data_func_range) {
+    // clang-format off
+    //! [ex_data_func_range]
+    //!
+    // Generates an array of [0, 4] along first dimension
+    array a = range(dim4(5));          // a = [0,
+                                       //      1,
+                                       //      2,
+                                       //      3,
+                                       //      4]
+
+    // Generates an array of [0, 4] along first dimension, tiled along second dimension
+    array b = range(dim4(5, 2));       // b = [0, 0,
+                                       //      1, 1,
+                                       //      2, 2,
+                                       //      3, 3,
+                                       //      4, 4]
+
+    // Generates an array of [0, 2] along second dimension, tiled along first dimension
+    array c = range(dim4(5, 3), 1);    // c = [0, 1, 2,
+                                       //      0, 1, 2,
+                                       //      0, 1, 2,
+                                       //      0, 1, 2,
+                                       //      0, 1, 2]
+
+    //! [ex_data_func_range]
+    // clang-format on
+
+    using std::vector;
+    vector<float> gold_a{0, 1, 2, 3, 4};
+    vector<float> gold_b{0, 1, 2, 3, 4, 0, 1, 2, 3, 4};
+    vector<float> gold_c{0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2};
+
+    ASSERT_VEC_ARRAY_EQ(gold_a, a.dims(), a);
+    ASSERT_VEC_ARRAY_EQ(gold_b, b.dims(), b);
+    ASSERT_VEC_ARRAY_EQ(gold_c, c.dims(), c);
+}
diff --git a/test/reduce.cpp b/test/reduce.cpp
index c6cc0d7d72..fc16e60716 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -2330,3 +2330,41 @@ TEST(Reduce, nanval_issue_3255) {
     }
     ASSERT_SUCCESS(af_release_array(ikeys));
 }
+
+TEST(Reduce, SNIPPET_algorithm_func_sum) {
+    // clang-format off
+    //! [ex_algorithm_func_sum]
+    //
+    // Create a, a 2x3 array
+    array a = iota(dim4(2, 3));           // a = [0, 2, 4,
+                                          //      1, 3, 5]
+
+    // Create b by summing across the first dimension
+    array b = sum(a);        // sum across the first dimension, same as sum(a, 0)
+
+    // Create c by summing across the second dimension
+    array c = sum(a, 1);     // sum across the second dimension
+
+    // Create d by summing across the third dimension
+    array d = sum(a, 2);     // sum across the third dimension
+
+    // Create e by summing across the fouth dimension
+    array e = sum(a, 3);     // sum acorss the fourth dimension
+
+    // Summing across higher dimensions fails due to stepping out of bounds. For example,
+    // array f = sum(a0, 4)  // fails due to stepping out of bounds
+
+    //! [ex_algorithm_func_sum]
+    // clang-format on
+
+    using std::vector;
+    vector<float> gold_a{0, 1, 2, 3, 4, 5};
+    vector<float> gold_b{1, 5, 9};
+    vector<float> gold_c{6, 9};
+
+    ASSERT_VEC_ARRAY_EQ(gold_a, a.dims(), a);
+    ASSERT_VEC_ARRAY_EQ(gold_b, b.dims(), b);
+    ASSERT_VEC_ARRAY_EQ(gold_c, c.dims(), c);
+    ASSERT_VEC_ARRAY_EQ(gold_a, d.dims(), d);
+    ASSERT_VEC_ARRAY_EQ(gold_a, e.dims(), e);
+}
diff --git a/test/transpose.cpp b/test/transpose.cpp
index 8bc0c1c6e9..72a32194fa 100644
--- a/test/transpose.cpp
+++ b/test/transpose.cpp
@@ -263,3 +263,25 @@ TEST(Transpose, GFOR) {
         ASSERT_EQ(max<double>(abs(c_ii - b_ii)) < 1E-5, true);
     }
 }
+
+TEST(Transpose, SNIPPET_blas_func_transpose) {
+    // clang-format off
+    //! [ex_blas_func_transpose]
+    //!
+    // Create a, a 2x3 array
+    array a = iota(dim4(2, 3));    // a = [0, 2, 4
+                                   //      1, 3, 5]
+
+    // Create b, the transpose of a
+    array b = transpose(a);        // b = [0, 1,
+                                   //      2, 3,
+                                   //      4, 5]
+
+    //! [ex_blas_func_transpose]
+    // clang-format on
+
+    using std::vector;
+    vector<float> gold_b{0, 2, 4, 1, 3, 5};
+
+    ASSERT_VEC_ARRAY_EQ(gold_b, b.dims(), b);
+}

From bac6b9302ad882756160d3f69868eebc80cd91df Mon Sep 17 00:00:00 2001
From: John Melonakos <john.melonakos@intel.com>
Date: Wed, 11 Jan 2023 15:30:39 -0500
Subject: [PATCH 190/473] improves documentation for arith functions

---
 docs/details/arith.dox | 294 ++++++++-----
 include/af/arith.h     | 919 +++++++++++++++++++++--------------------
 2 files changed, 662 insertions(+), 551 deletions(-)

diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index ca3968db68..84f9a5c451 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -21,51 +21,39 @@
 
 \ingroup arith_mat
 
-Addition of two inputs.
+Add.
 
+Add two arrays.
 
 
-\defgroup arith_func_sub sub
-
-\ingroup arith_mat
-
-Subtract one input from another
 
-
-
-\defgroup arith_func_mul mul
+\defgroup arith_func_sub sub
 
 \ingroup arith_mat
 
-Multiply two inputs element wise
-
-
-
-\defgroup arith_func_div div
-
-\ingroup arith_mat
+Subtract.
 
-Divide one input by another
+Subtract one array from another array.
 
 
 
-\defgroup arith_func_shiftl bitshiftl
+\defgroup arith_func_mul mul
 
 \ingroup arith_mat
 
-Left shift an input
+Multiply.
 
-\copydoc arith_int_only
+Multiply two arrays.
 
 
 
-\defgroup arith_func_shiftr bitshiftr
+\defgroup arith_func_div div
 
 \ingroup arith_mat
 
-Right shift an input
+Divide.
 
-\copydoc arith_int_only
+Divide one array by another array.
 
 
 
@@ -73,7 +61,9 @@ Right shift an input
 
 \ingroup logic_mat
 
-Check if input is less than another
+Is less than.
+
+Check if the elements of one array are less than those of another array.
 
 
 
@@ -81,7 +71,9 @@ Check if input is less than another
 
 \ingroup logic_mat
 
-Check if input is greater than another
+Is greater than.
+
+Check if the elements of one array are greater than those of another array.
 
 
 
@@ -89,7 +81,9 @@ Check if input is greater than another
 
 \ingroup logic_mat
 
-Check if input is less than or equal to another
+Is less than or equal.
+
+Check if the elements of one array are less than or equal to those of another array.
 
 
 
@@ -97,7 +91,9 @@ Check if input is less than or equal to another
 
 \ingroup logic_mat
 
-Check if input is greater than or equal to another
+Is greater than or equal.
+
+Check if the elements of one array are greater than or equal to those of another array.
 
 
 
@@ -105,7 +101,9 @@ Check if input is greater than or equal to another
 
 \ingroup logic_mat
 
-Check if input two inputs are equal
+Is equal.
+
+Check if the elements of one array are equal to those of another array.
 
 
 
@@ -113,7 +111,9 @@ Check if input two inputs are equal
 
 \ingroup logic_mat
 
-Check if input two inputs are not equal
+Is not equal.
+
+Check if the elements of one array are not equal to those of another array.
 
 
 
@@ -122,13 +122,17 @@ Check if input two inputs are not equal
 
 \ingroup logic_mat
 
-Logical and of two inputs
+Logical AND.
+
+Evaluate the logical AND of two arrays.
 
 \defgroup arith_func_or or
 
 \ingroup logic_mat
 
-Logical or of two inputs
+Logical OR.
+
+Evaluate the logical OR of two arrays.
 
 
 
@@ -136,7 +140,9 @@ Logical or of two inputs
 
 \ingroup logic_mat
 
-Logical not of an input
+Logical NOT.
+
+Evaluate the logical NOT of an array.
 
 
 
@@ -144,14 +150,18 @@ Logical not of an input
 
 \ingroup numeric_mat
 
-Negative of an input
+Negative of an array.
+
+Negate an array.
 
 
 \defgroup arith_func_bitnot bitnot
 
 \ingroup logic_mat
 
-Bitwise not on the input
+Bitwise NOT.
+
+Evaluate the bitwise NOT of an array.
 
 \copydoc arith_int_only
 
@@ -160,7 +170,9 @@ Bitwise not on the input
 
 \ingroup logic_mat
 
-Bitwise and operation of two inputs
+Bitwise AND.
+
+Evaluate the bitwise AND of two arrays.
 
 \copydoc arith_int_only
 
@@ -169,7 +181,9 @@ Bitwise and operation of two inputs
 
 \ingroup logic_mat
 
-Bitwise or operation of two inputs
+Bitwise OR.
+
+Evaluate the bitwise OR of two arrays.
 
 \copydoc arith_int_only
 
@@ -178,17 +192,49 @@ Bitwise or operation of two inputs
 
 \ingroup logic_mat
 
-Bitwise xor operation of two inputs
+Bitwise XOR.
+
+Evaluate the bitwise XOR of two arrays.
 
 \copydoc arith_int_only
 
 
+\defgroup arith_func_shiftl bitshiftl
+
+\ingroup arith_mat
+
+Left shift on integer arrays.
+
+Shift the bits of integer arrays left.
+
+\copydoc arith_int_only
+
+
+\defgroup arith_func_shiftr bitshiftr
+
+\ingroup arith_mat
+
+Right shift on integer arrays.
+
+Shift the bits of integer arrays right.
+
+\copydoc arith_int_only
+
+
+\defgroup arith_func_cast cast
+
+\ingroup helper_mat
+
+Cast an array from one type to another.
+
+
 \defgroup arith_func_min min
 
 \ingroup numeric_mat
 
 Minimum of two inputs.
 
+Find the elementwise minimum between two arrays.
 
 
 \defgroup arith_func_max max
@@ -197,13 +243,16 @@ Minimum of two inputs.
 
 Maximum of two inputs.
 
+Find the elementwise maximum between two arrays.
 
 
 \defgroup arith_func_rem rem
 
 \ingroup numeric_mat
 
-Remainder operation
+Remainder.
+
+Find the remainder of a division.
 
 \copydoc arith_real_only
 
@@ -212,34 +261,41 @@ Remainder operation
 
 \ingroup numeric_mat
 
-Compute \f$x - n * y\f$ where n is quotient of \f$x / y\f$
+Modulus.
 
-\copydoc arith_real_only
+Find the modulus.
 
+\copydoc arith_real_only
 
 
 \defgroup arith_func_abs abs
 
-\brief Absolute value
+Absolute value.
 
-\ingroup numeric_mat
+Find the absolute value.
 
-Absolute value
+__Examples:__
+
+\snippet test/math.cpp ex_arith_func_abs
 
+\ingroup numeric_mat
 
 
 \defgroup arith_func_arg arg
 \ingroup numeric_mat
 
-\brief Phase of a number in the complex plane
+Phase angle.
 
+Find the phase angle (in radians) of a complex array.
 
 
 \defgroup arith_func_sign sign
 
 \ingroup numeric_mat
 
-Check if input is negative
+Sign.
+
+Find the sign of elements in an array.
 
 \copydoc arith_real_only
 
@@ -248,7 +304,9 @@ Check if input is negative
 
 \ingroup numeric_mat
 
-Round to nearest integer
+Round.
+
+Round numbers to the nearest integer.
 
 \copydoc arith_real_only
 
@@ -257,7 +315,9 @@ Round to nearest integer
 
 \ingroup numeric_mat
 
-Truncate to nearest integer
+Truncate.
+
+Truncate numbers to nearest integer.
 
 \copydoc arith_real_only
 
@@ -266,7 +326,9 @@ Truncate to nearest integer
 
 \ingroup numeric_mat
 
-Round to integer less than equal to current value
+Floor.
+
+Round to the integer less than or equal to the magnitude of the input value.
 
 \copydoc arith_real_only
 
@@ -275,7 +337,9 @@ Round to integer less than equal to current value
 
 \ingroup numeric_mat
 
-Round to integer greater than equal to current value
+Ceil.
+
+Round to the integer greater than or equal to the magnitude of the input value.
 
 \copydoc arith_real_only
 
@@ -284,7 +348,9 @@ Round to integer greater than equal to current value
 
 \ingroup numeric_mat
 
-Hypotenuse of the two inputs
+Hypotenuse.
+
+Find the length of the hypotenuse of two inputs.
 
 \copydoc arith_real_only
 
@@ -293,87 +359,114 @@ Hypotenuse of the two inputs
 
 \ingroup trig_mat
 
-sin of input
+Sine.
+
+Evaluate the sine function.
 
 
 \defgroup arith_func_cos cos
 
 \ingroup trig_mat
 
-cos of input
+Cosine.
 
+Evaluate the cosine function.
 
 
 \defgroup arith_func_tan tan/tan2
 
 \ingroup trig_mat
 
-tan of input
+Tangent.
+
+Evaluate the tangent function.
 
 
 \defgroup arith_func_asin asin
 
 \ingroup trig_mat
 
-arc sin of input
+Inverse sine (arc sine).
+
+Evaluate the inverse sine function.
 
 
 \defgroup arith_func_acos acos
-\brief Inverse cosine.
 
-\ingroup trig_mat
+Inverse cosine (arc cosine).
+
+Evaluate the inverse cosine function.
 
-arc cos of input
+The inverse of cosine so that, if `y = cos(x)`, then `x = arccos(y)`.
+
+__Examples:__
+
+\snippet test/math.cpp ex_arith_func_acos
+
+\ingroup trig_mat
 
 
 \defgroup arith_func_atan atan/atan2
 
 \ingroup trig_mat
 
-arc tan of input
+Inverse tangent (arc tangent).
+
+Evaluate the inverse tangent function.
 
 
 \defgroup arith_func_sinh sinh
 
 \ingroup hyper_mat
 
-sinh of input
+Hyperbolic sine.
+
+Evaluate the hyperbolic sine function.
 
 
 \defgroup arith_func_cosh cosh
 
 \ingroup hyper_mat
 
-cosh of input
+Hyperbolic cosine.
+
+Evaluate the hyperbolic cosine function.
 
 
 \defgroup arith_func_tanh tanh
 
 \ingroup hyper_mat
 
-tanh of input
+Hyperbolic tangent.
+
+Evaluate the hyperbolic tangent function.
 
 
 \defgroup arith_func_asinh asinh
 
 \ingroup hyper_mat
 
-asinh of input
+Inverse hyperbolic sine (area hyperbolic sine).
+
+Evaluate the inverse hyperbolic sine function.
 
 
 \defgroup arith_func_acosh acosh
-\brief Inverse hyperbolic cosine
 
 \ingroup hyper_mat
 
-acosh of input
+Inverse hyperbolic cosine (area hyperbolic cosine).
+
+Evaluate the inverse hyperbolic cosine function.
 
 
 \defgroup arith_func_atanh atanh
 
 \ingroup hyper_mat
 
-atanh of input
+Inverse hyperbolic tangent (area hyperbolic tangent).
+
+Evaluate the inverse hyperbolic tangent function.
 
 
 \defgroup arith_func_cplx complex
@@ -394,44 +487,41 @@ __Examples:__
 \snippet test/complex.cpp ex_arith_func_complex
 
 
-
 \defgroup arith_func_real real
 
 \ingroup complex_mat
 
-Get real part of complex arrays
-
+Find the real part of a complex array.
 
 
 \defgroup arith_func_imag imag
 
 \ingroup complex_mat
 
-Get imaginary part of complex arrays
-
+Find the imaginary part of a complex array.
 
 
 \defgroup arith_func_conjg conjg
 
 \ingroup complex_mat
 
-Get complex conjugate
-
+Complex conjugate.
 
+Find the complex conjugate of an input array.
 
 
 \defgroup arith_func_root root
 
 \ingroup explog_mat
 
-Find root of an input
+Find the nth root.
 
 
 \defgroup arith_func_pow pow
 
 \ingroup explog_mat
 
-Raise an array to a power
+Raise a base to a power (or exponent).
 
 If the input array has values beyond what a floating point type can represent, then there is no
 guarantee that the results will be accurate. The exact type mapping from integral types to floating
@@ -450,19 +540,26 @@ point types used to compute power is given below.
 The output array will be of the same type as input.
 
 
+\defgroup arith_func_sigmoid sigmoid
+
+Sigmoid function (logistical).
+
+Evaluate the logistical sigmoid function.
+
+
 
 \defgroup arith_func_exp exp
 
 \ingroup explog_mat
 
-Exponential of input
+Evaluate the exponential.
 
 
 \defgroup arith_func_expm1 expm1
 
 \ingroup explog_mat
 
-Exponential of input - 1
+Evaluate the exponential of an array minus 1, `exp(in) - 1`.
 
 \copydoc arith_real_only
 
@@ -471,7 +568,7 @@ Exponential of input - 1
 
 \ingroup explog_mat
 
-Error function value
+Evaluate the error function.
 
 \copydoc arith_real_only
 
@@ -481,7 +578,7 @@ Error function value
 
 \ingroup explog_mat
 
-Complementary Error function value
+Evaluate the complementary error function.
 
 \copydoc arith_real_only
 
@@ -490,14 +587,14 @@ Complementary Error function value
 
 \ingroup explog_mat
 
-Natural logarithm
+Evaluate the natural logarithm.
 
 
 \defgroup arith_func_log1p log1p
 
 \ingroup explog_mat
 
-Natural logarithm of (1 + in)
+Evaluate the natural logarithm of 1 + input, `ln(1+in)`.
 
 \copydoc arith_real_only
 
@@ -506,7 +603,16 @@ Natural logarithm of (1 + in)
 
 \ingroup explog_mat
 
-logarithm base 10
+Evaluate the base 10 logarithm.
+
+\copydoc arith_real_only
+
+
+\defgroup arith_func_log2 log2
+
+\ingroup explog_mat
+
+Evaluate the base 2 logarithm.
 
 \copydoc arith_real_only
 
@@ -515,23 +621,25 @@ logarithm base 10
 
 \ingroup explog_mat
 
-Square root of input arrays
+Find the square root.
+
 
 \defgroup arith_func_rsqrt rsqrt
 
 \ingroup explog_mat
 
-The reciprocal or inverse square root of input arrays
+Find the reciprocal square root.
 
 \f[ \frac{1}{\sqrt{x}} \f]
 
 \copydoc arith_real_only
 
+
 \defgroup arith_func_cbrt cbrt
 
 \ingroup explog_mat
 
-Cube root of input arrays
+Find the cube root.
 
 \copydoc arith_real_only
 
@@ -540,7 +648,7 @@ Cube root of input arrays
 
 \ingroup explog_mat
 
-Factorial function
+Find the factorial.
 
 \copydoc arith_real_only
 
@@ -549,7 +657,7 @@ Factorial function
 
 \ingroup explog_mat
 
-Gamma function
+Evaluate the gamma function.
 
 \copydoc arith_real_only
 
@@ -558,7 +666,7 @@ Gamma function
 
 \ingroup explog_mat
 
-Logarithm of absolute values of Gamma function
+Evaluate the logarithm of the absolute value of the gamma function.
 
 \copydoc arith_real_only
 
@@ -567,28 +675,22 @@ Logarithm of absolute values of Gamma function
 
 \ingroup helper_mat
 
-Check if values are zero
+Check if values are zero.
 
 
 \defgroup arith_func_isinf isinf
 
 \ingroup helper_mat
 
-Check if values are infinite
+Check if values are infinite.
 
 
 \defgroup arith_func_isnan isNan
 
 \ingroup helper_mat
 
-Check if values are Nan
-
-
-\defgroup arith_func_cast cast
-
-\ingroup helper_mat
+Check if values are NaN.
 
-Casting inputs from one type to another
 
 @}
 */
diff --git a/include/af/arith.h b/include/af/arith.h
index e2f695601d..789e54aab5 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -14,48 +14,70 @@ namespace af
 {
     class array;
 
-    /// \ingroup arith_func_min
-    /// @{
-    /// \brief C++ interface for min of two arrays
+    /// C++ Interface to find the elementwise minimum between two arrays.
     ///
-    /// \param[in] lhs first input
-    /// \param[in] rhs second input
+    /// \param[in] lhs input array
+    /// \param[in] rhs input array
     /// \return minimum of \p lhs and \p rhs
     ///
+    /// \ingroup arith_func_min
     AFAPI array min    (const array &lhs, const array &rhs);
 
-    /// \copydoc min(const array&, const array &)
+    /// C++ Interface to find the elementwise minimum between an array and a scalar value.
+    ///
+    /// \param[in] lhs input array
+    /// \param[in] rhs scalar value
+    /// \return minimum of \p lhs and \p rhs
+    ///
+    /// \ingroup arith_func_min
     AFAPI array min    (const array &lhs, const double rhs);
 
-    /// \copydoc min(const array&, const array &)
+    /// C++ Interface to find the elementwise minimum between an array and a scalar value.
+    ///
+    /// \param[in] lhs scalar value
+    /// \param[in] rhs input array
+    /// \return minimum of \p lhs and \p rhs
+    ///
+    /// \ingroup arith_func_min
     AFAPI array min    (const double lhs, const array &rhs);
-    /// @}
 
-    /// \ingroup arith_func_max
-    /// @{
-    /// C++ Interface for max of two arrays or an array and a scalar
+    /// C++ Interface to find the elementwise maximum between two arrays.
     ///
-    /// \param[in] lhs first input
-    /// \param[in] rhs second input
+    /// \param[in] lhs input array
+    /// \param[in] rhs input array
     /// \return maximum of \p lhs and \p rhs
+    ///
+    /// \ingroup arith_func_max
     AFAPI array max    (const array &lhs, const array &rhs);
 
-    /// \copydoc max(const array&, const array&)
+    /// C++ Interface to find the elementwise maximum between an array and a scalar value.
+    ///
+    /// \param[in] lhs input array
+    /// \param[in] rhs scalar value
+    /// \return maximum of \p lhs and \p rhs
+    ///
+    /// \ingroup arith_func_max
     AFAPI array max    (const array &lhs, const double rhs);
 
-    /// \copydoc max(const array&, const array&)
+    /// C++ Interface to find the elementwise maximum between an array and a scalar value.
+    ///
+    /// \param[in] lhs input array
+    /// \param[in] rhs scalar value
+    /// \return maximum of \p lhs and \p rhs
+    ///
+    /// \ingroup arith_func_max
     AFAPI array max    (const double lhs, const array &rhs);
-    /// @}
 
 #if AF_API_VERSION >= 34
-    /// \ingroup arith_func_clamp
     /// @{
-    /// C++ Interface for clamping an array between two values
+    /// C++ Interface to clamp an array between an upper and a lower limit.
     ///
-    /// \param[in] in Input array
-    /// \param[in] lo Value for lower limit
-    /// \param[in] hi  Value for upper limit
+    /// \param[in] in input array
+    /// \param[in] lo lower limit; can be an array or a scalar
+    /// \param[in] hi upper limit; can be an array or a scalar
     /// \return array containing values from \p in clamped between \p lo and \p hi
+    /// 
+    /// \ingroup arith_func_clamp
     AFAPI array clamp(const array &in, const array &lo, const array &hi);
 #endif
 
@@ -75,14 +97,14 @@ namespace af
 #endif
     /// @}
 
-    /// \ingroup arith_func_rem
     /// @{
-    /// C++ Interface for remainder when array divides array,
-    /// scalar divides array or array divides scalar
+    /// C++ Interface to find the remainder.
     ///
-    /// \param[in] lhs is numerator
-    /// \param[in] rhs is denominator
-    /// \return remainder when \p rhs divides \p lhs
+    /// \param[in] lhs numerator; can be an array or a scalar
+    /// \param[in] rhs denominator; can be an array or a scalar
+    /// \return remainder of \p lhs divided by \p rhs
+    /// 
+    /// \ingroup arith_func_rem
     AFAPI array rem    (const array &lhs, const array &rhs);
 
     /// \copydoc rem(const array&, const array&)
@@ -92,14 +114,14 @@ namespace af
     AFAPI array rem    (const double lhs, const array &rhs);
     /// @}
 
-    /// \ingroup arith_func_mod
     /// @{
-    /// C++ Interface for modulus when dividend and divisor are arrays
-    /// or one of them is scalar
+    /// C++ Interface to find the modulus.
     ///
-    /// \param[in] lhs is dividend
-    /// \param[in] rhs is divisor
+    /// \param[in] lhs dividend; can be an array or a scalar
+    /// \param[in] rhs divisor; can be an array or a scalar
     /// \return \p lhs modulo \p rhs
+    /// 
+    /// \ingroup arith_func_mod
     AFAPI array mod    (const array &lhs, const array &rhs);
 
     /// \copydoc mod(const array&, const array&)
@@ -109,68 +131,57 @@ namespace af
     AFAPI array mod    (const double lhs, const array &rhs);
     /// @}
 
-    /// C++ Interface for absolute value
+    /// C++ Interface to find the absolute value.
     ///
-    /// \param[in] in is input array
-    /// \return absolute value of \p in
+    /// \param[in] in input array
+    /// \return absolute value
     ///
     /// \ingroup arith_func_abs
     AFAPI array abs    (const array &in);
 
-    /**
-       C++ Interface for arg
-
-       \param[in] in is input array
-       \return phase of \p in
-
-       \ingroup arith_func_arg
-    */
+    /// C++ Interface to find the phase angle (in radians) of a complex array.
+    ///
+    /// \param[in] in input array, typically complex
+    /// \return phase angle (in radians)
+    /// 
+    /// \ingroup arith_func_arg
     AFAPI array arg    (const array &in);
 
-    /**
-       C++ Interface for getting the sign of input
-
-       \param[in] in is input array
-       \return the sign of each element of input
-
-       \note output is 1 for negative numbers and 0 for positive numbers
-
-       \ingroup arith_func_sign
-    */
+    /// C++ Interface to find the sign of elements in an array.
+    ///
+    /// \param[in] in input array
+    /// \return array containing 1's for negative values; 0's otherwise
+    /// 
+    /// \ingroup arith_func_sign
     AFAPI array sign  (const array &in);
 
-    ///C++ Interface for rounding an array of numbers
-    ///
-    ///\param[in] in is input array
-    ///\return values rounded to nearest integer
+    /// C++ Interface to round numbers.
     ///
-    ///\note The values are rounded to nearest integer
+    /// \param[in] in input array
+    /// \return numbers rounded to nearest integer
     ///
-    ///\ingroup arith_func_round
+    /// \ingroup arith_func_round
     AFAPI array round  (const array &in);
 
-    /**
-       C++ Interface for truncating an array of numbers
-
-       \param[in] in is input array
-       \return values truncated to nearest integer not greater than input values
-
-       \ingroup arith_func_trunc
-    */
+    /// C++ Interface to truncate numbers.
+    ///
+    /// \param[in] in input array
+    /// \return nearest integer not greater in magnitude than \p in
+    /// 
+    /// \ingroup arith_func_trunc
     AFAPI array trunc  (const array &in);
 
-
-    /// C++ Interface for flooring an array of numbers
+    /// C++ Interface to floor numbers.
     ///
-    /// \param[in] in is input array
+    /// \param[in] in input array
     /// \return values rounded to nearest integer less than or equal to current value
     ///
     /// \ingroup arith_func_floor
     AFAPI array floor  (const array &in);
 
-    /// C++ Interface for ceiling an array of numbers
+    /// C++ Interface to ceil numbers.
     ///
-    /// \param[in] in is input array
+    /// \param[in] in input array
     /// \return values rounded to nearest integer greater than or equal to current value
     ///
     /// \ingroup arith_func_ceil
@@ -178,14 +189,14 @@ namespace af
 
     /// \ingroup arith_func_hypot
     /// @{
-    /// \brief C++ Interface for getting length of hypotenuse of two inputs
+    /// C++ Interface to find the length of the hypotenuse of two inputs.
     ///
     /// Calculates the hypotenuse of two inputs. The inputs can be both arrays
     /// or an array and a scalar.
     ///
-    /// \param[in] lhs is the length of first side
-    /// \param[in] rhs is the length of second side
-    /// \return the length of the hypotenuse
+    /// \param[in] lhs length of first side
+    /// \param[in] rhs length of second side
+    /// \return length of the hypotenuse
     AFAPI array hypot  (const array &lhs, const array &rhs);
 
     /// \copydoc hypot(const array&, const array&)
@@ -195,61 +206,61 @@ namespace af
     AFAPI array hypot  (const double lhs, const array &rhs);
     /// @}
 
-    /// C++ Interface for sin
+    /// C++ Interface to evaluate the sine function.
     ///
-    /// \param[in] in is input array
-    /// \return sin of input
+    /// \param[in] in input array
+    /// \return sine
     ///
     /// \ingroup arith_func_sin
     AFAPI array sin    (const array &in);
 
-    /// C++ Interface for cos
+    /// C++ Interface to evaluate the cosine function.
     ///
-    /// \param[in] in is input array
-    /// \return cos of input
+    /// \param[in] in input array
+    /// \return cosine
     ///
     /// \ingroup arith_func_cos
     AFAPI array cos    (const array &in);
 
-    /// C++ Interface for tan
+    /// C++ Interface to evaluate the tangent function.
     ///
-    /// \param[in] in is input array
-    /// \return tan of input
+    /// \param[in] in input array
+    /// \return tangent
     ///
     /// \ingroup arith_func_tan
     AFAPI array tan    (const array &in);
 
-    /// C++ Interface for arc sin (sin inverse)
+    /// C++ Interface to evaluate the inverse sine function.
     ///
-    /// \param[in] in is input array
-    /// \return arc sin of input
+    /// \param[in] in input array
+    /// \return inverse sine
     ///
     /// \ingroup arith_func_asin
     AFAPI array asin   (const array &in);
 
-    /// C++ Interface for arc cos (cos inverse)
+    /// C++ Interface to evaluate the inverse cosine function.
     ///
-    /// \param[in] in is input array
-    /// \return arc cos of input
+    /// \param[in] in input array
+    /// \return inverse cosine
     ///
     /// \ingroup arith_func_acos
     AFAPI array acos   (const array &in);
 
-    /// C++ Interface for arc tan (tan inverse)
+    /// C++ Interface to evaluate the inverse tangent function.
     ///
-    /// \param[in] in is input array
-    /// \return arc tan of input
+    /// \param[in] in input array
+    /// \return inverse tangent
     ///
     /// \ingroup arith_func_atan
     AFAPI array atan   (const array &in);
 
     /// \ingroup arith_func_atan
     /// @{
-    /// C++ Interface for arc tan of two arrays
+    /// C++ Interface to evaluate the inverse tangent of two arrays.
     ///
     /// \param[in] lhs value of numerator
     /// \param[in] rhs value of denominator
-    /// \return arc tan of the inputs
+    /// \return inverse tangent of the inputs
     AFAPI array atan2  (const array &lhs, const array &rhs);
 
     /// \copydoc atan2(const array&, const array&)
@@ -259,29 +270,77 @@ namespace af
     AFAPI array atan2  (const double lhs, const array &rhs);
     /// @}
 
+    /// C++ Interface to evaluate the hyperbolic sine function.
+    ///
+    /// \param[in] in input array
+    /// \return hyperbolic sine
+    ///
+    /// \ingroup arith_func_sinh
+    AFAPI array sinh(const array& in);
+
+    /// C++ Interface to evaluate the hyperbolic cosine function.
+    ///
+    /// \param[in] in input array
+    /// \return hyperbolic cosine
+    ///
+    /// \ingroup arith_func_cosh
+    AFAPI array cosh(const array& in);
+
+    /// C++ Interface to evaluate the hyperbolic tangent function.
+    ///
+    /// \param[in] in input array
+    /// \return hyperbolic tangent
+    ///
+    /// \ingroup arith_func_tanh
+    AFAPI array tanh(const array& in);
+
+    /// C++ Interface to evaluate the inverse hyperbolic sine function.
+    ///
+    /// \param[in] in input array
+    /// \return inverse hyperbolic sine
+    ///
+    /// \ingroup arith_func_asinh
+    AFAPI array asinh(const array& in);
+
+    /// C++ Interface to evaluate the inverse hyperbolic cosine function.
+    ///
+    /// \param[in] in input array
+    /// \return inverse hyperbolic cosine
+    ///
+    /// \ingroup arith_func_acosh
+    AFAPI array acosh(const array& in);
+
+    /// C++ Interface to evaluate the inverse hyperbolic tangent function.
+    ///
+    /// \param[in] in input array
+    /// \return inverse hyperbolic tangent
+    ///
+    /// \ingroup arith_func_atanh
+    AFAPI array atanh(const array& in);
+
     /// \ingroup arith_func_cplx
     /// @{
-    /// C++ Interface for creating a complex array from a single real array.
+    /// C++ Interface to create a complex array from a single real array.
     ///
     /// \param[in] in a real array
     /// \return the returned complex array
     AFAPI array complex(const array& in);
  
-    /// C++ Interface for creating a complex array from two real arrays.
+    /// C++ Interface to create a complex array from two real arrays.
     ///
     /// \param[in] real_ a real array to be assigned as the real component of the returned complex array
     /// \param[in] imag_ a real array to be assigned as the imaginary component of the returned complex array
     /// \return the returned complex array
     AFAPI array complex(const array &real_, const array &imag_);
 
-    /// C++ Interface for creating a complex array from a single real array for the real component and a single scalar for each imaginary component.
+    /// C++ Interface to create a complex array from a single real array for the real component and a single scalar for each imaginary component.
     ///
     /// \param[in] real_ a real array to be assigned as the real component of the returned complex array
     /// \param[in] imag_ a single scalar to be assigned as the imaginary component of each value of the returned complex array
     /// \return the returned complex array
     AFAPI array complex(const array &real_, const double imag_);
 
-    /// C++ Interface for creating a complex array from a single scalar for each real component and a single real array for the imaginary component.
+    /// C++ Interface to create a complex array from a single scalar for each real component and a single real array for the imaginary component.
     ///
     /// \param[in] real_ a single scalar to be assigned as the real component of each value of the returned complex array
     /// \param[in] imag_ a real array to be assigned as the imaginary component of the returned complex array
@@ -289,100 +348,52 @@ namespace af
     AFAPI array complex(const double real_, const array &imag_);
     /// @}
 
-    /// C++ Interface for getting real part from complex array
+    /// C++ Interface to find the real part of a complex array.
     ///
-    /// \param[in] in is complex array
-    /// \return the real part of \p in
+    /// \param[in] in input complex array
+    /// \return real part
     ///
     /// \ingroup arith_func_real
     AFAPI array real   (const array &in);
 
-    /// C++ Interface for getting imaginary part from complex array
+    /// C++ Interface to find the imaginary part of a complex array.
     ///
-    /// \param[in] in is complex array
-    /// \return the imaginary part of \p in
+    /// \param[in] in input complex array
+    /// \return imaginary part
     ///
     /// \ingroup arith_func_imag
     AFAPI array imag   (const array &in);
 
-    /// C++ Interface for getting the complex conjugate of input array
+    /// C++ Interface to find the complex conjugate of an input array.
     ///
-    /// \param[in] in is complex array
-    /// \return the complex conjugate of \p in
+    /// \param[in] in input complex array
+    /// \return complex conjugate
     ///
     /// \ingroup arith_func_conjg
     AFAPI array conjg  (const array &in);
 
-    /// C++ Interface for sinh
-    ///
-    /// \param[in] in is input array
-    /// \return sinh of input
-    ///
-    /// \ingroup arith_func_sinh
-    AFAPI array sinh    (const array &in);
-
-    /// C++ Interface for cosh
-    ///
-    /// \param[in] in is input array
-    /// \return cosh of input
-    ///
-    /// \ingroup arith_func_cosh
-    AFAPI array cosh    (const array &in);
-
-    /// C++ Interface for tanh
-    ///
-    /// \param[in] in is input array
-    /// \return tanh of input
-    ///
-    /// \ingroup arith_func_tanh
-    AFAPI array tanh    (const array &in);
-
-    /// C++ Interface for sinh inverse
-    ///
-    /// \param[in] in is input array
-    /// \return sinh inverse of input
-    ///
-    /// \ingroup arith_func_asinh
-    AFAPI array asinh   (const array &in);
-
-    /// C++ Interface for cosh inverse
+    /// C++ Interface to find the nth root.
     ///
-    /// \param[in] in is input array
-    /// \return cosh inverse of input
-    ///
-    /// \ingroup arith_func_acosh
-    AFAPI array acosh   (const array &in);
-
-    /// C++ Interface for tanh inverse
-    ///
-    /// \param[in] in is input array
-    /// \return tanh inverse of input
-    ///
-    /// \ingroup arith_func_atanh
-    AFAPI array atanh   (const array &in);
-
-    /// C++ Interface for nth root
-    ///
-    /// \param[in] lhs is nth root
-    /// \param[in] rhs is value
+    /// \param[in] lhs nth root
+    /// \param[in] rhs value
     /// \return \p lhs th root of \p rhs
     ///
     /// \ingroup arith_func_root
     AFAPI array root    (const array &lhs, const array &rhs);
 
-    /// C++ Interface for nth root
+    /// C++ Interface to find the nth root.
     ///
-    /// \param[in] lhs is nth root
-    /// \param[in] rhs is value
+    /// \param[in] lhs nth root
+    /// \param[in] rhs value
     /// \return \p lhs th root of \p rhs
     ///
     /// \ingroup arith_func_root
     AFAPI array root    (const array &lhs, const double rhs);
 
-    /// C++ Interface for nth root
+    /// C++ Interface to find the nth root.
     ///
-    /// \param[in] lhs is nth root
-    /// \param[in] rhs is value
+    /// \param[in] lhs nth root
+    /// \param[in] rhs value
     /// \return \p lhs th root of \p rhs
     ///
     /// \ingroup arith_func_root
@@ -391,14 +402,13 @@ namespace af
 
     /// \ingroup arith_func_pow
     /// @{
-    /// \brief C++ Interface for power
+    /// C++ Interface to raise a base to a power (or exponent).
     ///
-    /// Computes the value of \p lhs raised to the power of \p rhs. The inputs
-    /// can be two arrays or an array and a scalar.
+    /// Computes the value of \p lhs raised to the power of \p rhs. The inputs can be two arrays or an array and a scalar.
     ///
-    /// \param[in] lhs is base
-    /// \param[in] rhs is exponent
-    /// \return \p lhs raised to power \p rhs
+    /// \param[in] lhs base
+    /// \param[in] rhs exponent
+    /// \return \p lhs raised to the power of \p rhs
     AFAPI array pow    (const array &lhs, const array &rhs);
 
     /// \copydoc pow(const array&, const array&)
@@ -407,161 +417,162 @@ namespace af
     /// \copydoc pow(const array&, const array&)
     AFAPI array pow    (const double lhs, const array &rhs);
 
-    /// C++ Interface for power of 2
+    /// C++ Interface to raise 2 to a power (or exponent).
     ///
-    /// \param[in] in is exponent
-    /// \return 2 raised to power of \p in
+    /// \param[in] in exponent
+    /// \return 2 raised to the power
     ///
     AFAPI array pow2    (const array &in);
     /// @}
 
 #if AF_API_VERSION >= 31
-    /// C++ Interface for calculating sigmoid function of an array
+    /// C++ Interface to evaluate the logistical sigmoid function.
     ///
-    /// \param[in] in is input
-    /// \return the sigmoid of \p in
+    /// \param[in] in input
+    /// \return sigmoid
+    /// 
+    /// \note Computes `1/(1+e^-x)`.
     ///
     /// \ingroup arith_func_sigmoid
     AFAPI array sigmoid (const array &in);
 #endif
 
-    /// C++ Interface for exponential of an array
+    /// C++ Interface to evaluate the exponential.
     ///
-    /// \param[in] in is exponent
-    /// \return the exponential of \p in
+    /// \param[in] in exponent
+    /// \return exponential
     ///
     /// \ingroup arith_func_exp
     AFAPI array exp    (const array &in);
 
-    /// C++ Interface for exponential of an array minus 1
+    /// C++ Interface to evaluate the exponential of an array minus 1, `exp(in) - 1`.
     ///
-    /// \param[in] in is exponent
-    /// \return the exponential of \p in - 1
+    /// \param[in] in exponent
+    /// \return the exponential minus 1
     ///
     /// \note This function is useful when \p in is small
     /// \ingroup arith_func_expm1
     AFAPI array expm1  (const array &in);
 
-    /// C++ Interface for error function value
+    /// C++ Interface to evaluate the error function.
     ///
-    /// \param[in] in is input
-    /// \return the error function value
+    /// \param[in] in input
+    /// \return error function
     ///
     /// \ingroup arith_func_erf
     AFAPI array erf    (const array &in);
 
-    /// C++ Interface for complementary error function value
+    /// C++ Interface to evaluate the complementary error function.
     ///
-    /// \param[in] in is input
-    /// \return the complementary error function value
+    /// \param[in] in input
+    /// \return complementary error function
     ///
     /// \ingroup arith_func_erfc
     AFAPI array erfc   (const array &in);
 
-    /// C++ Interface for natural logarithm
+    /// C++ Interface to evaluate the natural logarithm.
     ///
-    /// \param[in] in is input
-    /// \return the natural logarithm of input
+    /// \param[in] in input
+    /// \return natural logarithm
     ///
     /// \ingroup arith_func_log
     AFAPI array log    (const array &in);
 
-    /// C++ Interface for natural logarithm of 1 + input
+    /// C++ Interface to evaluate the natural logarithm of 1 + input, `ln(1+in)`.
     ///
-    /// \param[in] in is input
-    /// \return the natural logarithm of (1 + input)
+    /// \param[in] in input
+    /// \return natural logarithm of `1 + input`
     ///
     /// \note This function is useful when \p in is small
     /// \ingroup arith_func_log1p
     AFAPI array log1p  (const array &in);
 
-    /// C++ Interface for logarithm base 10
+    /// C++ Interface to evaluate the base 10 logarithm.
     ///
-    /// \param[in] in is input
-    /// \return the logarithm of input in base 10
+    /// \param[in] in input
+    /// \return base 10 logarithm
     ///
     /// \ingroup arith_func_log10
     AFAPI array log10  (const array &in);
 
-    /// C++ Interface for logarithm base 2
+    /// C++ Interface to evaluate the base 2 logarithm.
     ///
-    /// \param[in] in is input
-    /// \return the logarithm of input \p in base 2
+    /// \param[in] in input
+    /// \return base 2 logarithm
     ///
     /// \ingroup explog_func_log2
     AFAPI array log2   (const array &in);
 
-    /// C++ Interface for square root of input
+    /// C++ Interface to find the square root.
     ///
-    /// \param[in] in is input
-    /// \return the square root of input
+    /// \param[in] in input
+    /// \return square root
     ///
     /// \ingroup arith_func_sqrt
     AFAPI array sqrt   (const array &in);
 
 #if AF_API_VERSION >= 37
-    /// C++ Interface for reciprocal square root of input
+    /// C++ Interface to find the reciprocal square root.
     ///
-    /// \param[in] in is input
-    /// \return the reciprocal square root of input
+    /// \param[in] in input
+    /// \return reciprocal square root
     ///
     /// \ingroup arith_func_rsqrt
     AFAPI array rsqrt   (const array &in);
 #endif
 
-    /// C++ Interface for cube root of input
+    /// C++ Interface to find the cube root.
     ///
-    /// \param[in] in is input
-    /// \return the cube root of input
+    /// \param[in] in input
+    /// \return cube root
     ///
     /// \ingroup arith_func_cbrt
     AFAPI array cbrt   (const array &in);
 
+    /// C++ Interface to find the factorial.
     ///
-    /// C++ Interface for factorial of input
-    ///
-    /// \param[in] in is input
-    /// \return the factorial function of input
+    /// \param[in] in input
+    /// \return the factorial function
     ///
     /// \ingroup arith_func_factorial
     AFAPI array factorial (const array &in);
 
-    /// C++ Interface for gamma function of input
+    /// C++ Interface to evaluate the gamma function.
     ///
-    /// \param[in] in is input
-    /// \return the gamma function of input
+    /// \param[in] in input
+    /// \return gamma function
     ///
     /// \ingroup arith_func_tgamma
     AFAPI array tgamma (const array &in);
 
-    /// C++ Interface for logarithm of absolute value of gamma function of input
+    /// C++ Interface to evaluate the logarithm of the absolute value of the gamma function.
     ///
-    /// \param[in] in is input
-    /// \return the logarithm of absolute value of gamma function of input
+    /// \param[in] in input
+    /// \return logarithm of the absolute value of the gamma function
     ///
-    /// \ingroup arith_func_tgamma
+    /// \ingroup arith_func_lgamma
     AFAPI array lgamma (const array &in);
 
-    /// C++ Interface for checking if values are zero
+    /// C++ Interface to check if values are zero.
     ///
-    /// \param[in] in is input
-    /// \return array containing 1's where input is 0, and 0 otherwise.
+    /// \param[in] in input
+    /// \return array containing 1's where input is 0; 0's otherwise
     ///
     /// \ingroup arith_func_iszero
     AFAPI array iszero (const array &in);
 
-    /// C++ Interface for checking if values are Infinities
+    /// C++ Interface to check if values are infinite.
     ///
-    /// \param[in] in is input
-    /// \return array containing 1's where input is Inf or -Inf, and 0 otherwise.
+    /// \param[in] in input
+    /// \return array containing 1's where input is Inf or -Inf; 0's otherwise
     ///
     /// \ingroup arith_func_isinf
     AFAPI array isInf  (const array &in);
 
-    /// C++ Interface for checking if values are NaNs
+    /// C++ Interface to check if values are NaN.
     ///
-    /// \param[in] in is input
-    /// \return array containing 1's where input is NaN, and 0 otherwise.
+    /// \param[in] in input
+    /// \return array containing 1's where input is NaN; 0's otherwise
     ///
     /// \ingroup arith_func_isnan
     AFAPI array isNaN  (const array &in);
@@ -573,9 +584,9 @@ extern "C" {
 #endif
 
     /**
-       C Interface for adding arrays
+       C Interface to add two arrays.
 
-       \param[out] out will contain sum of \p lhs and \p rhs
+       \param[out] out sum of \p lhs and \p rhs
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -586,9 +597,9 @@ extern "C" {
     AFAPI af_err af_add   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for subtracting an array from another
+       C Interface to subtract one array from another array.
 
-       \param[out] out will contain result of \p lhs - \p rhs
+       \param[out] out subtraction of \p lhs - \p rhs
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -599,9 +610,9 @@ extern "C" {
     AFAPI af_err af_sub   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for multiplying two arrays
+       C Interface to multiply two arrays.
 
-       \param[out] out will contain the product of \p lhs and  \p rhs
+       \param[out] out product of \p lhs and \p rhs
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -612,9 +623,9 @@ extern "C" {
     AFAPI af_err af_mul   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for dividing an array by another
+       C Interface to divide one array by another array.
 
-       \param[out] out will contain result of \p lhs / \p rhs.
+       \param[out] out result of \p lhs / \p rhs.
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -625,9 +636,9 @@ extern "C" {
     AFAPI af_err af_div   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for checking if an array is less than another
+       C Interface to check if the elements of one array are less than those of another array.
 
-       \param[out] out will contain result of \p lhs < \p rhs. out is of type b8
+       \param[out] out result of \p lhs < \p rhs; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -638,9 +649,9 @@ extern "C" {
     AFAPI af_err af_lt    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for checking if an array is greater than another
+       C Interface to check if the elements of one array are greater than those of another array.
 
-       \param[out] out will contain result of \p lhs > \p rhs. out is of type b8
+       \param[out] out result of \p lhs > \p rhs; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -651,9 +662,9 @@ extern "C" {
     AFAPI af_err af_gt    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for checking if an array is less or equal to another
+       C Interface to check if the elements of one array are less than or equal to those of another array.
 
-       \param[out] out will contain result of \p lhs <= \p rhs. out is of type b8
+       \param[out] out result of \p lhs <= \p rhs; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -664,9 +675,9 @@ extern "C" {
     AFAPI af_err af_le    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for checking if an array is greater or equal to another
+       C Interface to check if the elements of one array are greater than or equal to those of another array.
 
-       \param[out] out will contain result of \p lhs >= \p rhs. out is of type b8
+       \param[out] out result of \p lhs >= \p rhs; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -677,9 +688,9 @@ extern "C" {
     AFAPI af_err af_ge    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for checking if an array is equal to another
+       C Interface to check if the elements of one array are equal to those of another array.
 
-       \param[out] out will contain result of \p lhs == \p rhs. out is of type b8
+       \param[out] out result of \p lhs == \p rhs; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -690,9 +701,9 @@ extern "C" {
     AFAPI af_err af_eq    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for checking if an array is not equal to another
+       C Interface to check if the elements of one array are not equal to those of another array.
 
-       \param[out] out will contain result of \p lhs != \p rhs. out is of type b8
+       \param[out] out result of \p lhs != \p rhs; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -703,9 +714,9 @@ extern "C" {
     AFAPI af_err af_neq   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for performing logical and on two arrays
+       C Interface to evaluate the logical AND of two arrays.
 
-       \param[out] out will contain result of \p lhs && \p rhs. out is of type b8
+       \param[out] out result of \p lhs && \p rhs; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -716,9 +727,9 @@ extern "C" {
     AFAPI af_err af_and   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for performing logical or on two arrays
+       C Interface the evaluate the logical OR of two arrays.
 
-       \param[out] out will contain result of \p lhs || \p rhs. out is of type b8
+       \param[out] out result of \p lhs || \p rhs; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -729,10 +740,10 @@ extern "C" {
     AFAPI af_err af_or    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for performing logical not on input
+       C Interface to evaluate the logical NOT of an array.
 
-       \param[out] out will contain result of logical not of \p in. out is of type b8
-       \param[in] in is the input
+       \param[out] out result of logical NOT; type is b8
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_not
@@ -741,10 +752,10 @@ extern "C" {
 
 #if AF_API_VERSION >= 38
     /**
-       C Interface for performing bitwise not on input
+       C Interface to evaluate the bitwise NOT of an array.
 
-       \param[out] out will contain result of bitwise not of \p in.
-       \param[in] in is the input
+       \param[out] out result of bitwise NOT
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_bitnot
@@ -753,9 +764,9 @@ extern "C" {
 #endif
 
     /**
-       C Interface for performing bitwise and on two arrays
+       C Interface to evaluate the bitwise AND of two arrays.
 
-       \param[out] out will contain result of \p lhs & \p rhs
+       \param[out] out result of \p lhs & \p rhs
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -766,9 +777,9 @@ extern "C" {
     AFAPI af_err af_bitand   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for performing bitwise or on two arrays
+       C Interface to evaluate the bitwise OR of two arrays.
 
-       \param[out] out will contain result of \p lhs & \p rhs
+       \param[out] out result of \p lhs | \p rhs
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -779,9 +790,9 @@ extern "C" {
     AFAPI af_err af_bitor    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for performing bitwise xor on two arrays
+       C Interface to evaluate the bitwise XOR of two arrays.
 
-       \param[out] out will contain result of \p lhs ^ \p rhs
+       \param[out] out result of \p lhs ^ \p rhs
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -792,9 +803,9 @@ extern "C" {
     AFAPI af_err af_bitxor   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for left shift on integer arrays
+       C Interface to shift the bits of integer arrays left.
 
-       \param[out] out will contain result of the left shift
+       \param[out] out result of the left shift
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -805,9 +816,9 @@ extern "C" {
     AFAPI af_err af_bitshiftl(af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for right shift on integer arrays
+       C Interface to shift the bits of integer arrays right.
 
-       \param[out] out will contain result of the right shift
+       \param[out] out result of the right shift
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -818,7 +829,7 @@ extern "C" {
     AFAPI af_err af_bitshiftr(af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for casting an array from one type to another
+       C Interface to cast an array from one type to another.
 
        This function casts an af_array object from one type to another. If the
        type of the original array is the same as \p type then the same array is
@@ -847,11 +858,11 @@ extern "C" {
        | f16     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
        If you want to avoid this behavior use af_eval after the first cast
        operation. This will ensure that the cast operation is performed on the
-       af_array
+       af_array.
 
-       \param[out] out will contain the values in the specified type
-       \param[in] in is the input
-       \param[in] type is the target data type \ref af_dtype
+       \param[out] out values in the specified type
+       \param[in] in input
+       \param[in] type target data type \ref af_dtype
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_cast
@@ -859,11 +870,11 @@ extern "C" {
     AFAPI af_err af_cast    (af_array *out, const af_array in, const af_dtype type);
 
     /**
-       C Interface for min of two arrays
+       C Interface to find the elementwise minimum between two arrays.
 
-       \param[out] out will contain minimum of \p lhs and \p rhs
-       \param[in] lhs first input
-       \param[in] rhs second input
+       \param[out] out minimum of \p lhs and \p rhs
+       \param[in] lhs input array
+       \param[in] rhs input array
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -872,11 +883,11 @@ extern "C" {
     AFAPI af_err af_minof (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for max of two arrays
+       C Interface to find the elementwise minimum between an array and a scalar value.
 
-       \param[out] out will contain maximum of \p lhs and \p rhs
-       \param[in] lhs first input
-       \param[in] rhs second input
+       \param[out] out maximum of \p lhs and \p rhs
+       \param[in] lhs input array
+       \param[in] rhs input array
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -886,27 +897,27 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for max of two arrays
+       C Interface to clamp an array between an upper and a lower limit.
 
-       \param[out] out will contain the values from \p clamped between \p lo and \p hi
-       \param[in] in Input array
-       \param[in] lo Value for lower limit
-       \param[in] hi Value for upper limit
+       \param[out] out array containing values from \p in clamped between \p lo and \p hi
+       \param[in] in input array
+       \param[in] lo lower limit array
+       \param[in] hi upper limit array
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_max
+       \ingroup arith_func_clamp
     */
     AFAPI af_err af_clamp(af_array *out, const af_array in,
                           const af_array lo, const af_array hi, const bool batch);
 #endif
 
     /**
-       C Interface for remainder
+       C Interface to find the remainder.
 
-       \param[out] out will contain the remainder of \p lhs divided by \p rhs
-       \param[in] lhs is numerator
-       \param[in] rhs is denominator
+       \param[out] out remainder of \p lhs divided by \p rhs
+       \param[in] lhs numerator
+       \param[in] rhs denominator
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -915,11 +926,11 @@ extern "C" {
     AFAPI af_err af_rem   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for modulus
+       C Interface to find the modulus.
 
-       \param[out] out will contain the output of \p lhs modulo \p rhs
-       \param[in] lhs is dividend
-       \param[in] rhs is divisor
+       \param[out] out \p lhs modulo \p rhs
+       \param[in] lhs dividend
+       \param[in] rhs divisor
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -928,10 +939,10 @@ extern "C" {
     AFAPI af_err af_mod   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for absolute value
+       C Interface to find the absolute value.
 
-       \param[out] out will contain the absolute value of \p in
-       \param[in] in is input array
+       \param[out] out absolute value
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_abs
@@ -939,10 +950,10 @@ extern "C" {
     AFAPI af_err af_abs     (af_array *out, const af_array in);
 
     /**
-       C Interface for finding the phase
+       C Interface to find the phase angle (in radians) of a complex array.
 
-       \param[out] out will the phase of \p in
-       \param[in] in is input array
+       \param[out] out phase angle (in radians)
+       \param[in] in input array, typically complex
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_arg
@@ -950,36 +961,32 @@ extern "C" {
     AFAPI af_err af_arg     (af_array *out, const af_array in);
 
     /**
-       C Interface for finding the sign of the input
+       C Interface to find the sign of elements in an array.
 
-       \param[out] out will contain the sign of each element of the input arrays
-       \param[in] in is input array
+       \param[out] out array containing 1's for negative values; 0's otherwise
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \note output is 1 for negative numbers and 0 for positive numbers
-
-       \ingroup arith_func_round
+       \ingroup arith_func_sign
     */
     AFAPI af_err af_sign   (af_array *out, const af_array in);
 
     /**
-       C Interface for rounding an array of numbers
+       C Interface to round numbers.
 
-       \param[out] out will contain values rounded to nearest integer
-       \param[in] in is input array
+       \param[out] out values rounded to nearest integer
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \note The values are rounded to nearest integer
-
        \ingroup arith_func_round
     */
     AFAPI af_err af_round   (af_array *out, const af_array in);
 
     /**
-       C Interface for truncating an array of numbers
+       C Interface to truncate numbers.
 
-       \param[out] out will contain values truncated to nearest integer not greater than input
-       \param[in] in is input array
+       \param[out] out nearest integer not greater in magnitude than \p in
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_trunc
@@ -987,10 +994,10 @@ extern "C" {
     AFAPI af_err af_trunc   (af_array *out, const af_array in);
 
     /**
-       C Interface for flooring an array of numbers
+       C Interface to floor numbers.
 
-       \param[out] out will contain values rounded to nearest integer less than or equal to in
-       \param[in] in is input array
+       \param[out] out values rounded to nearest integer less than or equal to \p in
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_floor
@@ -998,10 +1005,10 @@ extern "C" {
     AFAPI af_err af_floor   (af_array *out, const af_array in);
 
     /**
-       C Interface for ceiling an array of numbers
+       C Interface to ceil numbers.
 
-       \param[out] out will contain values rounded to nearest integer greater than or equal to in
-       \param[in] in is input array
+       \param[out] out values rounded to nearest integer greater than or equal to \p in
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_ceil
@@ -1009,11 +1016,11 @@ extern "C" {
     AFAPI af_err af_ceil    (af_array *out, const af_array in);
 
     /**
-       C Interface for getting length of hypotenuse of two arrays
+       C Interface to find the length of the hypotenuse of two inputs.
 
-       \param[out] out will contain the length of the hypotenuse
-       \param[in] lhs is the length of first side
-       \param[in] rhs is the length of second side
+       \param[out] out length of the hypotenuse
+       \param[in] lhs length of first side
+       \param[in] rhs length of second side
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -1022,10 +1029,10 @@ extern "C" {
     AFAPI af_err af_hypot (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for sin
+       C Interface to evaluate the sine function.
 
-       \param[out] out will contain sin of input
-       \param[in] in is input array
+       \param[out] out sine
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_sin
@@ -1033,10 +1040,10 @@ extern "C" {
     AFAPI af_err af_sin     (af_array *out, const af_array in);
 
     /**
-       C Interface for cos
+       C Interface to evaluate the cosine function.
 
-       \param[out] out will contain cos of input
-       \param[in] in is input array
+       \param[out] out cosine
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_cos
@@ -1044,10 +1051,10 @@ extern "C" {
     AFAPI af_err af_cos     (af_array *out, const af_array in);
 
     /**
-       C Interface for tan
+       C Interface to evaluate the tangent function.
 
-       \param[out] out will contain tan of input
-       \param[in] in is input array
+       \param[out] out tangent
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_tan
@@ -1055,10 +1062,10 @@ extern "C" {
     AFAPI af_err af_tan     (af_array *out, const af_array in);
 
     /**
-       C Interface for arc sin
+       C Interface to evaluate the inverse sine function.
 
-       \param[out] out will contain arc sin of input
-       \param[in] in is input array
+       \param[out] out inverse sine
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_asin
@@ -1066,10 +1073,10 @@ extern "C" {
     AFAPI af_err af_asin    (af_array *out, const af_array in);
 
     /**
-       C Interface for arc cos
+       C Interface to evaluate the inverse cosine function.
 
-       \param[out] out will contain arc cos of input
-       \param[in] in is input array
+       \param[out] out inverse cos
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_acos
@@ -1077,10 +1084,10 @@ extern "C" {
     AFAPI af_err af_acos    (af_array *out, const af_array in);
 
     /**
-       C Interface for arc tan
+       C Interface to evaluate the inverse tangent function.
 
-       \param[out] out will contain arc tan of input
-       \param[in] in is input array
+       \param[out] out inverse tangent
+       \param[in] in input array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_atan
@@ -1088,11 +1095,11 @@ extern "C" {
     AFAPI af_err af_atan    (af_array *out, const af_array in);
 
     /**
-       C Interface for arc tan of two inputs
+       C Interface to evaluate the inverse tangent of two arrays.
 
-       \param[out] out will arc tan of the inputs
-       \param[in] lhs value of numerator
-       \param[in] rhs value of denominator
+       \param[out] out inverse tangent of two arrays
+       \param[in] lhs numerator
+       \param[in] rhs denominator
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -1101,10 +1108,10 @@ extern "C" {
     AFAPI af_err af_atan2 (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for creating a complex array from a single real array.
+       C Interface to create a complex array from a single real array.
 
-       \param[out] out the returned complex array
-       \param[in] in a real array
+       \param[out] out complex array
+       \param[in] in real array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_cplx
@@ -1112,11 +1119,11 @@ extern "C" {
     AFAPI af_err af_cplx(af_array* out, const af_array in);
 
     /**
-       C Interface for creating a complex array from two real arrays.
+       C Interface to create a complex array from two real arrays.
 
-       \param[out] out the returned complex array
-       \param[in] real a real array to be assigned as the real component of the returned complex array
-       \param[in] imag a real array to be assigned as the imaginary component of the returned complex array
+       \param[out] out complex array
+       \param[in] real real array to be assigned as the real component of the returned complex array
+       \param[in] imag real array to be assigned as the imaginary component of the returned complex array
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -1125,10 +1132,10 @@ extern "C" {
     AFAPI af_err af_cplx2 (af_array *out, const af_array real, const af_array imag, const bool batch);
 
     /**
-       C Interface for getting real part from complex array
+       C Interface to find the real part of a complex array.
 
-       \param[out] out will contain the real part of \p in
-       \param[in] in is complex array
+       \param[out] out real part
+       \param[in] in complex array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_real
@@ -1136,10 +1143,10 @@ extern "C" {
     AFAPI af_err af_real    (af_array *out, const af_array in);
 
     /**
-       C Interface for getting imaginary part from complex array
+       C Interface to find the imaginary part of a complex array.
 
-       \param[out] out will contain the imaginary part of \p in
-       \param[in] in is complex array
+       \param[out] out imaginary part
+       \param[in] in complex array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_imag
@@ -1147,10 +1154,10 @@ extern "C" {
     AFAPI af_err af_imag    (af_array *out, const af_array in);
 
     /**
-       C Interface for getting the complex conjugate of input array
+       C Interface to find the complex conjugate of an input array.
 
-       \param[out] out will contain the complex conjugate of \p in
-       \param[in] in is complex array
+       \param[out] out complex conjugate
+       \param[in] in complex array
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_conjg
@@ -1158,10 +1165,10 @@ extern "C" {
     AFAPI af_err af_conjg   (af_array *out, const af_array in);
 
     /**
-       C Interface for sinh
+       C Interface to evaluate the hyperbolic sine function.
 
-       \param[out] out will contain sinh of input
-       \param[in] in is input array
+       \param[out] out hyperbolic sine
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_sinh
@@ -1169,10 +1176,10 @@ extern "C" {
     AFAPI af_err af_sinh    (af_array *out, const af_array in);
 
     /**
-       C Interface for cosh
+       C Interface to evaluate the hyperbolic cosine function.
 
-       \param[out] out will contain cosh of input
-       \param[in] in is input array
+       \param[out] out hyperbolic cosine
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_cosh
@@ -1180,10 +1187,10 @@ extern "C" {
     AFAPI af_err af_cosh    (af_array *out, const af_array in);
 
     /**
-       C Interface for tanh
+       C Interface to evaluate the hyperbolic tangent function.
 
-       \param[out] out will contain tanh of input
-       \param[in] in is input array
+       \param[out] out hyperbolic tangent
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_tanh
@@ -1191,10 +1198,10 @@ extern "C" {
     AFAPI af_err af_tanh    (af_array *out, const af_array in);
 
     /**
-       C Interface for asinh
+       C Interface to evaluate the inverse hyperbolic sine function.
 
-       \param[out] out will contain inverse sinh of input
-       \param[in] in is input array
+       \param[out] out inverse hyperbolic sine
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_asinh
@@ -1202,10 +1209,10 @@ extern "C" {
     AFAPI af_err af_asinh   (af_array *out, const af_array in);
 
     /**
-       C Interface for acosh
+       C Interface to evaluate the inverse hyperbolic cosine function.
 
-       \param[out] out will contain inverse cosh of input
-       \param[in] in is input array
+       \param[out] out inverse hyperbolic cosine
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_acosh
@@ -1213,10 +1220,10 @@ extern "C" {
     AFAPI af_err af_acosh   (af_array *out, const af_array in);
 
     /**
-       C Interface for atanh
+       C Interface to evaluate the inverse hyperbolic tangent function.
 
-       \param[out] out will contain inverse tanh of input
-       \param[in] in is input array
+       \param[out] out inverse hyperbolic tangent
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_atanh
@@ -1224,11 +1231,11 @@ extern "C" {
     AFAPI af_err af_atanh   (af_array *out, const af_array in);
 
     /**
-       C Interface for root
+       C Interface to find the nth root.
 
-       \param[out] out will contain \p lhs th root of \p rhs
-       \param[in] lhs is nth root
-       \param[in] rhs is value
+       \param[out] out \p lhs th root of \p rhs
+       \param[in] lhs nth root
+       \param[in] rhs value
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -1238,11 +1245,11 @@ extern "C" {
 
 
     /**
-       C Interface for power
+       C Interface to raise a base to a power (or exponent).
 
-       \param[out] out will contain \p lhs raised to power \p rhs
-       \param[in] lhs is base
-       \param[in] rhs is exponent
+       \param[out] out \p lhs raised to the power of \p rhs
+       \param[in] lhs base
+       \param[in] rhs exponent
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -1251,45 +1258,47 @@ extern "C" {
     AFAPI af_err af_pow   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface for power of two
+       C Interface to raise 2 to a power (or exponent).
 
-       \param[out] out will contain the values of 2 to the power \p in
-       \param[in] in is exponent
+       \param[out] out 2 raised to the power of \p in
+       \param[in] in exponent
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_pow2
     */
     AFAPI af_err af_pow2     (af_array *out, const af_array in);
 
+#if AF_API_VERSION >= 31
     /**
-       C Interface for exponential of an array
+       C Interface to evaluate the logistical sigmoid function.
 
-       \param[out] out will contain the exponential of \p in
-       \param[in] in is exponent
+       \param[out] out output of the logistic sigmoid function
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_exp
+       \note Computes `1/(1+e^-x)`.
+
+       \ingroup arith_func_sigmoid
     */
-    AFAPI af_err af_exp     (af_array *out, const af_array in);
+    AFAPI af_err af_sigmoid(af_array* out, const af_array in);
+#endif
 
-#if AF_API_VERSION >= 31
     /**
-       C Interface for calculating sigmoid function of an array
+       C Interface to evaluate the exponential.
 
-       \param[out] out will contain the sigmoid of \p in
-       \param[in] in is input
+       \param[out] out e raised to the power of \p in
+       \param[in] in exponent
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_sigmoid
+       \ingroup arith_func_exp
     */
-    AFAPI af_err af_sigmoid (af_array *out, const af_array in);
-#endif
+    AFAPI af_err af_exp     (af_array *out, const af_array in);
 
     /**
-       C Interface for exponential of an array minus 1
+       C Interface to evaluate the exponential of an array minus 1, `exp(in) - 1`.
 
-       \param[out] out will contain the exponential of \p in - 1
-       \param[in] in is input
+       \param[out] out exponential of `in - 1`
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_expm1
@@ -1297,10 +1306,10 @@ extern "C" {
     AFAPI af_err af_expm1   (af_array *out, const af_array in);
 
     /**
-       C Interface for error function value
+       C Interface to evaluate the error function.
 
-       \param[out] out will contain the error function value of \p in
-       \param[in] in is input
+       \param[out] out error function value
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_erf
@@ -1308,10 +1317,10 @@ extern "C" {
     AFAPI af_err af_erf     (af_array *out, const af_array in);
 
     /**
-       C Interface for complementary error function value
+       C Interface to evaluate the complementary error function.
 
-       \param[out] out will contain the complementary error function value of \p in
-       \param[in] in is input
+       \param[out] out complementary error function
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_erfc
@@ -1319,10 +1328,10 @@ extern "C" {
     AFAPI af_err af_erfc    (af_array *out, const af_array in);
 
     /**
-       C Interface for natural logarithm
+       C Interface to evaluate the natural logarithm.
 
-       \param[out] out will contain the natural logarithm of \p in
-       \param[in] in is input
+       \param[out] out natural logarithm
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_log
@@ -1330,10 +1339,10 @@ extern "C" {
     AFAPI af_err af_log     (af_array *out, const af_array in);
 
     /**
-       C Interface for logarithm of (in + 1)
+       C Interface to evaluate the natural logarithm of 1 + input, `ln(1+in)`.
 
-       \param[out] out will contain the logarithm of of (in + 1)
-       \param[in] in is input
+       \param[out] out logarithm of `in + 1`
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_log1p
@@ -1341,10 +1350,10 @@ extern "C" {
     AFAPI af_err af_log1p   (af_array *out, const af_array in);
 
     /**
-       C Interface for logarithm base 10
+       C Interface to evaluate the base 10 logarithm.
 
-       \param[out] out will contain the base 10 logarithm of \p in
-       \param[in] in is input
+       \param[out] out base 10 logarithm
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_log10
@@ -1352,10 +1361,10 @@ extern "C" {
     AFAPI af_err af_log10   (af_array *out, const af_array in);
 
     /**
-       C Interface for logarithm base 2
+       C Interface to evaluate the base 2 logarithm.
 
-       \param[out] out will contain the base 2 logarithm of \p in
-       \param[in] in is input
+       \param[out] out base 2 logarithm
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup explog_func_log2
@@ -1363,10 +1372,10 @@ extern "C" {
     AFAPI af_err af_log2   (af_array *out, const af_array in);
 
     /**
-       C Interface for square root
+       C Interface to find the square root.
 
-       \param[out] out will contain the square root of \p in
-       \param[in] in is input
+       \param[out] out square root
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_sqrt
@@ -1375,10 +1384,10 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-      C Interface for reciprocal  square root
+      C Interface to find the reciprocal square root.
 
-      \param[out] out will contain the reciprocal square root of \p in
-      \param[in] in is input
+      \param[out] out reciprocal square root
+      \param[in] in input
       \return \ref AF_SUCCESS if the execution completes properly
 
       \ingroup arith_func_rsqrt
@@ -1386,10 +1395,10 @@ extern "C" {
     AFAPI af_err af_rsqrt    (af_array *out, const af_array in);
 #endif
     /**
-       C Interface for cube root
+       C Interface to find the cube root.
 
-       \param[out] out will contain the cube root of \p in
-       \param[in] in is input
+       \param[out] out cube root
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_cbrt
@@ -1397,10 +1406,10 @@ extern "C" {
     AFAPI af_err af_cbrt    (af_array *out, const af_array in);
 
     /**
-       C Interface for the factorial
+       C Interface to find the factorial.
 
-       \param[out] out will contain the result of factorial of \p in
-       \param[in] in is input
+       \param[out] out factorial
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_factorial
@@ -1408,10 +1417,10 @@ extern "C" {
     AFAPI af_err af_factorial   (af_array *out, const af_array in);
 
     /**
-       C Interface for the gamma function
+       C Interface to evaluate the gamma function.
 
-       \param[out] out will contain the result of gamma function of \p in
-       \param[in] in is input
+       \param[out] out gamma function
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_tgamma
@@ -1419,10 +1428,10 @@ extern "C" {
     AFAPI af_err af_tgamma   (af_array *out, const af_array in);
 
     /**
-       C Interface for the logarithm of absolute values of gamma function
+       C Interface to evaluate the logarithm of the absolute value of the gamma function.
 
-       \param[out] out will contain the result of logarithm of absolute values of gamma function of \p in
-       \param[in] in is input
+       \param[out] out logarithm of the absolute value of the gamma function
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
        \ingroup arith_func_lgamma
@@ -1430,10 +1439,10 @@ extern "C" {
     AFAPI af_err af_lgamma   (af_array *out, const af_array in);
 
     /**
-        C Interface for checking if values are zero
+        C Interface to check if values are zero.
 
-        \param[out] out will contain 1's where input is 0, and 0 otherwise.
-        \param[in] in is input
+        \param[out] out array containing 1's where input is 0; 0's otherwise
+        \param[in] in input
         \return \ref AF_SUCCESS if the execution completes properly
 
         \ingroup arith_func_iszero
@@ -1441,10 +1450,10 @@ extern "C" {
     AFAPI af_err af_iszero  (af_array *out, const af_array in);
 
     /**
-        C Interface for checking if values are infinities
+        C Interface to check if values are infinite.
 
-        \param[out] out will contain 1's where input is Inf or -Inf, and 0 otherwise.
-        \param[in] in is input
+        \param[out] out array containing 1's where input is Inf or -Inf; 0's otherwise
+        \param[in] in input
         \return \ref AF_SUCCESS if the execution completes properly
 
         \ingroup arith_func_isinf
@@ -1452,10 +1461,10 @@ extern "C" {
     AFAPI af_err af_isinf   (af_array *out, const af_array in);
 
     /**
-        C Interface for checking if values are NaNs
+        C Interface to check if values are NaN.
 
-        \param[out] out will contain 1's where input is NaN, and 0 otherwise.
-        \param[in] in is input
+        \param[out] out array containing 1's where input is NaN; 0's otherwise
+        \param[in] in input
         \return \ref AF_SUCCESS if the execution completes properly
 
         \ingroup arith_func_isnan

From 1af82bf2fb3e6c39bd5058cb919541309e05909e Mon Sep 17 00:00:00 2001
From: John Melonakos <john.melonakos@intel.com>
Date: Thu, 12 Jan 2023 17:19:25 -0500
Subject: [PATCH 191/473] improves documentation for arith functions, round 2

---
 docs/details/arith.dox | 153 ++++++++---------------------------------
 include/af/arith.h     | 116 +++++++++++++++----------------
 2 files changed, 87 insertions(+), 182 deletions(-)

diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index 84f9a5c451..ac8d265628 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -21,137 +21,109 @@
 
 \ingroup arith_mat
 
-Add.
-
 Add two arrays.
 
 
-
 \defgroup arith_func_sub sub
 
 \ingroup arith_mat
 
-Subtract.
-
 Subtract one array from another array.
 
 
-
 \defgroup arith_func_mul mul
 
 \ingroup arith_mat
 
-Multiply.
-
 Multiply two arrays.
 
 
-
 \defgroup arith_func_div div
 
 \ingroup arith_mat
 
-Divide.
-
 Divide one array by another array.
 
 
-
 \defgroup arith_func_lt lt
 
 \ingroup logic_mat
 
-Is less than.
+Less than, an elementwise comparison of two arrays.
 
 Check if the elements of one array are less than those of another array.
 
 
-
 \defgroup arith_func_gt gt
 
 \ingroup logic_mat
 
-Is greater than.
+Greater than comparison, an elementwise comparison of two arrays.
 
 Check if the elements of one array are greater than those of another array.
 
 
-
 \defgroup arith_func_le le
 
 \ingroup logic_mat
 
-Is less than or equal.
+Less than or equal to, an elementwise comparison of two arrays.
 
 Check if the elements of one array are less than or equal to those of another array.
 
 
-
 \defgroup arith_func_ge ge
 
 \ingroup logic_mat
 
-Is greater than or equal.
+Greater than or equal to, an elementwise comparison of two arrays.
 
 Check if the elements of one array are greater than or equal to those of another array.
 
 
-
 \defgroup arith_func_eq eq
 
 \ingroup logic_mat
 
-Is equal.
+\brief Equal to, an elementwise comparison of two arrays.
 
 Check if the elements of one array are equal to those of another array.
 
 
-
 \defgroup arith_func_neq neq
 
 \ingroup logic_mat
 
-Is not equal.
+\brief Not equal to, an elementwise comparison of two arrays.
 
 Check if the elements of one array are not equal to those of another array.
 
 
-
 \defgroup arith_func_and and
-\brief Logical AND
 
 \ingroup logic_mat
 
-Logical AND.
-
 Evaluate the logical AND of two arrays.
 
+
 \defgroup arith_func_or or
 
 \ingroup logic_mat
 
-Logical OR.
-
 Evaluate the logical OR of two arrays.
 
 
-
 \defgroup arith_func_not not
 
 \ingroup logic_mat
 
-Logical NOT.
-
 Evaluate the logical NOT of an array.
 
 
-
 \defgroup arith_func_neg neg
 
 \ingroup numeric_mat
 
-Negative of an array.
-
 Negate an array.
 
 
@@ -159,8 +131,6 @@ Negate an array.
 
 \ingroup logic_mat
 
-Bitwise NOT.
-
 Evaluate the bitwise NOT of an array.
 
 \copydoc arith_int_only
@@ -170,8 +140,6 @@ Evaluate the bitwise NOT of an array.
 
 \ingroup logic_mat
 
-Bitwise AND.
-
 Evaluate the bitwise AND of two arrays.
 
 \copydoc arith_int_only
@@ -181,8 +149,6 @@ Evaluate the bitwise AND of two arrays.
 
 \ingroup logic_mat
 
-Bitwise OR.
-
 Evaluate the bitwise OR of two arrays.
 
 \copydoc arith_int_only
@@ -192,8 +158,6 @@ Evaluate the bitwise OR of two arrays.
 
 \ingroup logic_mat
 
-Bitwise XOR.
-
 Evaluate the bitwise XOR of two arrays.
 
 \copydoc arith_int_only
@@ -203,8 +167,6 @@ Evaluate the bitwise XOR of two arrays.
 
 \ingroup arith_mat
 
-Left shift on integer arrays.
-
 Shift the bits of integer arrays left.
 
 \copydoc arith_int_only
@@ -214,8 +176,6 @@ Shift the bits of integer arrays left.
 
 \ingroup arith_mat
 
-Right shift on integer arrays.
-
 Shift the bits of integer arrays right.
 
 \copydoc arith_int_only
@@ -232,8 +192,6 @@ Cast an array from one type to another.
 
 \ingroup numeric_mat
 
-Minimum of two inputs.
-
 Find the elementwise minimum between two arrays.
 
 
@@ -241,16 +199,19 @@ Find the elementwise minimum between two arrays.
 
 \ingroup numeric_mat
 
-Maximum of two inputs.
-
 Find the elementwise maximum between two arrays.
 
 
-\defgroup arith_func_rem rem
+\defgroup arith_func_clamp clamp
 
 \ingroup numeric_mat
 
-Remainder.
+Clamp an array between an upper and a lower limit.
+
+
+\defgroup arith_func_rem rem
+
+\ingroup numeric_mat
 
 Find the remainder of a division.
 
@@ -261,8 +222,6 @@ Find the remainder of a division.
 
 \ingroup numeric_mat
 
-Modulus.
-
 Find the modulus.
 
 \copydoc arith_real_only
@@ -270,8 +229,6 @@ Find the modulus.
 
 \defgroup arith_func_abs abs
 
-Absolute value.
-
 Find the absolute value.
 
 __Examples:__
@@ -282,9 +239,8 @@ __Examples:__
 
 
 \defgroup arith_func_arg arg
-\ingroup numeric_mat
 
-Phase angle.
+\ingroup numeric_mat
 
 Find the phase angle (in radians) of a complex array.
 
@@ -293,8 +249,6 @@ Find the phase angle (in radians) of a complex array.
 
 \ingroup numeric_mat
 
-Sign.
-
 Find the sign of elements in an array.
 
 \copydoc arith_real_only
@@ -304,8 +258,6 @@ Find the sign of elements in an array.
 
 \ingroup numeric_mat
 
-Round.
-
 Round numbers to the nearest integer.
 
 \copydoc arith_real_only
@@ -315,8 +267,6 @@ Round numbers to the nearest integer.
 
 \ingroup numeric_mat
 
-Truncate.
-
 Truncate numbers to nearest integer.
 
 \copydoc arith_real_only
@@ -326,8 +276,6 @@ Truncate numbers to nearest integer.
 
 \ingroup numeric_mat
 
-Floor.
-
 Round to the integer less than or equal to the magnitude of the input value.
 
 \copydoc arith_real_only
@@ -337,8 +285,6 @@ Round to the integer less than or equal to the magnitude of the input value.
 
 \ingroup numeric_mat
 
-Ceil.
-
 Round to the integer greater than or equal to the magnitude of the input value.
 
 \copydoc arith_real_only
@@ -348,8 +294,6 @@ Round to the integer greater than or equal to the magnitude of the input value.
 
 \ingroup numeric_mat
 
-Hypotenuse.
-
 Find the length of the hypotenuse of two inputs.
 
 \copydoc arith_real_only
@@ -359,8 +303,6 @@ Find the length of the hypotenuse of two inputs.
 
 \ingroup trig_mat
 
-Sine.
-
 Evaluate the sine function.
 
 
@@ -368,17 +310,13 @@ Evaluate the sine function.
 
 \ingroup trig_mat
 
-Cosine.
-
 Evaluate the cosine function.
 
 
-\defgroup arith_func_tan tan/tan2
+\defgroup arith_func_tan tan
 
 \ingroup trig_mat
 
-Tangent.
-
 Evaluate the tangent function.
 
 
@@ -386,16 +324,12 @@ Evaluate the tangent function.
 
 \ingroup trig_mat
 
-Inverse sine (arc sine).
-
-Evaluate the inverse sine function.
+Evaluate the inverse sine function (arc sine).
 
 
 \defgroup arith_func_acos acos
 
-Inverse cosine (arc cosine).
-
-Evaluate the inverse cosine function.
+Evaluate the inverse cosine function (arc cosine).
 
 The inverse of cosine so that, if `y = cos(x)`, then `x = arccos(y)`.
 
@@ -410,17 +344,13 @@ __Examples:__
 
 \ingroup trig_mat
 
-Inverse tangent (arc tangent).
-
-Evaluate the inverse tangent function.
+Evaluate the inverse tangent function (arc tangent).
 
 
 \defgroup arith_func_sinh sinh
 
 \ingroup hyper_mat
 
-Hyperbolic sine.
-
 Evaluate the hyperbolic sine function.
 
 
@@ -428,8 +358,6 @@ Evaluate the hyperbolic sine function.
 
 \ingroup hyper_mat
 
-Hyperbolic cosine.
-
 Evaluate the hyperbolic cosine function.
 
 
@@ -437,8 +365,6 @@ Evaluate the hyperbolic cosine function.
 
 \ingroup hyper_mat
 
-Hyperbolic tangent.
-
 Evaluate the hyperbolic tangent function.
 
 
@@ -446,27 +372,21 @@ Evaluate the hyperbolic tangent function.
 
 \ingroup hyper_mat
 
-Inverse hyperbolic sine (area hyperbolic sine).
-
-Evaluate the inverse hyperbolic sine function.
+Evaluate the inverse hyperbolic sine function (area hyperbolic sine).
 
 
 \defgroup arith_func_acosh acosh
 
 \ingroup hyper_mat
 
-Inverse hyperbolic cosine (area hyperbolic cosine).
-
-Evaluate the inverse hyperbolic cosine function.
+Evaluate the inverse hyperbolic cosine function (area hyperbolic cosine).
 
 
 \defgroup arith_func_atanh atanh
 
 \ingroup hyper_mat
 
-Inverse hyperbolic tangent (area hyperbolic tangent).
-
-Evaluate the inverse hyperbolic tangent function.
+Evaluate the inverse hyperbolic tangent function (area hyperbolic tangent).
 
 
 \defgroup arith_func_cplx complex
@@ -505,8 +425,6 @@ Find the imaginary part of a complex array.
 
 \ingroup complex_mat
 
-Complex conjugate.
-
 Find the complex conjugate of an input array.
 
 
@@ -523,43 +441,31 @@ Find the nth root.
 
 Raise a base to a power (or exponent).
 
-If the input array has values beyond what a floating point type can represent, then there is no
-guarantee that the results will be accurate. The exact type mapping from integral types to floating
-point types used to compute power is given below.
 
-| Input Type         | Compute Type   |
-| :------------------| :--------------|
-| unsigned long long | double         |
-| long long          | double         |
-| unsigned int       | double         |
-| int                | double         |
-| unsigned short     | float          |
-| short              | float          |
-| unsigned char      | float          |
+\defgroup arith_func_pow pow2
 
-The output array will be of the same type as input.
+\ingroup explog_mat
 
+Raise 2 to a power (or exponent).
 
-\defgroup arith_func_sigmoid sigmoid
 
-Sigmoid function (logistical).
+\defgroup arith_func_sigmoid sigmoid
 
 Evaluate the logistical sigmoid function.
 
 
-
 \defgroup arith_func_exp exp
 
 \ingroup explog_mat
 
-Evaluate the exponential.
+Evaluate the exponential function.
 
 
 \defgroup arith_func_expm1 expm1
 
 \ingroup explog_mat
 
-Evaluate the exponential of an array minus 1, `exp(in) - 1`.
+Evaluate the exponential function of an array minus 1, `exp(in) - 1`.
 
 \copydoc arith_real_only
 
@@ -573,7 +479,6 @@ Evaluate the error function.
 \copydoc arith_real_only
 
 
-
 \defgroup arith_func_erfc erfc
 
 \ingroup explog_mat
@@ -685,7 +590,7 @@ Check if values are zero.
 Check if values are infinite.
 
 
-\defgroup arith_func_isnan isNan
+\defgroup arith_func_isnan isnan
 
 \ingroup helper_mat
 
diff --git a/include/af/arith.h b/include/af/arith.h
index 789e54aab5..f6f190f199 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -690,7 +690,7 @@ extern "C" {
     /**
        C Interface to check if the elements of one array are equal to those of another array.
 
-       \param[out] out result of \p lhs == \p rhs; type is b8
+       \param[out] out result of `lhs == rhs`; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -703,7 +703,7 @@ extern "C" {
     /**
        C Interface to check if the elements of one array are not equal to those of another array.
 
-       \param[out] out result of \p lhs != \p rhs; type is b8
+       \param[out] out result of `lhs != rhs`; type is b8
        \param[in] lhs first input
        \param[in] rhs second input
        \param[in] batch specifies if operations need to be performed in batch mode
@@ -1108,127 +1108,127 @@ extern "C" {
     AFAPI af_err af_atan2 (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to create a complex array from a single real array.
+       C Interface to evaluate the hyperbolic sine function.
 
-       \param[out] out complex array
-       \param[in] in real array
+       \param[out] out hyperbolic sine
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_cplx
+       \ingroup arith_func_sinh
     */
-    AFAPI af_err af_cplx(af_array* out, const af_array in);
+    AFAPI af_err af_sinh    (af_array *out, const af_array in);
 
     /**
-       C Interface to create a complex array from two real arrays.
+       C Interface to evaluate the hyperbolic cosine function.
 
-       \param[out] out complex array
-       \param[in] real real array to be assigned as the real component of the returned complex array
-       \param[in] imag real array to be assigned as the imaginary component of the returned complex array
-       \param[in] batch specifies if operations need to be performed in batch mode
+       \param[out] out hyperbolic cosine
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_cplx
+       \ingroup arith_func_cosh
     */
-    AFAPI af_err af_cplx2 (af_array *out, const af_array real, const af_array imag, const bool batch);
+    AFAPI af_err af_cosh    (af_array *out, const af_array in);
 
     /**
-       C Interface to find the real part of a complex array.
+       C Interface to evaluate the hyperbolic tangent function.
 
-       \param[out] out real part
-       \param[in] in complex array
+       \param[out] out hyperbolic tangent
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_real
+       \ingroup arith_func_tanh
     */
-    AFAPI af_err af_real    (af_array *out, const af_array in);
+    AFAPI af_err af_tanh    (af_array *out, const af_array in);
 
     /**
-       C Interface to find the imaginary part of a complex array.
+       C Interface to evaluate the inverse hyperbolic sine function.
 
-       \param[out] out imaginary part
-       \param[in] in complex array
+       \param[out] out inverse hyperbolic sine
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_imag
+       \ingroup arith_func_asinh
     */
-    AFAPI af_err af_imag    (af_array *out, const af_array in);
+    AFAPI af_err af_asinh   (af_array *out, const af_array in);
 
     /**
-       C Interface to find the complex conjugate of an input array.
+       C Interface to evaluate the inverse hyperbolic cosine function.
 
-       \param[out] out complex conjugate
-       \param[in] in complex array
+       \param[out] out inverse hyperbolic cosine
+       \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_conjg
+       \ingroup arith_func_acosh
     */
-    AFAPI af_err af_conjg   (af_array *out, const af_array in);
+    AFAPI af_err af_acosh   (af_array *out, const af_array in);
 
     /**
-       C Interface to evaluate the hyperbolic sine function.
+       C Interface to evaluate the inverse hyperbolic tangent function.
 
-       \param[out] out hyperbolic sine
+       \param[out] out inverse hyperbolic tangent
        \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_sinh
+       \ingroup arith_func_atanh
     */
-    AFAPI af_err af_sinh    (af_array *out, const af_array in);
+    AFAPI af_err af_atanh   (af_array *out, const af_array in);
 
     /**
-       C Interface to evaluate the hyperbolic cosine function.
+       C Interface to create a complex array from a single real array.
 
-       \param[out] out hyperbolic cosine
-       \param[in] in input
+       \param[out] out complex array
+       \param[in] in real array
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_cosh
+       \ingroup arith_func_cplx
     */
-    AFAPI af_err af_cosh    (af_array *out, const af_array in);
+    AFAPI af_err af_cplx(af_array* out, const af_array in);
 
     /**
-       C Interface to evaluate the hyperbolic tangent function.
+       C Interface to create a complex array from two real arrays.
 
-       \param[out] out hyperbolic tangent
-       \param[in] in input
+       \param[out] out complex array
+       \param[in] real real array to be assigned as the real component of the returned complex array
+       \param[in] imag real array to be assigned as the imaginary component of the returned complex array
+       \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_tanh
+       \ingroup arith_func_cplx
     */
-    AFAPI af_err af_tanh    (af_array *out, const af_array in);
+    AFAPI af_err af_cplx2(af_array* out, const af_array real, const af_array imag, const bool batch);
 
     /**
-       C Interface to evaluate the inverse hyperbolic sine function.
+       C Interface to find the real part of a complex array.
 
-       \param[out] out inverse hyperbolic sine
-       \param[in] in input
+       \param[out] out real part
+       \param[in] in complex array
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_asinh
+       \ingroup arith_func_real
     */
-    AFAPI af_err af_asinh   (af_array *out, const af_array in);
+    AFAPI af_err af_real(af_array* out, const af_array in);
 
     /**
-       C Interface to evaluate the inverse hyperbolic cosine function.
+       C Interface to find the imaginary part of a complex array.
 
-       \param[out] out inverse hyperbolic cosine
-       \param[in] in input
+       \param[out] out imaginary part
+       \param[in] in complex array
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_acosh
+       \ingroup arith_func_imag
     */
-    AFAPI af_err af_acosh   (af_array *out, const af_array in);
+    AFAPI af_err af_imag(af_array* out, const af_array in);
 
     /**
-       C Interface to evaluate the inverse hyperbolic tangent function.
+       C Interface to find the complex conjugate of an input array.
 
-       \param[out] out inverse hyperbolic tangent
-       \param[in] in input
+       \param[out] out complex conjugate
+       \param[in] in complex array
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \ingroup arith_func_atanh
+       \ingroup arith_func_conjg
     */
-    AFAPI af_err af_atanh   (af_array *out, const af_array in);
+    AFAPI af_err af_conjg(af_array* out, const af_array in);
 
     /**
        C Interface to find the nth root.

From 2333815b524abaf8623f3b407cd68e96cc681c69 Mon Sep 17 00:00:00 2001
From: John Melonakos <john.melonakos@intel.com>
Date: Fri, 13 Jan 2023 14:35:46 -0500
Subject: [PATCH 192/473] improves formatting of arith.dox

---
 docs/details/arith.dox | 148 ++++++++++++++++++++---------------------
 1 file changed, 73 insertions(+), 75 deletions(-)

diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index ac8d265628..4d0fee8ae3 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -1,52 +1,50 @@
 /*!
 \page arith_real_only arith_real
-
 \note This function supports real inputs only. Complex inputs are not yet supported.
-
 */
 
 /*!
 \page arith_int_only arith_int
-
 \note This function supports integer only.
-
 */
 
 
+
 /**
 \addtogroup arrayfire_func
 @{
 
-\defgroup arith_func_add add
 
+
+\defgroup arith_func_add add
 \ingroup arith_mat
 
 Add two arrays.
 
 
-\defgroup arith_func_sub sub
 
+\defgroup arith_func_sub sub
 \ingroup arith_mat
 
 Subtract one array from another array.
 
 
-\defgroup arith_func_mul mul
 
+\defgroup arith_func_mul mul
 \ingroup arith_mat
 
 Multiply two arrays.
 
 
-\defgroup arith_func_div div
 
+\defgroup arith_func_div div
 \ingroup arith_mat
 
 Divide one array by another array.
 
 
-\defgroup arith_func_lt lt
 
+\defgroup arith_func_lt lt
 \ingroup logic_mat
 
 Less than, an elementwise comparison of two arrays.
@@ -54,8 +52,8 @@ Less than, an elementwise comparison of two arrays.
 Check if the elements of one array are less than those of another array.
 
 
-\defgroup arith_func_gt gt
 
+\defgroup arith_func_gt gt
 \ingroup logic_mat
 
 Greater than comparison, an elementwise comparison of two arrays.
@@ -63,8 +61,8 @@ Greater than comparison, an elementwise comparison of two arrays.
 Check if the elements of one array are greater than those of another array.
 
 
-\defgroup arith_func_le le
 
+\defgroup arith_func_le le
 \ingroup logic_mat
 
 Less than or equal to, an elementwise comparison of two arrays.
@@ -73,7 +71,6 @@ Check if the elements of one array are less than or equal to those of another ar
 
 
 \defgroup arith_func_ge ge
-
 \ingroup logic_mat
 
 Greater than or equal to, an elementwise comparison of two arrays.
@@ -81,8 +78,8 @@ Greater than or equal to, an elementwise comparison of two arrays.
 Check if the elements of one array are greater than or equal to those of another array.
 
 
-\defgroup arith_func_eq eq
 
+\defgroup arith_func_eq eq
 \ingroup logic_mat
 
 \brief Equal to, an elementwise comparison of two arrays.
@@ -90,8 +87,8 @@ Check if the elements of one array are greater than or equal to those of another
 Check if the elements of one array are equal to those of another array.
 
 
-\defgroup arith_func_neq neq
 
+\defgroup arith_func_neq neq
 \ingroup logic_mat
 
 \brief Not equal to, an elementwise comparison of two arrays.
@@ -99,36 +96,36 @@ Check if the elements of one array are equal to those of another array.
 Check if the elements of one array are not equal to those of another array.
 
 
-\defgroup arith_func_and and
 
+\defgroup arith_func_and and
 \ingroup logic_mat
 
 Evaluate the logical AND of two arrays.
 
 
-\defgroup arith_func_or or
 
+\defgroup arith_func_or or
 \ingroup logic_mat
 
 Evaluate the logical OR of two arrays.
 
 
-\defgroup arith_func_not not
 
+\defgroup arith_func_not not
 \ingroup logic_mat
 
 Evaluate the logical NOT of an array.
 
 
-\defgroup arith_func_neg neg
 
+\defgroup arith_func_neg neg
 \ingroup numeric_mat
 
 Negate an array.
 
 
-\defgroup arith_func_bitnot bitnot
 
+\defgroup arith_func_bitnot bitnot
 \ingroup logic_mat
 
 Evaluate the bitwise NOT of an array.
@@ -136,8 +133,8 @@ Evaluate the bitwise NOT of an array.
 \copydoc arith_int_only
 
 
-\defgroup arith_func_bitand bitand
 
+\defgroup arith_func_bitand bitand
 \ingroup logic_mat
 
 Evaluate the bitwise AND of two arrays.
@@ -145,8 +142,8 @@ Evaluate the bitwise AND of two arrays.
 \copydoc arith_int_only
 
 
-\defgroup arith_func_bitor bitor
 
+\defgroup arith_func_bitor bitor
 \ingroup logic_mat
 
 Evaluate the bitwise OR of two arrays.
@@ -154,8 +151,8 @@ Evaluate the bitwise OR of two arrays.
 \copydoc arith_int_only
 
 
-\defgroup arith_func_bitxor bitxor
 
+\defgroup arith_func_bitxor bitxor
 \ingroup logic_mat
 
 Evaluate the bitwise XOR of two arrays.
@@ -163,8 +160,8 @@ Evaluate the bitwise XOR of two arrays.
 \copydoc arith_int_only
 
 
-\defgroup arith_func_shiftl bitshiftl
 
+\defgroup arith_func_shiftl bitshiftl
 \ingroup arith_mat
 
 Shift the bits of integer arrays left.
@@ -172,8 +169,8 @@ Shift the bits of integer arrays left.
 \copydoc arith_int_only
 
 
-\defgroup arith_func_shiftr bitshiftr
 
+\defgroup arith_func_shiftr bitshiftr
 \ingroup arith_mat
 
 Shift the bits of integer arrays right.
@@ -181,36 +178,36 @@ Shift the bits of integer arrays right.
 \copydoc arith_int_only
 
 
-\defgroup arith_func_cast cast
 
+\defgroup arith_func_cast cast
 \ingroup helper_mat
 
 Cast an array from one type to another.
 
 
-\defgroup arith_func_min min
 
+\defgroup arith_func_min min
 \ingroup numeric_mat
 
 Find the elementwise minimum between two arrays.
 
 
-\defgroup arith_func_max max
 
+\defgroup arith_func_max max
 \ingroup numeric_mat
 
 Find the elementwise maximum between two arrays.
 
 
-\defgroup arith_func_clamp clamp
 
+\defgroup arith_func_clamp clamp
 \ingroup numeric_mat
 
 Clamp an array between an upper and a lower limit.
 
 
-\defgroup arith_func_rem rem
 
+\defgroup arith_func_rem rem
 \ingroup numeric_mat
 
 Find the remainder of a division.
@@ -218,8 +215,8 @@ Find the remainder of a division.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_mod mod
 
+\defgroup arith_func_mod mod
 \ingroup numeric_mat
 
 Find the modulus.
@@ -227,7 +224,9 @@ Find the modulus.
 \copydoc arith_real_only
 
 
+
 \defgroup arith_func_abs abs
+\ingroup numeric_mat
 
 Find the absolute value.
 
@@ -235,18 +234,16 @@ __Examples:__
 
 \snippet test/math.cpp ex_arith_func_abs
 
-\ingroup numeric_mat
 
 
 \defgroup arith_func_arg arg
-
 \ingroup numeric_mat
 
 Find the phase angle (in radians) of a complex array.
 
 
-\defgroup arith_func_sign sign
 
+\defgroup arith_func_sign sign
 \ingroup numeric_mat
 
 Find the sign of elements in an array.
@@ -254,8 +251,8 @@ Find the sign of elements in an array.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_round round
 
+\defgroup arith_func_round round
 \ingroup numeric_mat
 
 Round numbers to the nearest integer.
@@ -263,8 +260,8 @@ Round numbers to the nearest integer.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_trunc trunc
 
+\defgroup arith_func_trunc trunc
 \ingroup numeric_mat
 
 Truncate numbers to nearest integer.
@@ -272,8 +269,8 @@ Truncate numbers to nearest integer.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_floor floor
 
+\defgroup arith_func_floor floor
 \ingroup numeric_mat
 
 Round to the integer less than or equal to the magnitude of the input value.
@@ -281,8 +278,8 @@ Round to the integer less than or equal to the magnitude of the input value.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_ceil ceil
 
+\defgroup arith_func_ceil ceil
 \ingroup numeric_mat
 
 Round to the integer greater than or equal to the magnitude of the input value.
@@ -290,8 +287,8 @@ Round to the integer greater than or equal to the magnitude of the input value.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_hypot hypot
 
+\defgroup arith_func_hypot hypot
 \ingroup numeric_mat
 
 Find the length of the hypotenuse of two inputs.
@@ -299,35 +296,37 @@ Find the length of the hypotenuse of two inputs.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_sin sin
 
+\defgroup arith_func_sin sin
 \ingroup trig_mat
 
 Evaluate the sine function.
 
 
-\defgroup arith_func_cos cos
 
+\defgroup arith_func_cos cos
 \ingroup trig_mat
 
 Evaluate the cosine function.
 
 
-\defgroup arith_func_tan tan
 
+\defgroup arith_func_tan tan
 \ingroup trig_mat
 
 Evaluate the tangent function.
 
 
-\defgroup arith_func_asin asin
 
+\defgroup arith_func_asin asin
 \ingroup trig_mat
 
 Evaluate the inverse sine function (arc sine).
 
 
+
 \defgroup arith_func_acos acos
+\ingroup trig_mat
 
 Evaluate the inverse cosine function (arc cosine).
 
@@ -337,60 +336,58 @@ __Examples:__
 
 \snippet test/math.cpp ex_arith_func_acos
 
-\ingroup trig_mat
 
 
 \defgroup arith_func_atan atan/atan2
-
 \ingroup trig_mat
 
 Evaluate the inverse tangent function (arc tangent).
 
 
-\defgroup arith_func_sinh sinh
 
+\defgroup arith_func_sinh sinh
 \ingroup hyper_mat
 
 Evaluate the hyperbolic sine function.
 
 
-\defgroup arith_func_cosh cosh
 
+\defgroup arith_func_cosh cosh
 \ingroup hyper_mat
 
 Evaluate the hyperbolic cosine function.
 
 
-\defgroup arith_func_tanh tanh
 
+\defgroup arith_func_tanh tanh
 \ingroup hyper_mat
 
 Evaluate the hyperbolic tangent function.
 
 
-\defgroup arith_func_asinh asinh
 
+\defgroup arith_func_asinh asinh
 \ingroup hyper_mat
 
 Evaluate the inverse hyperbolic sine function (area hyperbolic sine).
 
 
-\defgroup arith_func_acosh acosh
 
+\defgroup arith_func_acosh acosh
 \ingroup hyper_mat
 
 Evaluate the inverse hyperbolic cosine function (area hyperbolic cosine).
 
 
-\defgroup arith_func_atanh atanh
 
+\defgroup arith_func_atanh atanh
 \ingroup hyper_mat
 
 Evaluate the inverse hyperbolic tangent function (area hyperbolic tangent).
 
 
-\defgroup arith_func_cplx complex
 
+\defgroup arith_func_cplx complex
 \ingroup complex_mat
 
 Create complex arrays.
@@ -407,62 +404,62 @@ __Examples:__
 \snippet test/complex.cpp ex_arith_func_complex
 
 
-\defgroup arith_func_real real
 
+\defgroup arith_func_real real
 \ingroup complex_mat
 
 Find the real part of a complex array.
 
 
-\defgroup arith_func_imag imag
 
+\defgroup arith_func_imag imag
 \ingroup complex_mat
 
 Find the imaginary part of a complex array.
 
 
-\defgroup arith_func_conjg conjg
 
+\defgroup arith_func_conjg conjg
 \ingroup complex_mat
 
 Find the complex conjugate of an input array.
 
 
-\defgroup arith_func_root root
 
+\defgroup arith_func_root root
 \ingroup explog_mat
 
 Find the nth root.
 
 
-\defgroup arith_func_pow pow
 
+\defgroup arith_func_pow pow
 \ingroup explog_mat
 
 Raise a base to a power (or exponent).
 
 
-\defgroup arith_func_pow pow2
 
+\defgroup arith_func_pow pow2
 \ingroup explog_mat
 
 Raise 2 to a power (or exponent).
 
 
-\defgroup arith_func_sigmoid sigmoid
 
+\defgroup arith_func_sigmoid sigmoid
 Evaluate the logistical sigmoid function.
 
 
-\defgroup arith_func_exp exp
 
+\defgroup arith_func_exp exp
 \ingroup explog_mat
 
 Evaluate the exponential function.
 
 
-\defgroup arith_func_expm1 expm1
 
+\defgroup arith_func_expm1 expm1
 \ingroup explog_mat
 
 Evaluate the exponential function of an array minus 1, `exp(in) - 1`.
@@ -470,8 +467,8 @@ Evaluate the exponential function of an array minus 1, `exp(in) - 1`.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_erf erf
 
+\defgroup arith_func_erf erf
 \ingroup explog_mat
 
 Evaluate the error function.
@@ -479,8 +476,8 @@ Evaluate the error function.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_erfc erfc
 
+\defgroup arith_func_erfc erfc
 \ingroup explog_mat
 
 Evaluate the complementary error function.
@@ -488,15 +485,15 @@ Evaluate the complementary error function.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_log log
 
+\defgroup arith_func_log log
 \ingroup explog_mat
 
 Evaluate the natural logarithm.
 
 
-\defgroup arith_func_log1p log1p
 
+\defgroup arith_func_log1p log1p
 \ingroup explog_mat
 
 Evaluate the natural logarithm of 1 + input, `ln(1+in)`.
@@ -504,8 +501,8 @@ Evaluate the natural logarithm of 1 + input, `ln(1+in)`.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_log10 log10
 
+\defgroup arith_func_log10 log10
 \ingroup explog_mat
 
 Evaluate the base 10 logarithm.
@@ -513,8 +510,8 @@ Evaluate the base 10 logarithm.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_log2 log2
 
+\defgroup arith_func_log2 log2
 \ingroup explog_mat
 
 Evaluate the base 2 logarithm.
@@ -522,15 +519,15 @@ Evaluate the base 2 logarithm.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_sqrt sqrt
 
+\defgroup arith_func_sqrt sqrt
 \ingroup explog_mat
 
 Find the square root.
 
 
-\defgroup arith_func_rsqrt rsqrt
 
+\defgroup arith_func_rsqrt rsqrt
 \ingroup explog_mat
 
 Find the reciprocal square root.
@@ -540,8 +537,8 @@ Find the reciprocal square root.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_cbrt cbrt
 
+\defgroup arith_func_cbrt cbrt
 \ingroup explog_mat
 
 Find the cube root.
@@ -549,8 +546,8 @@ Find the cube root.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_factorial factorial
 
+\defgroup arith_func_factorial factorial
 \ingroup explog_mat
 
 Find the factorial.
@@ -558,8 +555,8 @@ Find the factorial.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_tgamma tgamma
 
+\defgroup arith_func_tgamma tgamma
 \ingroup explog_mat
 
 Evaluate the gamma function.
@@ -567,8 +564,8 @@ Evaluate the gamma function.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_lgamma lgamma
 
+\defgroup arith_func_lgamma lgamma
 \ingroup explog_mat
 
 Evaluate the logarithm of the absolute value of the gamma function.
@@ -576,26 +573,27 @@ Evaluate the logarithm of the absolute value of the gamma function.
 \copydoc arith_real_only
 
 
-\defgroup arith_func_iszero iszero
 
+\defgroup arith_func_iszero iszero
 \ingroup helper_mat
 
 Check if values are zero.
 
 
-\defgroup arith_func_isinf isinf
 
+\defgroup arith_func_isinf isinf
 \ingroup helper_mat
 
 Check if values are infinite.
 
 
-\defgroup arith_func_isnan isnan
 
+\defgroup arith_func_isnan isnan
 \ingroup helper_mat
 
 Check if values are NaN.
 
 
+
 @}
 */

From 2fb3c9e13f41436c030f59337122375004fe2167 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 20 Jan 2023 20:35:46 -0500
Subject: [PATCH 193/473] upgrade doxygen.mk to 1.9.6 for better compatibility
 with theme

---
 docs/details/examples.dox | 116 +++++++++++++++++++-------------------
 docs/doxygen.mk           |  49 +++++++++-------
 2 files changed, 88 insertions(+), 77 deletions(-)

diff --git a/docs/details/examples.dox b/docs/details/examples.dox
index a61ffbc271..1fd4451335 100644
--- a/docs/details/examples.dox
+++ b/docs/details/examples.dox
@@ -1,58 +1,58 @@
-/**
-\example benchmarks/blas.cpp
-\example benchmarks/cg.cpp
-\example benchmarks/fft.cpp
-\example benchmarks/pi.cpp
-\example computer_vision/fast.cpp
-\example computer_vision/harris.cpp
-\example computer_vision/matching.cpp
-\example computer_vision/susan.cpp
-\example financial/black_scholes_options.cpp
-\example financial/heston_model.cpp
-\example financial/monte_carlo_options.cpp
-\example getting_started/convolve.cpp
-\example getting_started/integer.cpp
-\example getting_started/rainfall.cpp
-\example getting_started/vectorize.cpp
-\example graphics/conway.cpp
-\example graphics/conway_pretty.cpp
-\example graphics/field.cpp
-\example graphics/fractal.cpp
-\example graphics/gravity_sim.cpp
-\example graphics/histogram.cpp
-\example graphics/plot2d.cpp
-\example graphics/plot3.cpp
-\example graphics/surface.cpp
-\example helloworld/helloworld.cpp
-\example image_processing/adaptive_thresholding.cpp
-\example image_processing/binary_thresholding.cpp
-\example image_processing/brain_segmentation.cpp
-\example image_processing/confidence_connected_components.cpp
-\example image_processing/deconvolution.cpp
-\example image_processing/edge.cpp
-\example image_processing/filters.cpp
-\example image_processing/gradient_diffusion.cpp
-\example image_processing/image_demo.cpp
-\example image_processing/image_editing.cpp
-\example image_processing/morphing.cpp
-\example image_processing/optical_flow.cpp
-\example image_processing/pyramids.cpp
-\example lin_algebra/cholesky.cpp
-\example lin_algebra/lu.cpp
-\example lin_algebra/qr.cpp
-\example lin_algebra/svd.cpp
-\example machine_learning/bagging.cpp
-\example machine_learning/deep_belief_net.cpp
-\example machine_learning/geneticalgorithm.cpp
-\example machine_learning/kmeans.cpp
-\example machine_learning/knn.cpp
-\example machine_learning/logistic_regression.cpp
-\example machine_learning/naive_bayes.cpp
-\example machine_learning/neural_network.cpp
-\example machine_learning/perceptron.cpp
-\example machine_learning/rbm.cpp
-\example machine_learning/softmax_regression.cpp
-\example pde/swe.cpp
-\example unified/basic.cpp
-
-*/
+/**
+\example benchmarks/blas.cpp
+\example benchmarks/cg.cpp
+\example benchmarks/fft.cpp
+\example benchmarks/pi.cpp
+\example computer_vision/fast.cpp
+\example computer_vision/harris.cpp
+\example computer_vision/matching.cpp
+\example computer_vision/susan.cpp
+\example financial/black_scholes_options.cpp
+\example financial/heston_model.cpp
+\example financial/monte_carlo_options.cpp
+\example getting_started/convolve.cpp
+\example getting_started/integer.cpp
+\example getting_started/rainfall.cpp
+\example getting_started/vectorize.cpp
+\example graphics/conway.cpp
+\example graphics/conway_pretty.cpp
+\example graphics/field.cpp
+\example graphics/fractal.cpp
+\example graphics/gravity_sim.cpp
+\example graphics/histogram.cpp
+\example graphics/plot2d.cpp
+\example graphics/plot3.cpp
+\example graphics/surface.cpp
+\example helloworld/helloworld.cpp
+\example image_processing/adaptive_thresholding.cpp
+\example image_processing/binary_thresholding.cpp
+\example image_processing/brain_segmentation.cpp
+\example image_processing/confidence_connected_components.cpp
+\example image_processing/deconvolution.cpp
+\example image_processing/edge.cpp
+\example image_processing/filters.cpp
+\example image_processing/gradient_diffusion.cpp
+\example image_processing/image_demo.cpp
+\example image_processing/image_editing.cpp
+\example image_processing/morphing.cpp
+\example image_processing/optical_flow.cpp
+\example image_processing/pyramids.cpp
+\example lin_algebra/cholesky.cpp
+\example lin_algebra/lu.cpp
+\example lin_algebra/qr.cpp
+\example lin_algebra/svd.cpp
+\example machine_learning/bagging.cpp
+\example machine_learning/deep_belief_net.cpp
+\example machine_learning/geneticalgorithm.cpp
+\example machine_learning/kmeans.cpp
+\example machine_learning/knn.cpp
+\example machine_learning/logistic_regression.cpp
+\example machine_learning/naive_bayes.cpp
+\example machine_learning/neural_network.cpp
+\example machine_learning/perceptron.cpp
+\example machine_learning/rbm.cpp
+\example machine_learning/softmax_regression.cpp
+\example pde/swe.cpp
+\example unified/basic.cpp
+
+*/
diff --git a/docs/doxygen.mk b/docs/doxygen.mk
index 2e4da59f66..914ebb35b4 100644
--- a/docs/doxygen.mk
+++ b/docs/doxygen.mk
@@ -1,4 +1,4 @@
-# Doxyfile 1.9.5
+# Doxyfile 1.9.6
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -86,7 +86,7 @@ CREATE_SUBDIRS         = NO
 # level increment doubles the number of directories, resulting in 4096
 # directories at level 8 which is the default and also the maximum value. The
 # sub-directories are organized in 2 levels, the first level always has a fixed
-# numer of 16 directories.
+# number of 16 directories.
 # Minimum value: 0, maximum value: 8, default value: 8.
 # This tag requires that the tag CREATE_SUBDIRS is set to YES.
 
@@ -582,7 +582,8 @@ HIDE_UNDOC_MEMBERS     = NO
 # If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
 # undocumented classes that are normally visible in the class hierarchy. If set
 # to NO, these classes will be included in the various overviews. This option
-# has no effect if EXTRACT_ALL is enabled.
+# will also hide undocumented C++ concepts if enabled. This option has no effect
+# if EXTRACT_ALL is enabled.
 # The default value is: NO.
 
 HIDE_UNDOC_CLASSES     = NO
@@ -873,6 +874,14 @@ WARN_IF_INCOMPLETE_DOC = YES
 
 WARN_NO_PARAMDOC       = YES
 
+# If WARN_IF_UNDOC_ENUM_VAL option is set to YES, doxygen will warn about
+# undocumented enumeration values. If set to NO, doxygen will accept
+# undocumented enumeration values. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: NO.
+
+WARN_IF_UNDOC_ENUM_VAL = NO
+
 # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
 # a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS
 # then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but
@@ -1246,10 +1255,11 @@ CLANG_DATABASE_PATH    =
 
 ALPHABETICAL_INDEX     = YES
 
-# In case all classes in a project start with a common prefix, all classes will
-# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
-# can be used to specify a prefix (or a list of prefixes) that should be ignored
-# while generating the index headers.
+# The IGNORE_PREFIX tag can be used to specify a prefix (or a list of prefixes)
+# that should be ignored while generating the index headers. The IGNORE_PREFIX
+# tag works for classes, function and member names. The entity will be placed in
+# the alphabetical list under the first letter of the entity name that remains
+# after removing the prefix.
 # This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
 
 IGNORE_PREFIX          = af_
@@ -1328,7 +1338,12 @@ HTML_STYLESHEET        =
 # Doxygen will copy the style sheet files to the output directory.
 # Note: The order of the extra style sheet files is of importance (e.g. the last
 # style sheet in the list overrules the setting of the previous ones in the
-# list). For an example see the documentation.
+# list).
+# Note: Since the styling of scrollbars can currently not be overruled in
+# Webkit/Chromium, the styling will be left out of the default doxygen.css if
+# one or more extra stylesheets have been specified. So if scrollbar
+# customization is desired it has to be added explicitly. For an example see the
+# documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
 HTML_EXTRA_STYLESHEET  = ${DOCS_DIR}/arrayfire.css \
@@ -1348,17 +1363,13 @@ HTML_EXTRA_FILES       = ${DOCS_DIR}/doxygen-awesome-darkmode-toggle.js \
                          ${DOCS_DIR}/doxygen-awesome-interactive-toc.js
 
 # The HTML_COLORSTYLE tag can be used to specify if the generated HTML output
-# should be rendered with a dark or light theme. Default setting AUTO_LIGHT
-# enables light output unless the user preference is dark output. Other options
-# are DARK to always use dark mode, LIGHT to always use light mode, AUTO_DARK to
-# default to dark mode unless the user prefers light mode, and TOGGLE to let the
-# user toggle between dark and light mode via a button.
-# Possible values are: LIGHT Always generate light output., DARK Always generate
-# dark output., AUTO_LIGHT Automatically set the mode according to the user
-# preference, use light mode if no preference is set (the default)., AUTO_DARK
-# Automatically set the mode according to the user preference, use dark mode if
-# no preference is set. and TOGGLE Allow to user to switch between light and
-# dark mode via a button..
+# should be rendered with a dark or light theme.
+# Possible values are: LIGHT always generate light mode output, DARK always
+# generate dark mode output, AUTO_LIGHT automatically set the mode according to
+# the user preference, use light mode if no preference is set (the default),
+# AUTO_DARK automatically set the mode according to the user preference, use
+# dark mode if no preference is set and TOGGLE allow to user to switch between
+# light and dark mode via a button.
 # The default value is: AUTO_LIGHT.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 

From 461b694d5dd1c494ddd04253950395f2421c9ad0 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 23 Jan 2023 18:42:32 -0500
Subject: [PATCH 194/473] remove doxygen warnings

---
 docs/details/arith.dox         | 12 +----
 docs/details/image.dox         |  8 ++--
 docs/details/lapack.dox        |  2 +-
 docs/details/signal.dox        |  2 +-
 docs/pages/getting_started.md  |  4 +-
 docs/pages/release_notes.md    | 84 +++++++++++++++++-----------------
 docs/pages/using_on_linux.md   |  4 +-
 docs/pages/using_on_osx.md     |  4 +-
 docs/pages/using_on_windows.md |  2 +-
 include/af/image.h             |  8 ++--
 include/af/ml.h                |  4 +-
 include/af/util.h              |  2 +-
 test/complex.cpp               | 40 ++++++++--------
 13 files changed, 85 insertions(+), 91 deletions(-)

diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index 4d0fee8ae3..a7130647df 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -230,11 +230,6 @@ Find the modulus.
 
 Find the absolute value.
 
-__Examples:__
-
-\snippet test/math.cpp ex_arith_func_abs
-
-
 
 \defgroup arith_func_arg arg
 \ingroup numeric_mat
@@ -332,11 +327,6 @@ Evaluate the inverse cosine function (arc cosine).
 
 The inverse of cosine so that, if `y = cos(x)`, then `x = arccos(y)`.
 
-__Examples:__
-
-\snippet test/math.cpp ex_arith_func_acos
-
-
 
 \defgroup arith_func_atan atan/atan2
 \ingroup trig_mat
@@ -440,7 +430,7 @@ Raise a base to a power (or exponent).
 
 
 
-\defgroup arith_func_pow pow2
+\defgroup arith_func_pow2 pow2
 \ingroup explog_mat
 
 Raise 2 to a power (or exponent).
diff --git a/docs/details/image.dox b/docs/details/image.dox
index 73ae3239eb..a93f1ebaed 100644
--- a/docs/details/image.dox
+++ b/docs/details/image.dox
@@ -855,7 +855,7 @@ is described above, but the effect should be the same.
 \defgroup image_func_wrap wrap
 \ingroup image_mod_mat
 
-Performs the opposite of \ref unwrap().
+Performs the opposite of \ref af::unwrap().
 
 More specifically, wrap takes each column (or row if `is_column` is false) of the
 \f$m \times n\f$ input array and reshapes them into `wx` \f$\times\f$ `wy`
@@ -935,7 +935,7 @@ is visualized above, but the effect should be the same.
 \defgroup image_func_moments moments
 \ingroup moments_mat
 
-The \ref moments() function allows for finding different
+The \ref af::moments() function allows for finding different
 properties of image regions. Currently, ArrayFire calculates all first order moments.
 The moments are defined within the \ref af_moment_type enum.
 
@@ -1059,8 +1059,8 @@ explicitly.
 
 \brief Segment image based on similar pixel characteristics
 
-This filter is similar to \ref regions() (connected components) with additional
-criteria for segmentation. In \ref regions(), all connected (\ref af_connectivity)
+This filter is similar to \ref af::regions() (connected components) with additional
+criteria for segmentation. In \ref af::regions(), all connected (\ref af_connectivity)
 pixels connected are considered to be a single component. In this
 variation of connected components, pixels having similar pixel statistics
 of the neighborhoods around a given set of seed points are grouped together.
diff --git a/docs/details/lapack.dox b/docs/details/lapack.dox
index 8bf5d5a5ea..bf977b0c0c 100644
--- a/docs/details/lapack.dox
+++ b/docs/details/lapack.dox
@@ -141,7 +141,7 @@ following code snippet can be used:
 
 \snippet test/svd_dense.cpp ex_svd_reg
 
-When memory is a concern, and \f$A\f$ is dispensable, \ref svdInPlace() can be
+When memory is a concern, and \f$A\f$ is dispensable, \ref af::svdInPlace() can be
 used. However, this in-place version is currently limited to input arrays where
 \f$M \geq N\f$.
 
diff --git a/docs/details/signal.dox b/docs/details/signal.dox
index fa1b3130c5..e77da4f968 100644
--- a/docs/details/signal.dox
+++ b/docs/details/signal.dox
@@ -274,7 +274,7 @@ Given below is an example of this batch mode.
 
 
 The batching behavior of convolve2NN functions(\ref af_convolve2_nn() and
-\ref convolve2NN() ) is different from convolve2. The new functions can perform 2D
+\ref af::convolve2NN() ) is different from convolve2. The new functions can perform 2D
 convolution on 3D signals and filters in a way that is more aligned with
 convolutional neural networks.
 
diff --git a/docs/pages/getting_started.md b/docs/pages/getting_started.md
index d10142269b..d958892c2e 100644
--- a/docs/pages/getting_started.md
+++ b/docs/pages/getting_started.md
@@ -18,7 +18,7 @@ achieve high throughput on most parallel architectures.
 
 ArrayFire provides one generic container object, the [array](\ref af::array)
 on which functions and mathematical operations are performed. The `array`
-can represent one of many different [basic data types](\ref af::af_dtype):
+can represent one of many different [basic data types](\ref af_dtype):
 
 * [f32](\ref f32) real single-precision (`float`)
 * [c32](\ref c32) complex single-precision (`cfloat`)
@@ -87,7 +87,7 @@ ArrayFire provides several functions to determine various aspects of arrays.
 This includes functions to print the contents, query the dimensions, and
 determine various other aspects of arrays.
 
-The [af_print](\ref af::af_print) function can be used to print arrays that
+The [af_print](\ref af_print) function can be used to print arrays that
 have already been generated or any expression involving arrays:
 
 \snippet test/getting_started.cpp ex_getting_started_print
diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index fe893c564c..bc40f2a7b7 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -1217,7 +1217,7 @@ Bug Fixes
       before returning pointer with asynchronous calls in CPU backend.
     * OpenCL Backend: [fix segfaults](https://github.com/arrayfire/arrayfire/issues/1324)
       when requested for device pointers on empty arrays.
-* Fixed \ref af::array::operator%() from using [rem to mod](https://github.com/arrayfire/arrayfire/issues/1318).
+* Fixed \ref af::operator%() from using [rem to mod](https://github.com/arrayfire/arrayfire/issues/1318).
 * Fixed [array destruction](https://github.com/arrayfire/arrayfire/issues/1321)
   when backends are switched in Unified API.
 * Fixed [indexing](https://github.com/arrayfire/arrayfire/issues/1331) after
@@ -1356,9 +1356,9 @@ Deprecations
 Documentation
 --------------
 
-* Fixes to documentation for \ref matchTemplate().
+* Fixes to documentation for \ref af::matchTemplate().
 * Improved documentation for deviceInfo.
-* Fixes to documentation for \ref exp().
+* Fixes to documentation for \ref af::exp().
 
 Known Issues
 ------------
@@ -1497,18 +1497,18 @@ Major Updates
 Function Additions
 ------------------
 * Unified Backend
-    * \ref setBackend() - Sets a backend as active
-    * \ref getBackendCount() - Gets the number of backends available for use
-    * \ref getAvailableBackends() - Returns information about available backends
-    * \ref getBackendId() - Gets the backend enum for an array
+    * \ref af::setBackend() - Sets a backend as active
+    * \ref af::getBackendCount() - Gets the number of backends available for use
+    * \ref af::getAvailableBackends() - Returns information about available backends
+    * \ref af::getBackendId() - Gets the backend enum for an array
 
 * Vision
-    * \ref homography() - Homography estimation
-    * \ref gloh() - GLOH Descriptor for SIFT
+    * \ref af::homography() - Homography estimation
+    * \ref af::gloh() - GLOH Descriptor for SIFT
 
 * Image Processing
-    * \ref loadImageNative() - Load an image as native data without modification
-    * \ref saveImageNative() - Save an image without modifying data or type
+    * \ref af::loadImageNative() - Load an image as native data without modification
+    * \ref af::saveImageNative() - Save an image without modifying data or type
 
 * Graphics
     * \ref af::Window::plot3() - 3-dimensional line plot
@@ -1522,26 +1522,26 @@ Function Additions
     * \ref af_release_indexers()
 
 * CUDA Backend Specific
-    * \ref setNativeId() - Set the CUDA device with given native id as active
+    * \ref afcu::setNativeId() - Set the CUDA device with given native id as active
         * ArrayFire uses a modified order for devices. The native id for a
           device can be retreived using `nvidia-smi`
 
 * OpenCL Backend Specific
-    * \ref setDeviceId() - Set the OpenCL device using the `clDeviceId`
+    * \ref afcl::setDeviceId() - Set the OpenCL device using the `clDeviceId`
 
 Other Improvements
 ------------------------
-* Added \ref c32 and \ref c64 support for \ref isNaN(), \ref isInf() and \ref iszero()
-* Added CPU information for `x86` and `x86_64` architectures in CPU backend's \ref info()
-* Batch support for \ref approx1() and \ref approx2()
+* Added \ref c32 and \ref c64 support for \ref af::isNaN(), \ref af::isInf() and \ref af::iszero()
+* Added CPU information for `x86` and `x86_64` architectures in CPU backend's \ref af::info()
+* Batch support for \ref af::approx1() and \ref af::approx2()
     * Now can be used with gfor as well
 * Added \ref s64 and \ref u64 support to:
-    * \ref sort() (along with sort index and sort by key)
-    * \ref setUnique(), \ref setUnion(), \ref setIntersect()
-    * \ref convolve() and \ref fftConvolve()
-    * \ref histogram() and \ref histEqual()
-    * \ref lookup()
-    * \ref mean()
+    * \ref af::sort() (along with sort index and sort by key)
+    * \ref af::setUnique(), \ref af::setUnion(), \ref af::setIntersect()
+    * \ref af::convolve() and \ref af::fftConvolve()
+    * \ref af::histogram() and \ref af::histEqual()
+    * \ref af::lookup()
+    * \ref af::mean()
 * Added \ref AF_MSG macro
 
 Build Improvements
@@ -1553,15 +1553,15 @@ Build Improvements
 
 Bug Fixes
 --------------
-* Fixed [memory leak](https://github.com/arrayfire/arrayfire/pull/1096) in \ref susan()
+* Fixed [memory leak](https://github.com/arrayfire/arrayfire/pull/1096) in \ref af::susan()
 * Fixed [failing test](https://github.com/arrayfire/arrayfire/commit/144a2db)
-  in \ref lower() and \ref upper() for CUDA compute 53
+  in \ref af::lower() and \ref af::upper() for CUDA compute 53
 * Fixed [bug](https://github.com/arrayfire/arrayfire/issues/1092) in CUDA for indexing out of bounds
-* Fixed [dims check](https://github.com/arrayfire/arrayfire/commit/6975da8) in \ref iota()
-* Fixed [out-of-bounds access](https://github.com/arrayfire/arrayfire/commit/7fc3856) in \ref sift()
-* Fixed [memory allocation](https://github.com/arrayfire/arrayfire/commit/5e88e4a) in \ref fast() OpenCL
+* Fixed [dims check](https://github.com/arrayfire/arrayfire/commit/6975da8) in \ref af::iota()
+* Fixed [out-of-bounds access](https://github.com/arrayfire/arrayfire/commit/7fc3856) in \ref af::sift()
+* Fixed [memory allocation](https://github.com/arrayfire/arrayfire/commit/5e88e4a) in \ref af::fast() OpenCL
 * Fixed [memory leak](https://github.com/arrayfire/arrayfire/pull/994) in image I/O functions
-* \ref dog() now returns float-point type arrays
+* \ref af::dog() now returns float-point type arrays
 
 Documentation Updates
 ---------------------
@@ -1664,10 +1664,10 @@ v3.1.0
 Function Additions
 ------------------
 * Computer Vision Functions
-    * \ref nearestNeighbour() - Nearest Neighbour with SAD, SSD and SHD distances
-    * \ref harris() - Harris Corner Detector
-    * \ref susan() - Susan Corner Detector
-    * \ref sift() - Scale Invariant Feature Transform (SIFT)
+    * \ref af::nearestNeighbour() - Nearest Neighbour with SAD, SSD and SHD distances
+    * \ref af::harris() - Harris Corner Detector
+    * \ref af::susan() - Susan Corner Detector
+    * \ref af::sift() - Scale Invariant Feature Transform (SIFT)
         * Method and apparatus for identifying scale invariant features"
           "in an image and use of same for locating an object in an image,\" David"
           "G. Lowe, US Patent 6,711,293 (March 23, 2004). Provisional application"
@@ -1677,7 +1677,7 @@ Function Additions
           "Columbia.")
         * SIFT is available for compiling but does not ship with ArrayFire
           hosted installers/pre-built libraries
-    * \ref dog() -  Difference of Gaussians
+    * \ref af::dog() -  Difference of Gaussians
 
 * Image Processing Functions
     * \ref ycbcr2rgb() and \ref rgb2ycbcr() - RGB <->YCbCr color space conversion
@@ -1803,20 +1803,20 @@ Bug Fixes
 --------------
 
 * Added missing symbols from the compatible API
-* Fixed a bug affecting corner rows and elements in \ref grad()
+* Fixed a bug affecting corner rows and elements in \ref af::grad()
 * Fixed linear interpolation bugs affecting large images in the following:
-    - \ref approx1()
-    - \ref approx2()
-    - \ref resize()
-    - \ref rotate()
-    - \ref scale()
-    - \ref skew()
-    - \ref transform()
+    - \ref af::approx1()
+    - \ref af::approx2()
+    - \ref af::resize()
+    - \ref af::rotate()
+    - \ref af::scale()
+    - \ref af::skew()
+    - \ref af::transform()
 
 Documentation
 -----------------
 
-* Added missing documentation for \ref constant()
+* Added missing documentation for \ref af::constant()
 * Added missing documentation for `array::scalar()`
 * Added supported input types for functions in `arith.h`
 
diff --git a/docs/pages/using_on_linux.md b/docs/pages/using_on_linux.md
index 4948763d77..0fcd23bba1 100644
--- a/docs/pages/using_on_linux.md
+++ b/docs/pages/using_on_linux.md
@@ -8,7 +8,7 @@ requirements are that you include the ArrayFire header directories and link with
 the ArrayFire library you intend to use i.e. CUDA, OpenCL, CPU, or Unified
 backends.
 
-## The big picture  {#big-picture}
+## The big picture  {#big-picture-linux}
 
 On Linux, we recommend installing ArrayFire to `/opt/arrayfire` directory. The
 installer will populate files in the following sub-directories:
@@ -57,7 +57,7 @@ apt install build-essential cmake cmake-curses-gui
 ## CMake
 
 We recommend that the CMake build system be used to create ArrayFire projects.
-As [discussed above](#big-picture), ArrayFire ships with a series of CMake
+As [discussed above](#big-picture-linux), ArrayFire ships with a series of CMake
 scripts to make finding and using our library easy.
 
 First create a file called `CMakeLists.txt` in your project directory:
diff --git a/docs/pages/using_on_osx.md b/docs/pages/using_on_osx.md
index 272898ec5e..e851509c4b 100644
--- a/docs/pages/using_on_osx.md
+++ b/docs/pages/using_on_osx.md
@@ -7,7 +7,7 @@ project using almost any editor, compiler, or build system. The only requirement
 is that you can include the ArrayFire header directory, and link with the
 ArrayFire library you intend to use.
 
-## <a name="big-picture"/> The big picture
+## The big picture {#big-picture-osx}
 
 By default, the ArrayFire OSX installer will place several files in your
 computer's `/opt/arrayfire` directory. The installer will populate this
@@ -33,7 +33,7 @@ CMake or Makefiles with CMake being our preferred build system.
 ## CMake {#CMake}
 
 The CMake build system can be used to create ArrayFire projects. As [discussed
-above](#big-picture), ArrayFire ships with a series of CMake scripts to make
+above](#big-picture-osx), ArrayFire ships with a series of CMake scripts to make
 finding and using our library easy.
 
 First create a file called `CMakeLists.txt` in your project directory:
diff --git a/docs/pages/using_on_windows.md b/docs/pages/using_on_windows.md
index 924fca2794..b178ad9c86 100644
--- a/docs/pages/using_on_windows.md
+++ b/docs/pages/using_on_windows.md
@@ -4,7 +4,7 @@ Using ArrayFire with Microsoft Windows and Visual Studio {#using_on_windows}
 If you have not already done so, please make sure you have installed,
 configured, and tested ArrayFire following the [installation instructions](#installing).
 
-# The big picture
+# The big picture {#big-picture-windows}
 
 The ArrayFire Windows installer creates the following:
 1. **AF_PATH** environment variable to point to the installation location. The
diff --git a/include/af/image.h b/include/af/image.h
index 5e32b551a9..b28d0b5395 100644
--- a/include/af/image.h
+++ b/include/af/image.h
@@ -602,7 +602,7 @@ AFAPI array unwrap(const array& in, const dim_t wx, const dim_t wy,
 
 #if AF_API_VERSION >= 31
 /**
-   C++ Interface for performing the opposite of \ref unwrap()
+   C++ Interface for performing the opposite of \ref unwrap
 
    \param[in]  in is the input array
    \param[in]  ox is the output's dimension 0 size
@@ -1487,7 +1487,7 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       C Interface for performing the opposite of \ref unwrap()
+       C Interface for performing the opposite of \ref af::unwrap()
 
        \param[out] out is an array with the input's columns (or rows) reshaped as
                    patches
@@ -1506,7 +1506,7 @@ extern "C" {
        otherwise an appropriate error code is returned.
 
        \note Wrap is typically used to recompose an unwrapped image. If this is the
-             case, use the same parameters that were used in \ref unwrap(). Also
+             case, use the same parameters that were used in \ref af::unwrap(). Also
              use the original image size (before unwrap) for \p ox and \p oy.
        \note The window/patch size, \p wx \f$\times\f$ \p wy, must equal
              `input.dims(0)` (or `input.dims(1)` if \p is_column is false).
@@ -1552,7 +1552,7 @@ extern "C" {
        otherwise an appropriate error code is returned.
 
        \note Wrap is typically used to recompose an unwrapped image. If this is the
-             case, use the same parameters that were used in \ref unwrap(). Also
+             case, use the same parameters that were used in \ref af::unwrap(). Also
              use the original image size (before unwrap) for \p ox and \p oy.
        \note The window/patch size, \p wx \f$\times\f$ \p wy, must equal
              `input.dims(0)` (or `input.dims(1)` if \p is_column is false).
diff --git a/include/af/ml.h b/include/af/ml.h
index c341fd9a43..33feff9112 100644
--- a/include/af/ml.h
+++ b/include/af/ml.h
@@ -20,7 +20,7 @@ class dim4;
     /**
         C++ interface for calculating backward pass gradient of 2D convolution
         This function calculates the gradient with respect to the output
-        of the \ref convolve2NN() function that uses the machine learning
+        of the \ref convolve2NN function that uses the machine learning
         formulation for the dimensions of the signals and filters
 
         \param[in]  incoming_gradient gradients to be distributed in backwards pass
@@ -60,7 +60,7 @@ extern "C" {
     /**
         C interface for calculating backward pass gradient of 2D convolution
         This function calculates the gradient with respect to the output
-        of the \ref convolve2NN() function that uses the machine learning
+        of the \ref af::convolve2NN() function that uses the machine learning
         formulation for the dimensions of the signals and filters
 
         \param[out] out gradient wrt/gradType
diff --git a/include/af/util.h b/include/af/util.h
index 6075625de5..49a16b43ec 100644
--- a/include/af/util.h
+++ b/include/af/util.h
@@ -184,7 +184,7 @@ extern "C" {
 #if AF_API_VERSION >= 31
     /**
         \param[out] index is the index location of the array in the file
-        \param[in] key is an expression used as tag/key for the array during \ref readArray()
+        \param[in] key is an expression used as tag/key for the array during \ref af::readArray()
         \param[in] arr is the array to be written
         \param[in] filename is the path to the location on disk
         \param[in] append is used to append to an existing file when true and create or
diff --git a/test/complex.cpp b/test/complex.cpp
index b63fd63bba..fe8a60c0f9 100644
--- a/test/complex.cpp
+++ b/test/complex.cpp
@@ -139,24 +139,28 @@ TEST(Complex, SNIPPET_arith_func_complex) {
     //! [ex_arith_func_complex]
     //!
     // Create a, a 2x3 array
-    array a = iota(dim4(2, 3));    // a = [0, 2, 4,
-                                   //      1, 3, 5]
-
-    // Create b from a single real array, returning zeros for the imaginary component
-    array b = complex(a);          // b = [(0, 0), (2, 0), (4, 0),
-                                   //      (1, 0), (3, 0), (5, 0)]
-
-    // Create c from two real arrays, one for the real component and one for the imaginary component
-    array c = complex(a, a);       // c = [(0, 0), (2, 2), (4, 4),
-                                   //      (1, 1), (3, 3), (5, 5)]
-
-    // Create d from a single real array for the real component and a single scalar for each imaginary component
-    array d = complex(a, 2);       // d = [(0, 2), (2, 2), (4, 2),
-                                   //      (1, 2), (3, 2), (5, 2)]
-
-    // Create e from a single scalar for each real component and a single real array for the imaginary component
-    array e = complex(2, a);       // e = [(2, 0), (2, 2), (2, 4),
-                                   //      (2, 1), (2, 3), (2, 5)]
+    array a = iota(dim4(2, 3));  // a = [0, 2, 4,
+                                 //      1, 3, 5]
+
+    // Create b from a single real array, returning zeros for the imaginary
+    // component
+    array b = complex(a);  // b = [(0, 0), (2, 0), (4, 0),
+                           //      (1, 0), (3, 0), (5, 0)]
+
+    // Create c from two real arrays, one for the real component and one for the
+    // imaginary component
+    array c = complex(a, a);  // c = [(0, 0), (2, 2), (4, 4),
+                              //      (1, 1), (3, 3), (5, 5)]
+
+    // Create d from a single real array for the real component and a single
+    // scalar for each imaginary component
+    array d = complex(a, 2);  // d = [(0, 2), (2, 2), (4, 2),
+                              //      (1, 2), (3, 2), (5, 2)]
+
+    // Create e from a single scalar for each real component and a single real
+    // array for the imaginary component
+    array e = complex(2, a);  // e = [(2, 0), (2, 2), (2, 4),
+                              //      (2, 1), (2, 3), (2, 5)]
 
     //! [ex_arith_func_complex]
 

From 33935abd8bf537b98ec264472d9089c8480ec6b8 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 23 Jan 2023 20:33:22 -0500
Subject: [PATCH 195/473] slight tweaks to documentation wording

---
 docs/details/arith.dox    |  44 +++++++-------
 docs/details/blas.dox     |  18 ++++--
 docs/details/examples.dox |  58 ------------------
 include/af/arith.h        | 124 +++++++++++++++++++-------------------
 4 files changed, 97 insertions(+), 147 deletions(-)
 delete mode 100644 docs/details/examples.dox

diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index a7130647df..2e123f7ba8 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -19,28 +19,28 @@
 \defgroup arith_func_add add
 \ingroup arith_mat
 
-Add two arrays.
+Elementwise addition
 
 
 
 \defgroup arith_func_sub sub
 \ingroup arith_mat
 
-Subtract one array from another array.
+Elementwise subtraction
 
 
 
 \defgroup arith_func_mul mul
 \ingroup arith_mat
 
-Multiply two arrays.
+Elementwise multiply
 
 
 
 \defgroup arith_func_div div
 \ingroup arith_mat
 
-Divide one array by another array.
+Elementwise division
 
 
 
@@ -189,14 +189,14 @@ Cast an array from one type to another.
 \defgroup arith_func_min min
 \ingroup numeric_mat
 
-Find the elementwise minimum between two arrays.
+Returns the elementwise minimum between two arrays.
 
 
 
 \defgroup arith_func_max max
 \ingroup numeric_mat
 
-Find the elementwise maximum between two arrays.
+Returns the elementwise maximum between two arrays.
 
 
 
@@ -210,7 +210,7 @@ Clamp an array between an upper and a lower limit.
 \defgroup arith_func_rem rem
 \ingroup numeric_mat
 
-Find the remainder of a division.
+Calculate the remainder of a division.
 
 \copydoc arith_real_only
 
@@ -219,7 +219,7 @@ Find the remainder of a division.
 \defgroup arith_func_mod mod
 \ingroup numeric_mat
 
-Find the modulus.
+Calculate the modulus.
 
 \copydoc arith_real_only
 
@@ -228,20 +228,20 @@ Find the modulus.
 \defgroup arith_func_abs abs
 \ingroup numeric_mat
 
-Find the absolute value.
+Calculate the absolute value.
 
 
 \defgroup arith_func_arg arg
 \ingroup numeric_mat
 
-Find the phase angle (in radians) of a complex array.
+Calculate the phase angle (in radians) of a complex array.
 
 
 
 \defgroup arith_func_sign sign
 \ingroup numeric_mat
 
-Find the sign of elements in an array.
+Return the sign of elements in an array.
 
 \copydoc arith_real_only
 
@@ -268,7 +268,7 @@ Truncate numbers to nearest integer.
 \defgroup arith_func_floor floor
 \ingroup numeric_mat
 
-Round to the integer less than or equal to the magnitude of the input value.
+Rounds down to the greatest integer less than or equal to x.
 
 \copydoc arith_real_only
 
@@ -277,7 +277,7 @@ Round to the integer less than or equal to the magnitude of the input value.
 \defgroup arith_func_ceil ceil
 \ingroup numeric_mat
 
-Round to the integer greater than or equal to the magnitude of the input value.
+Rounds up to the least integer greater than or equal to x.
 
 \copydoc arith_real_only
 
@@ -286,7 +286,7 @@ Round to the integer greater than or equal to the magnitude of the input value.
 \defgroup arith_func_hypot hypot
 \ingroup numeric_mat
 
-Find the length of the hypotenuse of two inputs.
+Evaluate the length of the hypotenuse of two inputs.
 
 \copydoc arith_real_only
 
@@ -398,28 +398,28 @@ __Examples:__
 \defgroup arith_func_real real
 \ingroup complex_mat
 
-Find the real part of a complex array.
+Returns the real part of a complex array.
 
 
 
 \defgroup arith_func_imag imag
 \ingroup complex_mat
 
-Find the imaginary part of a complex array.
+Returns the imaginary part of a complex array.
 
 
 
 \defgroup arith_func_conjg conjg
 \ingroup complex_mat
 
-Find the complex conjugate of an input array.
+Evaluate the complex conjugate of an input array.
 
 
 
 \defgroup arith_func_root root
 \ingroup explog_mat
 
-Find the nth root.
+Evaluate the nth root.
 
 
 
@@ -513,14 +513,14 @@ Evaluate the base 2 logarithm.
 \defgroup arith_func_sqrt sqrt
 \ingroup explog_mat
 
-Find the square root.
+Evaluate the square root.
 
 
 
 \defgroup arith_func_rsqrt rsqrt
 \ingroup explog_mat
 
-Find the reciprocal square root.
+Evaluate the reciprocal square root.
 
 \f[ \frac{1}{\sqrt{x}} \f]
 
@@ -531,7 +531,7 @@ Find the reciprocal square root.
 \defgroup arith_func_cbrt cbrt
 \ingroup explog_mat
 
-Find the cube root.
+Evaluate the cube root.
 
 \copydoc arith_real_only
 
@@ -540,7 +540,7 @@ Find the cube root.
 \defgroup arith_func_factorial factorial
 \ingroup explog_mat
 
-Find the factorial.
+Evaluate the factorial.
 
 \copydoc arith_real_only
 
diff --git a/docs/details/blas.dox b/docs/details/blas.dox
index 3765ed446c..b8757d81fb 100644
--- a/docs/details/blas.dox
+++ b/docs/details/blas.dox
@@ -52,11 +52,19 @@ and restrictions.
 
 \brief Transpose a matrix.
 
-Reverse or permute the dimensions of an array; returns the modified array. For an array a with two dimensions, `transpose(a)` gives the matrix transpose. For an array with more than two dimensions, the first two dimensions are transposed across higher dimensions.
-
-Set `conjugate=true` to perform the complex conjugate transpose of a matrix which interchanges the row and column index for each element, reflecting the elements across the main diagonal and negating the imaginary part of any complex numbers. For example, if `b = transpose(a, true)` and element `a(2, 1)` is `(1, 2)`, then element `b(1, 2)` is `(1, -2)`.
-
-In-place versions perform matrix transposition by reordering the input, reducing memory footprint.
+Reverse or permute the dimensions of an array; returns the modified array.
+For an array a with two dimensions, `transpose(a)` gives the matrix transpose.
+For an array with more than two dimensions, the first two dimensions are
+transposed across higher dimensions.
+
+Set `conjugate=true` to perform the complex conjugate transpose of a matrix
+which interchanges the row and column index for each element, reflecting the
+elements across the main diagonal and negating the imaginary part of any
+complex numbers. For example, if `b = transpose(a, true)` and element
+`a(2, 1)` is `(1, 2)`, then element `b(1, 2)` is `(1, -2)`.
+
+In-place versions perform matrix transposition by reordering the input,
+reducing memory footprint.
 
 __Examples:__
 
diff --git a/docs/details/examples.dox b/docs/details/examples.dox
deleted file mode 100644
index 1fd4451335..0000000000
--- a/docs/details/examples.dox
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
-\example benchmarks/blas.cpp
-\example benchmarks/cg.cpp
-\example benchmarks/fft.cpp
-\example benchmarks/pi.cpp
-\example computer_vision/fast.cpp
-\example computer_vision/harris.cpp
-\example computer_vision/matching.cpp
-\example computer_vision/susan.cpp
-\example financial/black_scholes_options.cpp
-\example financial/heston_model.cpp
-\example financial/monte_carlo_options.cpp
-\example getting_started/convolve.cpp
-\example getting_started/integer.cpp
-\example getting_started/rainfall.cpp
-\example getting_started/vectorize.cpp
-\example graphics/conway.cpp
-\example graphics/conway_pretty.cpp
-\example graphics/field.cpp
-\example graphics/fractal.cpp
-\example graphics/gravity_sim.cpp
-\example graphics/histogram.cpp
-\example graphics/plot2d.cpp
-\example graphics/plot3.cpp
-\example graphics/surface.cpp
-\example helloworld/helloworld.cpp
-\example image_processing/adaptive_thresholding.cpp
-\example image_processing/binary_thresholding.cpp
-\example image_processing/brain_segmentation.cpp
-\example image_processing/confidence_connected_components.cpp
-\example image_processing/deconvolution.cpp
-\example image_processing/edge.cpp
-\example image_processing/filters.cpp
-\example image_processing/gradient_diffusion.cpp
-\example image_processing/image_demo.cpp
-\example image_processing/image_editing.cpp
-\example image_processing/morphing.cpp
-\example image_processing/optical_flow.cpp
-\example image_processing/pyramids.cpp
-\example lin_algebra/cholesky.cpp
-\example lin_algebra/lu.cpp
-\example lin_algebra/qr.cpp
-\example lin_algebra/svd.cpp
-\example machine_learning/bagging.cpp
-\example machine_learning/deep_belief_net.cpp
-\example machine_learning/geneticalgorithm.cpp
-\example machine_learning/kmeans.cpp
-\example machine_learning/knn.cpp
-\example machine_learning/logistic_regression.cpp
-\example machine_learning/naive_bayes.cpp
-\example machine_learning/neural_network.cpp
-\example machine_learning/perceptron.cpp
-\example machine_learning/rbm.cpp
-\example machine_learning/softmax_regression.cpp
-\example pde/swe.cpp
-\example unified/basic.cpp
-
-*/
diff --git a/include/af/arith.h b/include/af/arith.h
index f6f190f199..ea9be6c328 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -98,7 +98,7 @@ namespace af
     /// @}
 
     /// @{
-    /// C++ Interface to find the remainder.
+    /// C++ Interface to calculate the remainder.
     ///
     /// \param[in] lhs numerator; can be an array or a scalar
     /// \param[in] rhs denominator; can be an array or a scalar
@@ -115,7 +115,7 @@ namespace af
     /// @}
 
     /// @{
-    /// C++ Interface to find the modulus.
+    /// C++ Interface to calculate the modulus.
     ///
     /// \param[in] lhs dividend; can be an array or a scalar
     /// \param[in] rhs divisor; can be an array or a scalar
@@ -131,7 +131,7 @@ namespace af
     AFAPI array mod    (const double lhs, const array &rhs);
     /// @}
 
-    /// C++ Interface to find the absolute value.
+    /// C++ Interface to calculate the absolute value.
     ///
     /// \param[in] in input array
     /// \return absolute value
@@ -139,7 +139,7 @@ namespace af
     /// \ingroup arith_func_abs
     AFAPI array abs    (const array &in);
 
-    /// C++ Interface to find the phase angle (in radians) of a complex array.
+    /// C++ Interface to calculate the phase angle (in radians) of a complex array.
     ///
     /// \param[in] in input array, typically complex
     /// \return phase angle (in radians)
@@ -147,7 +147,7 @@ namespace af
     /// \ingroup arith_func_arg
     AFAPI array arg    (const array &in);
 
-    /// C++ Interface to find the sign of elements in an array.
+    /// C++ Interface to return the sign of elements in an array.
     ///
     /// \param[in] in input array
     /// \return array containing 1's for negative values; 0's otherwise
@@ -189,7 +189,7 @@ namespace af
 
     /// \ingroup arith_func_hypot
     /// @{
-    /// C++ Interface to find the length of the hypotenuse of two inputs.
+    /// C++ Interface to calculate the length of the hypotenuse of two inputs.
     ///
     /// Calculates the hypotenuse of two inputs. The inputs can be both arrays
     /// or an array and a scalar.
@@ -348,7 +348,7 @@ namespace af
     AFAPI array complex(const double real_, const array &imag_);
     /// @}
 
-    /// C++ Interface to find the real part of a complex array.
+    /// C++ Interface to return the real part of a complex array.
     ///
     /// \param[in] in input complex array
     /// \return real part
@@ -356,7 +356,7 @@ namespace af
     /// \ingroup arith_func_real
     AFAPI array real   (const array &in);
 
-    /// C++ Interface to find the imaginary part of a complex array.
+    /// C++ Interface to return the imaginary part of a complex array.
     ///
     /// \param[in] in input complex array
     /// \return imaginary part
@@ -364,7 +364,7 @@ namespace af
     /// \ingroup arith_func_imag
     AFAPI array imag   (const array &in);
 
-    /// C++ Interface to find the complex conjugate of an input array.
+    /// C++ Interface to calculate the complex conjugate of an input array.
     ///
     /// \param[in] in input complex array
     /// \return complex conjugate
@@ -372,50 +372,50 @@ namespace af
     /// \ingroup arith_func_conjg
     AFAPI array conjg  (const array &in);
 
-    /// C++ Interface to find the nth root.
+    /// C++ Interface to evaluate the nth root.
     ///
-    /// \param[in] lhs nth root
-    /// \param[in] rhs value
-    /// \return \p lhs th root of \p rhs
+    /// \param[in] nth_root nth root
+    /// \param[in] value value
+    /// \return \p nth_root th root of \p value
     ///
     /// \ingroup arith_func_root
-    AFAPI array root    (const array &lhs, const array &rhs);
+    AFAPI array root    (const array &nth_root, const array &value);
 
-    /// C++ Interface to find the nth root.
+    /// C++ Interface to evaluate the nth root.
     ///
-    /// \param[in] lhs nth root
-    /// \param[in] rhs value
-    /// \return \p lhs th root of \p rhs
+    /// \param[in] nth_root nth root
+    /// \param[in] value value
+    /// \return \p nth_root th root of \p value
     ///
     /// \ingroup arith_func_root
-    AFAPI array root    (const array &lhs, const double rhs);
+    AFAPI array root    (const array &nth_root, const double value);
 
-    /// C++ Interface to find the nth root.
+    /// C++ Interface to evaluate the nth root.
     ///
-    /// \param[in] lhs nth root
-    /// \param[in] rhs value
-    /// \return \p lhs th root of \p rhs
+    /// \param[in] nth_root nth root
+    /// \param[in] value value
+    /// \return \p nth_root th root of \p value
     ///
     /// \ingroup arith_func_root
-    AFAPI array root    (const double lhs, const array &rhs);
+    AFAPI array root    (const double nth_root, const array &value);
 
 
     /// \ingroup arith_func_pow
     /// @{
     /// C++ Interface to raise a base to a power (or exponent).
     ///
-    /// Computes the value of \p lhs raised to the power of \p rhs. The inputs can be two arrays or an array and a scalar.
+    /// Computes the value of \p base raised to the power of \p exponent. The inputs can be two arrays or an array and a scalar.
     ///
-    /// \param[in] lhs base
-    /// \param[in] rhs exponent
-    /// \return \p lhs raised to the power of \p rhs
-    AFAPI array pow    (const array &lhs, const array &rhs);
+    /// \param[in] base base
+    /// \param[in] exponent exponent
+    /// \return \p base raised to the power of \p exponent
+    AFAPI array pow    (const array &base, const array &exponent);
 
     /// \copydoc pow(const array&, const array&)
-    AFAPI array pow    (const array &lhs, const double rhs);
+    AFAPI array pow    (const array &base, const double exponent);
 
     /// \copydoc pow(const array&, const array&)
-    AFAPI array pow    (const double lhs, const array &rhs);
+    AFAPI array pow    (const double base, const array &exponent);
 
     /// C++ Interface to raise 2 to a power (or exponent).
     ///
@@ -503,7 +503,7 @@ namespace af
     /// \ingroup explog_func_log2
     AFAPI array log2   (const array &in);
 
-    /// C++ Interface to find the square root.
+    /// C++ Interface to evaluate the square root.
     ///
     /// \param[in] in input
     /// \return square root
@@ -512,7 +512,7 @@ namespace af
     AFAPI array sqrt   (const array &in);
 
 #if AF_API_VERSION >= 37
-    /// C++ Interface to find the reciprocal square root.
+    /// C++ Interface to evaluate the reciprocal square root.
     ///
     /// \param[in] in input
     /// \return reciprocal square root
@@ -521,7 +521,7 @@ namespace af
     AFAPI array rsqrt   (const array &in);
 #endif
 
-    /// C++ Interface to find the cube root.
+    /// C++ Interface to evaluate the cube root.
     ///
     /// \param[in] in input
     /// \return cube root
@@ -529,7 +529,7 @@ namespace af
     /// \ingroup arith_func_cbrt
     AFAPI array cbrt   (const array &in);
 
-    /// C++ Interface to find the factorial.
+    /// C++ Interface to calculate the factorial.
     ///
     /// \param[in] in input
     /// \return the factorial function
@@ -553,7 +553,7 @@ namespace af
     /// \ingroup arith_func_lgamma
     AFAPI array lgamma (const array &in);
 
-    /// C++ Interface to check if values are zero.
+    /// C++ Interface to check which values are zero.
     ///
     /// \param[in] in input
     /// \return array containing 1's where input is 0; 0's otherwise
@@ -636,7 +636,7 @@ extern "C" {
     AFAPI af_err af_div   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to check if the elements of one array are less than those of another array.
+       C Interface to perform a less-than comparison between corresponding elements of two arrays.
 
        \param[out] out result of \p lhs < \p rhs; type is b8
        \param[in] lhs first input
@@ -649,7 +649,7 @@ extern "C" {
     AFAPI af_err af_lt    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to check if the elements of one array are greater than those of another array.
+       C Interface to perform a greater-than comparison between corresponding elements of two arrays.
 
        \param[out] out result of \p lhs > \p rhs; type is b8
        \param[in] lhs first input
@@ -662,7 +662,7 @@ extern "C" {
     AFAPI af_err af_gt    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to check if the elements of one array are less than or equal to those of another array.
+       C Interface to perform a less-than-or-equal comparison between corresponding elements of two arrays.
 
        \param[out] out result of \p lhs <= \p rhs; type is b8
        \param[in] lhs first input
@@ -675,7 +675,7 @@ extern "C" {
     AFAPI af_err af_le    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to check if the elements of one array are greater than or equal to those of another array.
+       C Interface to perform a greater-than-or-equal comparison between corresponding elements of two arrays.
 
        \param[out] out result of \p lhs >= \p rhs; type is b8
        \param[in] lhs first input
@@ -688,7 +688,7 @@ extern "C" {
     AFAPI af_err af_ge    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to check if the elements of one array are equal to those of another array.
+       C Interface to check if corresponding elements of two arrays are equal
 
        \param[out] out result of `lhs == rhs`; type is b8
        \param[in] lhs first input
@@ -701,7 +701,7 @@ extern "C" {
     AFAPI af_err af_eq    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to check if the elements of one array are not equal to those of another array.
+       C Interface to check if corresponding elements of two arrays are not equal
 
        \param[out] out result of `lhs != rhs`; type is b8
        \param[in] lhs first input
@@ -806,8 +806,8 @@ extern "C" {
        C Interface to shift the bits of integer arrays left.
 
        \param[out] out result of the left shift
-       \param[in] lhs first input
-       \param[in] rhs second input
+       \param[in] lhs values to shift
+       \param[in] rhs n bits to shift
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -819,8 +819,8 @@ extern "C" {
        C Interface to shift the bits of integer arrays right.
 
        \param[out] out result of the right shift
-       \param[in] lhs first input
-       \param[in] rhs second input
+       \param[in] lhs values to shift
+       \param[in] rhs n bits to shift
        \param[in] batch specifies if operations need to be performed in batch mode
        \return \ref AF_SUCCESS if the execution completes properly
 
@@ -913,7 +913,7 @@ extern "C" {
 #endif
 
     /**
-       C Interface to find the remainder.
+       C Interface to calculate the remainder.
 
        \param[out] out remainder of \p lhs divided by \p rhs
        \param[in] lhs numerator
@@ -926,7 +926,7 @@ extern "C" {
     AFAPI af_err af_rem   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to find the modulus.
+       C Interface to calculate the modulus.
 
        \param[out] out \p lhs modulo \p rhs
        \param[in] lhs dividend
@@ -939,7 +939,7 @@ extern "C" {
     AFAPI af_err af_mod   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to find the absolute value.
+       C Interface to calculate the absolute value.
 
        \param[out] out absolute value
        \param[in] in input array
@@ -950,7 +950,7 @@ extern "C" {
     AFAPI af_err af_abs     (af_array *out, const af_array in);
 
     /**
-       C Interface to find the phase angle (in radians) of a complex array.
+       C Interface to calculate the phase angle (in radians) of a complex array.
 
        \param[out] out phase angle (in radians)
        \param[in] in input array, typically complex
@@ -961,7 +961,7 @@ extern "C" {
     AFAPI af_err af_arg     (af_array *out, const af_array in);
 
     /**
-       C Interface to find the sign of elements in an array.
+       C Interface to calculate the sign of elements in an array.
 
        \param[out] out array containing 1's for negative values; 0's otherwise
        \param[in] in input array
@@ -1016,7 +1016,7 @@ extern "C" {
     AFAPI af_err af_ceil    (af_array *out, const af_array in);
 
     /**
-       C Interface to find the length of the hypotenuse of two inputs.
+       C Interface to calculate the length of the hypotenuse of two inputs.
 
        \param[out] out length of the hypotenuse
        \param[in] lhs length of first side
@@ -1198,7 +1198,7 @@ extern "C" {
     AFAPI af_err af_cplx2(af_array* out, const af_array real, const af_array imag, const bool batch);
 
     /**
-       C Interface to find the real part of a complex array.
+       C Interface to return the real part of a complex array.
 
        \param[out] out real part
        \param[in] in complex array
@@ -1209,7 +1209,7 @@ extern "C" {
     AFAPI af_err af_real(af_array* out, const af_array in);
 
     /**
-       C Interface to find the imaginary part of a complex array.
+       C Interface to return the imaginary part of a complex array.
 
        \param[out] out imaginary part
        \param[in] in complex array
@@ -1220,7 +1220,7 @@ extern "C" {
     AFAPI af_err af_imag(af_array* out, const af_array in);
 
     /**
-       C Interface to find the complex conjugate of an input array.
+       C Interface to evaluate the complex conjugate of an input array.
 
        \param[out] out complex conjugate
        \param[in] in complex array
@@ -1231,7 +1231,7 @@ extern "C" {
     AFAPI af_err af_conjg(af_array* out, const af_array in);
 
     /**
-       C Interface to find the nth root.
+       C Interface to evaluate the nth root.
 
        \param[out] out \p lhs th root of \p rhs
        \param[in] lhs nth root
@@ -1272,12 +1272,12 @@ extern "C" {
     /**
        C Interface to evaluate the logistical sigmoid function.
 
+       Computes `1/(1+e^-x)`.
+
        \param[out] out output of the logistic sigmoid function
        \param[in] in input
        \return \ref AF_SUCCESS if the execution completes properly
 
-       \note Computes `1/(1+e^-x)`.
-
        \ingroup arith_func_sigmoid
     */
     AFAPI af_err af_sigmoid(af_array* out, const af_array in);
@@ -1372,7 +1372,7 @@ extern "C" {
     AFAPI af_err af_log2   (af_array *out, const af_array in);
 
     /**
-       C Interface to find the square root.
+       C Interface to evaluate the square root.
 
        \param[out] out square root
        \param[in] in input
@@ -1384,7 +1384,7 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-      C Interface to find the reciprocal square root.
+      C Interface to evaluate the reciprocal square root.
 
       \param[out] out reciprocal square root
       \param[in] in input
@@ -1395,7 +1395,7 @@ extern "C" {
     AFAPI af_err af_rsqrt    (af_array *out, const af_array in);
 #endif
     /**
-       C Interface to find the cube root.
+       C Interface to evaluate the cube root.
 
        \param[out] out cube root
        \param[in] in input
@@ -1406,7 +1406,7 @@ extern "C" {
     AFAPI af_err af_cbrt    (af_array *out, const af_array in);
 
     /**
-       C Interface to find the factorial.
+       C Interface to calculate the factorial.
 
        \param[out] out factorial
        \param[in] in input

From 635718a121892719f4bb87e4b48552deba66f0bf Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Wed, 25 Jan 2023 12:02:04 -0500
Subject: [PATCH 196/473] pinned memory oneapi (#3356)

* supports pinned memory allocation on oneapi backend through USM

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
---
 src/backend/oneapi/memory.cpp | 111 ++++++++--------------------------
 1 file changed, 26 insertions(+), 85 deletions(-)
 mode change 100644 => 100755 src/backend/oneapi/memory.cpp

diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
old mode 100644
new mode 100755
index e87812e5b4..80c589a5b0
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -48,8 +48,7 @@ void signalMemoryCleanup() { memoryManager().signalMemoryCleanup(); }
 
 void shutdownMemoryManager() { memoryManager().shutdown(); }
 
-void shutdownPinnedMemoryManager() { /*pinnedMemoryManager().shutdown();*/
-}
+void shutdownPinnedMemoryManager() { pinnedMemoryManager().shutdown(); }
 
 void printMemInfo(const char *msg, const int device) {
     memoryManager().printInfo(msg, device);
@@ -62,18 +61,6 @@ std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>>
 memAlloc(const size_t &elements) {
     return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>(
         new sycl::buffer<T>(sycl::range(elements)), bufferFree<T>);
-    // // TODO: make memAlloc aware of array shapes
-    // if (elements) {
-    //     dim4 dims(elements);
-    //     void *ptr = memoryManager().alloc(false, 1, dims.get(), sizeof(T));
-    //     auto buf  = static_cast<cl_mem>(ptr);
-    //     cl::Buffer *bptr = new cl::Buffer(buf, true);
-    //     return unique_ptr<cl::Buffer, function<void(cl::Buffer *)>>(bptr,
-    //                                                                 bufferFree);
-    // } else {
-    //     return unique_ptr<cl::Buffer, function<void(cl::Buffer *)>>(nullptr,
-    //                                                                 bufferFree);
-    // }
 }
 
 void *memAllocUser(const size_t &bytes) {
@@ -159,17 +146,15 @@ void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
 
 template<typename T>
 T *pinnedAlloc(const size_t &elements) {
-    ONEAPI_NOT_SUPPORTED("pinnedAlloc Not supported");
-
-    // // TODO: make pinnedAlloc aware of array shapes
-    // dim4 dims(elements);
-    // void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), sizeof(T));
-    return static_cast<T *>(nullptr);
+    // TODO: make pinnedAlloc aware of array shapes
+    dim4 dims(elements);
+    void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), sizeof(T));
+    return static_cast<T *>(ptr);
 }
 
 template<typename T>
 void pinnedFree(T *ptr) {
-    // pinnedMemoryManager().unlock(static_cast<void *>(ptr), false);
+    pinnedMemoryManager().unlock(static_cast<void *>(ptr), false);
 }
 
 // template unique_ptr<int, function<void(int *)>> memAlloc<T>(
@@ -257,80 +242,36 @@ void Allocator::nativeFree(void *ptr) {
     // }
 }
 
-AllocatorPinned::AllocatorPinned() : pinnedMaps(oneapi::getDeviceCount()) {
-    logger = common::loggerFactory("mem");
-}
+AllocatorPinned::AllocatorPinned() { logger = common::loggerFactory("mem"); }
 
-void AllocatorPinned::shutdown() {
-    ONEAPI_NOT_SUPPORTED("AllocatorPinned::shutdown Not supported");
+void AllocatorPinned::shutdown() { shutdownPinnedMemoryManager(); }
 
-    //     for (int n = 0; n < opencl::getDeviceCount(); n++) {
-    //         opencl::setDevice(n);
-    //         shutdownPinnedMemoryManager();
-    //         auto currIterator = pinnedMaps[n].begin();
-    //         auto endIterator  = pinnedMaps[n].end();
-    //         while (currIterator != endIterator) {
-    //             pinnedMaps[n].erase(currIterator++);
-    //         }
-    //     }
-}
-
-int AllocatorPinned::getActiveDeviceId() {
-    ONEAPI_NOT_SUPPORTED("AllocatorPinned::getActiveDeviceId Not supported");
-    return 0;
-
-    // opencl::getActiveDeviceId();
-}
+int AllocatorPinned::getActiveDeviceId() { oneapi::getActiveDeviceId(); }
 
 size_t AllocatorPinned::getMaxMemorySize(int id) {
-    ONEAPI_NOT_SUPPORTED("AllocatorPinned::getMaxMemorySize Not supported");
-    return 0;
-    // return opencl::getDeviceMemorySize(id);
+    return oneapi::getDeviceMemorySize(id);
 }
 
 void *AllocatorPinned::nativeAlloc(const size_t bytes) {
-    ONEAPI_NOT_SUPPORTED("AllocatorPinned::nativeAlloc Not supported");
-    return nullptr;
-    //     void *ptr = NULL;
-
-    //     cl_int err = CL_SUCCESS;
-    //     auto buf   = clCreateBuffer(getContext()(), CL_MEM_ALLOC_HOST_PTR,
-    //     bytes,
-    //                               nullptr, &err);
-    //     if (err != CL_SUCCESS) {
-    //         AF_ERROR("Failed to allocate pinned memory.", AF_ERR_NO_MEM);
-    //     }
-
-    //     ptr = clEnqueueMapBuffer(getQueue()(), buf, CL_TRUE,
-    //                              CL_MAP_READ | CL_MAP_WRITE, 0, bytes, 0,
-    //                              nullptr, nullptr, &err);
-    //     if (err != CL_SUCCESS) {
-    //         AF_ERROR("Failed to map pinned memory", AF_ERR_RUNTIME);
-    //     }
-    //     AF_TRACE("Pinned::nativeAlloc: {:>7} {}", bytesToString(bytes), ptr);
-    //     pinnedMaps[opencl::getActiveDeviceId()].emplace(ptr, new
-    //     cl::Buffer(buf)); return ptr;
+    void *ptr = NULL;
+    try {
+        ptr = sycl::malloc_host<unsigned char>(bytes, getQueue());
+    } catch (...) {
+        auto str = fmt::format("Failed to allocate device memory of size {}",
+                               bytesToString(bytes));
+        AF_ERROR(str, AF_ERR_NO_MEM);
+    }
+    AF_TRACE("Pinned::nativeAlloc: {:>7} {}", bytesToString(bytes), ptr);
+    return ptr;
 }
 
 void AllocatorPinned::nativeFree(void *ptr) {
-    ONEAPI_NOT_SUPPORTED("AllocatorPinned::nativeFree Not supported");
-
-    // AF_TRACE("Pinned::nativeFree:          {}", ptr);
-    // int n     = opencl::getActiveDeviceId();
-    // auto &map = pinnedMaps[n];
-    // auto iter = map.find(ptr);
-
-    // if (iter != map.end()) {
-    //     cl::Buffer *buf = map[ptr];
-    //     if (cl_int err = getQueue().enqueueUnmapMemObject(*buf, ptr)) {
-    //         getLogger()->warn(
-    //             "Pinned::nativeFree: Error unmapping pinned memory({}:{}). "
-    //             "Ignoring",
-    //             err, getErrorMessage(err));
-    //     }
-    //     delete buf;
-    //     map.erase(iter);
-    // }
+    AF_TRACE("Pinned::nativeFree:          {}", ptr);
+    try {
+        sycl::free(ptr, getQueue());
+    } catch (...) {
+        AF_ERROR("Failed to release device memory.", AF_ERR_RUNTIME);
+    }
 }
 }  // namespace oneapi
 }  // namespace arrayfire

From 12f63fadaa42f0690b5c6c459a77c8e0bcafe3b6 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 25 Jan 2023 14:32:47 -0500
Subject: [PATCH 197/473] fix spdlog when external fmt found

---
 CMakeLists.txt | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8985c797ff..b049258552 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -60,7 +60,7 @@ find_package(CBLAS)
 find_package(LAPACKE)
 find_package(Doxygen)
 find_package(MKL)
-find_package(spdlog QUIET ${AF_REQUIRED})
+find_package(spdlog QUIET ${AF_REQUIRED} NO_CMAKE_PACKAGE_REGISTRY)
 find_package(fmt QUIET ${AF_REQUIRED})
 find_package(span-lite QUIET)
 find_package(GTest)
@@ -228,14 +228,13 @@ else()
     URI https://github.com/gabime/spdlog.git
     REF v1.9.2
   )
-  add_subdirectory(${${spdlog_prefix}_SOURCE_DIR} ${${spdlog_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
 
   if(TARGET fmt::fmt)
-    set_target_properties(af_spdlog
-      PROPERTIES
-        INTERFACE_COMPILE_DEFINITIONS "SPDLOG_FMT_EXTERNAL")
+    set(SPDLOG_FMT_EXTERNAL ON)
   endif()
 
+  add_subdirectory(${${spdlog_prefix}_SOURCE_DIR} ${${spdlog_prefix}_BINARY_DIR} EXCLUDE_FROM_ALL)
+
   if(AF_WITH_SPDLOG_HEADER_ONLY)
     set_target_properties(af_spdlog
       PROPERTIES

From 715e21fcd6e989793d01c5781908f221720e7d48 Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Wed, 1 Feb 2023 14:54:51 -0500
Subject: [PATCH 198/473] opencl to oneapi function port batch 1 (#3358)

* gradient ported to oneapi. tests pass but MaxDims b/c missing jit
* meanshift passes tests up to missing ImageIO, JIT
* lookup compiles for oneapi. spot checked. no test in harness
* restore standard device selection in device manager
* select compiles. tests pass up to jit, randu, and maxdims
* updates tile.hpp
* rotate functions for float inputs
* resize
* approx1. mods to interp to use only float's not double's
* approx2 now passes on a770 (float only) gpu
* reorder ported again to ensure compatibility with a770
* transform
---------

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Co-authored-by: Umar Arshad <umar@arrayfire.com>
---
 src/backend/oneapi/CMakeLists.txt       |   0
 src/backend/oneapi/gradient.cpp         |   4 +-
 src/backend/oneapi/kernel/convolve1.hpp |  10 +-
 src/backend/oneapi/kernel/convolve2.hpp |  10 +-
 src/backend/oneapi/kernel/convolve3.hpp |  10 +-
 src/backend/oneapi/kernel/gradient.hpp  | 163 +++++++++++++
 src/backend/oneapi/kernel/interp.hpp    |  28 +--
 src/backend/oneapi/kernel/lookup.hpp    | 133 ++++++++++
 src/backend/oneapi/kernel/meanshift.hpp | 229 ++++++++++++++++++
 src/backend/oneapi/kernel/reorder.hpp   |  46 ++--
 src/backend/oneapi/kernel/resize.hpp    | 230 ++++++++++++++++++
 src/backend/oneapi/kernel/rotate.hpp    | 217 +++++++++++++++++
 src/backend/oneapi/kernel/select.hpp    | 258 ++++++++++++++++++++
 src/backend/oneapi/kernel/tile.hpp      | 112 +++++++++
 src/backend/oneapi/kernel/transform.hpp | 307 ++++++++++++++++++++++++
 src/backend/oneapi/lookup.cpp           |  14 +-
 src/backend/oneapi/meanshift.cpp        |   9 +-
 src/backend/oneapi/memory.cpp           |   0
 src/backend/oneapi/resize.cpp           |   6 +-
 src/backend/oneapi/rotate.cpp           |  34 ++-
 src/backend/oneapi/select.cpp           |  13 +-
 src/backend/oneapi/tile.cpp             |   5 +-
 src/backend/oneapi/transform.cpp        |  12 +-
 23 files changed, 1741 insertions(+), 109 deletions(-)
 mode change 100755 => 100644 src/backend/oneapi/CMakeLists.txt
 mode change 100755 => 100644 src/backend/oneapi/kernel/convolve1.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/convolve2.hpp
 mode change 100755 => 100644 src/backend/oneapi/kernel/convolve3.hpp
 create mode 100644 src/backend/oneapi/kernel/gradient.hpp
 create mode 100644 src/backend/oneapi/kernel/lookup.hpp
 create mode 100644 src/backend/oneapi/kernel/meanshift.hpp
 create mode 100644 src/backend/oneapi/kernel/resize.hpp
 create mode 100644 src/backend/oneapi/kernel/rotate.hpp
 create mode 100644 src/backend/oneapi/kernel/select.hpp
 create mode 100644 src/backend/oneapi/kernel/tile.hpp
 create mode 100644 src/backend/oneapi/kernel/transform.hpp
 mode change 100755 => 100644 src/backend/oneapi/memory.cpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
old mode 100755
new mode 100644
diff --git a/src/backend/oneapi/gradient.cpp b/src/backend/oneapi/gradient.cpp
index dc45b67cc6..0ab39d7e8d 100644
--- a/src/backend/oneapi/gradient.cpp
+++ b/src/backend/oneapi/gradient.cpp
@@ -10,7 +10,7 @@
 #include <Array.hpp>
 #include <err_oneapi.hpp>
 #include <gradient.hpp>
-//#include <kernel/gradient.hpp>
+#include <kernel/gradient.hpp>
 #include <math.hpp>
 #include <stdexcept>
 
@@ -18,7 +18,7 @@ namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void gradient(Array<T> &grad0, Array<T> &grad1, const Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("");
+    kernel::gradient<T>(grad0, grad1, in);
 }
 
 #define INSTANTIATE(T)                                            \
diff --git a/src/backend/oneapi/kernel/convolve1.hpp b/src/backend/oneapi/kernel/convolve1.hpp
old mode 100755
new mode 100644
index 1383bb4591..1d3df7ef3b
--- a/src/backend/oneapi/kernel/convolve1.hpp
+++ b/src/backend/oneapi/kernel/convolve1.hpp
@@ -107,12 +107,10 @@ void conv1Helper(const conv_kparam_t<aT> &param, Param<T> &out,
                  const int rank, const bool expand) {
     auto Q = getQueue();
     Q.submit([&](auto &h) {
-        sycl::accessor<aT, 1, sycl::access::mode::read_write,
-                       sycl::access::target::local>
-            localMem(param.loc_size, h);
-        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
-        sycl::accessor signalAcc{*signal.data, h, sycl::read_only};
-        sycl::accessor impulseAcc{*param.impulse, h, sycl::read_only};
+        local_accessor<aT> localMem(param.loc_size, h);
+        write_accessor<T> outAcc{*out.data, h};
+        read_accessor<T> signalAcc{*signal.data, h};
+        read_accessor<aT> impulseAcc{*param.impulse, h};
         h.parallel_for(
             sycl::nd_range{param.global, param.local},
             conv1HelperCreateKernel<T, aT>(
diff --git a/src/backend/oneapi/kernel/convolve2.hpp b/src/backend/oneapi/kernel/convolve2.hpp
old mode 100755
new mode 100644
index 5232b225ff..173405bdb8
--- a/src/backend/oneapi/kernel/convolve2.hpp
+++ b/src/backend/oneapi/kernel/convolve2.hpp
@@ -131,12 +131,10 @@ void conv2Helper(const conv_kparam_t<aT> &param, Param<T> out,
 
     auto Q = getQueue();
     Q.submit([&](auto &h) {
-        sycl::accessor<aT, 1, sycl::access::mode::read_write,
-                       sycl::access::target::local>
-            localMem(LOC_SIZE, h);
-        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
-        sycl::accessor signalAcc{*signal.data, h, sycl::read_only};
-        sycl::accessor impulseAcc{*param.impulse, h, sycl::read_only};
+        local_accessor<aT> localMem(LOC_SIZE, h);
+        write_accessor<T> outAcc{*out.data, h};
+        read_accessor<T> signalAcc{*signal.data, h};
+        read_accessor<aT> impulseAcc{*param.impulse, h};
         h.parallel_for(
             sycl::nd_range{param.global, param.local},
             conv2HelperCreateKernel<T, aT>(
diff --git a/src/backend/oneapi/kernel/convolve3.hpp b/src/backend/oneapi/kernel/convolve3.hpp
old mode 100755
new mode 100644
index d9a93affef..57f1538ddc
--- a/src/backend/oneapi/kernel/convolve3.hpp
+++ b/src/backend/oneapi/kernel/convolve3.hpp
@@ -143,12 +143,10 @@ void conv3Helper(const conv_kparam_t<aT> &param, Param<T> &out,
                  const int rank, const bool EXPAND) {
     auto Q = getQueue();
     Q.submit([&](auto &h) {
-        sycl::accessor<aT, 1, sycl::access::mode::read_write,
-                       sycl::access::target::local>
-            localMem(param.loc_size, h);
-        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
-        sycl::accessor signalAcc{*signal.data, h, sycl::read_only};
-        sycl::accessor impulseAcc{*param.impulse, h, sycl::read_only};
+        local_accessor<aT> localMem(param.loc_size, h);
+        write_accessor<T> outAcc{*out.data, h};
+        read_accessor<T> signalAcc{*signal.data, h};
+        read_accessor<aT> impulseAcc{*param.impulse, h};
         h.parallel_for(
             sycl::nd_range{param.global, param.local},
             conv3HelperCreateKernel<T, aT>(
diff --git a/src/backend/oneapi/kernel/gradient.hpp b/src/backend/oneapi/kernel/gradient.hpp
new file mode 100644
index 0000000000..fbaae20b51
--- /dev/null
+++ b/src/backend/oneapi/kernel/gradient.hpp
@@ -0,0 +1,163 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <kernel/default_config.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
+                                      sycl::access::target::local>;
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+#define sidx(y, x) scratch_[((y + 1) * (TX + 2)) + (x + 1)]
+
+template<typename T, int TX, int TY>
+class gradientCreateKernel {
+   public:
+    gradientCreateKernel(write_accessor<T> d_grad0, const KParam grad0,
+                         write_accessor<T> d_grad1, const KParam grad1,
+                         read_accessor<T> d_in, const KParam in,
+                         const int blocksPerMatX, const int blocksPerMatY,
+                         local_accessor<T> scratch)
+        : d_grad0_(d_grad0)
+        , grad0_(grad0)
+        , d_grad1_(d_grad1)
+        , grad1_(grad1)
+        , d_in_(d_in)
+        , in_(in)
+        , blocksPerMatX_(blocksPerMatX)
+        , blocksPerMatY_(blocksPerMatY)
+        , scratch_(scratch) {}
+    void operator()(sycl::nd_item<2> it) const {
+        auto g = it.get_group();
+
+        const int idz = g.get_group_id(0) / blocksPerMatX_;
+        const int idw = g.get_group_id(1) / blocksPerMatY_;
+
+        const int blockIdx_x = g.get_group_id(0) - idz * blocksPerMatX_;
+        const int blockIdx_y = g.get_group_id(1) - idw * blocksPerMatY_;
+
+        const int xB = blockIdx_x * g.get_local_range(0);
+        const int yB = blockIdx_y * g.get_local_range(1);
+
+        const int tx = it.get_local_id(0);
+        const int ty = it.get_local_id(1);
+
+        const int idx = tx + xB;
+        const int idy = ty + yB;
+
+        const bool cond = (idx >= in_.dims[0] || idy >= in_.dims[1] ||
+                           idz >= in_.dims[2] || idw >= in_.dims[3]);
+
+        int xmax = (TX > (in_.dims[0] - xB)) ? (in_.dims[0] - xB) : TX;
+        int ymax = (TY > (in_.dims[1] - yB)) ? (in_.dims[1] - yB) : TY;
+
+        int iIdx = in_.offset + idw * in_.strides[3] + idz * in_.strides[2] +
+                   idy * in_.strides[1] + idx;
+
+        int g0dx = idw * grad0_.strides[3] + idz * grad0_.strides[2] +
+                   idy * grad0_.strides[1] + idx;
+
+        int g1dx = idw * grad1_.strides[3] + idz * grad1_.strides[2] +
+                   idy * grad1_.strides[1] + idx;
+
+        // Multipliers - 0.5 for interior, 1 for edge cases
+        typename std::conditional<std::is_same<T, std::complex<double>>::value,
+                                  double, float>::type
+            xf = 0.5 * (1 + (idx == 0 || idx >= (in_.dims[0] - 1))),
+            yf = 0.5 * (1 + (idy == 0 || idy >= (in_.dims[1] - 1)));
+
+        // Copy data to scratch space
+        T zero = (T)(0);
+        if (cond) {
+            sidx(ty, tx) = zero;
+        } else {
+            sidx(ty, tx) = d_in_[iIdx];
+        }
+
+        it.barrier();
+
+        // Copy buffer zone data. Corner (0,0) etc, are not used.
+        // Cols
+        if (ty == 0) {
+            // Y-1
+            sidx(-1, tx) =
+                (cond || idy == 0) ? sidx(0, tx) : d_in_[iIdx - in_.strides[1]];
+            sidx(ymax, tx) = (cond || (idy + ymax) >= in_.dims[1])
+                                 ? sidx(ymax - 1, tx)
+                                 : d_in_[iIdx + ymax * in_.strides[1]];
+        }
+        // Rows
+        if (tx == 0) {
+            sidx(ty, -1)   = (cond || idx == 0) ? sidx(ty, 0) : d_in_[iIdx - 1];
+            sidx(ty, xmax) = (cond || (idx + xmax) >= in_.dims[0])
+                                 ? sidx(ty, xmax - 1)
+                                 : d_in_[iIdx + xmax];
+        }
+
+        it.barrier();
+
+        if (cond) return;
+
+        d_grad0_[g0dx] = xf * (sidx(ty, tx + 1) - sidx(ty, tx - 1));
+        d_grad1_[g1dx] = yf * (sidx(ty + 1, tx) - sidx(ty - 1, tx));
+    }
+
+   private:
+    write_accessor<T> d_grad0_;
+    const KParam grad0_;
+    write_accessor<T> d_grad1_;
+    const KParam grad1_;
+    read_accessor<T> d_in_;
+    const KParam in_;
+    const int blocksPerMatX_;
+    const int blocksPerMatY_;
+    local_accessor<T> scratch_;
+};
+
+template<typename T>
+void gradient(Param<T> grad0, Param<T> grad1, const Param<T> in) {
+    constexpr int TX = 32;
+    constexpr int TY = 8;
+
+    auto local = sycl::range{TX, TY};
+
+    int blocksPerMatX = divup(in.info.dims[0], TX);
+    int blocksPerMatY = divup(in.info.dims[1], TY);
+    auto global       = sycl::range{local[0] * blocksPerMatX * in.info.dims[2],
+                              local[1] * blocksPerMatY * in.info.dims[3]};
+
+    getQueue().submit([&](sycl::handler &h) {
+        write_accessor<T> grad0Acc{*grad0.data, h};
+        write_accessor<T> grad1Acc{*grad1.data, h};
+        read_accessor<T> inAcc{*in.data, h};
+        auto scratch = local_accessor<T>((TY + 2) * (TX + 2), h);
+        h.parallel_for(sycl::nd_range{global, local},
+                       gradientCreateKernel<T, TX, TY>(
+                           grad0Acc, grad0.info, grad1Acc, grad1.info, inAcc,
+                           in.info, blocksPerMatX, blocksPerMatY, scratch));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
index af430ca031..cefd67c992 100644
--- a/src/backend/oneapi/kernel/interp.hpp
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -110,7 +110,7 @@ struct Interp1<Ty, Tp, 1> {
         const int x_lim    = iInfo.dims[xdim];
         const int x_stride = iInfo.strides[xdim];
 
-        int xid   = (method == AF_INTERP_LOWER ? floor(x) : round(x));
+        int xid = (method == AF_INTERP_LOWER ? sycl::floor(x) : sycl::round(x));
         bool cond = xid >= 0 && xid < x_lim;
         if (clamp) xid = std::max((int)0, std::min(xid, x_lim));
 
@@ -133,8 +133,8 @@ struct Interp1<Ty, Tp, 2> {
         typedef typename itype_t<Tp>::wtype WT;
         typedef typename itype_t<Ty>::vtype VT;
 
-        const int grid_x = floor(x);    // nearest grid
-        const WT off_x   = x - grid_x;  // fractional offset
+        const int grid_x = sycl::floor(x);  // nearest grid
+        const WT off_x   = x - grid_x;      // fractional offset
 
         const int x_lim    = iInfo.dims[xdim];
         const int x_stride = iInfo.strides[xdim];
@@ -145,7 +145,7 @@ struct Interp1<Ty, Tp, 2> {
         WT ratio     = off_x;
         if (method == AF_INTERP_LINEAR_COSINE) {
             // Smooth the factional part with cosine
-            ratio = (1 - cos(ratio * af::Pi)) / 2;
+            ratio = (1 - sycl::cospi(ratio)) / 2;
         }
 
         Ty zero = scalar<Ty>(0);
@@ -170,8 +170,8 @@ struct Interp1<Ty, Tp, 3> {
         typedef typename itype_t<Tp>::wtype WT;
         typedef typename itype_t<Ty>::vtype VT;
 
-        const int grid_x = floor(x);    // nearest grid
-        const WT off_x   = x - grid_x;  // fractional offset
+        const int grid_x = sycl::floor(x);  // nearest grid
+        const WT off_x   = x - grid_x;      // fractional offset
 
         const int x_lim    = iInfo.dims[xdim];
         const int x_stride = iInfo.strides[xdim];
@@ -206,8 +206,8 @@ struct Interp2<Ty, Tp, 1> {
                     read_accessor<Ty> in, KParam iInfo, int ioff, Tp x, Tp y,
                     int xdim, int ydim, af::interpType method, int batch,
                     bool clamp, int batch_dim = 2) {
-        int xid = (method == AF_INTERP_LOWER ? floor(x) : round(x));
-        int yid = (method == AF_INTERP_LOWER ? floor(y) : round(y));
+        int xid = (method == AF_INTERP_LOWER ? sycl::floor(x) : sycl::round(x));
+        int yid = (method == AF_INTERP_LOWER ? sycl::floor(y) : sycl::round(y));
 
         const int x_lim    = iInfo.dims[xdim];
         const int y_lim    = iInfo.dims[ydim];
@@ -244,10 +244,10 @@ struct Interp2<Ty, Tp, 2> {
         typedef typename itype_t<Tp>::wtype WT;
         typedef typename itype_t<Ty>::vtype VT;
 
-        const int grid_x = floor(x);
+        const int grid_x = sycl::floor(x);
         const WT off_x   = x - grid_x;
 
-        const int grid_y = floor(y);
+        const int grid_y = sycl::floor(y);
         const WT off_y   = y - grid_y;
 
         const int x_lim    = iInfo.dims[xdim];
@@ -265,8 +265,8 @@ struct Interp2<Ty, Tp, 2> {
         if (method == AF_INTERP_LINEAR_COSINE ||
             method == AF_INTERP_BILINEAR_COSINE) {
             // Smooth the factional part with cosine
-            xratio = (1 - cos(xratio * af::Pi)) / 2;
-            yratio = (1 - cos(yratio * af::Pi)) / 2;
+            xratio = (1 - sycl::cospi(xratio)) / 2;
+            yratio = (1 - sycl::cospi(yratio)) / 2;
         }
 
         Ty zero = scalar<Ty>(0);
@@ -296,10 +296,10 @@ struct Interp2<Ty, Tp, 3> {
         typedef typename itype_t<Tp>::wtype WT;
         typedef typename itype_t<Ty>::vtype VT;
 
-        const int grid_x = floor(x);
+        const int grid_x = sycl::floor(x);
         const WT off_x   = x - grid_x;
 
-        const int grid_y = floor(y);
+        const int grid_y = sycl::floor(y);
         const WT off_y   = y - grid_y;
 
         const int x_lim    = iInfo.dims[xdim];
diff --git a/src/backend/oneapi/kernel/lookup.hpp b/src/backend/oneapi/kernel/lookup.hpp
new file mode 100644
index 0000000000..8baf14ad21
--- /dev/null
+++ b/src/backend/oneapi/kernel/lookup.hpp
@@ -0,0 +1,133 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+int trimIndex(int idx, const int len) {
+    int ret_val = idx;
+    if (ret_val < 0) {
+        int offset = (abs(ret_val) - 1) % len;
+        ret_val    = offset;
+    } else if (ret_val >= len) {
+        int offset = abs(ret_val) % len;
+        ret_val    = len - offset - 1;
+    }
+    return ret_val;
+}
+
+template<typename in_t, typename idx_t>
+class lookupNDCreateKernel {
+   public:
+    lookupNDCreateKernel(write_accessor<in_t> out, KParam oInfo,
+                         read_accessor<in_t> in, KParam iInfo,
+                         read_accessor<idx_t> indices, KParam idxInfo,
+                         int nBBS0, int nBBS1, const int DIM)
+        : out_(out)
+        , oInfo_(oInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , indices_(indices)
+        , idxInfo_(idxInfo)
+        , nBBS0_(nBBS0)
+        , nBBS1_(nBBS1)
+        , DIM_(DIM) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        int lx = it.get_local_id(0);
+        int ly = it.get_local_id(1);
+
+        int gz = g.get_group_id(0) / nBBS0_;
+        int gw = g.get_group_id(1) / nBBS1_;
+
+        int gx = g.get_local_range(0) * (g.get_group_id(0) - gz * nBBS0_) + lx;
+        int gy = g.get_local_range(1) * (g.get_group_id(1) - gw * nBBS1_) + ly;
+
+        const idx_t *idxPtr = indices_.get_pointer();
+
+        int i = iInfo_.strides[0] *
+                (DIM_ == 0 ? trimIndex((int)idxPtr[gx], iInfo_.dims[0]) : gx);
+        int j = iInfo_.strides[1] *
+                (DIM_ == 1 ? trimIndex((int)idxPtr[gy], iInfo_.dims[1]) : gy);
+        int k = iInfo_.strides[2] *
+                (DIM_ == 2 ? trimIndex((int)idxPtr[gz], iInfo_.dims[2]) : gz);
+        int l = iInfo_.strides[3] *
+                (DIM_ == 3 ? trimIndex((int)idxPtr[gw], iInfo_.dims[3]) : gw);
+
+        const in_t *inPtr = in_.get_pointer() + (i + j + k + l) + iInfo_.offset;
+        in_t *outPtr =
+            out_.get_pointer() +
+            (gx * oInfo_.strides[0] + gy * oInfo_.strides[1] +
+             gz * oInfo_.strides[2] + gw * oInfo_.strides[3] + oInfo_.offset);
+
+        if (gx < oInfo_.dims[0] && gy < oInfo_.dims[1] && gz < oInfo_.dims[2] &&
+            gw < oInfo_.dims[3]) {
+            outPtr[0] = inPtr[0];
+        }
+    }
+
+   private:
+    write_accessor<in_t> out_;
+    KParam oInfo_;
+    read_accessor<in_t> in_;
+    KParam iInfo_;
+    read_accessor<idx_t> indices_;
+    KParam idxInfo_;
+    int nBBS0_;
+    int nBBS1_;
+    const int DIM_;
+};
+
+template<typename in_t, typename idx_t>
+void lookup(Param<in_t> out, const Param<in_t> in, const Param<idx_t> indices,
+            const unsigned dim) {
+    constexpr int THREADS_X = 32;
+    constexpr int THREADS_Y = 8;
+
+    auto local = sycl::range(THREADS_X, THREADS_Y);
+
+    int blk_x = divup(out.info.dims[0], THREADS_X);
+    int blk_y = divup(out.info.dims[1], THREADS_Y);
+
+    auto global = sycl::range(blk_x * out.info.dims[2] * THREADS_X,
+                              blk_y * out.info.dims[3] * THREADS_Y);
+
+    getQueue().submit([&](auto &h) {
+        write_accessor<in_t> d_out{*out.data, h};
+        read_accessor<in_t> d_in{*in.data, h};
+        read_accessor<idx_t> d_indices{*indices.data, h};
+        h.parallel_for(sycl::nd_range{global, local},
+                       lookupNDCreateKernel<in_t, idx_t>(
+                           d_out, out.info, d_in, in.info, d_indices,
+                           indices.info, blk_x, blk_y, dim));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/meanshift.hpp b/src/backend/oneapi/kernel/meanshift.hpp
new file mode 100644
index 0000000000..8dfb96a3b7
--- /dev/null
+++ b/src/backend/oneapi/kernel/meanshift.hpp
@@ -0,0 +1,229 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+inline int convert_int_rtz(float number) { return ((int)(number)); }
+
+template<typename T, typename AccType, const int MAX_CHANNELS>
+class meanshiftCreateKernel {
+   public:
+    meanshiftCreateKernel(write_accessor<T> d_dst, KParam oInfo,
+                          read_accessor<T> d_src, KParam iInfo, int radius,
+                          float cvar, unsigned numIters, int nBBS0, int nBBS1)
+        : d_dst_(d_dst)
+        , oInfo_(oInfo)
+        , d_src_(d_src)
+        , iInfo_(iInfo)
+        , radius_(radius)
+        , cvar_(cvar)
+        , numIters_(numIters)
+        , nBBS0_(nBBS0)
+        , nBBS1_(nBBS1) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        unsigned b2 = g.get_group_id(0) / nBBS0_;
+        unsigned b3 = g.get_group_id(1) / nBBS1_;
+        const int gx =
+            g.get_local_range(0) * (g.get_group_id(0) - b2 * nBBS0_) +
+            it.get_local_id(0);
+        const int gy =
+            g.get_local_range(1) * (g.get_group_id(1) - b3 * nBBS1_) +
+            it.get_local_id(1);
+
+        if (gx < iInfo_.dims[0] && gy < iInfo_.dims[1]) {
+            const T* iptr =
+                d_src_.get_pointer() + (b2 * iInfo_.strides[2] +
+                                        b3 * iInfo_.strides[3] + iInfo_.offset);
+            T* optr = d_dst_.get_pointer() +
+                      (b2 * oInfo_.strides[2] + b3 * oInfo_.strides[3]);
+
+            int meanPosI = gx;
+            int meanPosJ = gy;
+
+            T currentCenterColors[MAX_CHANNELS];
+            T tempColors[MAX_CHANNELS];
+
+            AccType currentMeanColors[MAX_CHANNELS];
+
+#pragma unroll
+            for (int ch = 0; ch < MAX_CHANNELS; ++ch)
+                currentCenterColors[ch] =
+                    iptr[gx * iInfo_.strides[0] + gy * iInfo_.strides[1] +
+                         ch * iInfo_.strides[2]];
+
+            const int dim0LenLmt = iInfo_.dims[0] - 1;
+            const int dim1LenLmt = iInfo_.dims[1] - 1;
+
+            // scope of meanshift iterationd begin
+            for (uint it = 0; it < numIters_; ++it) {
+                int oldMeanPosJ = meanPosJ;
+                int oldMeanPosI = meanPosI;
+                unsigned count  = 0;
+
+                int shift_x = 0;
+                int shift_y = 0;
+
+                for (int ch = 0; ch < MAX_CHANNELS; ++ch)
+                    currentMeanColors[ch] = 0;
+
+                for (int wj = -radius_; wj <= radius_; ++wj) {
+                    int hit_count = 0;
+                    int tj        = meanPosJ + wj;
+
+                    if (tj < 0 || tj > dim1LenLmt) continue;
+
+                    for (int wi = -radius_; wi <= radius_; ++wi) {
+                        int ti = meanPosI + wi;
+
+                        if (ti < 0 || ti > dim0LenLmt) continue;
+
+                        AccType norm = 0;
+#pragma unroll
+                        for (int ch = 0; ch < MAX_CHANNELS; ++ch) {
+                            unsigned idx = ti * iInfo_.strides[0] +
+                                           tj * iInfo_.strides[1] +
+                                           ch * iInfo_.strides[2];
+                            tempColors[ch] = iptr[idx];
+                            AccType diff   = (AccType)currentCenterColors[ch] -
+                                           (AccType)tempColors[ch];
+                            norm += (diff * diff);
+                        }
+
+                        if (norm <= cvar_) {
+#pragma unroll
+                            for (int ch = 0; ch < MAX_CHANNELS; ++ch)
+                                currentMeanColors[ch] +=
+                                    (AccType)tempColors[ch];
+
+                            shift_x += ti;
+                            ++hit_count;
+                        }
+                    }
+                    count += hit_count;
+                    shift_y += tj * hit_count;
+                }
+
+                if (count == 0) break;
+
+                const AccType fcount = 1 / (AccType)count;
+
+                meanPosI = convert_int_rtz(shift_x * fcount);
+                meanPosJ = convert_int_rtz(shift_y * fcount);
+
+#pragma unroll
+                for (int ch = 0; ch < MAX_CHANNELS; ++ch)
+                    currentMeanColors[ch] =
+                        convert_int_rtz(currentMeanColors[ch] * fcount);
+
+                AccType norm = 0;
+#pragma unroll
+                for (int ch = 0; ch < MAX_CHANNELS; ++ch) {
+                    AccType diff = (AccType)currentCenterColors[ch] -
+                                   currentMeanColors[ch];
+                    norm += (diff * diff);
+                }
+
+                bool stop =
+                    (meanPosJ == oldMeanPosJ && meanPosI == oldMeanPosI) ||
+                    ((abs(oldMeanPosJ - meanPosJ) +
+                      abs(oldMeanPosI - meanPosI)) +
+                     norm) <= 1;
+
+#pragma unroll
+                for (int ch = 0; ch < MAX_CHANNELS; ++ch)
+                    currentCenterColors[ch] = (T)(currentMeanColors[ch]);
+
+                if (stop) break;
+            }  // scope of meanshift iterations end
+
+#pragma unroll
+            for (int ch = 0; ch < MAX_CHANNELS; ++ch)
+                optr[gx * oInfo_.strides[0] + gy * oInfo_.strides[1] +
+                     ch * oInfo_.strides[2]] = currentCenterColors[ch];
+        }
+    }
+
+   private:
+    write_accessor<T> d_dst_;
+    KParam oInfo_;
+    read_accessor<T> d_src_;
+    KParam iInfo_;
+    int radius_;
+    float cvar_;
+    unsigned numIters_;
+    int nBBS0_;
+    int nBBS1_;
+};
+
+template<typename T>
+void meanshift(Param<T> out, const Param<T> in, const float spatialSigma,
+               const float chromaticSigma, const uint numIters,
+               const bool is_color) {
+    using AccType = typename std::conditional<std::is_same<T, double>::value,
+                                              double, float>::type;
+    constexpr int THREADS_X = 16;
+    constexpr int THREADS_Y = 16;
+
+    const int MAX_CHANNELS = (is_color ? 3 : 1);
+
+    auto local = sycl::range(THREADS_X, THREADS_Y);
+
+    int blk_x = divup(in.info.dims[0], THREADS_X);
+    int blk_y = divup(in.info.dims[1], THREADS_Y);
+
+    const int bCount = (is_color ? 1 : in.info.dims[2]);
+
+    auto global = sycl::range(bCount * blk_x * THREADS_X,
+                              in.info.dims[3] * blk_y * THREADS_Y);
+
+    // clamp spatial and chromatic sigma's
+    int radius = std::max((int)(spatialSigma * 1.5f), 1);
+
+    const float cvar = chromaticSigma * chromaticSigma;
+
+    getQueue().submit([&](auto& h) {
+        read_accessor<T> d_src{*in.data, h};
+        write_accessor<T> d_dst{*out.data, h};
+        if (MAX_CHANNELS == 3) {
+            h.parallel_for(sycl::nd_range{global, local},
+                           meanshiftCreateKernel<T, AccType, 3>(
+                               d_dst, out.info, d_src, in.info, radius, cvar,
+                               numIters, blk_x, blk_y));
+        } else {
+            h.parallel_for(sycl::nd_range{global, local},
+                           meanshiftCreateKernel<T, AccType, 1>(
+                               d_dst, out.info, d_src, in.info, radius, cvar,
+                               numIters, blk_x, blk_y));
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/reorder.hpp b/src/backend/oneapi/kernel/reorder.hpp
index 6aa6cd39c0..c39ff556b7 100644
--- a/src/backend/oneapi/kernel/reorder.hpp
+++ b/src/backend/oneapi/kernel/reorder.hpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2022, ArrayFire
+ * Copyright (c) 2023, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -11,9 +11,8 @@
 
 #include <Param.hpp>
 #include <common/dispatch.hpp>
-#include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
-// #include <traits.hpp>
+#include <traits.hpp>
 
 #include <string>
 #include <vector>
@@ -22,9 +21,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
-                                      sycl::access::target::local>;
 template<typename T>
 using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
 template<typename T>
@@ -47,9 +43,8 @@ class reorderCreateKernel {
         , d3_(d3)
         , blocksPerMatX_(blocksPerMatX)
         , blocksPerMatY_(blocksPerMatY) {}
-
     void operator()(sycl::nd_item<2> it) const {
-        auto g = it.get_group();
+        sycl::group g = it.get_group();
 
         const int oz = g.get_group_id(0) / blocksPerMatX_;
         const int ow = g.get_group_id(1) / blocksPerMatY_;
@@ -66,10 +61,10 @@ class reorderCreateKernel {
         const int incy = blocksPerMatY_ * g.get_local_range(1);
         const int incx = blocksPerMatX_ * g.get_local_range(0);
 
-        const int o_off    = ow * op_.strides[3] + oz * op_.strides[2];
-        const int rdims[4] = {d0_, d1_, d2_, d3_};
-        int ods[4]         = {xx, yy, oz, ow};
-        int ids[4]         = {0};
+        const int o_off   = ow * op_.strides[3] + oz * op_.strides[2];
+        const int rdims[] = {d0_, d1_, d2_, d3_};
+        int ods[]         = {xx, yy, oz, ow};
+        int ids[4]        = {0};
 
         ids[rdims[3]] = ow;
         ids[rdims[2]] = oz;
@@ -110,22 +105,25 @@ void reorder(Param<T> out, const Param<T> in, const dim_t* rdims) {
     constexpr int TILEX = 512;
     constexpr int TILEY = 32;
 
-    auto local = sycl::range{TX, TY};
+    auto local = sycl::range(TX, TY);
 
     int blocksPerMatX = divup(out.info.dims[0], TILEX);
     int blocksPerMatY = divup(out.info.dims[1], TILEY);
-    auto global       = sycl::range{local[0] * blocksPerMatX * out.info.dims[2],
-                              local[1] * blocksPerMatY * out.info.dims[3]};
-
-    getQueue().submit([&](sycl::handler& h) {
-        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
-        sycl::accessor inAcc{*in.data, h, sycl::read_only};
-
-        h.parallel_for(sycl::nd_range{global, local},
-                       reorderCreateKernel<T>(
-                           outAcc, inAcc, out.info, in.info, rdims[0], rdims[1],
-                           rdims[2], rdims[3], blocksPerMatX, blocksPerMatY));
+    auto global       = sycl::range(local[0] * blocksPerMatX * out.info.dims[2],
+                                    local[1] * blocksPerMatY * out.info.dims[3]);
+
+    getQueue().submit([&](auto& h) {
+        read_accessor<T> d_in{*in.data, h};
+        write_accessor<T> d_out{*out.data, h};
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            reorderCreateKernel<T>(
+                d_out, d_in, out.info, in.info, static_cast<int>(rdims[0]),
+                static_cast<int>(rdims[1]), static_cast<int>(rdims[2]),
+                static_cast<int>(rdims[3]), blocksPerMatX, blocksPerMatY));
     });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 }  // namespace kernel
 }  // namespace oneapi
diff --git a/src/backend/oneapi/kernel/resize.hpp b/src/backend/oneapi/kernel/resize.hpp
new file mode 100644
index 0000000000..b44d878818
--- /dev/null
+++ b/src/backend/oneapi/kernel/resize.hpp
@@ -0,0 +1,230 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/complex.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename AT, typename BT>
+BT mul(AT a, BT b) {
+    return a * b;
+}
+template<typename AT>
+std::complex<double> mul(AT a, std::complex<double> b) {
+    return std::complex<double>(a * b.real(), a * b.imag());
+}
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T>
+using wtype_t = typename std::conditional<std::is_same<T, double>::value,
+                                          double, float>::type;
+
+template<typename T>
+using vtype_t = typename std::conditional<common::is_complex<T>::value, T,
+                                          wtype_t<T>>::type;
+
+////////////////////////////////////////////////////////////////////////////////////
+// nearest-neighbor resampling
+template<typename T>
+void resize_n_(T* d_out, const KParam out, const T* d_in, const KParam in,
+               const int blockIdx_x, const int blockIdx_y, const float xf,
+               const float yf, sycl::nd_item<2>& it) {
+    sycl::group g = it.get_group();
+    int const ox  = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+    int const oy  = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+    // int ix = convert_int_rtp(ox * xf);
+    // int iy = convert_int_rtp(oy * yf);
+    int ix = sycl::round(ox * xf);
+    int iy = sycl::round(oy * yf);
+
+    if (ox >= out.dims[0] || oy >= out.dims[1]) { return; }
+    if (ix >= in.dims[0]) { ix = in.dims[0] - 1; }
+    if (iy >= in.dims[1]) { iy = in.dims[1] - 1; }
+
+    d_out[ox + oy * out.strides[1]] = d_in[ix + iy * in.strides[1]];
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+// bilinear resampling
+template<typename T, typename VT>
+void resize_b_(T* d_out, const KParam out, const T* d_in, const KParam in,
+               const int blockIdx_x, const int blockIdx_y, const float xf_,
+               const float yf_, sycl::nd_item<2>& it) {
+    sycl::group g = it.get_group();
+
+    int const ox = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+    int const oy = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+    float xf = ox * xf_;
+    float yf = oy * yf_;
+
+    int ix = sycl::floor(xf);
+
+    int iy = sycl::floor(yf);
+
+    if (ox >= out.dims[0] || oy >= out.dims[1]) { return; }
+    if (ix >= in.dims[0]) { ix = in.dims[0] - 1; }
+    if (iy >= in.dims[1]) { iy = in.dims[1] - 1; }
+
+    float b = xf - ix;
+    float a = yf - iy;
+
+    const int ix2 = (ix + 1) < in.dims[0] ? (ix + 1) : ix;
+    const int iy2 = (iy + 1) < in.dims[1] ? (iy + 1) : iy;
+
+    const VT p1 = d_in[ix + in.strides[1] * iy];
+    const VT p2 = d_in[ix + in.strides[1] * iy2];
+    const VT p3 = d_in[ix2 + in.strides[1] * iy];
+    const VT p4 = d_in[ix2 + in.strides[1] * iy2];
+
+    d_out[ox + oy * out.strides[1]] =
+        mul(((1.0f - a) * (1.0f - b)), p1) + mul(((a) * (1.0f - b)), p2) +
+        mul(((1.0f - a) * (b)), p3) + mul(((a) * (b)), p4);
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+// lower resampling
+template<typename T>
+void resize_l_(T* d_out, const KParam out, const T* d_in, const KParam in,
+               const int blockIdx_x, const int blockIdx_y, const float xf,
+               const float yf, sycl::nd_item<2>& it) {
+    sycl::group g = it.get_group();
+
+    int const ox = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+    int const oy = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+    int ix = (ox * xf);
+    int iy = (oy * yf);
+
+    if (ox >= out.dims[0] || oy >= out.dims[1]) { return; }
+    if (ix >= in.dims[0]) { ix = in.dims[0] - 1; }
+    if (iy >= in.dims[1]) { iy = in.dims[1] - 1; }
+
+    d_out[ox + oy * out.strides[1]] = d_in[ix + iy * in.strides[1]];
+}
+
+template<typename T, int method>
+class resizeCreateKernel {
+   public:
+    resizeCreateKernel(write_accessor<T> d_out, const KParam out,
+                       read_accessor<T> d_in, const KParam in, const int b0,
+                       const int b1, const float xf, const float yf)
+        : d_out_(d_out)
+        , out_(out)
+        , d_in_(d_in)
+        , in_(in)
+        , b0_(b0)
+        , b1_(b1)
+        , xf_(xf)
+        , yf_(yf) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        int bIdx = g.get_group_id(0) / b0_;
+        int bIdy = g.get_group_id(1) / b1_;
+        // batch adjustment
+        int i_off = bIdy * in_.strides[3] + bIdx * in_.strides[2] + in_.offset;
+        int o_off = bIdy * out_.strides[3] + bIdx * out_.strides[2];
+        int blockIdx_x = g.get_group_id(0) - bIdx * b0_;
+        int blockIdx_y = g.get_group_id(1) - bIdy * b1_;
+
+        switch (method) {
+            case AF_INTERP_NEAREST:
+                resize_n_<T>(d_out_.get_pointer() + o_off, out_,
+                             d_in_.get_pointer() + i_off, in_, blockIdx_x,
+                             blockIdx_y, xf_, yf_, it);
+                break;
+            case AF_INTERP_BILINEAR:
+                resize_b_<T, vtype_t<T>>(d_out_.get_pointer() + o_off, out_,
+                                         d_in_.get_pointer() + i_off, in_,
+                                         blockIdx_x, blockIdx_y, xf_, yf_, it);
+                break;
+            case AF_INTERP_LOWER:
+                resize_l_<T>(d_out_.get_pointer() + o_off, out_,
+                             d_in_.get_pointer() + i_off, in_, blockIdx_x,
+                             blockIdx_y, xf_, yf_, it);
+                break;
+        }
+    }
+
+   private:
+    write_accessor<T> d_out_;
+    const KParam out_;
+    read_accessor<T> d_in_;
+    const KParam in_;
+    const int b0_;
+    const int b1_;
+    const float xf_;
+    const float yf_;
+};
+
+template<typename T>
+void resize(Param<T> out, const Param<T> in, const af_interp_type method) {
+    constexpr int RESIZE_TX = 16;
+    constexpr int RESIZE_TY = 16;
+
+    auto local = sycl::range(RESIZE_TX, RESIZE_TY);
+
+    int blocksPerMatX = divup(out.info.dims[0], local[0]);
+    int blocksPerMatY = divup(out.info.dims[1], local[1]);
+    auto global       = sycl::range(local[0] * blocksPerMatX * in.info.dims[2],
+                                    local[1] * blocksPerMatY * in.info.dims[3]);
+
+    double xd = (double)in.info.dims[0] / (double)out.info.dims[0];
+    double yd = (double)in.info.dims[1] / (double)out.info.dims[1];
+
+    float xf = (float)xd, yf = (float)yd;
+
+    getQueue().submit([&](auto& h) {
+        read_accessor<T> d_in{*in.data, h};
+        write_accessor<T> d_out{*out.data, h};
+        switch (method) {
+            case AF_INTERP_NEAREST:
+                h.parallel_for(sycl::nd_range{global, local},
+                               resizeCreateKernel<T, AF_INTERP_NEAREST>(
+                                   d_out, out.info, d_in, in.info,
+                                   blocksPerMatX, blocksPerMatY, xf, yf));
+                break;
+            case AF_INTERP_BILINEAR:
+                h.parallel_for(sycl::nd_range{global, local},
+                               resizeCreateKernel<T, AF_INTERP_BILINEAR>(
+                                   d_out, out.info, d_in, in.info,
+                                   blocksPerMatX, blocksPerMatY, xf, yf));
+                break;
+            case AF_INTERP_LOWER:
+                h.parallel_for(sycl::nd_range{global, local},
+                               resizeCreateKernel<T, AF_INTERP_LOWER>(
+                                   d_out, out.info, d_in, in.info,
+                                   blocksPerMatX, blocksPerMatY, xf, yf));
+                break;
+            default: break;
+        }
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/rotate.hpp b/src/backend/oneapi/kernel/rotate.hpp
new file mode 100644
index 0000000000..61d736763a
--- /dev/null
+++ b/src/backend/oneapi/kernel/rotate.hpp
@@ -0,0 +1,217 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/complex.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <kernel/interp.hpp>
+#include <math.hpp>
+#include <traits.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+typedef struct {
+    float tmat[6];
+} tmat_t;
+
+template<typename T>
+using wtype_t = typename std::conditional<std::is_same<T, double>::value,
+                                          double, float>::type;
+
+template<typename T>
+using vtype_t = typename std::conditional<common::is_complex<T>::value, T,
+                                          wtype_t<T>>::type;
+
+template<typename T, typename InterpInTy, typename InterpPosTy,
+         int INTERP_ORDER>
+class rotateCreateKernel {
+   public:
+    rotateCreateKernel(write_accessor<T> d_out, const KParam out,
+                       read_accessor<T> d_in, const KParam in, const tmat_t t,
+                       const int nimages, const int batches,
+                       const int blocksXPerImage, const int blocksYPerImage,
+                       af::interpType method)
+        : d_out_(d_out)
+        , out_(out)
+        , d_in_(d_in)
+        , in_(in)
+        , t_(t)
+        , nimages_(nimages)
+        , batches_(batches)
+        , blocksXPerImage_(blocksXPerImage)
+        , blocksYPerImage_(blocksYPerImage)
+        , method_(method)
+        , INTERP_ORDER_(INTERP_ORDER) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        // Compute which image set
+        const int setId      = g.get_group_id(0) / blocksXPerImage_;
+        const int blockIdx_x = g.get_group_id(0) - setId * blocksXPerImage_;
+
+        const int batch      = g.get_group_id(1) / blocksYPerImage_;
+        const int blockIdx_y = g.get_group_id(1) - batch * blocksYPerImage_;
+
+        // Get thread indices
+        const int xido = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+        const int yido = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+        const int limages =
+            std::min((int)out_.dims[2] - setId * nimages_, nimages_);
+
+        if (xido >= out_.dims[0] || yido >= out_.dims[1]) return;
+
+        InterpPosTy xidi = xido * t_.tmat[0] + yido * t_.tmat[1] + t_.tmat[2];
+        InterpPosTy yidi = xido * t_.tmat[3] + yido * t_.tmat[4] + t_.tmat[5];
+
+        int outoff = out_.offset + setId * nimages_ * out_.strides[2] +
+                     batch * out_.strides[3];
+        int inoff = in_.offset + setId * nimages_ * in_.strides[2] +
+                    batch * in_.strides[3];
+
+        const int loco = outoff + (yido * out_.strides[1] + xido);
+
+        InterpInTy zero = (InterpInTy)0;
+        if (INTERP_ORDER_ > 1) {
+            // Special conditions to deal with boundaries for bilinear and
+            // bicubic
+            // FIXME: Ideally this condition should be removed or be present for
+            // all  methods But tests are expecting a different behavior for
+            // bilinear and nearest
+            if (xidi < (InterpPosTy)-0.0001 || yidi < (InterpPosTy)-0.0001 ||
+                in_.dims[0] <= xidi || in_.dims[1] <= yidi) {
+                for (int i = 0; i < nimages_; i++) {
+                    d_out_[loco + i * out_.strides[2]] = zero;
+                }
+                return;
+            }
+        }
+
+        // FIXME: Nearest and lower do not do clamping, but other methods do
+        // Make it consistent
+        const bool doclamp = INTERP_ORDER_ != 1;
+        Interp2<T, InterpPosTy, 1> interp2;  // INTERP_ORDER> interp2;
+        interp2(d_out_, out_, loco, d_in_, in_, inoff, xidi, yidi, 0, 1,
+                method_, limages, doclamp, 2);
+    }
+
+   private:
+    write_accessor<T> d_out_;
+    const KParam out_;
+    read_accessor<T> d_in_;
+    const KParam in_;
+    const tmat_t t_;
+    const int nimages_;
+    const int batches_;
+    const int blocksXPerImage_;
+    const int blocksYPerImage_;
+    af::interpType method_;
+    const int INTERP_ORDER_;
+};
+
+template<typename T>
+void rotate(Param<T> out, const Param<T> in, const float theta,
+            af_interp_type method, int order) {
+    using std::string;
+
+    using BT = typename dtype_traits<T>::base_type;
+
+    constexpr int TX = 16;
+    constexpr int TY = 16;
+
+    // Used for batching images
+    constexpr int TI = 4;
+    constexpr bool isComplex =
+        static_cast<af_dtype>(dtype_traits<T>::af_type) == c32 ||
+        static_cast<af_dtype>(dtype_traits<T>::af_type) == c64;
+
+    const float c = cos(-theta), s = sin(-theta);
+    float tx, ty;
+    {
+        const float nx = 0.5 * (in.info.dims[0] - 1);
+        const float ny = 0.5 * (in.info.dims[1] - 1);
+        const float mx = 0.5 * (out.info.dims[0] - 1);
+        const float my = 0.5 * (out.info.dims[1] - 1);
+        const float sx = (mx * c + my * -s);
+        const float sy = (mx * s + my * c);
+        tx             = -(sx - nx);
+        ty             = -(sy - ny);
+    }
+
+    // Rounding error. Anything more than 3 decimal points wont make a diff
+    tmat_t t;
+    t.tmat[0] = round(c * 1000) / 1000.0f;
+    t.tmat[1] = round(-s * 1000) / 1000.0f;
+    t.tmat[2] = round(tx * 1000) / 1000.0f;
+    t.tmat[3] = round(s * 1000) / 1000.0f;
+    t.tmat[4] = round(c * 1000) / 1000.0f;
+    t.tmat[5] = round(ty * 1000) / 1000.0f;
+
+    auto local = sycl::range(TX, TY);
+
+    int nimages               = in.info.dims[2];
+    int nbatches              = in.info.dims[3];
+    int global_x              = local[0] * divup(out.info.dims[0], local[0]);
+    int global_y              = local[1] * divup(out.info.dims[1], local[1]);
+    const int blocksXPerImage = global_x / local[0];
+    const int blocksYPerImage = global_y / local[1];
+
+    if (nimages > TI) {
+        int tile_images = divup(nimages, TI);
+        nimages         = TI;
+        global_x        = global_x * tile_images;
+    }
+    global_y *= nbatches;
+
+    auto global = sycl::range(global_x, global_y);
+
+    getQueue().submit([&](auto &h) {
+        read_accessor<T> d_in{*in.data, h};
+        write_accessor<T> d_out{*out.data, h};
+        switch (order) {
+            case 1:
+                h.parallel_for(
+                    sycl::nd_range{global, local},
+                    rotateCreateKernel<T, T, wtype_t<BT>, 1>(
+                        d_out, out.info, d_in, in.info, t, nimages, nbatches,
+                        blocksXPerImage, blocksYPerImage, method));
+                break;
+            case 2:
+                h.parallel_for(
+                    sycl::nd_range{global, local},
+                    rotateCreateKernel<T, T, wtype_t<BT>, 2>(
+                        d_out, out.info, d_in, in.info, t, nimages, nbatches,
+                        blocksXPerImage, blocksYPerImage, method));
+                break;
+            case 3:
+                h.parallel_for(
+                    sycl::nd_range{global, local},
+                    rotateCreateKernel<T, T, wtype_t<BT>, 3>(
+                        d_out, out.info, d_in, in.info, t, nimages, nbatches,
+                        blocksXPerImage, blocksYPerImage, method));
+                break;
+            default: throw std::string("invalid interpolation order");
+        }
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/select.hpp b/src/backend/oneapi/kernel/select.hpp
new file mode 100644
index 0000000000..618cea3437
--- /dev/null
+++ b/src/backend/oneapi/kernel/select.hpp
@@ -0,0 +1,258 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <math.hpp>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+constexpr uint DIMX  = 32;
+constexpr uint DIMY  = 8;
+constexpr int REPEAT = 64;
+
+int getOffset(const dim_t *dims, const dim_t *strides, const dim_t *refdims,
+              int ids[4]) {
+    int off = 0;
+    off += ids[3] * (dims[3] == refdims[3]) * strides[3];
+    off += ids[2] * (dims[2] == refdims[2]) * strides[2];
+    off += ids[1] * (dims[1] == refdims[1]) * strides[1];
+    return off;
+}
+
+template<typename T>
+class selectKernelCreateKernel {
+   public:
+    selectKernelCreateKernel(write_accessor<T> optr, KParam oinfo,
+                             read_accessor<char> cptr_, KParam cinfo,
+                             read_accessor<T> aptr_, KParam ainfo,
+                             read_accessor<T> bptr_, KParam binfo, int groups_0,
+                             int groups_1, const bool is_same)
+        : optr_(optr)
+        , oinfo_(oinfo)
+        , cptr__(cptr_)
+        , cinfo_(cinfo)
+        , aptr__(aptr_)
+        , ainfo_(ainfo)
+        , bptr__(bptr_)
+        , binfo_(binfo)
+        , groups_0_(groups_0)
+        , groups_1_(groups_1)
+        , is_same_(is_same) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        char *cptr = cptr__.get_pointer() + cinfo_.offset;
+        T *aptr    = aptr__.get_pointer() + ainfo_.offset;
+        T *bptr    = bptr__.get_pointer() + binfo_.offset;
+
+        const int idz = g.get_group_id(0) / groups_0_;
+        const int idw = g.get_group_id(1) / groups_1_;
+
+        const int group_id_0 = g.get_group_id(0) - idz * groups_0_;
+        const int group_id_1 = g.get_group_id(1) - idw * groups_1_;
+
+        const int idx0 = group_id_0 * g.get_local_range(0) + it.get_local_id(0);
+        const int idy  = group_id_1 * g.get_local_range(1) + it.get_local_id(1);
+
+        const int off = idw * oinfo_.strides[3] + idz * oinfo_.strides[2] +
+                        idy * oinfo_.strides[1];
+
+        const bool valid = (idw < oinfo_.dims[3] && idz < oinfo_.dims[2] &&
+                            idy < oinfo_.dims[1]);
+
+        int ids[] = {idx0, idy, idz, idw};
+
+        T *optr_pointer = optr_.get_pointer();
+        optr_pointer += off;
+        aptr += getOffset(ainfo_.dims, ainfo_.strides, oinfo_.dims, ids);
+        bptr += getOffset(binfo_.dims, binfo_.strides, oinfo_.dims, ids);
+        cptr += getOffset(cinfo_.dims, cinfo_.strides, oinfo_.dims, ids);
+
+        if (is_same_) {
+            for (int idx = idx0; idx < oinfo_.dims[0];
+                 idx += g.get_local_range(0) * groups_0_) {
+                if (valid)
+                    optr_pointer[idx] = (cptr[idx]) ? aptr[idx] : bptr[idx];
+            }
+        } else {
+            bool csame = cinfo_.dims[0] == oinfo_.dims[0];
+            bool asame = ainfo_.dims[0] == oinfo_.dims[0];
+            bool bsame = binfo_.dims[0] == oinfo_.dims[0];
+            for (int idx = idx0; idx < oinfo_.dims[0];
+                 idx += g.get_local_range(0) * groups_0_) {
+                if (valid)
+                    optr_pointer[idx] = (cptr[csame * idx]) ? aptr[asame * idx]
+                                                            : bptr[bsame * idx];
+            }
+        }
+    }
+
+   private:
+    write_accessor<T> optr_;
+    KParam oinfo_;
+    read_accessor<char> cptr__;
+    KParam cinfo_;
+    read_accessor<T> aptr__;
+    KParam ainfo_;
+    read_accessor<T> bptr__;
+    KParam binfo_;
+    int groups_0_;
+    int groups_1_;
+    const bool is_same_;
+};
+
+template<typename T>
+void selectLauncher(Param<T> out, Param<char> cond, Param<T> a, Param<T> b,
+                    const int ndims, const bool is_same) {
+    int threads[] = {DIMX, DIMY};
+
+    if (ndims == 1) {
+        threads[0] *= threads[1];
+        threads[1] = 1;
+    }
+
+    auto local = sycl::range(threads[0], threads[1]);
+
+    int groups_0 = divup(out.info.dims[0], REPEAT * local[0]);
+    int groups_1 = divup(out.info.dims[1], local[1]);
+
+    auto global = sycl::range(groups_0 * out.info.dims[2] * local[0],
+                              groups_1 * out.info.dims[3] * local[1]);
+
+    getQueue().submit([&](auto &h) {
+        write_accessor<T> d_out{*out.data, h};
+        read_accessor<char> d_cond{*cond.data, h};
+        read_accessor<T> d_a{*a.data, h};
+        read_accessor<T> d_b{*b.data, h};
+        h.parallel_for(sycl::nd_range{global, local},
+                       selectKernelCreateKernel<T>(
+                           d_out, out.info, d_cond, cond.info, d_a, a.info, d_b,
+                           b.info, groups_0, groups_1, is_same));
+    });
+}
+
+template<typename T>
+class selectScalarCreateKernel {
+   public:
+    selectScalarCreateKernel(write_accessor<T> optr, KParam oinfo,
+                             read_accessor<char> cptr_, KParam cinfo,
+                             read_accessor<T> aptr_, KParam ainfo, T b,
+                             int groups_0, int groups_1, const bool flip)
+        : optr_(optr)
+        , oinfo_(oinfo)
+        , cptr__(cptr_)
+        , cinfo_(cinfo)
+        , aptr__(aptr_)
+        , ainfo_(ainfo)
+        , b_(b)
+        , groups_0_(groups_0)
+        , groups_1_(groups_1)
+        , flip_(flip) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        char *cptr = cptr__.get_pointer() + cinfo_.offset;
+        T *aptr    = aptr__.get_pointer() + ainfo_.offset;
+
+        const int idz = g.get_group_id(0) / groups_0_;
+        const int idw = g.get_group_id(1) / groups_1_;
+
+        const int group_id_0 = g.get_group_id(0) - idz * groups_0_;
+        const int group_id_1 = g.get_group_id(1) - idw * groups_1_;
+
+        const int idx0 = group_id_0 * g.get_local_range(0) + it.get_local_id(0);
+        const int idy  = group_id_1 * g.get_local_range(1) + it.get_local_id(1);
+
+        const int off = idw * oinfo_.strides[3] + idz * oinfo_.strides[2] +
+                        idy * oinfo_.strides[1];
+
+        int ids[] = {idx0, idy, idz, idw};
+        optr_.get_pointer() += off;
+        aptr += getOffset(ainfo_.dims, ainfo_.strides, oinfo_.dims, ids);
+        cptr += getOffset(cinfo_.dims, cinfo_.strides, oinfo_.dims, ids);
+
+        if (idw >= oinfo_.dims[3] || idz >= oinfo_.dims[2] ||
+            idy >= oinfo_.dims[1]) {
+            return;
+        }
+
+        for (int idx = idx0; idx < oinfo_.dims[0];
+             idx += g.get_local_range(0) * groups_0_) {
+            optr_.get_pointer()[idx] = (cptr[idx] ^ flip_) ? aptr[idx] : b_;
+        }
+    }
+
+   private:
+    write_accessor<T> optr_;
+    KParam oinfo_;
+    read_accessor<char> cptr__;
+    KParam cinfo_;
+    read_accessor<T> aptr__;
+    KParam ainfo_;
+    T b_;
+    int groups_0_;
+    int groups_1_;
+    const bool flip_;
+};
+
+template<typename T>
+void select(Param<T> out, Param<char> cond, Param<T> a, Param<T> b, int ndims) {
+    bool is_same = true;
+    for (int i = 0; i < 4; i++) {
+        is_same &= (a.info.dims[i] == b.info.dims[i]);
+    }
+    selectLauncher<T>(out, cond, a, b, ndims, is_same);
+}
+
+template<typename T>
+void select_scalar(Param<T> out, Param<char> cond, Param<T> a, const T b,
+                   const int ndims, const bool flip) {
+    int threads[] = {DIMX, DIMY};
+
+    if (ndims == 1) {
+        threads[0] *= threads[1];
+        threads[1] = 1;
+    }
+
+    auto local = sycl::range(threads[0], threads[1]);
+
+    int groups_0 = divup(out.info.dims[0], REPEAT * local[0]);
+    int groups_1 = divup(out.info.dims[1], local[1]);
+
+    auto global = sycl::range(groups_0 * out.info.dims[2] * local[0],
+                              groups_1 * out.info.dims[3] * local[1]);
+
+    getQueue().submit([&](auto &h) {
+        write_accessor<T> d_out{*out.data, h};
+        read_accessor<char> d_cond{*cond.data, h};
+        read_accessor<T> d_a{*a.data, h};
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            selectScalarCreateKernel<T>(d_out, out.info, d_cond, cond.info, d_a,
+                                        a.info, b, groups_0, groups_1, flip));
+    });
+}
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/tile.hpp b/src/backend/oneapi/kernel/tile.hpp
new file mode 100644
index 0000000000..24112442a9
--- /dev/null
+++ b/src/backend/oneapi/kernel/tile.hpp
@@ -0,0 +1,112 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T>
+class tileCreateKernel {
+   public:
+    tileCreateKernel(write_accessor<T> out, read_accessor<T> in,
+                     const KParam op, const KParam ip, const int blocksPerMatX,
+                     const int blocksPerMatY)
+        : out_(out)
+        , in_(in)
+        , op_(op)
+        , ip_(ip)
+        , blocksPerMatX_(blocksPerMatX)
+        , blocksPerMatY_(blocksPerMatY) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        const int oz = g.get_group_id(0) / blocksPerMatX_;
+        const int ow = g.get_group_id(1) / blocksPerMatY_;
+
+        const int blockIdx_x = g.get_group_id(0) - oz * blocksPerMatX_;
+        const int blockIdx_y = g.get_group_id(1) - ow * blocksPerMatY_;
+
+        const int xx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+        const int yy = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+        const bool valid = (xx < op_.dims[0] && yy < op_.dims[1] &&
+                            oz < op_.dims[2] && ow < op_.dims[3]);
+
+        const int iz  = oz % ip_.dims[2];
+        const int iw  = ow % ip_.dims[3];
+        const int izw = iw * ip_.strides[3] + iz * ip_.strides[2];
+        const int ozw = ow * op_.strides[3] + oz * op_.strides[2];
+
+        const int incy = blocksPerMatY_ * g.get_local_range(1);
+        const int incx = blocksPerMatX_ * g.get_local_range(0);
+
+        for (int oy = yy; oy < op_.dims[1]; oy += incy) {
+            const int iy = oy % ip_.dims[1];
+            for (int ox = xx; ox < op_.dims[0]; ox += incx) {
+                const int ix = ox % ip_.dims[0];
+
+                int iMem = izw + iy * ip_.strides[1] + ix;
+                int oMem = ozw + oy * op_.strides[1] + ox;
+
+                if (valid) out_[oMem] = in_[ip_.offset + iMem];
+            }
+        }
+    }
+
+   private:
+    write_accessor<T> out_;
+    read_accessor<T> in_;
+    const KParam op_;
+    const KParam ip_;
+    const int blocksPerMatX_;
+    const int blocksPerMatY_;
+};
+
+template<typename T>
+void tile(Param<T> out, const Param<T> in) {
+    constexpr int TX    = 32;
+    constexpr int TY    = 8;
+    constexpr int TILEX = 512;
+    constexpr int TILEY = 32;
+
+    auto local = sycl::range(TX, TY);
+
+    int blocksPerMatX = divup(out.info.dims[0], TILEX);
+    int blocksPerMatY = divup(out.info.dims[1], TILEY);
+    auto global       = sycl::range(local[0] * blocksPerMatX * out.info.dims[2],
+                                    local[1] * blocksPerMatY * out.info.dims[3]);
+
+    getQueue().submit([&](auto &h) {
+        write_accessor<T> d_out{*out.data, h};
+        read_accessor<T> d_in{*in.data, h};
+        h.parallel_for(sycl::nd_range{global, local},
+                       tileCreateKernel<T>(d_out, d_in, out.info, in.info,
+                                           blocksPerMatX, blocksPerMatY));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/transform.hpp b/src/backend/oneapi/kernel/transform.hpp
new file mode 100644
index 0000000000..b67a11c660
--- /dev/null
+++ b/src/backend/oneapi/kernel/transform.hpp
@@ -0,0 +1,307 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/complex.hpp>
+#include <common/dispatch.hpp>
+// #include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+// #include <kernel/config.hpp>
+#include <kernel/interp.hpp>
+#include <math.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T>
+using wtype_t = typename std::conditional<std::is_same<T, double>::value,
+                                          double, float>::type;
+
+template<typename T>
+using vtype_t = typename std::conditional<common::is_complex<T>::value, T,
+                                          wtype_t<T>>::type;
+
+template<bool PERSPECTIVE>
+void calc_transf_inverse(float *txo, const float *txi) {
+    if constexpr (PERSPECTIVE) {
+        txo[0] = txi[4] * txi[8] - txi[5] * txi[7];
+        txo[1] = -(txi[1] * txi[8] - txi[2] * txi[7]);
+        txo[2] = txi[1] * txi[5] - txi[2] * txi[4];
+
+        txo[3] = -(txi[3] * txi[8] - txi[5] * txi[6]);
+        txo[4] = txi[0] * txi[8] - txi[2] * txi[6];
+        txo[5] = -(txi[0] * txi[5] - txi[2] * txi[3]);
+
+        txo[6] = txi[3] * txi[7] - txi[4] * txi[6];
+        txo[7] = -(txi[0] * txi[7] - txi[1] * txi[6]);
+        txo[8] = txi[0] * txi[4] - txi[1] * txi[3];
+
+        float det = txi[0] * txo[0] + txi[1] * txo[3] + txi[2] * txo[6];
+
+        txo[0] /= det;
+        txo[1] /= det;
+        txo[2] /= det;
+        txo[3] /= det;
+        txo[4] /= det;
+        txo[5] /= det;
+        txo[6] /= det;
+        txo[7] /= det;
+        txo[8] /= det;
+    } else {
+        float det = txi[0] * txi[4] - txi[1] * txi[3];
+
+        txo[0] = txi[4] / det;
+        txo[1] = txi[3] / det;
+        txo[3] = txi[1] / det;
+        txo[4] = txi[0] / det;
+
+        txo[2] = txi[2] * -txo[0] + txi[5] * -txo[1];
+        txo[5] = txi[2] * -txo[3] + txi[5] * -txo[4];
+    }
+}
+
+template<typename T, typename InterpPosTy, bool PERSPECTIVE, int INTERP_ORDER>
+class transformCreateKernel {
+   public:
+    transformCreateKernel(write_accessor<T> d_out, const KParam out,
+                          read_accessor<T> d_in, const KParam in,
+                          read_accessor<float> c_tmat, const KParam tf,
+                          const int nImg2, const int nImg3, const int nTfs2,
+                          const int nTfs3, const int batchImg2,
+                          const int blocksXPerImage, const int blocksYPerImage,
+                          const af::interpType method, const bool INVERSE)
+        : d_out_(d_out)
+        , out_(out)
+        , d_in_(d_in)
+        , in_(in)
+        , c_tmat_(c_tmat)
+        , tf_(tf)
+        , nImg2_(nImg2)
+        , nImg3_(nImg3)
+        , nTfs2_(nTfs2)
+        , nTfs3_(nTfs3)
+        , batchImg2_(batchImg2)
+        , blocksXPerImage_(blocksXPerImage)
+        , blocksYPerImage_(blocksYPerImage)
+        , method_(method)
+        , INVERSE_(INVERSE) {}
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g = it.get_group();
+
+        // Image Ids
+        const int imgId2 = g.get_group_id(0) / blocksXPerImage_;
+        const int imgId3 = g.get_group_id(1) / blocksYPerImage_;
+
+        // Block in_ local image
+        const int blockIdx_x = g.get_group_id(0) - imgId2 * blocksXPerImage_;
+        const int blockIdx_y = g.get_group_id(1) - imgId3 * blocksYPerImage_;
+
+        // Get thread indices in_ local image
+        const int xido = blockIdx_x * g.get_local_range(0) + it.get_local_id(0);
+        const int yido = blockIdx_y * g.get_local_range(1) + it.get_local_id(1);
+
+        // Image iteration loop count for image batching
+        int limages = sycl::min(
+            sycl::max((int)(out_.dims[2] - imgId2 * nImg2_), 1), batchImg2_);
+
+        if (xido >= out_.dims[0] || yido >= out_.dims[1]) return;
+
+        // Index of transform
+        const int eTfs2 = sycl::max((nTfs2_ / nImg2_), 1);
+        const int eTfs3 = sycl::max((nTfs3_ / nImg3_), 1);
+
+        int t_idx3        = -1;  // init
+        int t_idx2        = -1;  // init
+        int t_idx2_offset = 0;
+
+        const int blockIdx_z = g.get_group_id(2);
+
+        if (nTfs3_ == 1) {
+            t_idx3 = 0;  // Always 0 as only 1 transform defined
+        } else {
+            if (nTfs3_ == nImg3_) {
+                t_idx3 =
+                    imgId3;  // One to one batch with all transforms defined
+            } else {
+                t_idx3 = blockIdx_z / eTfs2;  // Transform batched, calculate
+                t_idx2_offset = t_idx3 * nTfs2_;
+            }
+        }
+
+        if (nTfs2_ == 1) {
+            t_idx2 = 0;  // Always 0 as only 1 transform defined
+        } else {
+            if (nTfs2_ == nImg2_) {
+                t_idx2 =
+                    imgId2;  // One to one batch with all transforms defined
+            } else {
+                t_idx2 =
+                    blockIdx_z - t_idx2_offset;  // Transform batched, calculate
+            }
+        }
+
+        // Linear transform index
+        const int t_idx = t_idx2 + t_idx3 * nTfs2_;
+
+        // Global outoff
+        int outoff = out_.offset;
+        int inoff  = imgId2 * batchImg2_ * in_.strides[2] +
+                    imgId3 * in_.strides[3] + in_.offset;
+        if (nImg2_ == nTfs2_ || nImg2_ > 1) {  // One-to-One or Image on dim2
+            outoff += imgId2 * batchImg2_ * out_.strides[2];
+        } else {  // Transform batched on dim2
+            outoff += t_idx2 * out_.strides[2];
+        }
+
+        if (nImg3_ == nTfs3_ || nImg3_ > 1) {  // One-to-One or Image on dim3
+            outoff += imgId3 * out_.strides[3];
+        } else {  // Transform batched on dim2
+            outoff += t_idx3 * out_.strides[3];
+        }
+
+        // Transform is in_ global memory.
+        // Needs outoff to correct transform being processed.
+        const int transf_len = PERSPECTIVE ? 9 : 6;
+        using TMatTy =
+            typename std::conditional<PERSPECTIVE, float[9], float[6]>::type;
+        TMatTy tmat;
+        const float *tmat_ptr = c_tmat_.get_pointer() + t_idx * transf_len;
+
+        // We expect a inverse transform matrix by default
+        // If it is an forward transform, then we need its inverse
+        if (INVERSE_ == 1) {
+#pragma unroll 3
+            for (int i = 0; i < transf_len; i++) tmat[i] = tmat_ptr[i];
+        } else {
+            calc_transf_inverse<PERSPECTIVE>(tmat, tmat_ptr);
+        }
+
+        InterpPosTy xidi = xido * tmat[0] + yido * tmat[1] + tmat[2];
+        InterpPosTy yidi = xido * tmat[3] + yido * tmat[4] + tmat[5];
+
+        if constexpr (PERSPECTIVE) {
+            const InterpPosTy W = xido * tmat[6] + yido * tmat[7] + tmat[8];
+            xidi /= W;
+            yidi /= W;
+        }
+        const int loco = outoff + (yido * out_.strides[1] + xido);
+        // FIXME: Nearest and lower do not do clamping, but other methods do
+        // Make it consistent
+        const bool doclamp = INTERP_ORDER != 1;
+
+        T zero = (T)0;
+        if (xidi < (InterpPosTy)-0.0001f || yidi < (InterpPosTy)-0.0001f ||
+            in_.dims[0] <= xidi || in_.dims[1] <= yidi) {
+            for (int n = 0; n < limages; n++) {
+                d_out_[loco + n * out_.strides[2]] = zero;
+            }
+            return;
+        }
+
+        Interp2<T, InterpPosTy, INTERP_ORDER> interp2;
+        interp2(d_out_, out_, loco, d_in_, in_, inoff, xidi, yidi, 0, 1,
+                method_, limages, doclamp, 2);
+    }
+
+   private:
+    write_accessor<T> d_out_;
+    const KParam out_;
+    read_accessor<T> d_in_;
+    const KParam in_;
+    read_accessor<float> c_tmat_;
+    const KParam tf_;
+    const int nImg2_;
+    const int nImg3_;
+    const int nTfs2_;
+    const int nTfs3_;
+    const int batchImg2_;
+    const int blocksXPerImage_;
+    const int blocksYPerImage_;
+    const af::interpType method_;
+    const bool INVERSE_;
+};
+
+template<typename T>
+void transform(Param<T> out, const Param<T> in, const Param<float> tf,
+               bool isInverse, bool isPerspective, af_interp_type method,
+               int order) {
+    static int counter = 0;
+
+    using std::string;
+
+    using BT = typename dtype_traits<T>::base_type;
+
+    constexpr int TX = 16;
+    constexpr int TY = 16;
+    // Used for batching images
+    constexpr int TI = 4;
+    constexpr bool isComplex =
+        static_cast<af_dtype>(dtype_traits<T>::af_type) == c32 ||
+        static_cast<af_dtype>(dtype_traits<T>::af_type) == c64;
+
+    const int nImg2 = in.info.dims[2];
+    const int nImg3 = in.info.dims[3];
+    const int nTfs2 = tf.info.dims[2];
+    const int nTfs3 = tf.info.dims[3];
+
+    auto local = sycl::range(TX, TY, 1);
+
+    int batchImg2 = 1;
+    if (nImg2 != nTfs2) batchImg2 = fmin(nImg2, TI);
+
+    const int blocksXPerImage = divup(out.info.dims[0], local[0]);
+    const int blocksYPerImage = divup(out.info.dims[1], local[1]);
+
+    int global_x = local[0] * blocksXPerImage * (nImg2 / batchImg2);
+    int global_y = local[1] * blocksYPerImage * nImg3;
+    int global_z =
+        local[2] * fmax((nTfs2 / nImg2), 1) * fmax((nTfs3 / nImg3), 1);
+
+    auto global = sycl::range(global_x, global_y, global_z);
+
+#define INVOKE(PERSPECTIVE, INTERP_ORDER)                                      \
+    h.parallel_for(                                                            \
+        sycl::nd_range{global, local},                                         \
+        transformCreateKernel<T, wtype_t<BT>, PERSPECTIVE, INTERP_ORDER>(      \
+            d_out, out.info, d_in, in.info, d_tf, tf.info, nImg2, nImg3,       \
+            nTfs2, nTfs3, batchImg2, blocksXPerImage, blocksYPerImage, method, \
+            isInverse));
+
+    getQueue().submit([&](auto &h) {
+        read_accessor<T> d_in{*in.data, h};
+        read_accessor<float> d_tf{*tf.data, h};
+        write_accessor<T> d_out{*out.data, h};
+
+        if (isPerspective == true && order == 1) INVOKE(true, 1);
+        if (isPerspective == true && order == 2) INVOKE(true, 2);
+        if (isPerspective == true && order == 3) INVOKE(true, 3);
+
+        if (isPerspective == false && order == 1) INVOKE(false, 1);
+        if (isPerspective == false && order == 2) INVOKE(false, 2);
+        if (isPerspective == false && order == 3) INVOKE(false, 3);
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/lookup.cpp b/src/backend/oneapi/lookup.cpp
index 101dc90c1d..9c87003375 100644
--- a/src/backend/oneapi/lookup.cpp
+++ b/src/backend/oneapi/lookup.cpp
@@ -12,6 +12,7 @@
 #include <Array.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/lookup.hpp>
 #include <af/dim4.hpp>
 
 using arrayfire::common::half;
@@ -21,8 +22,17 @@ namespace oneapi {
 template<typename in_t, typename idx_t>
 Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                    const unsigned dim) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<in_t> out = createEmptyArray<in_t>(af::dim4(1));
+    const dim4 &iDims = input.dims();
+
+    dim4 oDims(1);
+    for (int d = 0; d < 4; ++d) {
+        oDims[d] = (d == int(dim) ? indices.elements() : iDims[d]);
+    }
+
+    Array<in_t> out = createEmptyArray<in_t>(oDims);
+
+    kernel::lookup<in_t, idx_t>(out, input, indices, dim);
+
     return out;
 }
 
diff --git a/src/backend/oneapi/meanshift.cpp b/src/backend/oneapi/meanshift.cpp
index de517e700f..1017b9074b 100644
--- a/src/backend/oneapi/meanshift.cpp
+++ b/src/backend/oneapi/meanshift.cpp
@@ -9,7 +9,7 @@
 
 #include <Array.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/meanshift.hpp>
+#include <kernel/meanshift.hpp>
 #include <meanshift.hpp>
 #include <af/dim4.hpp>
 
@@ -21,13 +21,10 @@ template<typename T>
 Array<T> meanshift(const Array<T> &in, const float &spatialSigma,
                    const float &chromaticSigma, const unsigned &numIterations,
                    const bool &isColor) {
-    ONEAPI_NOT_SUPPORTED("meanshift Not supported");
-
     const dim4 &dims = in.dims();
     Array<T> out     = createEmptyArray<T>(dims);
-    // kernel::meanshift<T>(out, in, spatialSigma, chromaticSigma,
-    // numIterations,
-    //                      isColor);
+    kernel::meanshift<T>(out, in, spatialSigma, chromaticSigma, numIterations,
+                         isColor);
     return out;
 }
 
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
old mode 100755
new mode 100644
diff --git a/src/backend/oneapi/resize.cpp b/src/backend/oneapi/resize.cpp
index 6d8d3307ab..005faf6b2b 100644
--- a/src/backend/oneapi/resize.cpp
+++ b/src/backend/oneapi/resize.cpp
@@ -9,7 +9,7 @@
 
 #include <Array.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/resize.hpp>
+#include <kernel/resize.hpp>
 #include <resize.hpp>
 #include <af/dim4.hpp>
 #include <stdexcept>
@@ -23,9 +23,7 @@ Array<T> resize(const Array<T> &in, const dim_t odim0, const dim_t odim1,
     af::dim4 oDims(odim0, odim1, iDims[2], iDims[3]);
     Array<T> out = createEmptyArray<T>(oDims);
 
-    ONEAPI_NOT_SUPPORTED("resize Not supported");
-
-    // kernel::resize<T>(out, in, method);
+    kernel::resize<T>(out, in, method);
     return out;
 }
 
diff --git a/src/backend/oneapi/rotate.cpp b/src/backend/oneapi/rotate.cpp
index b5cd2fa6e3..10f1f93480 100644
--- a/src/backend/oneapi/rotate.cpp
+++ b/src/backend/oneapi/rotate.cpp
@@ -10,32 +10,30 @@
 #include <err_oneapi.hpp>
 #include <rotate.hpp>
 
-// #include <kernel/rotate.hpp>
+#include <kernel/rotate.hpp>
 
 namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> rotate(const Array<T> &in, const float theta, const af::dim4 &odims,
                 const af_interp_type method) {
-    ONEAPI_NOT_SUPPORTED("rotate Not supported");
-
     Array<T> out = createEmptyArray<T>(odims);
 
-    // switch (method) {
-    //     case AF_INTERP_NEAREST:
-    //     case AF_INTERP_LOWER:
-    //         kernel::rotate<T>(out, in, theta, method, 1);
-    //         break;
-    //     case AF_INTERP_BILINEAR:
-    //     case AF_INTERP_BILINEAR_COSINE:
-    //         kernel::rotate<T>(out, in, theta, method, 2);
-    //         break;
-    //     case AF_INTERP_BICUBIC:
-    //     case AF_INTERP_BICUBIC_SPLINE:
-    //         kernel::rotate<T>(out, in, theta, method, 3);
-    //         break;
-    //     default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG);
-    // }
+    switch (method) {
+        case AF_INTERP_NEAREST:
+        case AF_INTERP_LOWER:
+            kernel::rotate<T>(out, in, theta, method, 1);
+            break;
+        case AF_INTERP_BILINEAR:
+        case AF_INTERP_BILINEAR_COSINE:
+            kernel::rotate<T>(out, in, theta, method, 2);
+            break;
+        case AF_INTERP_BICUBIC:
+        case AF_INTERP_BICUBIC_SPLINE:
+            kernel::rotate<T>(out, in, theta, method, 3);
+            break;
+        default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG);
+    }
     return out;
 }
 
diff --git a/src/backend/oneapi/select.cpp b/src/backend/oneapi/select.cpp
index 08458b9778..8cb80c919d 100644
--- a/src/backend/oneapi/select.cpp
+++ b/src/backend/oneapi/select.cpp
@@ -13,6 +13,7 @@
 #include <common/half.hpp>
 #include <common/jit/NaryNode.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/select.hpp>
 #include <scalar.hpp>
 
 #include <nonstd/span.hpp>
@@ -31,8 +32,6 @@ namespace oneapi {
 template<typename T>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
                           const Array<T> &b, const dim4 &odims) {
-    ONEAPI_NOT_SUPPORTED("createSelectNode Not supported");
-
     auto cond_node   = cond.getNode();
     auto a_node      = a.getNode();
     auto b_node      = b.getNode();
@@ -61,8 +60,6 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
 template<typename T, bool flip>
 Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
                           const T &b_val, const dim4 &odims) {
-    ONEAPI_NOT_SUPPORTED("createSelectNode Not supported");
-
     auto cond_node   = cond.getNode();
     auto a_node      = a.getNode();
     Array<T> b       = createScalarNode<T>(odims, b_val);
@@ -94,17 +91,13 @@ Array<T> createSelectNode(const Array<char> &cond, const Array<T> &a,
 template<typename T>
 void select(Array<T> &out, const Array<char> &cond, const Array<T> &a,
             const Array<T> &b) {
-    ONEAPI_NOT_SUPPORTED("select Not supported");
-
-    // kernel::select<T>(out, cond, a, b, out.ndims());
+    kernel::select<T>(out, cond, a, b, out.ndims());
 }
 
 template<typename T, bool flip>
 void select_scalar(Array<T> &out, const Array<char> &cond, const Array<T> &a,
                    const T &b) {
-    ONEAPI_NOT_SUPPORTED("select_scalar Not supported");
-
-    // kernel::select_scalar<T>(out, cond, a, b, out.ndims(), flip);
+    kernel::select_scalar<T>(out, cond, a, b, out.ndims(), flip);
 }
 
 #define INSTANTIATE(T)                                                   \
diff --git a/src/backend/oneapi/tile.cpp b/src/backend/oneapi/tile.cpp
index 5f2c38c475..aca96e4ec6 100644
--- a/src/backend/oneapi/tile.cpp
+++ b/src/backend/oneapi/tile.cpp
@@ -6,8 +6,8 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
-//#include <kernel/tile.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/tile.hpp>
 #include <tile.hpp>
 
 #include <Array.hpp>
@@ -26,8 +26,7 @@ Array<T> tile(const Array<T> &in, const af::dim4 &tileDims) {
 
     Array<T> out = createEmptyArray<T>(oDims);
 
-    ONEAPI_NOT_SUPPORTED("tile Not supported");
-    // kernel::tile<T>(out, in);
+    kernel::tile<T>(out, in);
 
     return out;
 }
diff --git a/src/backend/oneapi/transform.cpp b/src/backend/oneapi/transform.cpp
index 720dfa1654..54b328f7fd 100644
--- a/src/backend/oneapi/transform.cpp
+++ b/src/backend/oneapi/transform.cpp
@@ -9,8 +9,8 @@
 
 #include <transform.hpp>
 
-// #include <kernel/transform.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/transform.hpp>
 
 namespace arrayfire {
 namespace oneapi {
@@ -19,22 +19,18 @@ template<typename T>
 void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const af_interp_type method, const bool inverse,
                const bool perspective) {
-    ONEAPI_NOT_SUPPORTED("transform Not supported");
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            // kernel::transform<T>(out, in, tf, inverse, perspective, method,
-            // 1);
+            kernel::transform<T>(out, in, tf, inverse, perspective, method, 1);
             break;
         case AF_INTERP_BILINEAR:
         case AF_INTERP_BILINEAR_COSINE:
-            // kernel::transform<T>(out, in, tf, inverse, perspective, method,
-            // 2);
+            kernel::transform<T>(out, in, tf, inverse, perspective, method, 2);
             break;
         case AF_INTERP_BICUBIC:
         case AF_INTERP_BICUBIC_SPLINE:
-            // kernel::transform<T>(out, in, tf, inverse, perspective, method,
-            // 3);
+            kernel::transform<T>(out, in, tf, inverse, perspective, method, 3);
             break;
         default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG);
     }

From 04e27256a7be20853c62629a24dc224ccfe6f646 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 7 Jul 2022 12:13:27 -0400
Subject: [PATCH 199/473] Create a getQueueHandle function to unify backend
 code

The getQueueHandle function is necessary for the creation of a more uniform API
for the backend code. This allow us to combine the getStream and getQueue APIs
so that the same function can be used for both
---
 src/backend/cpu/platform.cpp          | 2 ++
 src/backend/cpu/platform.hpp          | 6 ++++++
 src/backend/cuda/platform.cpp         | 2 ++
 src/backend/cuda/platform.hpp         | 6 ++++++
 src/backend/oneapi/device_manager.hpp | 2 ++
 src/backend/oneapi/platform.cpp       | 8 ++++++++
 src/backend/oneapi/platform.hpp       | 6 ++++++
 src/backend/opencl/device_manager.hpp | 2 ++
 src/backend/opencl/platform.cpp       | 8 ++++++++
 src/backend/opencl/platform.hpp       | 6 ++++++
 10 files changed, 48 insertions(+)

diff --git a/src/backend/cpu/platform.cpp b/src/backend/cpu/platform.cpp
index dc73e76f17..a1dd7cd67b 100644
--- a/src/backend/cpu/platform.cpp
+++ b/src/backend/cpu/platform.cpp
@@ -148,6 +148,8 @@ queue& getQueue(int device) {
     return DeviceManager::getInstance().queues[device];
 }
 
+queue* getQueueHandle(int device) { return &getQueue(device); }
+
 void sync(int device) { getQueue(device).sync(); }
 
 bool& evalFlag() {
diff --git a/src/backend/cpu/platform.hpp b/src/backend/cpu/platform.hpp
index b02a1ca118..1f86639188 100644
--- a/src/backend/cpu/platform.hpp
+++ b/src/backend/cpu/platform.hpp
@@ -50,6 +50,12 @@ int setDevice(int device);
 
 queue& getQueue(int device = 0);
 
+/// Return a handle to the queue for the device.
+///
+/// \param[in] device The device of the returned queue
+/// \returns The handle to the queue
+queue* getQueueHandle(int device);
+
 void sync(int device);
 
 bool& evalFlag();
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 5ad8c27a7f..4b311f9808 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -396,6 +396,8 @@ cudaStream_t getStream(int device) {
 
 cudaStream_t getActiveStream() { return getStream(getActiveDeviceId()); }
 
+cudaStream_t getQueueHandle(int device) { return getStream(device); }
+
 size_t getDeviceMemorySize(int device) {
     return getDeviceProp(device).totalGlobalMem;
 }
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index 946c6addf1..cac1281b59 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -88,6 +88,12 @@ cudaStream_t getStream(int device);
 
 cudaStream_t getActiveStream();
 
+/// Return a handle to the stream for the device.
+///
+/// \param[in] device The device of the returned stream
+/// \returns The handle to the queue/stream
+cudaStream_t getQueueHandle(int device);
+
 size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
index df14603147..36824539b2 100644
--- a/src/backend/oneapi/device_manager.hpp
+++ b/src/backend/oneapi/device_manager.hpp
@@ -81,6 +81,8 @@ class DeviceManager {
 
     friend sycl::queue& getQueue();
 
+    friend sycl::queue* getQueueHandle(int device_id);
+
     friend const sycl::device& getDevice(int id);
 
     friend size_t getDeviceMemorySize(int device);
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index c0f3a0d08e..ce3ad2e099 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -285,6 +285,14 @@ sycl::queue& getQueue() {
     return *(devMngr.mQueues[get<1>(devId)]);
 }
 
+sycl::queue* getQueueHandle(int device_id) {
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return devMngr.mQueues[device_id].get();
+}
+
 const sycl::device& getDevice(int id) {
     device_id_t& devId = tlocalActiveDeviceId();
 
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index aa58ea5a7e..af579573d8 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -54,6 +54,12 @@ const sycl::context& getContext();
 
 sycl::queue& getQueue();
 
+/// Return a handle to the queue for the device.
+///
+/// \param[in] device The device of the returned queue
+/// \returns The handle to the queue
+sycl::queue* getQueueHandle(int device);
+
 const sycl::device& getDevice(int id = -1);
 
 size_t getDeviceMemorySize(int device);
diff --git a/src/backend/opencl/device_manager.hpp b/src/backend/opencl/device_manager.hpp
index 4e06582da3..432758bd87 100644
--- a/src/backend/opencl/device_manager.hpp
+++ b/src/backend/opencl/device_manager.hpp
@@ -107,6 +107,8 @@ class DeviceManager {
 
     friend cl::CommandQueue& getQueue();
 
+    friend cl_command_queue getQueueHandle(int device_id);
+
     friend const cl::Device& getDevice(int id);
 
     friend const std::string& getActiveDeviceBaseBuildFlags();
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 7e94cb0bde..165eded95f 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -288,6 +288,14 @@ const Context& getContext() {
     return *(devMngr.mContexts[get<0>(devId)]);
 }
 
+cl_command_queue getQueueHandle(int device_id) {
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return (*(devMngr.mQueues[device_id]))();
+}
+
 CommandQueue& getQueue() {
     device_id_t& devId = tlocalActiveDeviceId();
 
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 050e44f8c3..94ab6dff52 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -67,6 +67,12 @@ const cl::Context& getContext();
 
 cl::CommandQueue& getQueue();
 
+/// Return a cl_command_queue handle to the queue for the device.
+///
+/// \param[in] device The device of the returned queue
+/// \returns The cl_command_queue handle to the queue
+cl_command_queue getQueueHandle(int device_id);
+
 const cl::Device& getDevice(int id = -1);
 
 const std::string& getActiveDeviceBaseBuildFlags();

From e5132b0fb63f650bcb43cc36585efae4ed1ec7f5 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 7 Jul 2022 12:16:47 -0400
Subject: [PATCH 200/473] Create a specilization for the pinnedAlloc function
 that returns void*

This function makes it easier to create a void* pointer to pinned memory. This
is necessary when you want to create type independent code that requires the use
of pinned memory.
---
 src/backend/cpu/memory.cpp    | 13 +++++++++++++
 src/backend/cuda/memory.cpp   | 13 +++++++++++++
 src/backend/oneapi/memory.cpp | 14 ++++++++++++++
 src/backend/opencl/memory.cpp | 13 +++++++++++++
 4 files changed, 53 insertions(+)

diff --git a/src/backend/cpu/memory.cpp b/src/backend/cpu/memory.cpp
index 440680b48d..7f0ba41965 100644
--- a/src/backend/cpu/memory.cpp
+++ b/src/backend/cpu/memory.cpp
@@ -121,6 +121,19 @@ INSTANTIATE(ushort)
 INSTANTIATE(short)
 INSTANTIATE(half)
 
+template<>
+void *pinnedAlloc<void>(const size_t &elements) {
+    // TODO: make pinnedAlloc aware of array shapes
+    dim4 dims(elements);
+    void *ptr = memoryManager().alloc(false, 1, dims.get(), 1);
+    return ptr;
+}
+
+template<>
+void pinnedFree(void *ptr) {
+    memoryManager().unlock(ptr, false);
+}
+
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
 void Allocator::shutdown() {
diff --git a/src/backend/cuda/memory.cpp b/src/backend/cuda/memory.cpp
index 6c86a6244a..13106fd5c1 100644
--- a/src/backend/cuda/memory.cpp
+++ b/src/backend/cuda/memory.cpp
@@ -132,6 +132,19 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(half)
 
+template<>
+void *pinnedAlloc<void>(const size_t &elements) {
+    // TODO: make pinnedAlloc aware of array shapes
+    dim4 dims(elements);
+    void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), 1);
+    return ptr;
+}
+
+template<>
+void pinnedFree(void *ptr) {
+    pinnedMemoryManager().unlock(ptr, false);
+}
+
 template void memFree(void *ptr);
 
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index 80c589a5b0..56efa95785 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -183,6 +183,20 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(arrayfire::common::half)
 
+template<>
+void *pinnedAlloc<void>(const size_t &elements) {
+    ONEAPI_NOT_SUPPORTED("pinnedAlloc Not supported");
+
+    // // TODO: make pinnedAlloc aware of array shapes
+    // dim4 dims(elements);
+    // void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), sizeof(T));
+    return static_cast<void *>(nullptr);
+}
+template<>
+void pinnedFree(void *ptr) {
+    // pinnedMemoryManager().unlock(ptr, false);
+}
+
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
 void Allocator::shutdown() {
diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index 6c37d873a2..f1158dd91f 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -167,6 +167,19 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(common::half)
 
+template<>
+void *pinnedAlloc<void>(const size_t &elements) {
+    // TODO: make pinnedAlloc aware of array shapes
+    dim4 dims(elements);
+    void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), 1);
+    return ptr;
+}
+
+template<>
+void pinnedFree(void *ptr) {
+    pinnedMemoryManager().unlock(ptr, false);
+}
+
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
 void Allocator::shutdown() {

From cd9686ee8e583d0e920eb929ce92cc2fbecf6d92 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 7 Jul 2022 16:00:09 -0400
Subject: [PATCH 201/473] Expose copy parameter to the createDeviceDataArray
 function

The copy parameter was not exposed ot the createDeviceDataArray, this
parameter determines weather we should use the pointer directly or
allocate a new array and copy data to it.
---
 src/backend/cpu/Array.cpp    |  8 +++--
 src/backend/cpu/Array.hpp    | 14 ++++++--
 src/backend/cuda/Array.cpp   | 18 +++++-----
 src/backend/cuda/Array.hpp   | 14 ++++++--
 src/backend/oneapi/Array.cpp | 64 ++++++++++++++++++------------------
 src/backend/oneapi/Array.hpp | 14 ++++++--
 src/backend/opencl/Array.cpp |  9 +++--
 src/backend/opencl/Array.hpp | 14 ++++++--
 8 files changed, 98 insertions(+), 57 deletions(-)

diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index 9498fa36aa..c190c9b51d 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -217,8 +217,9 @@ Array<T> createHostDataArray(const dim4 &dims, const T *const data) {
 }
 
 template<typename T>
-Array<T> createDeviceDataArray(const dim4 &dims, void *data) {
-    return Array<T>(dims, static_cast<T *>(data), true);
+Array<T> createDeviceDataArray(const dim4 &dims, void *data, bool copy) {
+    bool is_device = true;
+    return Array<T>(dims, static_cast<T *>(data), is_device, copy);
 }
 
 template<typename T>
@@ -330,7 +331,8 @@ void Array<T>::setDataDims(const dim4 &new_dims) {
 #define INSTANTIATE(T)                                                        \
     template Array<T> createHostDataArray<T>(const dim4 &dims,                \
                                              const T *const data);            \
-    template Array<T> createDeviceDataArray<T>(const dim4 &dims, void *data); \
+    template Array<T> createDeviceDataArray<T>(const dim4 &dims, void *data,  \
+                                               bool copy);                    \
     template Array<T> createValueArray<T>(const dim4 &dims, const T &value);  \
     template Array<T> createEmptyArray<T>(const dim4 &dims);                  \
     template Array<T> createSubArray<T>(                                      \
diff --git a/src/backend/cpu/Array.hpp b/src/backend/cpu/Array.hpp
index 120d24b373..3c7b54c5ec 100644
--- a/src/backend/cpu/Array.hpp
+++ b/src/backend/cpu/Array.hpp
@@ -69,8 +69,17 @@ Array<T> createValueArray(const af::dim4 &dims, const T &value);
 template<typename T>
 Array<T> createHostDataArray(const af::dim4 &dims, const T *const data);
 
+/// Creates an Array<T> object from a device pointer.
+///
+/// \param[in] dims The shape of the resulting Array.
+/// \param[in] data The device pointer to the data
+/// \param[in] copy If true, memory will be allocated and the data will be
+///                 copied to the device. If false the data will be used
+///                 directly
+/// \returns The new Array<T> object based on the device pointer.
 template<typename T>
-Array<T> createDeviceDataArray(const af::dim4 &dims, void *data);
+Array<T> createDeviceDataArray(const af::dim4 &dims, void *data,
+                               bool copy = false);
 
 template<typename T>
 Array<T> createStridedArray(af::dim4 dims, af::dim4 strides, dim_t offset,
@@ -269,7 +278,8 @@ class Array {
     friend Array<T> createValueArray<T>(const af::dim4 &dims, const T &value);
     friend Array<T> createHostDataArray<T>(const af::dim4 &dims,
                                            const T *const data);
-    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data);
+    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data,
+                                             bool copy);
     friend Array<T> createStridedArray<T>(af::dim4 dims, af::dim4 strides,
                                           dim_t offset, T *const in_data,
                                           bool is_device);
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index ea5a7e971a..2ced1ea214 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -71,10 +71,10 @@ Array<T>::Array(const af::dim4 &dims, const T *const in_data, bool is_device,
                 bool copy_device)
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(
-          ((is_device & !copy_device) ? const_cast<T *>(in_data)
-                                      : memAlloc<T>(dims.elements()).release()),
-          memFree<T>)
+    , data(((is_device && !copy_device)
+                ? const_cast<T *>(in_data)
+                : memAlloc<T>(dims.elements()).release()),
+           memFree<T>)
     , data_dims(dims)
     , node()
     , owner(true) {
@@ -338,11 +338,10 @@ Array<T> createHostDataArray(const dim4 &dims, const T *const data) {
 }
 
 template<typename T>
-Array<T> createDeviceDataArray(const dim4 &dims, void *data) {
+Array<T> createDeviceDataArray(const dim4 &dims, void *data, bool copy) {
     verifyTypeSupport<T>();
-    bool is_device   = true;
-    bool copy_device = false;
-    return Array<T>(dims, static_cast<T *>(data), is_device, copy_device);
+    bool is_device = true;
+    return Array<T>(dims, static_cast<T *>(data), is_device, copy);
 }
 
 template<typename T>
@@ -432,7 +431,8 @@ void Array<T>::setDataDims(const dim4 &new_dims) {
 #define INSTANTIATE(T)                                                        \
     template Array<T> createHostDataArray<T>(const dim4 &size,                \
                                              const T *const data);            \
-    template Array<T> createDeviceDataArray<T>(const dim4 &size, void *data); \
+    template Array<T> createDeviceDataArray<T>(const dim4 &size, void *data,  \
+                                               bool copy);                    \
     template Array<T> createValueArray<T>(const dim4 &size, const T &value);  \
     template Array<T> createEmptyArray<T>(const dim4 &size);                  \
     template Array<T> createParamArray<T>(Param<T> & tmp, bool owner);        \
diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp
index 07e06f0681..6c00910c9d 100644
--- a/src/backend/cuda/Array.hpp
+++ b/src/backend/cuda/Array.hpp
@@ -56,8 +56,17 @@ Array<T> createValueArray(const af::dim4 &dims, const T &value);
 template<typename T>
 Array<T> createHostDataArray(const af::dim4 &dims, const T *const data);
 
+/// Creates an Array<T> object from a device pointer.
+///
+/// \param[in] dims The shape of the resulting Array.
+/// \param[in] data The device pointer to the data
+/// \param[in] copy If true, memory will be allocated and the data will be
+///                 copied to the device. If false the data will be used
+///                 directly
+/// \returns The new Array<T> object based on the device pointer.
 template<typename T>
-Array<T> createDeviceDataArray(const af::dim4 &dims, void *data);
+Array<T> createDeviceDataArray(const af::dim4 &dims, void *data,
+                               bool copy = false);
 
 template<typename T>
 Array<T> createStridedArray(const af::dim4 &dims, const af::dim4 &strides,
@@ -268,7 +277,8 @@ class Array {
     friend Array<T> createValueArray<T>(const af::dim4 &size, const T &value);
     friend Array<T> createHostDataArray<T>(const af::dim4 &dims,
                                            const T *const data);
-    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data);
+    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data,
+                                             bool copy);
     friend Array<T> createStridedArray<T>(const af::dim4 &dims,
                                           const af::dim4 &strides, dim_t offset,
                                           const T *const in_data,
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 16ab7e5b5a..ab880732e3 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -453,11 +453,10 @@ Array<T> createHostDataArray(const dim4 &dims, const T *const data) {
 }
 
 template<typename T>
-Array<T> createDeviceDataArray(const dim4 &dims, void *data) {
+Array<T> createDeviceDataArray(const dim4 &dims, void *data, bool copy) {
     verifyTypeSupport<T>();
 
-    bool copy_device = false;
-    return Array<T>(dims, static_cast<buffer<T> *>(data), 0, copy_device);
+    return Array<T>(dims, static_cast<buffer<T> *>(data), 0, copy);
 }
 
 template<typename T>
@@ -530,35 +529,36 @@ size_t Array<T>::getAllocatedBytes() const {
     return bytes;
 }
 
-#define INSTANTIATE(T)                                                        \
-    template Array<T> createHostDataArray<T>(const dim4 &dims,                \
-                                             const T *const data);            \
-    template Array<T> createDeviceDataArray<T>(const dim4 &dims, void *data); \
-    template Array<T> createValueArray<T>(const dim4 &dims, const T &value);  \
-    template Array<T> createEmptyArray<T>(const dim4 &dims);                  \
-    template Array<T> createParamArray<T>(Param<T> & tmp, bool owner);        \
-    template Array<T> createSubArray<T>(                                      \
-        const Array<T> &parent, const vector<af_seq> &index, bool copy);      \
-    template void destroyArray<T>(Array<T> * A);                              \
-    template Array<T> createNodeArray<T>(const dim4 &dims, Node_ptr node);    \
-    template Array<T>::Array(const dim4 &dims, const dim4 &strides,           \
-                             dim_t offset, const T *const in_data,            \
-                             bool is_device);                                 \
-    template Array<T>::Array(const dim4 &dims, buffer<T> *mem,                \
-                             size_t src_offset, bool copy);                   \
-    template Node_ptr Array<T>::getNode();                                    \
-    template Node_ptr Array<T>::getNode() const;                              \
-    template void Array<T>::eval();                                           \
-    template void Array<T>::eval() const;                                     \
-    template buffer<T> *Array<T>::device();                                   \
-    template void writeHostDataArray<T>(Array<T> & arr, const T *const data,  \
-                                        const size_t bytes);                  \
-    template void writeDeviceDataArray<T>(                                    \
-        Array<T> & arr, const void *const data, const size_t bytes);          \
-    template void evalMultiple<T>(vector<Array<T> *> arrays);                 \
-    template kJITHeuristics passesJitHeuristics<T>(span<Node *> node);        \
-    template void *getDevicePtr<T>(const Array<T> &arr);                      \
-    template void Array<T>::setDataDims(const dim4 &new_dims);                \
+#define INSTANTIATE(T)                                                       \
+    template Array<T> createHostDataArray<T>(const dim4 &dims,               \
+                                             const T *const data);           \
+    template Array<T> createDeviceDataArray<T>(const dim4 &dims, void *data, \
+                                               bool copy);                   \
+    template Array<T> createValueArray<T>(const dim4 &dims, const T &value); \
+    template Array<T> createEmptyArray<T>(const dim4 &dims);                 \
+    template Array<T> createParamArray<T>(Param<T> & tmp, bool owner);       \
+    template Array<T> createSubArray<T>(                                     \
+        const Array<T> &parent, const vector<af_seq> &index, bool copy);     \
+    template void destroyArray<T>(Array<T> * A);                             \
+    template Array<T> createNodeArray<T>(const dim4 &dims, Node_ptr node);   \
+    template Array<T>::Array(const dim4 &dims, const dim4 &strides,          \
+                             dim_t offset, const T *const in_data,           \
+                             bool is_device);                                \
+    template Array<T>::Array(const dim4 &dims, buffer<T> *mem,               \
+                             size_t src_offset, bool copy);                  \
+    template Node_ptr Array<T>::getNode();                                   \
+    template Node_ptr Array<T>::getNode() const;                             \
+    template void Array<T>::eval();                                          \
+    template void Array<T>::eval() const;                                    \
+    template buffer<T> *Array<T>::device();                                  \
+    template void writeHostDataArray<T>(Array<T> & arr, const T *const data, \
+                                        const size_t bytes);                 \
+    template void writeDeviceDataArray<T>(                                   \
+        Array<T> & arr, const void *const data, const size_t bytes);         \
+    template void evalMultiple<T>(vector<Array<T> *> arrays);                \
+    template kJITHeuristics passesJitHeuristics<T>(span<Node *> node);       \
+    template void *getDevicePtr<T>(const Array<T> &arr);                     \
+    template void Array<T>::setDataDims(const dim4 &new_dims);               \
     template size_t Array<T>::getAllocatedBytes() const;
 
 INSTANTIATE(float)
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index c3e0d38b98..3d74a897ba 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -66,8 +66,17 @@ Array<T> createValueArray(const af::dim4 &dims, const T &value);
 template<typename T>
 Array<T> createHostDataArray(const af::dim4 &dims, const T *const data);
 
+/// Creates an Array<T> object from a device pointer.
+///
+/// \param[in] dims The shape of the resulting Array.
+/// \param[in] data The device pointer to the data
+/// \param[in] copy If true, memory will be allocated and the data will be
+///                 copied to the device. If false the data will be used
+///                 directly
+/// \returns The new Array<T> object based on the device pointer.
 template<typename T>
-Array<T> createDeviceDataArray(const af::dim4 &dims, void *data);
+Array<T> createDeviceDataArray(const af::dim4 &dims, void *data,
+                               bool copy = false);
 
 template<typename T>
 Array<T> createStridedArray(const af::dim4 &dims, const af::dim4 &strides,
@@ -306,7 +315,8 @@ class Array {
     friend Array<T> createValueArray<T>(const af::dim4 &dims, const T &value);
     friend Array<T> createHostDataArray<T>(const af::dim4 &dims,
                                            const T *const data);
-    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data);
+    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data,
+                                             bool copy);
     friend Array<T> createStridedArray<T>(const af::dim4 &dims,
                                           const af::dim4 &strides, dim_t offset,
                                           const T *const in_data,
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 811f5551e3..810666b9a6 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -435,11 +435,9 @@ Array<T> createHostDataArray(const dim4 &dims, const T *const data) {
 }
 
 template<typename T>
-Array<T> createDeviceDataArray(const dim4 &dims, void *data) {
+Array<T> createDeviceDataArray(const dim4 &dims, void *data, bool copy) {
     verifyTypeSupport<T>();
-
-    bool copy_device = false;
-    return Array<T>(dims, static_cast<cl_mem>(data), 0, copy_device);
+    return Array<T>(dims, static_cast<cl_mem>(data), 0, copy);
 }
 
 template<typename T>
@@ -507,7 +505,8 @@ size_t Array<T>::getAllocatedBytes() const {
 #define INSTANTIATE(T)                                                        \
     template Array<T> createHostDataArray<T>(const dim4 &dims,                \
                                              const T *const data);            \
-    template Array<T> createDeviceDataArray<T>(const dim4 &dims, void *data); \
+    template Array<T> createDeviceDataArray<T>(const dim4 &dims, void *data,  \
+                                               bool copy);                    \
     template Array<T> createValueArray<T>(const dim4 &dims, const T &value);  \
     template Array<T> createEmptyArray<T>(const dim4 &dims);                  \
     template Array<T> createParamArray<T>(Param & tmp, bool owner);           \
diff --git a/src/backend/opencl/Array.hpp b/src/backend/opencl/Array.hpp
index 2d2ca97c94..6951021f19 100644
--- a/src/backend/opencl/Array.hpp
+++ b/src/backend/opencl/Array.hpp
@@ -60,8 +60,17 @@ Array<T> createValueArray(const af::dim4 &dims, const T &value);
 template<typename T>
 Array<T> createHostDataArray(const af::dim4 &dims, const T *const data);
 
+/// Creates an Array<T> object from a device pointer.
+///
+/// \param[in] dims The shape of the resulting Array.
+/// \param[in] data The device pointer to the data
+/// \param[in] copy If true, memory will be allocated and the data will be
+///                 copied to the device. If false the data will be used
+///                 directly
+/// \returns The new Array<T> object based on the device pointer.
 template<typename T>
-Array<T> createDeviceDataArray(const af::dim4 &dims, void *data);
+Array<T> createDeviceDataArray(const af::dim4 &dims, void *data,
+                               bool copy = false);
 
 template<typename T>
 Array<T> createStridedArray(const af::dim4 &dims, const af::dim4 &strides,
@@ -295,7 +304,8 @@ class Array {
     friend Array<T> createValueArray<T>(const af::dim4 &dims, const T &value);
     friend Array<T> createHostDataArray<T>(const af::dim4 &dims,
                                            const T *const data);
-    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data);
+    friend Array<T> createDeviceDataArray<T>(const af::dim4 &dims, void *data,
+                                             bool copy);
     friend Array<T> createStridedArray<T>(const af::dim4 &dims,
                                           const af::dim4 &strides, dim_t offset,
                                           const T *const in_data,

From 76a08fdcda579465766d8159caf6e998ee777417 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 8 Jul 2022 15:38:30 -0400
Subject: [PATCH 202/473] Call getActiveStream to create a stream in the init
 function.

This is necessary because when creating a new event before an operaiton is
performed in ArrayFire, the cuda driver API will throw an error.
---
 src/backend/cuda/platform.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 4b311f9808..3fab99bb7f 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -360,7 +360,9 @@ int getDeviceCount() {
 void init() {
     thread_local auto err =
         cudaSetDevice(getDeviceNativeId(getActiveDeviceId()));
+    thread_local auto queue2 = getActiveStream();
     UNUSED(err);
+    UNUSED(queue2);
 }
 
 int getActiveDeviceId() { return tlocalActiveDeviceId(); }

From c83bcdf9fd3e36e97f20f81d0ee137dfa5ac2a99 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 8 Jul 2022 19:19:49 -0400
Subject: [PATCH 203/473] Add function to create an af_array from a device
 pointer

---
 src/api/c/array.cpp  |  1 +
 src/api/c/handle.cpp | 23 +++++++++++++++++++++++
 src/api/c/handle.hpp |  9 +++++++++
 3 files changed, 33 insertions(+)

diff --git a/src/api/c/array.cpp b/src/api/c/array.cpp
index e9a0f68603..173c52171c 100644
--- a/src/api/c/array.cpp
+++ b/src/api/c/array.cpp
@@ -27,6 +27,7 @@ using arrayfire::common::half;
 using arrayfire::common::SparseArrayBase;
 using detail::cdouble;
 using detail::cfloat;
+using detail::createDeviceDataArray;
 using detail::intl;
 using detail::uchar;
 using detail::uint;
diff --git a/src/api/c/handle.cpp b/src/api/c/handle.cpp
index 392e120fca..0d9f3d2aec 100644
--- a/src/api/c/handle.cpp
+++ b/src/api/c/handle.cpp
@@ -18,6 +18,7 @@ using af::dim4;
 using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
+using detail::createDeviceDataArray;
 using detail::intl;
 using detail::uchar;
 using detail::uint;
@@ -100,6 +101,28 @@ af_array createHandleFromValue(const dim4 &d, double val, af_dtype dtype) {
     // clang-format on
 }
 
+af_array createHandleFromDeviceData(const af::dim4 &d, af_dtype dtype,
+                                    void *data) {
+    // clang-format off
+    switch (dtype) {
+        case f32: return getHandle(createDeviceDataArray<float  >(d, data, false));
+        case c32: return getHandle(createDeviceDataArray<cfloat >(d, data, false));
+        case f64: return getHandle(createDeviceDataArray<double >(d, data, false));
+        case c64: return getHandle(createDeviceDataArray<cdouble>(d, data, false));
+        case b8:  return getHandle(createDeviceDataArray<char   >(d, data, false));
+        case s32: return getHandle(createDeviceDataArray<int    >(d, data, false));
+        case u32: return getHandle(createDeviceDataArray<uint   >(d, data, false));
+        case u8:  return getHandle(createDeviceDataArray<uchar  >(d, data, false));
+        case s64: return getHandle(createDeviceDataArray<intl   >(d, data, false));
+        case u64: return getHandle(createDeviceDataArray<uintl  >(d, data, false));
+        case s16: return getHandle(createDeviceDataArray<short  >(d, data, false));
+        case u16: return getHandle(createDeviceDataArray<ushort >(d, data, false));
+        case f16: return getHandle(createDeviceDataArray<half   >(d, data, false));
+        default: TYPE_ERROR(2, dtype);
+    }
+    // clang-format on
+}
+
 dim4 verifyDims(const unsigned ndims, const dim_t *const dims) {
     DIM_ASSERT(1, ndims >= 1);
 
diff --git a/src/api/c/handle.hpp b/src/api/c/handle.hpp
index 4b73293cb3..b19de9c143 100644
--- a/src/api/c/handle.hpp
+++ b/src/api/c/handle.hpp
@@ -30,6 +30,15 @@ af_array createHandle(const af::dim4 &d, af_dtype dtype);
 
 af_array createHandleFromValue(const af::dim4 &d, double val, af_dtype dtype);
 
+/// This function creates an af_array handle from memory handle on the device.
+///
+/// \param[in] d The shape of the new af_array
+/// \param[in] dtype The type of the new af_array
+/// \param[in] data The handle to the device memory
+/// \returns a new af_array with a view to the \p data pointer
+af_array createHandleFromDeviceData(const af::dim4 &d, af_dtype dtype,
+                                    void *data);
+
 namespace common {
 const ArrayInfo &getInfo(const af_array arr, bool sparse_check = true,
                          bool device_check = true);

From e93f000335494a8207b0ede5e09f4320c545fda4 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 4 Aug 2022 15:12:21 -0400
Subject: [PATCH 204/473] Create EXPECT macros to the internal ArrayFire test
 macros

---
 test/testHelpers.hpp | 65 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 69240883ac..2382060ebf 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -406,6 +406,34 @@ ::testing::AssertionResult assertRefEq(std::string hA_name,
     ASSERT_PRED_FORMAT3(assertArrayEq, EXPECTED_VEC, EXPECTED_ARR_DIMS,  \
                         ACTUAL_ARR)
 
+/// Compares two af::array or af_arrays for their types, dims, and values
+/// (strict equality).
+///
+/// \param[in] EXPECTED The expected array of the assertion
+/// \param[in] ACTUAL The actual resulting array from the calculation
+#define EXPECT_ARRAYS_EQ(EXPECTED, ACTUAL) \
+    EXPECT_PRED_FORMAT2(assertArrayEq, EXPECTED, ACTUAL)
+
+/// Same as EXPECT_ARRAYS_EQ, but for cases when a "special" output array is
+/// given to the function.
+/// The special array can be null, a full-sized array, a subarray, or reordered
+/// Can only be used for testing C-API functions currently
+///
+/// \param[in] EXPECTED The expected array of the assertion
+/// \param[in] ACTUAL The actual resulting array from the calculation
+#define EXPECT_SPECIAL_ARRAYS_EQ(EXPECTED, ACTUAL, META) \
+    EXPECT_PRED_FORMAT3(assertArrayEq, EXPECTED, ACTUAL, META)
+
+/// Compares a std::vector with an af::/af_array for their types, dims, and
+/// values (strict equality).
+///
+/// \param[in] EXPECTED_VEC The vector that represents the expected array
+/// \param[in] EXPECTED_ARR_DIMS The dimensions of the expected array
+/// \param[in] ACTUAL_ARR The actual resulting array from the calculation
+#define EXPECT_VEC_ARRAY_EQ(EXPECTED_VEC, EXPECTED_ARR_DIMS, ACTUAL_ARR) \
+    EXPECT_PRED_FORMAT3(assertArrayEq, EXPECTED_VEC, EXPECTED_ARR_DIMS,  \
+                        ACTUAL_ARR)
+
 /// Compares two af::array or af_arrays for their type, dims, and values (with a
 /// given tolerance).
 ///
@@ -443,6 +471,43 @@ ::testing::AssertionResult assertRefEq(std::string hA_name,
     ASSERT_PRED_FORMAT4(assertArrayNear, EXPECTED_VEC, EXPECTED_ARR_DIMS,  \
                         ACTUAL_ARR, MAX_ABSDIFF)
 
+/// Compares two af::array or af_arrays for their type, dims, and values (with a
+/// given tolerance).
+///
+/// \param[in] EXPECTED Expected value of the assertion
+/// \param[in] ACTUAL Actual value of the calculation
+/// \param[in] MAX_ABSDIFF Expected maximum absolute difference between
+///            elements of EXPECTED and ACTUAL
+///
+/// \NOTE: This macro will deallocate the af_arrays after the call
+#define EXPECT_ARRAYS_NEAR(EXPECTED, ACTUAL, MAX_ABSDIFF) \
+    EXPECT_PRED_FORMAT3(assertArrayNear, EXPECTED, ACTUAL, MAX_ABSDIFF)
+
+/// Compares two af::array or af_arrays for their type, dims, and values (with a
+/// given tolerance).
+///
+/// \param[in] EXPECTED Expected value of the assertion
+/// \param[in] ACTUAL Actual value of the calculation
+/// \param[in] MAX_ABSDIFF Expected maximum absolute difference between
+///            elements of EXPECTED and ACTUAL
+///
+/// \NOTE: This macro will deallocate the af_arrays after the call
+#define EXPECT_IMAGES_NEAR(EXPECTED, ACTUAL, MAX_ABSDIFF) \
+    EXPECT_PRED_FORMAT3(assertImageNear, EXPECTED, ACTUAL, MAX_ABSDIFF)
+
+/// Compares a std::vector with an af::array for their dims and values (with a
+/// given tolerance).
+///
+/// \param[in] EXPECTED_VEC The vector that represents the expected array
+/// \param[in] EXPECTED_ARR_DIMS The dimensions of the expected array
+/// \param[in] ACTUAL_ARR The actual array from the calculation
+/// \param[in] MAX_ABSDIFF Expected maximum absolute difference between
+///            elements of EXPECTED and ACTUAL
+#define EXPECT_VEC_ARRAY_NEAR(EXPECTED_VEC, EXPECTED_ARR_DIMS, ACTUAL_ARR, \
+                              MAX_ABSDIFF)                                 \
+    EXPECT_PRED_FORMAT4(assertArrayNear, EXPECTED_VEC, EXPECTED_ARR_DIMS,  \
+                        ACTUAL_ARR, MAX_ABSDIFF)
+
 #define ASSERT_REF(arr, expected) \
     ASSERT_PRED_FORMAT2(assertRefEq, arr, expected)
 

From e7aa327c3442fceccc3974351e37aded1ff79a40 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 4 Aug 2022 15:58:47 -0400
Subject: [PATCH 205/473] Convert memFree and pinnedFree function to
 non-templated functions

---
 src/api/c/memory.cpp                 |  2 +-
 src/backend/cpu/Array.cpp            |  9 ++++-----
 src/backend/cpu/memory.cpp           | 29 ++++++++--------------------
 src/backend/cpu/memory.hpp           |  9 ++++-----
 src/backend/cpu/susan.cpp            | 12 ++++++------
 src/backend/cuda/Array.cpp           | 12 ++++++------
 src/backend/cuda/ThrustAllocator.cuh |  2 +-
 src/backend/cuda/memory.cpp          | 23 ++++------------------
 src/backend/cuda/memory.hpp          |  6 ++----
 src/backend/cuda/solve.cu            | 14 ++++++--------
 src/backend/oneapi/memory.cpp        | 14 +++-----------
 src/backend/oneapi/memory.hpp        |  7 +++----
 src/backend/opencl/memory.cpp        |  8 +++++---
 src/backend/opencl/memory.hpp        |  4 ++--
 14 files changed, 55 insertions(+), 96 deletions(-)

diff --git a/src/api/c/memory.cpp b/src/api/c/memory.cpp
index a689f92a91..17ea0a4d73 100644
--- a/src/api/c/memory.cpp
+++ b/src/api/c/memory.cpp
@@ -308,7 +308,7 @@ af_err af_free_device_v2(void *ptr) {
 
 af_err af_free_pinned(void *ptr) {
     try {
-        pinnedFree<char>(static_cast<char *>(ptr));
+        pinnedFree(ptr);
     }
     CATCHALL;
     return AF_SUCCESS;
diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index c190c9b51d..88f4bcabee 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -67,7 +67,7 @@ template<typename T>
 Array<T>::Array(dim4 dims)
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(memAlloc<T>(dims.elements()).release(), memFree<T>)
+    , data(memAlloc<T>(dims.elements()).release(), memFree)
     , data_dims(dims)
     , node()
     , owner(true) {}
@@ -79,7 +79,7 @@ Array<T>::Array(const dim4 &dims, T *const in_data, bool is_device,
            static_cast<af_dtype>(dtype_traits<T>::af_type))
     , data((is_device & !copy_device) ? in_data
                                       : memAlloc<T>(dims.elements()).release(),
-           memFree<T>)
+           memFree)
     , data_dims(dims)
     , node()
     , owner(true) {
@@ -123,8 +123,7 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
                 T *const in_data, bool is_device)
     : info(getActiveDeviceId(), dims, offset_, strides,
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(is_device ? in_data : memAlloc<T>(info.total()).release(),
-           memFree<T>)
+    , data(is_device ? in_data : memAlloc<T>(info.total()).release(), memFree)
     , data_dims(dims)
     , node()
     , owner(true) {
@@ -180,7 +179,7 @@ void evalMultiple(vector<Array<T> *> array_ptrs) {
 
         array->setId(getActiveDeviceId());
         array->data =
-            shared_ptr<T>(memAlloc<T>(array->elements()).release(), memFree<T>);
+            shared_ptr<T>(memAlloc<T>(array->elements()).release(), memFree);
 
         outputs.push_back(array);
         params.emplace_back(array->getData().get(), array->dims(),
diff --git a/src/backend/cpu/memory.cpp b/src/backend/cpu/memory.cpp
index 7f0ba41965..9bbb41d458 100644
--- a/src/backend/cpu/memory.cpp
+++ b/src/backend/cpu/memory.cpp
@@ -54,12 +54,12 @@ void printMemInfo(const char *msg, const int device) {
 }
 
 template<typename T>
-unique_ptr<T[], function<void(T *)>> memAlloc(const size_t &elements) {
+unique_ptr<T[], function<void(void *)>> memAlloc(const size_t &elements) {
     // TODO: make memAlloc aware of array shapes
     dim4 dims(elements);
     T *ptr = static_cast<T *>(
         memoryManager().alloc(false, 1, dims.get(), sizeof(T)));
-    return unique_ptr<T[], function<void(T *)>>(ptr, memFree<T>);
+    return unique_ptr<T[], function<void(void *)>>(ptr, memFree);
 }
 
 void *memAllocUser(const size_t &bytes) {
@@ -68,10 +68,7 @@ void *memAllocUser(const size_t &bytes) {
     return ptr;
 }
 
-template<typename T>
-void memFree(T *ptr) {
-    return memoryManager().unlock(static_cast<void *>(ptr), false);
-}
+void memFree(void *ptr) { return memoryManager().unlock(ptr, false); }
 
 void memFreeUser(void *ptr) { memoryManager().unlock(ptr, true); }
 
@@ -95,17 +92,12 @@ T *pinnedAlloc(const size_t &elements) {
     return static_cast<T *>(ptr);
 }
 
-template<typename T>
-void pinnedFree(T *ptr) {
-    memoryManager().unlock(static_cast<void *>(ptr), false);
-}
+void pinnedFree(void *ptr) { memoryManager().unlock(ptr, false); }
 
-#define INSTANTIATE(T)                                                \
-    template std::unique_ptr<T[], std::function<void(T *)>> memAlloc( \
-        const size_t &elements);                                      \
-    template void memFree(T *ptr);                                    \
-    template T *pinnedAlloc(const size_t &elements);                  \
-    template void pinnedFree(T *ptr);
+#define INSTANTIATE(T)                                                   \
+    template std::unique_ptr<T[], std::function<void(void *)>> memAlloc( \
+        const size_t &elements);                                         \
+    template T *pinnedAlloc(const size_t &elements);
 
 INSTANTIATE(float)
 INSTANTIATE(cfloat)
@@ -129,11 +121,6 @@ void *pinnedAlloc<void>(const size_t &elements) {
     return ptr;
 }
 
-template<>
-void pinnedFree(void *ptr) {
-    memoryManager().unlock(ptr, false);
-}
-
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
 void Allocator::shutdown() {
diff --git a/src/backend/cpu/memory.hpp b/src/backend/cpu/memory.hpp
index a45ca06ec1..908136d094 100644
--- a/src/backend/cpu/memory.hpp
+++ b/src/backend/cpu/memory.hpp
@@ -20,14 +20,14 @@ template<typename T>
 using uptr = std::unique_ptr<T[], std::function<void(T[])>>;
 
 template<typename T>
-std::unique_ptr<T[], std::function<void(T *)>> memAlloc(const size_t &elements);
+std::unique_ptr<T[], std::function<void(void *)>> memAlloc(
+    const size_t &elements);
 void *memAllocUser(const size_t &bytes);
 
 // Need these as 2 separate function and not a default argument
 // This is because it is used as the deleter in shared pointer
 // which cannot support default arguments
-template<typename T>
-void memFree(T *ptr);
+void memFree(void *ptr);
 void memFreeUser(void *ptr);
 
 void memLock(const void *ptr);
@@ -36,8 +36,7 @@ bool isLocked(const void *ptr);
 
 template<typename T>
 T *pinnedAlloc(const size_t &elements);
-template<typename T>
-void pinnedFree(T *ptr);
+void pinnedFree(void *ptr);
 
 void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
                       size_t *lock_bytes, size_t *lock_buffers);
diff --git a/src/backend/cpu/susan.cpp b/src/backend/cpu/susan.cpp
index 0d79078988..6ab2bfba78 100644
--- a/src/backend/cpu/susan.cpp
+++ b/src/backend/cpu/susan.cpp
@@ -30,12 +30,12 @@ unsigned susan(Array<float> &x_out, Array<float> &y_out, Array<float> &resp_out,
     dim4 idims                = in.dims();
     const unsigned corner_lim = in.elements() * feature_ratio;
 
-    auto x_corners     = createEmptyArray<float>(dim4(corner_lim));
-    auto y_corners     = createEmptyArray<float>(dim4(corner_lim));
-    auto resp_corners  = createEmptyArray<float>(dim4(corner_lim));
-    auto response      = createEmptyArray<T>(dim4(in.elements()));
-    auto corners_found = std::shared_ptr<unsigned>(
-        memAlloc<unsigned>(1).release(), memFree<unsigned>);
+    auto x_corners    = createEmptyArray<float>(dim4(corner_lim));
+    auto y_corners    = createEmptyArray<float>(dim4(corner_lim));
+    auto resp_corners = createEmptyArray<float>(dim4(corner_lim));
+    auto response     = createEmptyArray<T>(dim4(in.elements()));
+    auto corners_found =
+        std::shared_ptr<unsigned>(memAlloc<unsigned>(1).release(), memFree);
     corners_found.get()[0] = 0;
 
     getQueue().enqueue(kernel::susan_responses<T>, response, in, idims[0],
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index 2ced1ea214..db03d1b3e5 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -61,7 +61,7 @@ Array<T>::Array(const af::dim4 &dims)
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
     , data((dims.elements() ? memAlloc<T>(dims.elements()).release() : nullptr),
-           memFree<T>)
+           memFree)
     , data_dims(dims)
     , node()
     , owner(true) {}
@@ -74,7 +74,7 @@ Array<T>::Array(const af::dim4 &dims, const T *const in_data, bool is_device,
     , data(((is_device && !copy_device)
                 ? const_cast<T *>(in_data)
                 : memAlloc<T>(dims.elements()).release()),
-           memFree<T>)
+           memFree)
     , data_dims(dims)
     , node()
     , owner(true) {
@@ -117,7 +117,7 @@ Array<T>::Array(Param<T> &tmp, bool owner_)
            af::dim4(tmp.strides[0], tmp.strides[1], tmp.strides[2],
                     tmp.strides[3]),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(tmp.ptr, owner_ ? std::function<void(T *)>(memFree<T>)
+    , data(tmp.ptr, owner_ ? std::function<void(T *)>(memFree)
                            : std::function<void(T *)>([](T * /*unused*/) {}))
     , data_dims(af::dim4(tmp.dims[0], tmp.dims[1], tmp.dims[2], tmp.dims[3]))
     , node()
@@ -143,7 +143,7 @@ Array<T>::Array(const af::dim4 &dims, const af::dim4 &strides, dim_t offset_,
            static_cast<af_dtype>(dtype_traits<T>::af_type))
     , data(is_device ? const_cast<T *>(in_data)
                      : memAlloc<T>(info.total()).release(),
-           memFree<T>)
+           memFree)
     , data_dims(dims)
     , node()
     , owner(true) {
@@ -161,7 +161,7 @@ void Array<T>::eval() {
     if (isReady()) { return; }
 
     this->setId(getActiveDeviceId());
-    this->data = shared_ptr<T>(memAlloc<T>(elements()).release(), memFree<T>);
+    this->data = shared_ptr<T>(memAlloc<T>(elements()).release(), memFree);
 
     Param<T> p(data.get(), dims().get(), strides().get());
     evalNodes<T>(p, node.get());
@@ -204,7 +204,7 @@ void evalMultiple(std::vector<Array<T> *> arrays) {
 
         array->setId(getActiveDeviceId());
         array->data =
-            shared_ptr<T>(memAlloc<T>(array->elements()).release(), memFree<T>);
+            shared_ptr<T>(memAlloc<T>(array->elements()).release(), memFree);
 
         output_params.emplace_back(array->getData().get(), array->dims().get(),
                                    array->strides().get());
diff --git a/src/backend/cuda/ThrustAllocator.cuh b/src/backend/cuda/ThrustAllocator.cuh
index 21152e6059..93a4a8fc6d 100644
--- a/src/backend/cuda/ThrustAllocator.cuh
+++ b/src/backend/cuda/ThrustAllocator.cuh
@@ -39,7 +39,7 @@ struct ThrustAllocator : thrust::device_malloc_allocator<T> {
 
     void deallocate(pointer p, size_type n) {
         UNUSED(n);
-        memFree<T>(p.get());  // delegate to ArrayFire allocator
+        memFree(p.get());  // delegate to ArrayFire allocator
     }
 };
 }  // namespace cuda
diff --git a/src/backend/cuda/memory.cpp b/src/backend/cuda/memory.cpp
index 13106fd5c1..dafbef1ce8 100644
--- a/src/backend/cuda/memory.cpp
+++ b/src/backend/cuda/memory.cpp
@@ -65,7 +65,7 @@ uptr<T> memAlloc(const size_t &elements) {
     // TODO: make memAlloc aware of array shapes
     dim4 dims(elements);
     void *ptr = memoryManager().alloc(false, 1, dims.get(), sizeof(T));
-    return uptr<T>(static_cast<T *>(ptr), memFree<T>);
+    return uptr<T>(static_cast<T *>(ptr), memFree);
 }
 
 void *memAllocUser(const size_t &bytes) {
@@ -74,10 +74,7 @@ void *memAllocUser(const size_t &bytes) {
     return ptr;
 }
 
-template<typename T>
-void memFree(T *ptr) {
-    memoryManager().unlock(static_cast<void *>(ptr), false);
-}
+void memFree(void *ptr) { memoryManager().unlock(ptr, false); }
 
 void memFreeUser(void *ptr) { memoryManager().unlock(ptr, true); }
 
@@ -107,16 +104,11 @@ T *pinnedAlloc(const size_t &elements) {
     return static_cast<T *>(ptr);
 }
 
-template<typename T>
-void pinnedFree(T *ptr) {
-    pinnedMemoryManager().unlock(static_cast<void *>(ptr), false);
-}
+void pinnedFree(void *ptr) { pinnedMemoryManager().unlock(ptr, false); }
 
 #define INSTANTIATE(T)                                 \
     template uptr<T> memAlloc(const size_t &elements); \
-    template void memFree(T *ptr);                     \
-    template T *pinnedAlloc(const size_t &elements);   \
-    template void pinnedFree(T *ptr);
+    template T *pinnedAlloc(const size_t &elements);
 
 INSTANTIATE(float)
 INSTANTIATE(cfloat)
@@ -140,13 +132,6 @@ void *pinnedAlloc<void>(const size_t &elements) {
     return ptr;
 }
 
-template<>
-void pinnedFree(void *ptr) {
-    pinnedMemoryManager().unlock(ptr, false);
-}
-
-template void memFree(void *ptr);
-
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
 void Allocator::shutdown() {
diff --git a/src/backend/cuda/memory.hpp b/src/backend/cuda/memory.hpp
index 935c788769..039879a90e 100644
--- a/src/backend/cuda/memory.hpp
+++ b/src/backend/cuda/memory.hpp
@@ -19,8 +19,7 @@ namespace cuda {
 float getMemoryPressure();
 float getMemoryPressureThreshold();
 
-template<typename T>
-void memFree(T *ptr);
+void memFree(void *ptr);
 
 template<typename T>
 using uptr = std::unique_ptr<T[], std::function<void(T[])>>;
@@ -42,8 +41,7 @@ bool isLocked(const void *ptr);
 
 template<typename T>
 T *pinnedAlloc(const size_t &elements);
-template<typename T>
-void pinnedFree(T *ptr);
+void pinnedFree(void *ptr);
 
 void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
                       size_t *lock_bytes, size_t *lock_buffers);
diff --git a/src/backend/cuda/solve.cu b/src/backend/cuda/solve.cu
index f762785818..884d7735b1 100644
--- a/src/backend/cuda/solve.cu
+++ b/src/backend/cuda/solve.cu
@@ -251,12 +251,12 @@ Array<T> generalSolveBatched(const Array<T> &a, const Array<T> &b) {
     int batch  = batchz * batchw;
 
     size_t bytes         = batch * sizeof(T *);
-    using unique_mem_ptr = std::unique_ptr<char, void (*)(char *)>;
+    using unique_mem_ptr = std::unique_ptr<char, void (*)(void *)>;
 
     unique_mem_ptr aBatched_host_mem(pinnedAlloc<char>(bytes),
-                                     pinnedFree<char>);
+                                     pinnedFree);
     unique_mem_ptr bBatched_host_mem(pinnedAlloc<char>(bytes),
-                                     pinnedFree<char>);
+                                     pinnedFree);
 
     T *a_ptr               = A.get();
     T *b_ptr               = B.get();
@@ -272,10 +272,8 @@ Array<T> generalSolveBatched(const Array<T> &a, const Array<T> &b) {
         }
     }
 
-    unique_mem_ptr aBatched_device_mem(pinnedAlloc<char>(bytes),
-                                       pinnedFree<char>);
-    unique_mem_ptr bBatched_device_mem(pinnedAlloc<char>(bytes),
-                                       pinnedFree<char>);
+    unique_mem_ptr aBatched_device_mem(pinnedAlloc<char>(bytes), pinnedFree);
+    unique_mem_ptr bBatched_device_mem(pinnedAlloc<char>(bytes), pinnedFree);
 
     T **aBatched_device_ptrs = (T **)aBatched_device_mem.get();
     T **bBatched_device_ptrs = (T **)bBatched_device_mem.get();
@@ -299,7 +297,7 @@ Array<T> generalSolveBatched(const Array<T> &a, const Array<T> &b) {
 
     // getrs requires info to be host pointer
     unique_mem_ptr info_host_mem(pinnedAlloc<char>(batch * sizeof(int)),
-                                 pinnedFree<char>);
+                                 pinnedFree);
     CUBLAS_CHECK(getrsBatched_func<T>()(
         blasHandle(), CUBLAS_OP_N, N, NRHS, (const T **)aBatched_device_ptrs,
         A.strides()[1], pivots.get(), bBatched_device_ptrs, B.strides()[1],
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index 56efa95785..17cfb37d32 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -73,8 +73,7 @@ void *memAllocUser(const size_t &bytes) {
     // return new cl::Buffer(buf, true);
 }
 
-template<typename T>
-void memFree(T *ptr) {
+void memFree(void *ptr) {
     ONEAPI_NOT_SUPPORTED("memFree Not supported");
 
     // cl::Buffer *buf = reinterpret_cast<cl::Buffer *>(ptr);
@@ -152,9 +151,8 @@ T *pinnedAlloc(const size_t &elements) {
     return static_cast<T *>(ptr);
 }
 
-template<typename T>
-void pinnedFree(T *ptr) {
-    pinnedMemoryManager().unlock(static_cast<void *>(ptr), false);
+void pinnedFree(void *ptr) {
+    pinnedMemoryManager().unlock(ptr, false);
 }
 
 // template unique_ptr<int, function<void(int *)>> memAlloc<T>(
@@ -162,9 +160,7 @@ void pinnedFree(T *ptr) {
     template std::unique_ptr<sycl::buffer<T>,                        \
                              std::function<void(sycl::buffer<T> *)>> \
     memAlloc(const size_t &elements);                                \
-    template void memFree(T *ptr);                                   \
     template T *pinnedAlloc(const size_t &elements);                 \
-    template void pinnedFree(T *ptr);                                \
     template void bufferFree(sycl::buffer<T> *buf);                  \
     template void memLock(const sycl::buffer<T> *buf);               \
     template void memUnlock(const sycl::buffer<T> *buf);
@@ -192,10 +188,6 @@ void *pinnedAlloc<void>(const size_t &elements) {
     // void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), sizeof(T));
     return static_cast<void *>(nullptr);
 }
-template<>
-void pinnedFree(void *ptr) {
-    // pinnedMemoryManager().unlock(ptr, false);
-}
 
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
diff --git a/src/backend/oneapi/memory.hpp b/src/backend/oneapi/memory.hpp
index bcb8c1dabf..809f219eb7 100644
--- a/src/backend/oneapi/memory.hpp
+++ b/src/backend/oneapi/memory.hpp
@@ -35,8 +35,7 @@ void *memAllocUser(const size_t &bytes);
 // Need these as 2 separate function and not a default argument
 // This is because it is used as the deleter in shared pointer
 // which cannot support default arguments
-template<typename T>
-void memFree(T *ptr);
+void memFree(void *ptr);
 void memFreeUser(void *ptr);
 
 template<typename T>
@@ -49,8 +48,8 @@ bool isLocked(const void *ptr);
 
 template<typename T>
 T *pinnedAlloc(const size_t &elements);
-template<typename T>
-void pinnedFree(T *ptr);
+
+void pinnedFree(void *ptr);
 
 void deviceMemoryInfo(size_t *alloc_bytes, size_t *alloc_buffers,
                       size_t *lock_bytes, size_t *lock_buffers);
diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index f1158dd91f..68ae43c5e8 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -78,14 +78,17 @@ void *memAllocUser(const size_t &bytes) {
     return new cl::Buffer(buf, true);
 }
 
-template<typename T>
-void memFree(T *ptr) {
+void memFree(cl::Buffer *ptr) {
     cl::Buffer *buf = reinterpret_cast<cl::Buffer *>(ptr);
     cl_mem mem      = static_cast<cl_mem>((*buf)());
     delete buf;
     return memoryManager().unlock(static_cast<void *>(mem), false);
 }
 
+void memFree(cl_mem ptr) {
+    return memoryManager().unlock(static_cast<void *>(ptr), false);
+}
+
 void memFreeUser(void *ptr) {
     cl::Buffer *buf = static_cast<cl::Buffer *>(ptr);
     cl_mem mem      = (*buf)();
@@ -149,7 +152,6 @@ void pinnedFree(T *ptr) {
 #define INSTANTIATE(T)                                                         \
     template unique_ptr<cl::Buffer, function<void(cl::Buffer *)>> memAlloc<T>( \
         const size_t &elements);                                               \
-    template void memFree(T *ptr);                                             \
     template T *pinnedAlloc(const size_t &elements);                           \
     template void pinnedFree(T *ptr);
 
diff --git a/src/backend/opencl/memory.hpp b/src/backend/opencl/memory.hpp
index 4f618d7956..447f80bb83 100644
--- a/src/backend/opencl/memory.hpp
+++ b/src/backend/opencl/memory.hpp
@@ -34,8 +34,8 @@ void *memAllocUser(const size_t &bytes);
 // Need these as 2 separate function and not a default argument
 // This is because it is used as the deleter in shared pointer
 // which cannot support default arguments
-template<typename T>
-void memFree(T *ptr);
+void memFree(cl::Buffer *ptr);
+void memFree(cl_mem ptr);
 void memFreeUser(void *ptr);
 
 void memLock(const cl::Buffer *ptr);

From be685a9a0d8e977ec6e71519c0ce54478492bd21 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 5 Aug 2022 11:45:35 -0400
Subject: [PATCH 206/473] Update threads library for event behavior. Update
 event docs

---
 src/api/c/events.cpp           | 7 +------
 src/api/c/events.hpp           | 3 +--
 src/backend/cpu/CMakeLists.txt | 2 +-
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/api/c/events.cpp b/src/api/c/events.cpp
index c3d7d5a773..112373672d 100644
--- a/src/api/c/events.cpp
+++ b/src/api/c/events.cpp
@@ -20,16 +20,11 @@ using detail::enqueueWaitOnActiveQueue;
 using detail::Event;
 using detail::markEventOnActiveQueue;
 
-Event &getEvent(af_event &handle) {
+Event &getEvent(af_event handle) {
     Event &event = *static_cast<Event *>(handle);
     return event;
 }
 
-const Event &getEvent(const af_event &handle) {
-    const Event &event = *static_cast<const Event *>(handle);
-    return event;
-}
-
 af_event getHandle(Event &event) { return static_cast<af_event>(&event); }
 
 af_err af_create_event(af_event *handle) {
diff --git a/src/api/c/events.hpp b/src/api/c/events.hpp
index b3d3eb398d..488cb204e4 100644
--- a/src/api/c/events.hpp
+++ b/src/api/c/events.hpp
@@ -15,5 +15,4 @@
 
 af_event getHandle(detail::Event& event);
 
-detail::Event& getEvent(af_event& eventHandle);
-const detail::Event& getEvent(const af_event& eventHandle);
+detail::Event& getEvent(af_event eventHandle);
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index fc84101de4..b8025d53a2 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -274,7 +274,7 @@ endif(AF_WITH_CPUID)
 
 af_dep_check_and_populate(${threads_prefix}
   URI https://github.com/arrayfire/threads.git
-  REF b666773940269179f19ef11c8f1eb77005e85d9a
+  REF 4d4a4f0384d1ac2f25b2c4fc1d57b9e25f4d6818
 )
 
 target_sources(afcpu

From 30676ee9d9826242fc1c46cbe8004d1fd3c79dcf Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sun, 7 Aug 2022 09:21:42 -0400
Subject: [PATCH 207/473] Fix minor warnings and update clang-tidy

---
 src/.clang-tidy                         | 2 +-
 src/backend/common/jit/NodeIterator.hpp | 2 +-
 src/backend/cuda/reduce_impl.hpp        | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/.clang-tidy b/src/.clang-tidy
index a3e8a261dd..549c784606 100644
--- a/src/.clang-tidy
+++ b/src/.clang-tidy
@@ -1,5 +1,5 @@
 ---
-Checks:          'clang-diagnostic-*,clang-analyzer-*,*,-fuchsia-*,-cppcoreguidelines-*,-misc-misplaced-const,-hicpp-no-array-decay,-readability-implicit-bool-conversion,bugprone-*,performance-*,modernize-*,-llvm-header-guard,-hicpp-use-auto,-modernize-use-trailing-return-type,-hicpp-uppercase-literal-suffix,-hicpp-use-nullptr,-modernize-use-nullptr,-google-runtime-int,-llvm-include-order,-google-runtime-references,-readability-magic-numbers,-readability-isolate-declaration,-hicpp-vararg,-google-readability-todo,-bugprone-macro-parentheses,-misc-unused-using-decls,-readability-else-after-return,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-hicpp-braces-around-statements,-hicpp-noexcept-move'
+Checks:          'clang-diagnostic-*,clang-analyzer-*,*,-fuchsia-*,-cppcoreguidelines-*,-misc-misplaced-const,-hicpp-no-array-decay,-readability-implicit-bool-conversion,bugprone-*,performance-*,modernize-*,-llvm-header-guard,-hicpp-use-auto,-modernize-use-trailing-return-type,-hicpp-uppercase-literal-suffix,-hicpp-use-nullptr,-modernize-use-nullptr,-google-runtime-int,-llvm-include-order,-google-runtime-references,-readability-magic-numbers,-readability-isolate-declaration,-hicpp-vararg,-google-readability-todo,-bugprone-macro-parentheses,-misc-unused-using-decls,-readability-else-after-return,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-hicpp-braces-around-statements,-hicpp-noexcept-move,-llvmlibc-*,-altera-*,-hicpp-explicit-conversions'
 WarningsAsErrors: ''
 HeaderFilterRegex: ''
 AnalyzeTemporaryDtors: true
diff --git a/src/backend/common/jit/NodeIterator.hpp b/src/backend/common/jit/NodeIterator.hpp
index 82e916c7ef..7359316c65 100644
--- a/src/backend/common/jit/NodeIterator.hpp
+++ b/src/backend/common/jit/NodeIterator.hpp
@@ -46,7 +46,7 @@ class NodeIterator {
     /// NodeIterator Constructor
     ///
     /// \param[in] root The root node of the tree
-    NodeIterator(pointer root) : tree{root}, index(0) {
+    NodeIterator(pointer root) : tree{root} {
         tree.reserve(root->getHeight() * 8);
     }
 
diff --git a/src/backend/cuda/reduce_impl.hpp b/src/backend/cuda/reduce_impl.hpp
index eb8a5b9a48..bbb91d79d9 100644
--- a/src/backend/cuda/reduce_impl.hpp
+++ b/src/backend/cuda/reduce_impl.hpp
@@ -172,8 +172,8 @@ void reduce_by_key_dim(Array<Tk> &keys_out, Array<To> &vals_out,
                         t_reduced_keys, t_reduced_vals, dim, folded_dim_sz);
             POST_LAUNCH_CHECK();
 
-            swap(t_reduced_keys, reduced_keys);
-            swap(t_reduced_vals, reduced_vals);
+            std::swap(t_reduced_keys, reduced_keys);
+            std::swap(t_reduced_vals, reduced_vals);
             reduce_host_event.block();
         }
     } while (needs_another_reduction_host ||
@@ -319,8 +319,8 @@ void reduce_by_key_first(Array<Tk> &keys_out, Array<To> &vals_out,
                         t_reduced_keys, t_reduced_vals, odims[2]);
             POST_LAUNCH_CHECK();
 
-            swap(t_reduced_keys, reduced_keys);
-            swap(t_reduced_vals, reduced_vals);
+            std::swap(t_reduced_keys, reduced_keys);
+            std::swap(t_reduced_vals, reduced_vals);
             reduce_host_event.block();
         }
     } while (needs_another_reduction_host ||

From bb5e46557caf3f3a4188bfc92f1d54efce61ede9 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 20 Feb 2023 20:21:17 -0500
Subject: [PATCH 208/473] Add function that returns basic OpenCL build flags
 for each device

---
 src/backend/oneapi/device_manager.cpp | 23 ++++++++++++++++++++---
 src/backend/oneapi/device_manager.hpp |  3 +++
 src/backend/oneapi/platform.cpp       |  8 ++++++++
 src/backend/oneapi/platform.hpp       |  2 ++
 4 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index 54878e3fea..c559fafbbd 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -10,6 +10,7 @@
 #include <common/graphics_common.hpp>
 
 #include <GraphicsResourceManager.hpp>
+#include <build_version.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
@@ -17,10 +18,8 @@
 #include <common/util.hpp>
 #include <device_manager.hpp>
 #include <err_oneapi.hpp>
-#include <platform.hpp>  //TODO: blas.hpp? y tho, also Array.hpp
-//#include <errorcodes.hpp>
-#include <build_version.hpp>
 #include <memory.hpp>
+#include <platform.hpp>
 #include <af/oneapi.h>
 #include <af/version.h>
 
@@ -44,6 +43,8 @@ using std::vector;
 using sycl::device;
 using sycl::platform;
 
+using af::dtype_traits;
+
 namespace arrayfire {
 namespace oneapi {
 
@@ -118,6 +119,22 @@ DeviceManager::DeviceManager()
                 // mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
                 // mPlatforms.push_back(getPlatformEnum(*devices[i]));
                 mDevices.emplace_back(std::move(devices[i]));
+
+                std::string options;
+#ifdef AF_WITH_FAST_MATH
+                options = fmt::format(" -D dim_t=CL3.0 -cl-fast-relaxed-math",
+                                      dtype_traits<dim_t>::getName());
+#else
+                options = fmt::format(" -cl-std=CL3.0 -D dim_t={}",
+                                      dtype_traits<dim_t>::getName());
+#endif
+                mBaseOpenCLBuildFlags.push_back(options);
+                if (mDevices.back()->has(sycl::aspect::fp64)) {
+                    mBaseOpenCLBuildFlags.back() += " -DUSE_DOUBLE";
+                }
+                if (mDevices.back()->has(sycl::aspect::fp16)) {
+                    mBaseOpenCLBuildFlags.back() += " -D USE_HALF";
+                }
             } catch (sycl::exception& err) {
                 AF_TRACE("Error creating context for device {} with error {}\n",
                          devices[i]->get_info<sycl::info::device::name>(),
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
index 36824539b2..37c5cbe087 100644
--- a/src/backend/oneapi/device_manager.hpp
+++ b/src/backend/oneapi/device_manager.hpp
@@ -85,6 +85,8 @@ class DeviceManager {
 
     friend const sycl::device& getDevice(int id);
 
+    friend const std::string& getActiveDeviceBaseBuildFlags();
+
     friend size_t getDeviceMemorySize(int device);
 
     friend bool isGLSharingSupported();
@@ -137,6 +139,7 @@ class DeviceManager {
     std::vector<std::unique_ptr<sycl::context>> mContexts;
     std::vector<std::unique_ptr<sycl::queue>> mQueues;
     std::vector<bool> mIsGLSharingOn;
+    std::vector<std::string> mBaseOpenCLBuildFlags;
     std::vector<int> mDeviceTypes;
     std::vector<int> mPlatforms;
     unsigned mUserDeviceOffset;
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index ce3ad2e099..dc2c8a9766 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -304,6 +304,14 @@ const sycl::device& getDevice(int id) {
     return *(devMngr.mDevices[id]);
 }
 
+const std::string& getActiveDeviceBaseBuildFlags() {
+    device_id_t& devId     = tlocalActiveDeviceId();
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+    return devMngr.mBaseOpenCLBuildFlags[get<1>(devId)];
+}
+
 size_t getDeviceMemorySize(int device) {
     DeviceManager& devMngr = DeviceManager::getInstance();
 
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index af579573d8..b508f6fc4e 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -62,6 +62,8 @@ sycl::queue* getQueueHandle(int device);
 
 const sycl::device& getDevice(int id = -1);
 
+const std::string& getActiveDeviceBaseBuildFlags();
+
 size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();

From 95d433717eb38841ae76cedb55d624d977dd9bd1 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 20 Feb 2023 20:40:50 -0500
Subject: [PATCH 209/473] Refactor some JIT tests to use new style asserts

---
 test/jit.cpp | 37 ++++++++++++-------------------------
 1 file changed, 12 insertions(+), 25 deletions(-)

diff --git a/test/jit.cpp b/test/jit.cpp
index 101580a488..3848a22242 100644
--- a/test/jit.cpp
+++ b/test/jit.cpp
@@ -89,12 +89,7 @@ TEST(JIT, CPP_JIT_Reset_Binary) {
     array g = d - c;
     g.eval();
 
-    vector<float> hf(f.elements());
-    vector<float> hg(g.elements());
-    f.host(&hf[0]);
-    g.host(&hg[0]);
-
-    for (int i = 0; i < (int)f.elements(); i++) { ASSERT_EQ(hf[i], -hg[i]); }
+    ASSERT_ARRAYS_NEAR(f, -g, 1e-5);
 }
 
 TEST(JIT, CPP_JIT_Reset_Unary) {
@@ -109,12 +104,7 @@ TEST(JIT, CPP_JIT_Reset_Unary) {
     array g = d - c;
     g.eval();
 
-    vector<float> hf(f.elements());
-    vector<float> hg(g.elements());
-    f.host(&hf[0]);
-    g.host(&hg[0]);
-
-    for (int i = 0; i < (int)f.elements(); i++) { ASSERT_EQ(hf[i], -hg[i]); }
+    ASSERT_ARRAYS_EQ(f, -g);
 }
 
 TEST(JIT, CPP_Multi_linear) {
@@ -142,7 +132,7 @@ TEST(JIT, CPP_Multi_linear) {
     ASSERT_VEC_ARRAY_EQ(goldy, dim4(num), y);
 }
 
-TEST(JIT, CPP_strided) {
+TEST(JIT, CPP_gforSet_strided) {
     const int num = 1024;
     gforSet(true);
     array a = randu(num, 1, s32);
@@ -155,23 +145,23 @@ TEST(JIT, CPP_strided) {
 
     vector<int> ha(num);
     vector<int> hb(num);
-    vector<int> hx(num * num);
-    vector<int> hy(num * num);
 
     a.host(&ha[0]);
     b.host(&hb[0]);
-    x.host(&hx[0]);
-    y.host(&hy[0]);
 
+    vector<int> hapb(num * num);
+    vector<int> hamb(num * num);
     for (int j = 0; j < num; j++) {
         for (int i = 0; i < num; i++) {
-            ASSERT_EQ((ha[i] + hb[j]), hx[j * num + i]);
-            ASSERT_EQ((ha[i] - hb[j]), hy[j * num + i]);
+            hapb[j * num + i] = ha[i] + hb[j];
+            hamb[j * num + i] = ha[i] - hb[j];
         }
     }
+    ASSERT_VEC_ARRAY_EQ(hapb, dim4(num, num), x);
+    ASSERT_VEC_ARRAY_EQ(hamb, dim4(num, num), y);
 }
 
-TEST(JIT, CPP_Multi_strided) {
+TEST(JIT, CPP_gforSet_Multi_strided) {
     const int num = 1024;
     gforSet(true);
     array a = randu(num, 1, s32);
@@ -285,14 +275,11 @@ TEST(JIT, NonLinearLargeY) {
 
     a.host(ha.data());
     b.host(hb.data());
-    c.host(hc.data());
 
     for (int j = 0; j < d1; j++) {
-        for (int i = 0; i < d0; i++) {
-            ASSERT_EQ(hc[i + j * d0], ha[i] + hb[j])
-                << " at " << i << " , " << j;
-        }
+        for (int i = 0; i < d0; i++) { hc[i + j * d0] = ha[i] + hb[j]; }
     }
+    ASSERT_VEC_ARRAY_EQ(hc, dim4(d0, d1), c);
 }
 
 TEST(JIT, NonLinearLargeX) {

From ef1823bd25d16d46d5e5b23cad230c2d4e4bbf7d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 21 Feb 2023 20:26:52 -0500
Subject: [PATCH 210/473] Improve compile times by using more specific headers,
 etc.

---
 include/af/oneapi.h                           | 11 -----
 src/api/c/handle.cpp                          | 46 +++++++++++++++++
 src/api/c/handle.hpp                          | 26 +---------
 src/api/c/plot.cpp                            |  1 +
 src/api/c/vector_field.cpp                    |  1 +
 src/backend/common/ArrayInfo.cpp              | 42 ++++++++++++++++
 src/backend/common/ArrayInfo.hpp              | 40 +--------------
 src/backend/common/MemoryManagerBase.hpp      |  2 +-
 src/backend/common/err_common.cpp             | 14 +++---
 src/backend/common/err_common.hpp             |  3 +-
 src/backend/common/half.hpp                   |  5 ++
 src/backend/common/jit/BufferNodeBase.hpp     |  4 +-
 src/backend/oneapi/Array.cpp                  |  2 +
 src/backend/oneapi/Array.hpp                  | 27 +++++-----
 src/backend/oneapi/Event.hpp                  |  5 +-
 src/backend/oneapi/Param.hpp                  |  8 ++-
 src/backend/oneapi/backend.hpp                |  6 ++-
 src/backend/oneapi/device_manager.cpp         | 11 ++++-
 src/backend/oneapi/device_manager.hpp         | 13 +++--
 src/backend/oneapi/kernel/approx1.hpp         |  1 -
 src/backend/oneapi/kernel/assign.hpp          | 11 ++---
 src/backend/oneapi/kernel/bilateral.hpp       |  3 +-
 src/backend/oneapi/kernel/diagonal.hpp        |  2 -
 src/backend/oneapi/kernel/diff.hpp            |  1 -
 src/backend/oneapi/kernel/histogram.hpp       |  3 +-
 src/backend/oneapi/kernel/interp.hpp          |  3 ++
 src/backend/oneapi/kernel/iota.hpp            | 10 ++--
 src/backend/oneapi/kernel/mean.hpp            | 39 ++++++---------
 src/backend/oneapi/kernel/memcopy.hpp         | 25 +++-------
 src/backend/oneapi/kernel/random_engine.hpp   |  2 +-
 .../oneapi/kernel/random_engine_write.hpp     | 38 +++++++-------
 src/backend/oneapi/kernel/reduce_all.hpp      | 17 ++++---
 src/backend/oneapi/kernel/reduce_dim.hpp      | 17 +++----
 src/backend/oneapi/kernel/reduce_first.hpp    | 49 +++++++++----------
 src/backend/oneapi/kernel/reorder.hpp         |  1 -
 src/backend/oneapi/kernel/resize.hpp          |  2 +
 src/backend/oneapi/kernel/scan_dim.hpp        | 31 +++++-------
 src/backend/oneapi/kernel/scan_first.hpp      | 40 ++++++---------
 src/backend/oneapi/kernel/transpose.hpp       | 14 ++----
 src/backend/oneapi/kernel/triangle.hpp        |  1 -
 src/backend/oneapi/kernel/where.hpp           |  9 ++--
 src/backend/oneapi/memory.cpp                 |  9 ++--
 src/backend/oneapi/memory.hpp                 |  2 +
 src/backend/oneapi/platform.cpp               | 27 ++++++++--
 src/backend/oneapi/platform.hpp               | 21 ++++++--
 src/backend/oneapi/print.hpp                  |  2 +
 src/backend/oneapi/types.hpp                  |  3 +-
 47 files changed, 343 insertions(+), 307 deletions(-)

diff --git a/include/af/oneapi.h b/include/af/oneapi.h
index baf28bf73b..b6a3da15fa 100644
--- a/include/af/oneapi.h
+++ b/include/af/oneapi.h
@@ -9,23 +9,12 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <af/defines.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#if AF_API_VERSION >= 39
-typedef enum
-{
-    AF_ONEAPI_DEVICE_TYPE_CPU     = (int)sycl::info::device_type::cpu,
-    AF_ONEAPI_DEVICE_TYPE_GPU     = (int)sycl::info::device_type::gpu,
-    AF_ONEAPI_DEVICE_TYPE_ACC     = (int)sycl::info::device_type::accelerator,
-    AF_ONEAPI_DEVICE_TYPE_UNKNOWN = -1
-} af_oneapi_device_type;
-#endif
-
 #if AF_API_VERSION >= 39
 typedef enum
 {
diff --git a/src/api/c/handle.cpp b/src/api/c/handle.cpp
index 0d9f3d2aec..a432d8a720 100644
--- a/src/api/c/handle.cpp
+++ b/src/api/c/handle.cpp
@@ -136,4 +136,50 @@ dim4 verifyDims(const unsigned ndims, const dim_t *const dims) {
     return d;
 }
 
+template<typename T>
+void releaseHandle(const af_array arr) {
+    auto &Arr      = getArray<T>(arr);
+    int old_device = detail::getActiveDeviceId();
+    int array_id   = Arr.getDevId();
+    if (array_id != old_device) {
+        detail::setDevice(array_id);
+        detail::destroyArray(static_cast<detail::Array<T> *>(arr));
+        detail::setDevice(old_device);
+    } else {
+        detail::destroyArray(static_cast<detail::Array<T> *>(arr));
+    }
+}
+
+template<typename T>
+detail::Array<T> &getCopyOnWriteArray(const af_array &arr) {
+    detail::Array<T> *A = static_cast<detail::Array<T> *>(arr);
+
+    if ((af_dtype)af::dtype_traits<T>::af_type != A->getType())
+        AF_ERROR("Invalid type for input array.", AF_ERR_INTERNAL);
+
+    ARG_ASSERT(0, A->isSparse() == false);
+
+    if (A->useCount() > 1) { *A = copyArray(*A); }
+
+    return *A;
+}
+
+#define INSTANTIATE(TYPE)                                  \
+    template void releaseHandle<TYPE>(const af_array arr); \
+    template detail::Array<TYPE> &getCopyOnWriteArray<TYPE>(const af_array &arr)
+
+INSTANTIATE(float);
+INSTANTIATE(double);
+INSTANTIATE(cfloat);
+INSTANTIATE(cdouble);
+INSTANTIATE(int);
+INSTANTIATE(uint);
+INSTANTIATE(intl);
+INSTANTIATE(uintl);
+INSTANTIATE(uchar);
+INSTANTIATE(char);
+INSTANTIATE(short);
+INSTANTIATE(ushort);
+INSTANTIATE(half);
+
 }  // namespace arrayfire
diff --git a/src/api/c/handle.hpp b/src/api/c/handle.hpp
index b19de9c143..97243ac353 100644
--- a/src/api/c/handle.hpp
+++ b/src/api/c/handle.hpp
@@ -104,32 +104,10 @@ af_array copyArray(const af_array in) {
 }
 
 template<typename T>
-void releaseHandle(const af_array arr) {
-    auto &Arr      = getArray<T>(arr);
-    int old_device = detail::getActiveDeviceId();
-    int array_id   = Arr.getDevId();
-    if (array_id != old_device) {
-        detail::setDevice(array_id);
-        detail::destroyArray(static_cast<detail::Array<T> *>(arr));
-        detail::setDevice(old_device);
-    } else {
-        detail::destroyArray(static_cast<detail::Array<T> *>(arr));
-    }
-}
+void releaseHandle(const af_array arr);
 
 template<typename T>
-detail::Array<T> &getCopyOnWriteArray(const af_array &arr) {
-    detail::Array<T> *A = static_cast<detail::Array<T> *>(arr);
-
-    if ((af_dtype)af::dtype_traits<T>::af_type != A->getType())
-        AF_ERROR("Invalid type for input array.", AF_ERR_INTERNAL);
-
-    ARG_ASSERT(0, A->isSparse() == false);
-
-    if (A->useCount() > 1) { *A = copyArray(*A); }
-
-    return *A;
-}
+detail::Array<T> &getCopyOnWriteArray(const af_array &arr);
 
 }  // namespace arrayfire
 
diff --git a/src/api/c/plot.cpp b/src/api/c/plot.cpp
index b60448593f..3cf03d05cf 100644
--- a/src/api/c/plot.cpp
+++ b/src/api/c/plot.cpp
@@ -17,6 +17,7 @@
 #include <common/graphics_common.hpp>
 #include <handle.hpp>
 #include <join.hpp>
+#include <platform.hpp>
 #include <plot.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
diff --git a/src/api/c/vector_field.cpp b/src/api/c/vector_field.cpp
index a6bd0e07cc..a46d1eed47 100644
--- a/src/api/c/vector_field.cpp
+++ b/src/api/c/vector_field.cpp
@@ -16,6 +16,7 @@
 #include <common/graphics_common.hpp>
 #include <handle.hpp>
 #include <join.hpp>
+#include <platform.hpp>
 #include <reduce.hpp>
 #include <transpose.hpp>
 #include <vector_field.hpp>
diff --git a/src/backend/common/ArrayInfo.cpp b/src/backend/common/ArrayInfo.cpp
index b83380fe88..d919c942f8 100644
--- a/src/backend/common/ArrayInfo.cpp
+++ b/src/backend/common/ArrayInfo.cpp
@@ -32,6 +32,48 @@ dim4 calcStrides(const dim4 &parentDim) {
     return out;
 }
 
+ArrayInfo::ArrayInfo(unsigned id, af::dim4 size, dim_t offset_, af::dim4 stride,
+                     af_dtype af_type)
+    : devId(id)
+    , type(af_type)
+    , dim_size(size)
+    , offset(offset_)
+    , dim_strides(stride)
+    , is_sparse(false) {
+    setId(id);
+    static_assert(std::is_move_assignable<ArrayInfo>::value,
+                  "ArrayInfo is not move assignable");
+    static_assert(std::is_move_constructible<ArrayInfo>::value,
+                  "ArrayInfo is not move constructible");
+    static_assert(
+        offsetof(ArrayInfo, devId) == 0,
+        "ArrayInfo::devId must be the first member variable of ArrayInfo. \
+                   devId is used to encode the backend into the integer. \
+                   This is then used in the unified backend to check mismatched arrays.");
+    static_assert(std::is_standard_layout<ArrayInfo>::value,
+                  "ArrayInfo must be a standard layout type");
+}
+
+ArrayInfo::ArrayInfo(unsigned id, af::dim4 size, dim_t offset_, af::dim4 stride,
+                     af_dtype af_type, bool sparse)
+    : devId(id)
+    , type(af_type)
+    , dim_size(size)
+    , offset(offset_)
+    , dim_strides(stride)
+    , is_sparse(sparse) {
+    setId(id);
+    static_assert(
+        offsetof(ArrayInfo, devId) == 0,
+        "ArrayInfo::devId must be the first member variable of ArrayInfo. \
+                   devId is used to encode the backend into the integer. \
+                   This is then used in the unified backend to check mismatched arrays.");
+    static_assert(std::is_nothrow_move_assignable<ArrayInfo>::value,
+                  "ArrayInfo is not nothrow move assignable");
+    static_assert(std::is_nothrow_move_constructible<ArrayInfo>::value,
+                  "ArrayInfo is not nothrow move constructible");
+}
+
 unsigned ArrayInfo::getDevId() const {
     // The actual device ID is only stored in the first 8 bits of devId
     // See ArrayInfo.hpp for more
diff --git a/src/backend/common/ArrayInfo.hpp b/src/backend/common/ArrayInfo.hpp
index f2a99c0b1e..aae9e7b6a7 100644
--- a/src/backend/common/ArrayInfo.hpp
+++ b/src/backend/common/ArrayInfo.hpp
@@ -49,44 +49,10 @@ class ArrayInfo {
 
    public:
     ArrayInfo(unsigned id, af::dim4 size, dim_t offset_, af::dim4 stride,
-              af_dtype af_type)
-        : devId(id)
-        , type(af_type)
-        , dim_size(size)
-        , offset(offset_)
-        , dim_strides(stride)
-        , is_sparse(false) {
-        setId(id);
-        static_assert(std::is_move_assignable<ArrayInfo>::value,
-                      "ArrayInfo is not move assignable");
-        static_assert(std::is_move_constructible<ArrayInfo>::value,
-                      "ArrayInfo is not move constructible");
-        static_assert(
-            offsetof(ArrayInfo, devId) == 0,
-            "ArrayInfo::devId must be the first member variable of ArrayInfo. \
-                   devId is used to encode the backend into the integer. \
-                   This is then used in the unified backend to check mismatched arrays.");
-    }
+              af_dtype af_type);
 
     ArrayInfo(unsigned id, af::dim4 size, dim_t offset_, af::dim4 stride,
-              af_dtype af_type, bool sparse)
-        : devId(id)
-        , type(af_type)
-        , dim_size(size)
-        , offset(offset_)
-        , dim_strides(stride)
-        , is_sparse(sparse) {
-        setId(id);
-        static_assert(
-            offsetof(ArrayInfo, devId) == 0,
-            "ArrayInfo::devId must be the first member variable of ArrayInfo. \
-                   devId is used to encode the backend into the integer. \
-                   This is then used in the unified backend to check mismatched arrays.");
-        static_assert(std::is_nothrow_move_assignable<ArrayInfo>::value,
-                      "ArrayInfo is not nothrow move assignable");
-        static_assert(std::is_nothrow_move_constructible<ArrayInfo>::value,
-                      "ArrayInfo is not nothrow move constructible");
-    }
+              af_dtype af_type, bool sparse);
 
     ArrayInfo()                       = default;
     ArrayInfo(const ArrayInfo& other) = default;
@@ -170,8 +136,6 @@ class ArrayInfo {
 
     bool isSparse() const;
 };
-static_assert(std::is_standard_layout<ArrayInfo>::value,
-              "ArrayInfo must be a standard layout type");
 
 af::dim4 toDims(const std::vector<af_seq>& seqs, const af::dim4& parentDims);
 
diff --git a/src/backend/common/MemoryManagerBase.hpp b/src/backend/common/MemoryManagerBase.hpp
index 569154695e..ceeb26c605 100644
--- a/src/backend/common/MemoryManagerBase.hpp
+++ b/src/backend/common/MemoryManagerBase.hpp
@@ -9,8 +9,8 @@
 
 #pragma once
 
-#include <Event.hpp>
 #include <common/AllocatorInterface.hpp>
+#include <af/defines.h>
 
 #include <cstddef>
 #include <memory>
diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index 68514bac29..c7dc95b8fd 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -40,16 +40,16 @@ AfError::AfError(const char *const func, const char *const file, const int line,
     : logic_error(message)
     , functionName(func)
     , fileName(file)
-    , st_(move(st))
+    , st_(std::move(st))
     , lineNumber(line)
     , error(err) {}
 
 AfError::AfError(string func, string file, const int line,
                  const string &message, af_err err, stacktrace st)
     : logic_error(message)
-    , functionName(move(func))
-    , fileName(move(file))
-    , st_(move(st))
+    , functionName(std::move(func))
+    , fileName(std::move(file))
+    , st_(std::move(st))
     , lineNumber(line)
     , error(err) {}
 
@@ -66,7 +66,7 @@ AfError::~AfError() noexcept = default;
 TypeError::TypeError(const char *const func, const char *const file,
                      const int line, const int index, const af_dtype type,
                      stacktrace st)
-    : AfError(func, file, line, "Invalid data type", AF_ERR_TYPE, move(st))
+    : AfError(func, file, line, "Invalid data type", AF_ERR_TYPE, std::move(st))
     , errTypeName(getName(type))
     , argIndex(index) {}
 
@@ -77,7 +77,7 @@ int TypeError::getArgIndex() const noexcept { return argIndex; }
 ArgumentError::ArgumentError(const char *const func, const char *const file,
                              const int line, const int index,
                              const char *const expectString, stacktrace st)
-    : AfError(func, file, line, "Invalid argument", AF_ERR_ARG, move(st))
+    : AfError(func, file, line, "Invalid argument", AF_ERR_ARG, std::move(st))
     , expected(expectString)
     , argIndex(index) {}
 
@@ -91,7 +91,7 @@ SupportError::SupportError(const char *const func, const char *const file,
                            const int line, const char *const back,
                            stacktrace st)
     : AfError(func, file, line, "Unsupported Error", AF_ERR_NOT_SUPPORTED,
-              move(st))
+              std::move(st))
     , backend(back) {}
 
 const string &SupportError::getBackendName() const noexcept { return backend; }
diff --git a/src/backend/common/err_common.hpp b/src/backend/common/err_common.hpp
index a2c55742e0..79c9d029d7 100644
--- a/src/backend/common/err_common.hpp
+++ b/src/backend/common/err_common.hpp
@@ -17,11 +17,10 @@
 #include <common/defines.hpp>
 #include <af/defines.h>
 
-#include <cassert>
 #include <sstream>
 #include <stdexcept>
 #include <string>
-#include <vector>
+#include <utility>
 
 class AfError : public std::logic_error {
     std::string functionName;
diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index f653024fb1..57545f4bcd 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -32,6 +32,10 @@
 #endif
 #endif
 
+#ifdef AF_ONEAPI
+#include <sycl/half_type.hpp>
+#endif
+
 #include <backend.hpp>
 
 #ifdef __CUDACC_RTC__
@@ -41,6 +45,7 @@ using uint16_t = unsigned short;
 #define AF_CONSTEXPR constexpr
 #else
 #include <af/compilers.h>
+#include <cmath>
 #include <cstring>
 #include <ostream>
 #include <string>
diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index 5af3a216d0..061aa37a8c 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -8,11 +8,13 @@
  ********************************************************/
 
 #pragma once
-#include <backend.hpp>
 #include <common/jit/Node.hpp>
 #include <jit/kernel_generators.hpp>
 
+#include <backend.hpp>
+
 #include <cstring>
+#include <memory>
 #include <sstream>
 
 namespace arrayfire {
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index ab880732e3..a55915edb8 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -9,6 +9,8 @@
 
 #include <Array.hpp>
 
+#include <Param.hpp>
+#include <common/MemoryManagerBase.hpp>
 #include <common/half.hpp>
 #include <common/jit/NodeIterator.hpp>
 #include <common/jit/ScalarNode.hpp>
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index 3d74a897ba..d907cad92f 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -9,20 +9,14 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
-#include <Param.hpp>
 #include <backend.hpp>
 #include <common/ArrayInfo.hpp>
-#include <common/MemoryManagerBase.hpp>
-#include <common/jit/Node.hpp>
-//#include <err_opencl.hpp>
-//#include <jit/BufferNode.hpp>
-//#include <memory.hpp>
-#include <platform.hpp>
+#include <kernel/KParam.hpp>
 #include <traits.hpp>
 #include <types.hpp>
+#include <af/dim4.hpp>
 
-//#include <af/dim4.hpp>
+#include <sycl/buffer.hpp>
 
 #include <nonstd/span.hpp>
 #include <algorithm>
@@ -30,14 +24,25 @@
 #include <memory>
 #include <vector>
 
+enum class kJITHeuristics;
+
+namespace arrayfire {
 namespace common {
 template<typename T>
 class SparseArray;
-}
 
-namespace arrayfire {
+class Node;
+
+using Node_ptr = std::shared_ptr<Node>;
+
+}  // namespace common
 namespace oneapi {
 
+template<typename T>
+struct Param;
+template<typename T>
+struct AParam;
+
 template<typename T>
 using Buffer_ptr = std::shared_ptr<sycl::buffer<T>>;
 using af::dim4;
diff --git a/src/backend/oneapi/Event.hpp b/src/backend/oneapi/Event.hpp
index 1bdedf34ad..90aaf1b2ca 100644
--- a/src/backend/oneapi/Event.hpp
+++ b/src/backend/oneapi/Event.hpp
@@ -8,10 +8,13 @@
  ********************************************************/
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <common/EventBase.hpp>
+
 #include <af/event.h>
 
+#include <sycl/event.hpp>
+#include <sycl/queue.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 class OneAPIEventPolicy {
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index 01088f86b7..cca1d519f6 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -9,8 +9,14 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <kernel/KParam.hpp>
+#include <types.hpp>
+
+#include <af/dim4.hpp>
+
+#include <sycl/accessor.hpp>
+#include <sycl/buffer.hpp>
+#include <sycl/handler.hpp>
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/backend.hpp b/src/backend/oneapi/backend.hpp
index 3366912b3b..2eb14151d8 100644
--- a/src/backend/oneapi/backend.hpp
+++ b/src/backend/oneapi/backend.hpp
@@ -7,16 +7,18 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include "types.hpp"
 #ifdef __DH__
 #undef __DH__
 #endif
 
 #ifdef __CUDACC__
-#include <opencl_runtime.h>
 #define __DH__ __device__ __host__
 #else
 #define __DH__
 #endif
 
+namespace arrayfire {
+namespace oneapi {}
+}  // namespace arrayfire
+
 namespace detail = arrayfire::oneapi;
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index c559fafbbd..aea4398c66 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -7,22 +7,29 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <common/graphics_common.hpp>
+#include <device_manager.hpp>
 
 #include <GraphicsResourceManager.hpp>
 #include <build_version.hpp>
 #include <common/DefaultMemoryManager.hpp>
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
+#include <common/graphics_common.hpp>
 #include <common/host_memory.hpp>
 #include <common/util.hpp>
-#include <device_manager.hpp>
 #include <err_oneapi.hpp>
 #include <memory.hpp>
 #include <platform.hpp>
 #include <af/oneapi.h>
 #include <af/version.h>
 
+#include <sycl/context.hpp>
+#include <sycl/device.hpp>
+#include <sycl/exception.hpp>
+#include <sycl/exception_list.hpp>
+#include <sycl/platform.hpp>
+#include <sycl/queue.hpp>
+
 #include <algorithm>
 #include <iterator>
 #include <sstream>
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
index 37c5cbe087..198ddd07e0 100644
--- a/src/backend/oneapi/device_manager.hpp
+++ b/src/backend/oneapi/device_manager.hpp
@@ -9,7 +9,10 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
+#include <sycl/context.hpp>
+#include <sycl/device.hpp>
+#include <sycl/queue.hpp>
+
 #include <memory>
 #include <mutex>
 #include <string>
@@ -100,12 +103,12 @@ class DeviceManager {
 
     friend int setDevice(int device);
 
-    friend void addDeviceContext(sycl::device dev, sycl::context ctx,
-                                 sycl::queue que);
+    friend void addDeviceContext(sycl::device& dev, sycl::context& ctx,
+                                 sycl::queue& que);
 
-    friend void setDeviceContext(sycl::device dev, sycl::context ctx);
+    friend void setDeviceContext(sycl::device& dev, sycl::context& ctx);
 
-    friend void removeDeviceContext(sycl::device dev, sycl::context ctx);
+    friend void removeDeviceContext(sycl::device& dev, sycl::context& ctx);
 
     friend int getActiveDeviceType();
 
diff --git a/src/backend/oneapi/kernel/approx1.hpp b/src/backend/oneapi/kernel/approx1.hpp
index f520719749..4d9d039f1b 100644
--- a/src/backend/oneapi/kernel/approx1.hpp
+++ b/src/backend/oneapi/kernel/approx1.hpp
@@ -151,7 +151,6 @@ void approx1(Param<Ty> yo, const Param<Ty> yi, const Param<Tp> xo,
         write_accessor<Ty> yoAcc{*yo.data, h};
         read_accessor<Ty> yiAcc{*yi.data, h};
         read_accessor<Tp> xoAcc{*xo.data, h};
-        sycl::stream debugStream(128, 128, h);
 
         h.parallel_for(sycl::nd_range{global, local},
                        approx1Kernel<Ty, Tp, order>(
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 162c1d5254..0876b9e16c 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -45,8 +45,7 @@ class assignKernel {
     assignKernel(sycl::accessor<T> out, KParam oInfo, sycl::accessor<T> in,
                  KParam iInfo, AssignKernelParam_t p, sycl::accessor<uint> ptr0,
                  sycl::accessor<uint> ptr1, sycl::accessor<uint> ptr2,
-                 sycl::accessor<uint> ptr3, const int nBBS0, const int nBBS1,
-                 sycl::stream debug)
+                 sycl::accessor<uint> ptr3, const int nBBS0, const int nBBS1)
         : out_(out)
         , in_(in)
         , oInfo_(oInfo)
@@ -57,8 +56,7 @@ class assignKernel {
         , ptr2_(ptr2)
         , ptr3_(ptr3)
         , nBBS0_(nBBS0)
-        , nBBS1_(nBBS1)
-        , debug_(debug) {}
+        , nBBS1_(nBBS1) {}
 
     void operator()(sycl::nd_item<2> it) const {
         // retrive booleans that tell us which index to use
@@ -108,7 +106,6 @@ class assignKernel {
     AssignKernelParam_t p_;
     sycl::accessor<uint> ptr0_, ptr1_, ptr2_, ptr3_;
     const int nBBS0_, nBBS1_;
-    sycl::stream debug_;
 };
 
 template<typename T>
@@ -134,12 +131,10 @@ void assign(Param<T> out, const Param<T> in, const AssignKernelParam_t& p,
         auto bptr2 = bPtr[2]->get_access(h);
         auto bptr3 = bPtr[3]->get_access(h);
 
-        sycl::stream debug_stream(2048, 128, h);
-
         h.parallel_for(
             sycl::nd_range<2>(global, local),
             assignKernel<T>(out_acc, out.info, in_acc, in.info, p, bptr0, bptr1,
-                            bptr2, bptr3, blk_x, blk_y, debug_stream));
+                            bptr2, bptr3, blk_x, blk_y));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/bilateral.hpp b/src/backend/oneapi/kernel/bilateral.hpp
index 3814084c1b..cb3d323f07 100644
--- a/src/backend/oneapi/kernel/bilateral.hpp
+++ b/src/backend/oneapi/kernel/bilateral.hpp
@@ -15,6 +15,8 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/builtins.hpp>
+
 #include <string>
 #include <vector>
 
@@ -200,7 +202,6 @@ void bilateral(Param<outType> out, const Param<inType> in, const float s_sigma,
     getQueue().submit([&](sycl::handler& h) {
         auto inAcc  = in.data->get_access(h);
         auto outAcc = out.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
 
         auto localMem = local_accessor<outType, 1>(num_shrd_elems, h);
         auto gauss2d  = local_accessor<outType, 1>(num_shrd_elems, h);
diff --git a/src/backend/oneapi/kernel/diagonal.hpp b/src/backend/oneapi/kernel/diagonal.hpp
index a21c1abd11..c49d9871e3 100644
--- a/src/backend/oneapi/kernel/diagonal.hpp
+++ b/src/backend/oneapi/kernel/diagonal.hpp
@@ -82,7 +82,6 @@ static void diagCreate(Param<T> out, Param<T> in, int num) {
     getQueue().submit([&](sycl::handler &h) {
         auto oData = out.data->get_access(h);
         auto iData = in.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
 
         h.parallel_for(sycl::nd_range{global, local},
                        diagCreateKernel<T>(oData, out.info, iData, in.info, num,
@@ -151,7 +150,6 @@ static void diagExtract(Param<T> out, Param<T> in, int num) {
     getQueue().submit([&](sycl::handler &h) {
         auto oData = out.data->get_access(h);
         auto iData = in.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
 
         h.parallel_for(sycl::nd_range{global, local},
                        diagExtractKernel<T>(oData, out.info, iData, in.info,
diff --git a/src/backend/oneapi/kernel/diff.hpp b/src/backend/oneapi/kernel/diff.hpp
index bd3d925d3b..f5a73c8c40 100644
--- a/src/backend/oneapi/kernel/diff.hpp
+++ b/src/backend/oneapi/kernel/diff.hpp
@@ -111,7 +111,6 @@ void diff(Param<T> out, const Param<T> in, const unsigned indims,
     getQueue().submit([&](sycl::handler &h) {
         auto inAcc  = in.data->get_access(h);
         auto outAcc = out.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
 
         h.parallel_for(
             sycl::nd_range{global, local},
diff --git a/src/backend/oneapi/kernel/histogram.hpp b/src/backend/oneapi/kernel/histogram.hpp
index 99ee437ae3..ea6c4c229a 100644
--- a/src/backend/oneapi/kernel/histogram.hpp
+++ b/src/backend/oneapi/kernel/histogram.hpp
@@ -15,6 +15,8 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/atomic_ref.hpp>
+
 #include <string>
 #include <vector>
 
@@ -152,7 +154,6 @@ void histogram(Param<uint> out, const Param<T> in, int nbins, float minval,
     getQueue().submit([&](sycl::handler &h) {
         auto inAcc  = in.data->get_access(h);
         auto outAcc = out.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
 
         auto localMem = local_accessor<uint, 1>(locSize, h);
 
diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
index cefd67c992..d6bb62b177 100644
--- a/src/backend/oneapi/kernel/interp.hpp
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -11,6 +11,9 @@
 #include <math.hpp>
 #include <types.hpp>
 #include <af/constants.h>
+
+#include <sycl/builtins.hpp>
+
 #include <algorithm>
 
 namespace arrayfire {
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index 956bbc401a..e326ff9416 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -28,7 +28,7 @@ class iotaKernel {
    public:
     iotaKernel(sycl::accessor<T> out, KParam oinfo, const int s0, const int s1,
                const int s2, const int s3, const int blocksPerMatX,
-               const int blocksPerMatY, sycl::stream debug)
+               const int blocksPerMatY)
         : out_(out)
         , oinfo_(oinfo)
         , s0_(s0)
@@ -36,8 +36,7 @@ class iotaKernel {
         , s2_(s2)
         , s3_(s3)
         , blocksPerMatX_(blocksPerMatX)
-        , blocksPerMatY_(blocksPerMatY)
-        , debug_(debug) {}
+        , blocksPerMatY_(blocksPerMatY) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group gg = it.get_group();
@@ -77,7 +76,6 @@ class iotaKernel {
     KParam oinfo_;
     int s0_, s1_, s2_, s3_;
     int blocksPerMatX_, blocksPerMatY_;
-    sycl::stream debug_;
 };
 
 template<typename T>
@@ -100,15 +98,13 @@ void iota(Param<T> out, const af::dim4& sdims) {
             .submit([=](sycl::handler& h) {
                 auto out_acc = out.data->get_access(h);
 
-                sycl::stream debug_stream(2048, 128, h);
-
                 h.parallel_for(
                     ndrange,
                     iotaKernel<T>(out_acc, out.info, static_cast<int>(sdims[0]),
                                   static_cast<int>(sdims[1]),
                                   static_cast<int>(sdims[2]),
                                   static_cast<int>(sdims[3]), blocksPerMatX,
-                                  blocksPerMatY, debug_stream));
+                                  blocksPerMatY));
             })
             .wait();
         ONEAPI_DEBUG_FINISH(getQueue());
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index d0361a18dc..3f3dbc378b 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -13,7 +13,6 @@
 #include <common/Transform.hpp>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
-//#include <copy.hpp>?
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
 #include <kernel/default_config.hpp>
@@ -21,7 +20,9 @@
 #include <math.hpp>
 #include <memory.hpp>
 
-#include <iostream>
+#include <sycl/builtins.hpp>
+#include <sycl/group_algorithm.hpp>
+
 #include <memory>
 #include <vector>
 
@@ -72,8 +73,8 @@ class meanDimKernelSMEM {
                       read_accessor<Ti> in, KParam iInfo, read_accessor<Tw> iwt,
                       KParam iwInfo, uint groups_x, uint groups_y,
                       uint offset_dim, local_accessor<compute_t<To>, 1> s_val,
-                      local_accessor<compute_t<Tw>, 1> s_idx,
-                      sycl::stream debug, bool input_weight, bool output_weight)
+                      local_accessor<compute_t<Tw>, 1> s_idx, bool input_weight,
+                      bool output_weight)
         : out_(out)
         , owt_(owt)
         , in_(in)
@@ -88,8 +89,7 @@ class meanDimKernelSMEM {
         , s_val_(s_val)
         , s_idx_(s_idx)
         , input_weight_(input_weight)
-        , output_weight_(output_weight)
-        , debug_(debug) {}
+        , output_weight_(output_weight) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -217,7 +217,6 @@ class meanDimKernelSMEM {
     local_accessor<compute_t<To>, 1> s_val_;
     local_accessor<compute_t<Tw>, 1> s_idx_;
     bool input_weight_, output_weight_;
-    sycl::stream debug_;
 };
 
 template<typename Ti, typename Tw, typename To, int dim>
@@ -233,8 +232,6 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        sycl::stream debug_stream(2048 * 2048, 2048, h);
-
         auto s_val = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
         auto s_idx = local_accessor<compute_t<Tw>, 1>(THREADS_PER_BLOCK, h);
 
@@ -254,7 +251,7 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                                    out_acc, out.info, owt_acc, owt.info, in_acc,
                                    in.info, iwt_acc, iwt.info, blocks_dim[0],
                                    blocks_dim[1], blocks_dim[dim], s_val, s_idx,
-                                   debug_stream, input_weight, output_weight));
+                                   input_weight, output_weight));
                 break;
             case 4:
                 h.parallel_for(sycl::nd_range<2>(global, local),
@@ -262,7 +259,7 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                                    out_acc, out.info, owt_acc, owt.info, in_acc,
                                    in.info, iwt_acc, iwt.info, blocks_dim[0],
                                    blocks_dim[1], blocks_dim[dim], s_val, s_idx,
-                                   debug_stream, input_weight, output_weight));
+                                   input_weight, output_weight));
                 break;
             case 2:
                 h.parallel_for(sycl::nd_range<2>(global, local),
@@ -270,7 +267,7 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                                    out_acc, out.info, owt_acc, owt.info, in_acc,
                                    in.info, iwt_acc, iwt.info, blocks_dim[0],
                                    blocks_dim[1], blocks_dim[dim], s_val, s_idx,
-                                   debug_stream, input_weight, output_weight));
+                                   input_weight, output_weight));
                 break;
             case 1:
                 h.parallel_for(sycl::nd_range<2>(global, local),
@@ -278,7 +275,7 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                                    out_acc, out.info, owt_acc, owt.info, in_acc,
                                    in.info, iwt_acc, iwt.info, blocks_dim[0],
                                    blocks_dim[1], blocks_dim[dim], s_val, s_idx,
-                                   debug_stream, input_weight, output_weight));
+                                   input_weight, output_weight));
                 break;
         }
     });
@@ -333,8 +330,7 @@ class meanFirstKernelSMEM {
                         const uint repeat,
                         local_accessor<compute_t<To>, 1> s_val,
                         local_accessor<compute_t<Tw>, 1> s_idx,
-                        sycl::stream debug, bool input_weight,
-                        bool output_weight)
+                        bool input_weight, bool output_weight)
         : out_(out)
         , owt_(owt)
         , in_(in)
@@ -350,8 +346,7 @@ class meanFirstKernelSMEM {
         , s_val_(s_val)
         , s_idx_(s_idx)
         , input_weight_(input_weight)
-        , output_weight_(output_weight)
-        , debug_(debug) {}
+        , output_weight_(output_weight) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -387,7 +382,7 @@ class meanFirstKernelSMEM {
         bool cond = (yid < iInfo_.dims[1] && zid < iInfo_.dims[2] &&
                      wid < iInfo_.dims[3]);
 
-        int lim = sycl::min((dim_t)(xid + repeat_ * DIMX_), iInfo_.dims[0]);
+        int lim = min((dim_t)(xid + repeat_ * DIMX_), iInfo_.dims[0]);
 
         common::Transform<Ti, compute_t<To>, af_add_t> transform;
 
@@ -411,7 +406,8 @@ class meanFirstKernelSMEM {
         } else {
             for (int id = xid + DIMX_; cond && id < lim; id += DIMX_) {
                 // Faster version of stable_mean when iwptr is NULL
-                val    = val + (transform(iptr[id]) - val) / (weight + (Tw)1);
+                val = val + (transform(iptr[id]) - compute_t<To>(val)) /
+                                (weight + (Tw)1);
                 weight = weight + (Tw)1;
             }
         }
@@ -493,7 +489,6 @@ class meanFirstKernelSMEM {
     local_accessor<compute_t<To>, 1> s_val_;
     local_accessor<compute_t<Tw>, 1> s_idx_;
     bool input_weight_, output_weight_;
-    sycl::stream debug_;
 };
 
 template<typename Ti, typename Tw, typename To>
@@ -511,8 +506,6 @@ void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        sycl::stream debug_stream(2048 * 2048, 2048, h);
-
         auto s_val = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
         auto s_idx = local_accessor<compute_t<Tw>, 1>(THREADS_PER_BLOCK, h);
 
@@ -530,7 +523,7 @@ void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
             meanFirstKernelSMEM<Ti, Tw, To>(
                 out_acc, out.info, owt_acc, owt.info, in_acc, in.info, iwt_acc,
                 iwt.info, threads_x, groups_x, groups_y, repeat, s_val, s_idx,
-                debug_stream, input_weight, output_weight));
+                input_weight, output_weight));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index efe577c9ce..294573b1bf 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -9,11 +9,9 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
-//#include <common/kernel_cache.hpp>
 #include <common/traits.hpp>
 #include <debug_oneapi.hpp>
 #include <traits.hpp>
@@ -35,7 +33,7 @@ class memCopy {
    public:
     memCopy(sycl::accessor<T> out, dims_t ostrides, sycl::accessor<T> in,
             dims_t idims, dims_t istrides, int offset, int groups_0,
-            int groups_1, sycl::stream debug)
+            int groups_1)
         : out_(out)
         , in_(in)
         , ostrides_(ostrides)
@@ -43,8 +41,7 @@ class memCopy {
         , istrides_(istrides)
         , offset_(offset)
         , groups_0_(groups_0)
-        , groups_1_(groups_1)
-        , debug_(debug) {}
+        , groups_1_(groups_1) {}
 
     void operator()(sycl::nd_item<2> it) const {
         const int lid0 = it.get_local_id(0);
@@ -79,7 +76,6 @@ class memCopy {
     sycl::accessor<T> out_, in_;
     dims_t ostrides_, idims_, istrides_;
     int offset_, groups_0_, groups_1_;
-    sycl::stream debug_;
 };
 
 constexpr uint DIM0 = 32;
@@ -111,11 +107,9 @@ void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
         auto out_acc = out->get_access(h);
         auto in_acc  = const_cast<sycl::buffer<T> *>(in)->get_access(h);
 
-        sycl::stream debug_stream(2048, 128, h);
-
         h.parallel_for(ndrange,
                        memCopy<T>(out_acc, _ostrides, in_acc, _idims, _istrides,
-                                  offset, groups_0, groups_1, debug_stream));
+                                  offset, groups_0, groups_1));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
@@ -204,8 +198,7 @@ class reshapeCopy {
    public:
     reshapeCopy(sycl::accessor<outType> dst, KParam oInfo,
                 sycl::accessor<inType> src, KParam iInfo, outType default_value,
-                float factor, dims_t trgt, int blk_x, int blk_y,
-                sycl::stream debug)
+                float factor, dims_t trgt, int blk_x, int blk_y)
         : dst_(dst)
         , src_(src)
         , oInfo_(oInfo)
@@ -214,8 +207,7 @@ class reshapeCopy {
         , factor_(factor)
         , trgt_(trgt)
         , blk_x_(blk_x)
-        , blk_y_(blk_y)
-        , debug_(debug) {}
+        , blk_y_(blk_y) {}
 
     void operator()(sycl::nd_item<2> it) const {
         const uint lx = it.get_local_id(0);
@@ -265,7 +257,6 @@ class reshapeCopy {
     float factor_;
     dims_t trgt_;
     int blk_x_, blk_y_;
-    sycl::stream debug_;
 };
 
 template<typename inType, typename outType>
@@ -305,18 +296,16 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
         auto src_acc =
             const_cast<sycl::buffer<inType> *>(src.data)->get_access(h);
 
-        sycl::stream debug_stream(2048, 128, h);
-
         if (same_dims) {
             h.parallel_for(ndrange, reshapeCopy<inType, outType, true>(
                                         dst_acc, dst.info, src_acc, src.info,
                                         default_value, (float)factor, trgt_dims,
-                                        blk_x, blk_y, debug_stream));
+                                        blk_x, blk_y));
         } else {
             h.parallel_for(ndrange, reshapeCopy<inType, outType, false>(
                                         dst_acc, dst.info, src_acc, src.info,
                                         default_value, (float)factor, trgt_dims,
-                                        blk_x, blk_y, debug_stream));
+                                        blk_x, blk_y));
         }
     });
     ONEAPI_DEBUG_FINISH(getQueue());
diff --git a/src/backend/oneapi/kernel/random_engine.hpp b/src/backend/oneapi/kernel/random_engine.hpp
index d86700a7fb..66e286fea9 100644
--- a/src/backend/oneapi/kernel/random_engine.hpp
+++ b/src/backend/oneapi/kernel/random_engine.hpp
@@ -8,8 +8,8 @@
  ********************************************************/
 #pragma once
 
+#include <Param.hpp>
 #include <common/dispatch.hpp>
-#include <common/half.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
 #include <kernel/random_engine_mersenne.hpp>
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index 426b518eba..9769285d2f 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -23,8 +23,8 @@ namespace kernel {
 //// above. This is done so that we can avoid unnecessary computations because
 /// the / __half datatype is not a constexprable type. This prevents the
 /// compiler from / peforming these operations at compile time.
-//#define HALF_FACTOR __ushort_as_half(0x100u)
-//#define HALF_HALF_FACTOR __ushort_as_half(0x80)
+// #define HALF_FACTOR __ushort_as_half(0x100u)
+// #define HALF_HALF_FACTOR __ushort_as_half(0x80)
 //
 //// Conversion to half adapted from Random123
 ////#define SIGNED_HALF_FACTOR                                \
@@ -35,8 +35,8 @@ namespace kernel {
 //// above. This is done so that we can avoid unnecessary computations because
 /// the / __half datatype is not a constexprable type. This prevents the
 /// compiler from / peforming these operations at compile time
-//#define SIGNED_HALF_FACTOR __ushort_as_half(0x200u)
-//#define SIGNED_HALF_HALF_FACTOR __ushort_as_half(0x100u)
+// #define SIGNED_HALF_FACTOR __ushort_as_half(0x200u)
+// #define SIGNED_HALF_HALF_FACTOR __ushort_as_half(0x100u)
 //
 ///// This is the largest integer representable by fp16. We need to
 ///// make sure that the value converted from ushort is smaller than this
@@ -47,15 +47,15 @@ namespace kernel {
 //__device__ static __half oneMinusGetHalf01(uint num) {
 //    // convert to ushort before the min operation
 //    ushort v = min(max_int_before_infinity, ushort(num));
-//#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 530
+// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 530
 //    return (1.0f - __half2float(__hfma(__ushort2half_rn(v), HALF_FACTOR,
 //                                       HALF_HALF_FACTOR)));
-//#else
+// #else
 //    __half out = __ushort_as_half(0x3c00u) /*1.0h*/ -
 //                 __hfma(__ushort2half_rn(v), HALF_FACTOR, HALF_HALF_FACTOR);
 //    if (__hisinf(out)) printf("val: %d ushort: %d\n", num, v);
 //    return out;
-//#endif
+// #endif
 //}
 //
 //// Generates rationals in (0, 1]
@@ -128,22 +128,22 @@ static double getDoubleNegative11(uint num1, uint num2) {
 
 namespace {
 //
-//#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
-//#define HALF_MATH_FUNC(OP, HALF_OP)    \
+// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
+// #define HALF_MATH_FUNC(OP, HALF_OP)    \
 //    template<>                         \
 //    __device__ __half OP(__half val) { \
 //        return ::HALF_OP(val);         \
 //    }
-//#else
-//#define HALF_MATH_FUNC(OP, HALF_OP)     \
+// #else
+// #define HALF_MATH_FUNC(OP, HALF_OP)     \
 //    template<>                          \
 //    __device__ __half OP(__half val) {  \
 //        float fval = __half2float(val); \
 //        return __float2half(OP(fval));  \
 //    }
-//#endif
+// #endif
 //
-//#define MATH_FUNC(OP, DOUBLE_OP, FLOAT_OP, HALF_OP) \
+// #define MATH_FUNC(OP, DOUBLE_OP, FLOAT_OP, HALF_OP) \
 //    template<typename T>                            \
 //    __device__ T OP(T val);                         \
 //    template<>                                      \
@@ -176,16 +176,16 @@ namespace {
 //
 // template<>
 //__device__ void sincos(__half val, __half *sptr, __half *cptr) {
-//#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
+// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
 //    *sptr = sin(val);
 //    *cptr = cos(val);
-//#else
+// #else
 //    float s, c;
 //    float fval = __half2float(val);
 //    sincos(fval, &s, &c);
 //    *sptr = __float2half(s);
 //    *cptr = __float2half(c);
-//#endif
+// #endif
 //}
 //
 template<typename T>
@@ -198,18 +198,18 @@ void sincospi(T val, T *sptr, T *cptr) {
 //__device__ void sincospi(__half val, __half *sptr, __half *cptr) {
 //    // CUDA cannot make __half into a constexpr as of CUDA 11 so we are
 //    // converting this offline
-//#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
+// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
 //    const __half pi_val = __ushort_as_half(0x4248);  // 0x4248 == 3.14062h
 //    val *= pi_val;
 //    *sptr = sin(val);
 //    *cptr = cos(val);
-//#else
+// #else
 //    float fval = __half2float(val);
 //    float s, c;
 //    sincospi(fval, &s, &c);
 //    *sptr = __float2half(s);
 //    *cptr = __float2half(c);
-//#endif
+// #endif
 //}
 //
 }  // namespace
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index eb8b206a02..14e5f757c5 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -19,6 +19,11 @@
 #include <math.hpp>
 #include <memory.hpp>
 
+#include <sycl/atomic_fence.hpp>
+#include <sycl/atomic_ref.hpp>
+#include <sycl/builtins.hpp>
+#include <sycl/group_algorithm.hpp>
+
 #include <algorithm>
 #include <climits>
 #include <complex>
@@ -55,7 +60,7 @@ class reduceAllKernelSMEM {
                         uint groups_x, uint groups_y, uint repeat,
                         bool change_nan, To nanval,
                         local_accessor<compute_t<To>, 1> s_ptr,
-                        local_accessor<bool, 1> amLast, sycl::stream debug)
+                        local_accessor<bool, 1> amLast)
         : out_(out)
         , retCount_(retCount)
         , tmp_(tmp)
@@ -70,8 +75,7 @@ class reduceAllKernelSMEM {
         , change_nan_(change_nan)
         , nanval_(nanval)
         , s_ptr_(s_ptr)
-        , amLast_(amLast)
-        , debug_(debug) {}
+        , amLast_(amLast) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -97,7 +101,7 @@ class reduceAllKernelSMEM {
                     (wid < iInfo_.dims[3]);
 
         dim_t last = (xid + repeat_ * DIMX_);
-        int lim    = sycl::min(last, iInfo_.dims[0]);
+        int lim    = min(last, iInfo_.dims[0]);
 
         compute_t<To> out_val = common::Binary<compute_t<To>, op>::init();
         for (int id = xid; cond && id < lim; id += DIMX_) {
@@ -238,7 +242,6 @@ class reduceAllKernelSMEM {
     To nanval_;
     local_accessor<compute_t<To>, 1> s_ptr_;
     local_accessor<bool, 1> amLast_;
-    sycl::stream debug_;
 };
 
 template<typename Ti, typename To, af_op_t op>
@@ -273,8 +276,6 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
         auto tmp_acc      = tmp.getData()->get_access(h);
         read_accessor<Ti> in_acc{*in.data, h};
 
-        sycl::stream debug_stream(2048 * 256, 128, h);
-
         auto shrdMem =
             local_accessor<compute_t<To>, 1>(creduce::THREADS_PER_BLOCK, h);
         auto amLast = local_accessor<bool, 1>(1, h);
@@ -283,7 +284,7 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
             reduceAllKernelSMEM<Ti, To, op>(
                 out_acc, out.info, retCount_acc, tmp_acc, (KParam)tmp, in_acc,
                 in.info, threads_x, groups_x, groups_y, repeat, change_nan,
-                scalar<To>(nanval), shrdMem, amLast, debug_stream));
+                scalar<To>(nanval), shrdMem, amLast));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index bfb4f808aa..99f0452785 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -46,8 +46,7 @@ class reduceDimKernelSMEM {
     reduceDimKernelSMEM(write_accessor<To> out, KParam oInfo,
                         read_accessor<Ti> in, KParam iInfo, uint groups_x,
                         uint groups_y, uint offset_dim, bool change_nan,
-                        To nanval, local_accessor<compute_t<To>, 1> s_val,
-                        sycl::stream debug)
+                        To nanval, local_accessor<compute_t<To>, 1> s_val)
         : out_(out)
         , oInfo_(oInfo)
         , iInfo_(iInfo)
@@ -57,8 +56,7 @@ class reduceDimKernelSMEM {
         , offset_dim_(offset_dim)
         , change_nan_(change_nan)
         , nanval_(nanval)
-        , s_val_(s_val)
-        , debug_(debug) {}
+        , s_val_(s_val) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -142,7 +140,6 @@ class reduceDimKernelSMEM {
     bool change_nan_;
     To nanval_;
     local_accessor<compute_t<To>, 1> s_val_;
-    sycl::stream debug_;
 };
 
 template<typename Ti, typename To, af_op_t op, uint dim>
@@ -158,8 +155,6 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        sycl::stream debug_stream(2048 * 256, 128, h);
-
         auto shrdMem =
             local_accessor<compute_t<To>, 1>(creduce::THREADS_X * threads_y, h);
 
@@ -170,7 +165,7 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                     reduceDimKernelSMEM<Ti, To, op, dim, 8>(
                         out_acc, out.info, in_acc, in.info, blocks_dim[0],
                         blocks_dim[1], blocks_dim[dim], change_nan,
-                        scalar<To>(nanval), shrdMem, debug_stream));
+                        scalar<To>(nanval), shrdMem));
                 break;
             case 4:
                 h.parallel_for(
@@ -178,7 +173,7 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                     reduceDimKernelSMEM<Ti, To, op, dim, 4>(
                         out_acc, out.info, in_acc, in.info, blocks_dim[0],
                         blocks_dim[1], blocks_dim[dim], change_nan,
-                        scalar<To>(nanval), shrdMem, debug_stream));
+                        scalar<To>(nanval), shrdMem));
                 break;
             case 2:
                 h.parallel_for(
@@ -186,7 +181,7 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                     reduceDimKernelSMEM<Ti, To, op, dim, 2>(
                         out_acc, out.info, in_acc, in.info, blocks_dim[0],
                         blocks_dim[1], blocks_dim[dim], change_nan,
-                        scalar<To>(nanval), shrdMem, debug_stream));
+                        scalar<To>(nanval), shrdMem));
                 break;
             case 1:
                 h.parallel_for(
@@ -194,7 +189,7 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                     reduceDimKernelSMEM<Ti, To, op, dim, 1>(
                         out_acc, out.info, in_acc, in.info, blocks_dim[0],
                         blocks_dim[1], blocks_dim[dim], change_nan,
-                        scalar<To>(nanval), shrdMem, debug_stream));
+                        scalar<To>(nanval), shrdMem));
                 break;
         }
     });
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index 94553f2b07..342f6f2530 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -46,8 +46,7 @@ class reduceFirstKernelSMEM {
     reduceFirstKernelSMEM(write_accessor<To> out, KParam oInfo,
                           read_accessor<Ti> in, KParam iInfo, uint groups_x,
                           uint groups_y, uint repeat, bool change_nan,
-                          To nanval, local_accessor<compute_t<To>, 1> s_val,
-                          sycl::stream debug)
+                          To nanval, local_accessor<compute_t<To>, 1> s_val)
         : out_(out)
         , oInfo_(oInfo)
         , iInfo_(iInfo)
@@ -57,8 +56,7 @@ class reduceFirstKernelSMEM {
         , repeat_(repeat)
         , change_nan_(change_nan)
         , nanval_(nanval)
-        , s_val_(s_val)
-        , debug_(debug) {}
+        , s_val_(s_val) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -147,7 +145,6 @@ class reduceFirstKernelSMEM {
     bool change_nan_;
     To nanval_;
     local_accessor<compute_t<To>, 1> s_val_;
-    sycl::stream debug_;
 };
 
 template<typename Ti, typename To, af_op_t op>
@@ -165,39 +162,37 @@ void reduce_first_launcher_default(Param<To> out, Param<Ti> in,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        sycl::stream debug_stream(2048 * 256, 128, h);
-
         auto shrdMem =
             local_accessor<compute_t<To>, 1>(creduce::THREADS_PER_BLOCK, h);
 
         switch (threads_x) {
             case 32:
-                h.parallel_for(sycl::nd_range<2>(global, local),
-                               reduceFirstKernelSMEM<Ti, To, op, 32>(
-                                   out_acc, out.info, in_acc, in.info, groups_x,
-                                   groups_y, repeat, change_nan,
-                                   scalar<To>(nanval), shrdMem, debug_stream));
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    reduceFirstKernelSMEM<Ti, To, op, 32>(
+                        out_acc, out.info, in_acc, in.info, groups_x, groups_y,
+                        repeat, change_nan, scalar<To>(nanval), shrdMem));
                 break;
             case 64:
-                h.parallel_for(sycl::nd_range<2>(global, local),
-                               reduceFirstKernelSMEM<Ti, To, op, 64>(
-                                   out_acc, out.info, in_acc, in.info, groups_x,
-                                   groups_y, repeat, change_nan,
-                                   scalar<To>(nanval), shrdMem, debug_stream));
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    reduceFirstKernelSMEM<Ti, To, op, 64>(
+                        out_acc, out.info, in_acc, in.info, groups_x, groups_y,
+                        repeat, change_nan, scalar<To>(nanval), shrdMem));
                 break;
             case 128:
-                h.parallel_for(sycl::nd_range<2>(global, local),
-                               reduceFirstKernelSMEM<Ti, To, op, 128>(
-                                   out_acc, out.info, in_acc, in.info, groups_x,
-                                   groups_y, repeat, change_nan,
-                                   scalar<To>(nanval), shrdMem, debug_stream));
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    reduceFirstKernelSMEM<Ti, To, op, 128>(
+                        out_acc, out.info, in_acc, in.info, groups_x, groups_y,
+                        repeat, change_nan, scalar<To>(nanval), shrdMem));
                 break;
             case 256:
-                h.parallel_for(sycl::nd_range<2>(global, local),
-                               reduceFirstKernelSMEM<Ti, To, op, 256>(
-                                   out_acc, out.info, in_acc, in.info, groups_x,
-                                   groups_y, repeat, change_nan,
-                                   scalar<To>(nanval), shrdMem, debug_stream));
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    reduceFirstKernelSMEM<Ti, To, op, 256>(
+                        out_acc, out.info, in_acc, in.info, groups_x, groups_y,
+                        repeat, change_nan, scalar<To>(nanval), shrdMem));
                 break;
         }
     });
diff --git a/src/backend/oneapi/kernel/reorder.hpp b/src/backend/oneapi/kernel/reorder.hpp
index c39ff556b7..b643bb6fc8 100644
--- a/src/backend/oneapi/kernel/reorder.hpp
+++ b/src/backend/oneapi/kernel/reorder.hpp
@@ -63,7 +63,6 @@ class reorderCreateKernel {
 
         const int o_off   = ow * op_.strides[3] + oz * op_.strides[2];
         const int rdims[] = {d0_, d1_, d2_, d3_};
-        int ods[]         = {xx, yy, oz, ow};
         int ids[4]        = {0};
 
         ids[rdims[3]] = ow;
diff --git a/src/backend/oneapi/kernel/resize.hpp b/src/backend/oneapi/kernel/resize.hpp
index b44d878818..5443815b75 100644
--- a/src/backend/oneapi/kernel/resize.hpp
+++ b/src/backend/oneapi/kernel/resize.hpp
@@ -15,6 +15,8 @@
 #include <debug_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/builtins.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp
index eb0683791c..b4a2678dac 100644
--- a/src/backend/oneapi/kernel/scan_dim.hpp
+++ b/src/backend/oneapi/kernel/scan_dim.hpp
@@ -17,6 +17,9 @@
 #include <kernel/default_config.hpp>
 #include <memory.hpp>
 
+#include <sycl/builtins.hpp>
+#include <sycl/group_algorithm.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 namespace kernel {
@@ -41,7 +44,7 @@ class scanDimKernel {
                   const uint groups_y, const uint blocks_dim, const uint lim,
                   const bool isFinalPass, const uint DIMY,
                   const bool inclusive_scan, local_accessor<To, 1> s_val,
-                  local_accessor<To, 1> s_tmp, sycl::stream debug)
+                  local_accessor<To, 1> s_tmp)
         : out_acc_(out_acc)
         , tmp_acc_(tmp_acc)
         , in_acc_(in_acc)
@@ -56,8 +59,7 @@ class scanDimKernel {
         , isFinalPass_(isFinalPass)
         , inclusive_scan_(inclusive_scan)
         , s_val_(s_val)
-        , s_tmp_(s_tmp)
-        , debug_(debug) {}
+        , s_tmp_(s_tmp) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -162,7 +164,6 @@ class scanDimKernel {
     const bool isFinalPass_, inclusive_scan_;
     local_accessor<To, 1> s_val_;
     local_accessor<To, 1> s_tmp_;
-    sycl::stream debug_;
 };
 
 template<typename To, af_op_t op, int dim>
@@ -172,7 +173,7 @@ class scanDimBcastKernel {
                        read_accessor<To> tmp_acc, KParam tInfo,
                        const uint groups_x, const uint groups_y,
                        const uint groups_dim, const uint lim,
-                       const bool inclusive_scan, sycl::stream debug)
+                       const bool inclusive_scan)
         : out_acc_(out_acc)
         , tmp_acc_(tmp_acc)
         , oInfo_(oInfo)
@@ -181,8 +182,7 @@ class scanDimBcastKernel {
         , groups_y_(groups_y)
         , groups_dim_(groups_dim)
         , lim_(lim)
-        , inclusive_scan_(inclusive_scan)
-        , debug_(debug) {}
+        , inclusive_scan_(inclusive_scan) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -245,7 +245,6 @@ class scanDimBcastKernel {
     KParam oInfo_, tInfo_;
     const uint groups_x_, groups_y_, groups_dim_, lim_;
     const bool inclusive_scan_;
-    sycl::stream debug_;
 };
 
 template<typename Ti, typename To, af_op_t op, int dim>
@@ -264,8 +263,6 @@ static void scan_dim_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
         write_accessor<To> tmp_acc{*tmp.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        sycl::stream debug_stream(2048 * 256, 128, h);
-
         auto s_val =
             local_accessor<compute_t<To>, 1>(THREADS_X * threads_y * 2, h);
         auto s_tmp = local_accessor<compute_t<To>, 1>(THREADS_X, h);
@@ -275,7 +272,7 @@ static void scan_dim_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
             scanDimKernel<Ti, To, op, dim>(
                 out_acc, out.info, tmp_acc, tmp.info, in_acc, in.info,
                 blocks_all[0], blocks_all[1], blocks_all[dim], lim, isFinalPass,
-                threads_y, inclusive_scan, s_val, s_tmp, debug_stream));
+                threads_y, inclusive_scan, s_val, s_tmp));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
@@ -294,13 +291,11 @@ static void bcast_dim_launcher(Param<To> out, Param<To> tmp,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<To> tmp_acc{*tmp.data, h};
 
-        sycl::stream debug_stream(2048 * 256, 128, h);
-
-        h.parallel_for(sycl::nd_range<2>(global, local),
-                       scanDimBcastKernel<To, op, dim>(
-                           out_acc, out.info, tmp_acc, tmp.info, blocks_all[0],
-                           blocks_all[1], blocks_all[dim], lim, inclusive_scan,
-                           debug_stream));
+        h.parallel_for(
+            sycl::nd_range<2>(global, local),
+            scanDimBcastKernel<To, op, dim>(
+                out_acc, out.info, tmp_acc, tmp.info, blocks_all[0],
+                blocks_all[1], blocks_all[dim], lim, inclusive_scan));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index 78039dd36d..777f8f205e 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -17,6 +17,9 @@
 #include <kernel/default_config.hpp>
 #include <memory.hpp>
 
+#include <sycl/builtins.hpp>
+#include <sycl/group_algorithm.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 namespace kernel {
@@ -40,8 +43,7 @@ class scanFirstKernel {
                     read_accessor<Ti> in_acc, KParam iInfo, const uint groups_x,
                     const uint groups_y, const uint lim, const bool isFinalPass,
                     const uint DIMX, const bool inclusive_scan,
-                    local_accessor<To, 1> s_val, local_accessor<To, 1> s_tmp,
-                    sycl::stream debug_stream)
+                    local_accessor<To, 1> s_val, local_accessor<To, 1> s_tmp)
         : out_acc_(out_acc)
         , tmp_acc_(tmp_acc)
         , in_acc_(in_acc)
@@ -55,8 +57,7 @@ class scanFirstKernel {
         , isFinalPass_(isFinalPass)
         , inclusive_scan_(inclusive_scan)
         , s_val_(s_val)
-        , s_tmp_(s_tmp)
-        , debug_stream_(debug_stream) {}
+        , s_tmp_(s_tmp) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -122,7 +123,6 @@ class scanFirstKernel {
                 if (cond_yzw && id == (oInfo_.dims[0] - 1)) {
                     optr[0] = init;
                 } else if (cond_yzw && id < (oInfo_.dims[0] - 1)) {
-                    // debug_stream_ << "oe0 ";
                     optr[id + 1] = val;
                 }
             }
@@ -130,10 +130,7 @@ class scanFirstKernel {
             group_barrier(g);
         }
 
-        if (!isFinalPass_ && isLast && cond_yzw) {
-            // debug_stream_ << "ot ";
-            tptr[groupId_x] = val;
-        }
+        if (!isFinalPass_ && isLast && cond_yzw) { tptr[groupId_x] = val; }
     }
 
    protected:
@@ -145,7 +142,6 @@ class scanFirstKernel {
     const bool isFinalPass_, inclusive_scan_;
     local_accessor<To, 1> s_val_;
     local_accessor<To, 1> s_tmp_;
-    sycl::stream debug_stream_;
 };
 
 template<typename To, af_op_t op>
@@ -154,8 +150,7 @@ class scanFirstBcastKernel {
     scanFirstBcastKernel(write_accessor<To> out_acc, KParam oInfo,
                          read_accessor<To> tmp_acc, KParam tInfo,
                          const uint groups_x, const uint groups_y,
-                         const uint lim, const bool inclusive_scan,
-                         sycl::stream debug_stream)
+                         const uint lim, const bool inclusive_scan)
         : out_acc_(out_acc)
         , tmp_acc_(tmp_acc)
         , oInfo_(oInfo)
@@ -163,8 +158,7 @@ class scanFirstBcastKernel {
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , lim_(lim)
-        , inclusive_scan_(inclusive_scan)
-        , debug_stream_(debug_stream) {}
+        , inclusive_scan_(inclusive_scan) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -209,7 +203,6 @@ class scanFirstBcastKernel {
     KParam oInfo_, tInfo_;
     const uint groups_x_, groups_y_, lim_;
     const bool inclusive_scan_;
-    sycl::stream debug_stream_;
 };
 
 template<typename Ti, typename To, af_op_t op>
@@ -227,20 +220,17 @@ static void scan_first_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
         write_accessor<To> tmp_acc{*tmp.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        sycl::stream debug_stream(2048 * 256, 128, h);
-
         const int DIMY            = THREADS_PER_BLOCK / threads_x;
         const int SHARED_MEM_SIZE = (2 * threads_x + 1) * (DIMY);
         auto s_val = local_accessor<compute_t<To>, 1>(SHARED_MEM_SIZE, h);
         auto s_tmp = local_accessor<compute_t<To>, 1>(DIMY, h);
 
         // TODO threads_x as template arg for #pragma unroll?
-        h.parallel_for(
-            sycl::nd_range<2>(global, local),
-            scanFirstKernel<Ti, To, op>(
-                out_acc, out.info, tmp_acc, tmp.info, in_acc, in.info, groups_x,
-                groups_y, lim, isFinalPass, threads_x, inclusive_scan, s_val,
-                s_tmp, debug_stream));
+        h.parallel_for(sycl::nd_range<2>(global, local),
+                       scanFirstKernel<Ti, To, op>(
+                           out_acc, out.info, tmp_acc, tmp.info, in_acc,
+                           in.info, groups_x, groups_y, lim, isFinalPass,
+                           threads_x, inclusive_scan, s_val, s_tmp));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
@@ -258,12 +248,10 @@ static void bcast_first_launcher(Param<To> out, Param<To> tmp,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<To> tmp_acc{*tmp.data, h};
 
-        sycl::stream debug_stream(2048 * 256, 128, h);
-
         h.parallel_for(sycl::nd_range<2>(global, local),
                        scanFirstBcastKernel<To, op>(
                            out_acc, out.info, tmp_acc, tmp.info, groups_x,
-                           groups_y, lim, inclusive_scan, debug_stream));
+                           groups_y, lim, inclusive_scan));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index 0fac0bacb7..d22a6f4475 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -54,7 +54,7 @@ class transposeKernel {
                     const sycl::accessor<T> iData, const KParam in,
                     const int blocksPerMatX, const int blocksPerMatY,
                     const bool conjugate, const bool IS32MULTIPLE,
-                    local_accessor<T, 1> shrdMem, sycl::stream debugStream)
+                    local_accessor<T, 1> shrdMem)
         : oData_(oData)
         , out_(out)
         , iData_(iData)
@@ -63,8 +63,7 @@ class transposeKernel {
         , blocksPerMatY_(blocksPerMatY)
         , conjugate_(conjugate)
         , IS32MULTIPLE_(IS32MULTIPLE)
-        , shrdMem_(shrdMem)
-        , debugStream_(debugStream) {}
+        , shrdMem_(shrdMem) {}
     void operator()(sycl::nd_item<2> it) const {
         const int shrdStride = TILE_DIM + 1;
 
@@ -134,7 +133,6 @@ class transposeKernel {
     bool conjugate_;
     bool IS32MULTIPLE_;
     local_accessor<T, 1> shrdMem_;
-    sycl::stream debugStream_;
 };
 
 template<typename T>
@@ -151,14 +149,12 @@ void transpose(Param<T> out, const Param<T> in, const bool conjugate,
     getQueue().submit([&](sycl::handler &h) {
         auto r = in.data->get_access(h);
         auto q = out.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
 
         auto shrdMem = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
 
-        h.parallel_for(
-            sycl::nd_range{global, local},
-            transposeKernel<T>(q, out.info, r, in.info, blk_x, blk_y, conjugate,
-                               IS32MULTIPLE, shrdMem, debugStream));
+        h.parallel_for(sycl::nd_range{global, local},
+                       transposeKernel<T>(q, out.info, r, in.info, blk_x, blk_y,
+                                          conjugate, IS32MULTIPLE, shrdMem));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/triangle.hpp b/src/backend/oneapi/kernel/triangle.hpp
index 96fdeb3d88..2f65abe20c 100644
--- a/src/backend/oneapi/kernel/triangle.hpp
+++ b/src/backend/oneapi/kernel/triangle.hpp
@@ -109,7 +109,6 @@ void triangle(Param<T> out, const Param<T> in, bool is_upper,
     getQueue().submit([&](sycl::handler &h) {
         auto iAcc = in.data->get_access(h);
         auto rAcc = out.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
 
         h.parallel_for(
             sycl::nd_range{global, local},
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index d9ee535eb6..3d8fe3324f 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -37,7 +37,7 @@ class whereKernel {
                 read_accessor<uint> otmp_acc, KParam otInfo,
                 read_accessor<uint> rtmp_acc, KParam rtInfo,
                 read_accessor<T> in_acc, KParam iInfo, uint groups_x,
-                uint groups_y, uint lim, sycl::stream debug)
+                uint groups_y, uint lim)
         : out_acc_(out_acc)
         , otmp_acc_(otmp_acc)
         , rtmp_acc_(rtmp_acc)
@@ -48,8 +48,7 @@ class whereKernel {
         , iInfo_(iInfo)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
-        , lim_(lim)
-        , debug_(debug) {}
+        , lim_(lim) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g   = it.get_group();
@@ -99,7 +98,6 @@ class whereKernel {
     read_accessor<T> in_acc_;
     KParam oInfo_, otInfo_, rtInfo_, iInfo_;
     uint groups_x_, groups_y_, lim_;
-    sycl::stream debug_;
 };
 
 template<typename T>
@@ -181,11 +179,10 @@ static void where(Param<uint> &out, Param<T> in) {
         read_accessor<uint> rtmp_acc{*rtmp.data, h};
         read_accessor<T> in_acc{*in.data, h};
 
-        sycl::stream debug_stream(2048 * 256, 128, h);
         h.parallel_for(sycl::nd_range<2>(global, local),
                        whereKernel<T>(out_acc, out.info, otmp_acc, otmp.info,
                                       rtmp_acc, rtmp.info, in_acc, in.info,
-                                      groups_x, groups_y, lim, debug_stream));
+                                      groups_x, groups_y, lim));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
     out_alloc.release();
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index 17cfb37d32..ee47082295 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -18,6 +18,9 @@
 #include <types.hpp>
 #include <af/dim4.hpp>
 
+#include <sycl/builtins.hpp>
+#include <sycl/usm.hpp>
+
 #include <utility>
 
 using arrayfire::common::bytesToString;
@@ -151,9 +154,7 @@ T *pinnedAlloc(const size_t &elements) {
     return static_cast<T *>(ptr);
 }
 
-void pinnedFree(void *ptr) {
-    pinnedMemoryManager().unlock(ptr, false);
-}
+void pinnedFree(void *ptr) { pinnedMemoryManager().unlock(ptr, false); }
 
 // template unique_ptr<int, function<void(int *)>> memAlloc<T>(
 #define INSTANTIATE(T)                                               \
@@ -252,7 +253,7 @@ AllocatorPinned::AllocatorPinned() { logger = common::loggerFactory("mem"); }
 
 void AllocatorPinned::shutdown() { shutdownPinnedMemoryManager(); }
 
-int AllocatorPinned::getActiveDeviceId() { oneapi::getActiveDeviceId(); }
+int AllocatorPinned::getActiveDeviceId() { return oneapi::getActiveDeviceId(); }
 
 size_t AllocatorPinned::getMaxMemorySize(int id) {
     return oneapi::getDeviceMemorySize(id);
diff --git a/src/backend/oneapi/memory.hpp b/src/backend/oneapi/memory.hpp
index 809f219eb7..462c1498f1 100644
--- a/src/backend/oneapi/memory.hpp
+++ b/src/backend/oneapi/memory.hpp
@@ -10,6 +10,8 @@
 
 #include <common/AllocatorInterface.hpp>
 
+#include <sycl/buffer.hpp>
+
 #include <cstdlib>
 #include <functional>
 #include <map>
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index dc2c8a9766..b95b5326bc 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -7,6 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <platform.hpp>
+
 #include <GraphicsResourceManager.hpp>
 #include <blas.hpp>
 #include <build_version.hpp>
@@ -19,12 +21,15 @@
 #include <err_oneapi.hpp>
 #include <errorcodes.hpp>
 #include <memory.hpp>
+#include <af/oneapi.h>
 #include <af/version.h>
 
 #ifdef OS_MAC
 #include <OpenGL/CGLCurrent.h>
 #endif
 
+#include <sycl/platform.hpp>
+
 #include <cctype>
 #include <cstdlib>
 #include <functional>
@@ -418,7 +423,7 @@ void sync(int device) {
     setDevice(currDevice);
 }
 
-void addDeviceContext(sycl::device dev, sycl::context ctx, sycl::queue que) {
+void addDeviceContext(sycl::device& dev, sycl::context& ctx, sycl::queue& que) {
     DeviceManager& devMngr = DeviceManager::getInstance();
 
     int nDevices = 0;
@@ -448,7 +453,7 @@ void addDeviceContext(sycl::device dev, sycl::context ctx, sycl::queue que) {
     memoryManager().addMemoryManagement(nDevices);
 }
 
-void setDeviceContext(sycl::device dev, sycl::context ctx) {
+void setDeviceContext(sycl::device& dev, sycl::context& ctx) {
     // FIXME: add OpenGL Interop for user provided contexts later
     DeviceManager& devMngr = DeviceManager::getInstance();
 
@@ -464,7 +469,7 @@ void setDeviceContext(sycl::device dev, sycl::context ctx) {
     AF_ERROR("No matching device found", AF_ERR_ARG);
 }
 
-void removeDeviceContext(sycl::device dev, sycl::context ctx) {
+void removeDeviceContext(sycl::device& dev, sycl::context& ctx) {
     if (getDevice() == dev && getContext() == ctx) {
         AF_ERROR("Cannot pop the device currently in use", AF_ERR_ARG);
     }
@@ -519,6 +524,22 @@ void removeDeviceContext(sycl::device dev, sycl::context ctx) {
     }
 }
 
+unsigned getMemoryBusWidth(const sycl::device& device) {
+    return device.get_info<sycl::info::device::global_mem_cache_line_size>();
+}
+
+size_t getL2CacheSize(const sycl::device& device) {
+    return device.get_info<sycl::info::device::global_mem_cache_line_size>();
+}
+
+unsigned getComputeUnits(const sycl::device& device) {
+    return device.get_info<sycl::info::device::max_compute_units>();
+}
+
+unsigned getMaxParallelThreads(const sycl::device& device) {
+    return getComputeUnits(device) * 2048;
+}
+
 bool synchronize_calls() {
     static const bool sync = getEnvVar("AF_SYNCHRONOUS_CALLS") == "1";
     return sync;
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index b508f6fc4e..de6ae498dc 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -9,9 +9,12 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <af/oneapi.h>
 
+#include <sycl/context.hpp>
+#include <sycl/device.hpp>
+#include <sycl/queue.hpp>
+
 #include <memory>
 #include <string>
 
@@ -68,6 +71,16 @@ size_t getDeviceMemorySize(int device);
 
 size_t getHostMemorySize();
 
+unsigned getMemoryBusWidth(const sycl::device& device);
+
+size_t getL2CacheSize(const sycl::device& device);
+
+unsigned getComputeUnits(const sycl::device& device);
+
+// maximum nr of threads the device really can run in parallel, without
+// scheduling
+unsigned getMaxParallelThreads(const sycl::device& device);
+
 // sycl::device::is_cpu,is_gpu,is_accelerator
 sycl::info::device_type getDeviceType();
 
@@ -88,11 +101,11 @@ std::string getPlatformName(const sycl::device& device);
 
 int setDevice(int device);
 
-void addDeviceContext(sycl::device dev, sycl::context ctx, sycl::queue que);
+void addDeviceContext(sycl::device& dev, sycl::context& ctx, sycl::queue& que);
 
-void setDeviceContext(sycl::device dev, sycl::context ctx);
+void setDeviceContext(sycl::device& dev, sycl::context& ctx);
 
-void removeDeviceContext(sycl::device dev, sycl::context ctx);
+void removeDeviceContext(sycl::device& dev, sycl::context& ctx);
 
 void sync(int device);
 
diff --git a/src/backend/oneapi/print.hpp b/src/backend/oneapi/print.hpp
index 0e487278d5..686445db49 100644
--- a/src/backend/oneapi/print.hpp
+++ b/src/backend/oneapi/print.hpp
@@ -9,6 +9,8 @@
 
 #pragma once
 #include <backend.hpp>
+#include <types.hpp>
+
 #include <ostream>
 
 namespace arrayfire {
diff --git a/src/backend/oneapi/types.hpp b/src/backend/oneapi/types.hpp
index dacfd85f01..74d117a491 100644
--- a/src/backend/oneapi/types.hpp
+++ b/src/backend/oneapi/types.hpp
@@ -9,12 +9,13 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <common/kernel_type.hpp>
 #include <common/traits.hpp>
 #include <af/compilers.h>
 #include <af/traits.hpp>
 
+#include <sycl/aliases.hpp>
+
 #include <algorithm>
 #include <array>
 #include <complex>

From aed0ff5c626988340ee04e8d4751aab394b760c6 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 6 Mar 2023 14:12:36 -0500
Subject: [PATCH 211/473] Update ToolkitDriverVersions for CUDA 12.1

---
 src/backend/cuda/device_manager.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 00d2e68ee3..8000f2f635 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -101,6 +101,7 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {12010, 9, 0, 0},
     {12000, 9, 0, 0},
     {11080, 9, 0, 0},
     {11070, 8, 7, 0},
@@ -138,6 +139,7 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
+        {12010, 525.60f, 527.41f},
         {12000, 525.60f, 527.41f},
         {11080, 450.80f, 452.39f},
         {11070, 450.80f, 452.39f},

From 0e06e99945264c6f1ac66b979b8320b6d2bc1670 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 3 Mar 2023 18:48:34 -0500
Subject: [PATCH 212/473] Improve half support for oneAPI

---
 src/backend/common/half.hpp                | 264 +++++++++++++--------
 src/backend/oneapi/Param.hpp               |   6 +
 src/backend/oneapi/kernel/reduce_all.hpp   |   6 +-
 src/backend/oneapi/kernel/reduce_dim.hpp   |  51 ++--
 src/backend/oneapi/kernel/reduce_first.hpp |  11 +-
 src/backend/oneapi/types.hpp               |   9 +-
 6 files changed, 208 insertions(+), 139 deletions(-)

diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index 57545f4bcd..0de986ceb5 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -60,61 +60,126 @@ namespace common {
 
 #if defined(__CUDA_ARCH__)
 using native_half_t = __half;
+#elif defined(AF_ONEAPI)
+using native_half_t = sycl::half;
 #else
 using native_half_t = uint16_t;
 #endif
 
 #ifdef __CUDACC_RTC__
-template<typename T>
-AF_CONSTEXPR __DH__ native_half_t float2half(T value) {
-    return __float2half(value);
+template<std::float_round_style R = std::round_to_nearest>
+AF_CONSTEXPR __DH__ native_half_t float2half_impl(float value) {
+    return __float2half_rn(value);
+}
+
+template<std::float_round_style R = std::round_to_nearest>
+AF_CONSTEXPR __DH__ native_half_t float2half_impl(double value) {
+    return __float2half_rn(value);
 }
 
-AF_CONSTEXPR __DH__ inline float half2float(native_half_t value) noexcept {
+AF_CONSTEXPR __DH__ inline float half2float_impl(native_half_t value) noexcept {
     return __half2float(value);
 }
 
 template<typename T>
-AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept;
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(T value) noexcept;
 
 template<>
-AF_CONSTEXPR __DH__ native_half_t int2half(int value) noexcept {
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(int value) noexcept {
     return __int2half_rn(value);
 }
 
 template<>
-AF_CONSTEXPR __DH__ native_half_t int2half(unsigned value) noexcept {
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(unsigned value) noexcept {
     return __uint2half_rn(value);
 }
 
 template<>
-AF_CONSTEXPR __DH__ native_half_t int2half(long long value) noexcept {
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(long long value) noexcept {
     return __ll2half_rn(value);
 }
 
 template<>
-AF_CONSTEXPR __DH__ native_half_t int2half(unsigned long long value) noexcept {
+AF_CONSTEXPR __DH__ native_half_t
+int2half_impl(unsigned long long value) noexcept {
     return __ull2half_rn(value);
 }
 
 template<>
-AF_CONSTEXPR __DH__ native_half_t int2half(short value) noexcept {
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(short value) noexcept {
     return __short2half_rn(value);
 }
 template<>
-AF_CONSTEXPR __DH__ native_half_t int2half(unsigned short value) noexcept {
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(unsigned short value) noexcept {
     return __ushort2half_rn(value);
 }
 
 template<>
-AF_CONSTEXPR __DH__ native_half_t int2half(char value) noexcept {
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(char value) noexcept {
     return __ull2half_rn(value);
 }
 template<>
-AF_CONSTEXPR __DH__ native_half_t int2half(unsigned char value) noexcept {
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(unsigned char value) noexcept {
     return __ull2half_rn(value);
 }
 
+#elif defined(AF_ONEAPI)
+
+template<std::float_round_style R = std::round_to_nearest>
+AF_CONSTEXPR native_half_t float2half_impl(float value) {
+    return static_cast<native_half_t>(value);
+}
+
+template<std::float_round_style R = std::round_to_nearest>
+AF_CONSTEXPR native_half_t float2half_impl(double value) {
+    return static_cast<native_half_t>(value);
+}
+
+AF_CONSTEXPR inline float half2float_impl(native_half_t value) noexcept {
+    return static_cast<float>(value);
+}
+
+template<typename T>
+AF_CONSTEXPR native_half_t int2half_impl(T value) noexcept;
+
+template<>
+AF_CONSTEXPR native_half_t int2half_impl(int value) noexcept {
+    return static_cast<native_half_t>(value);
+}
+
+template<>
+AF_CONSTEXPR native_half_t int2half_impl(unsigned value) noexcept {
+    return static_cast<native_half_t>(value);
+}
+
+template<>
+AF_CONSTEXPR native_half_t int2half_impl(long long value) noexcept {
+    return static_cast<native_half_t>(value);
+}
+
+template<>
+AF_CONSTEXPR native_half_t int2half_impl(unsigned long long value) noexcept {
+    return static_cast<native_half_t>(value);
+}
+
+template<>
+AF_CONSTEXPR native_half_t int2half_impl(short value) noexcept {
+    return static_cast<native_half_t>(value);
+}
+template<>
+AF_CONSTEXPR native_half_t int2half_impl(unsigned short value) noexcept {
+    return static_cast<native_half_t>(value);
+}
+
+template<>
+AF_CONSTEXPR native_half_t int2half_impl(char value) noexcept {
+    return static_cast<native_half_t>(value);
+}
+template<>
+AF_CONSTEXPR native_half_t int2half_impl(unsigned char value) noexcept {
+    return static_cast<native_half_t>(value);
+}
+
 #else
 
 /// Convert integer to half-precision floating point.
@@ -162,22 +227,6 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(T value) noexcept {
     return bits;
 }
 
-template<typename T, std::float_round_style R = std::round_to_nearest,
-         typename std::enable_if_t<std::is_integral<T>::value &&
-                                   std::is_signed<T>::value>* = nullptr>
-AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
-    uint16_t out = (value < 0) ? int2half_impl<R, true, T>(value)
-                               : int2half_impl<R, false, T>(value);
-    return out;
-}
-
-template<typename T, std::float_round_style R = std::round_to_nearest,
-         typename std::enable_if_t<std::is_integral<T>::value &&
-                                   std::is_unsigned<T>::value>* = nullptr>
-AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
-    return int2half_impl<R, false, T>(value);
-}
-
 /// Convert IEEE single-precision to half-precision.
 /// Credit for this goes to [Jeroen van der
 /// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf).
@@ -186,7 +235,7 @@ AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
 ///
 /// \param value single-precision value
 /// \return binary representation of half-precision value
-template<std::float_round_style R = std::round_indeterminate>
+template<std::float_round_style R = std::round_to_nearest>
 __DH__ native_half_t float2half_impl(float value) noexcept {
     uint32_t bits = 0;  // = *reinterpret_cast<uint32*>(&value);
                         // //violating strict aliasing!
@@ -366,23 +415,7 @@ __DH__ native_half_t float2half_impl(double value) {
     return hbits;
 }
 
-template<typename T, std::float_round_style R = std::round_to_nearest>
-#ifdef __CUDA_ARCH__
-AF_CONSTEXPR
-#endif
-    __DH__ native_half_t
-    float2half(T val) {
-#ifdef __CUDA_ARCH__
-    return __float2half(val);
-#else
-    return float2half_impl<R>(val);
-#endif
-}
-
-__DH__ inline float half2float(native_half_t value) noexcept {
-#ifdef __CUDA_ARCH__
-    return __half2float(value);
-#else
+__DH__ inline float half2float_impl(native_half_t value) noexcept {
     // return _cvtsh_ss(data.data_);
     constexpr uint32_t mantissa_table[2048] = {
         0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000,
@@ -749,12 +782,52 @@ __DH__ inline float half2float(native_half_t value) noexcept {
         1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
         1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024};
 
+    uint16_t value_bits = 0;
+    std::memcpy(&value_bits, &value, sizeof(uint16_t));
     uint32_t bits =
-        mantissa_table[offset_table[value >> 10] + (value & 0x3FF)] +
-        exponent_table[value >> 10];
+        mantissa_table[offset_table[value_bits >> 10] + (value_bits & 0x3FF)] +
+        exponent_table[value_bits >> 10];
     float out = 0.0f;
     std::memcpy(&out, &bits, sizeof(float));
     return out;
+}
+
+#endif  // __CUDACC_RTC__
+
+template<typename T, std::float_round_style R = std::round_to_nearest>
+#ifdef __CUDA_ARCH__
+AF_CONSTEXPR
+#endif
+    __DH__ native_half_t
+    float2half(T val) {
+    return float2half_impl<R>(val);
+}
+
+__DH__ inline float half2float(native_half_t value) noexcept {
+    return half2float_impl(value);
+}
+
+template<typename T, std::float_round_style R = std::round_to_nearest,
+         typename std::enable_if_t<std::is_integral<T>::value &&
+                                   std::is_signed<T>::value>* = nullptr>
+AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
+#if defined(__CUDACC_RTC__) || defined(AF_ONEAPI)
+    native_half_t out = int2half_impl(value);
+#else
+    uint16_t out = (value < 0) ? int2half_impl<R, true, T>(value)
+                               : int2half_impl<R, false, T>(value);
+#endif
+    return out;
+}
+
+template<typename T, std::float_round_style R = std::round_to_nearest,
+         typename std::enable_if_t<std::is_integral<T>::value &&
+                                   std::is_unsigned<T>::value>* = nullptr>
+AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
+#if defined(__CUDACC_RTC__) || defined(AF_ONEAPI)
+    return int2half_impl(value);
+#else
+    return int2half_impl<R, false, T>(value);
 #endif
 }
 
@@ -771,6 +844,24 @@ __DH__ inline float half2float(native_half_t value) noexcept {
 /// \param value The value to convert to integer
 template<std::float_round_style R, bool E, typename T>
 AF_CONSTEXPR T half2int(native_half_t value) {
+#ifdef __CUDA_ARCH__
+    if constexpr (std::is_same_v<T, short> || std::is_same_v<T, char> ||
+                  std::is_same_v<T, unsigned char>) {
+        return __half2short_rn(value);
+    } else if constexpr (std::is_same_v<T, unsigned short>) {
+        return __half2ushort_rn(value);
+    } else if constexpr (std::is_same_v<T, long long>) {
+        return __half2ll_rn(value);
+    } else if constexpr (std::is_same_v<T, unsigned long long>) {
+        return __half2ull_rn(value);
+    } else if constexpr (std::is_same_v<T, int>) {
+        return __half2int_rn(value);
+    } else if constexpr (std::is_same_v<T, unsigned>) {
+        return __half2uint_rn(value);
+    }
+#elif defined(AF_ONEAPI)
+    return static_cast<T>(value);
+#else
     static_assert(std::is_integral<T>::value,
                   "half to int conversion only supports builtin integer types");
     unsigned int e = value & 0x7FFF;
@@ -797,10 +888,9 @@ AF_CONSTEXPR T half2int(native_half_t value) {
     } else
         m <<= e - 25;
     return (value & 0x8000) ? -static_cast<T>(m) : static_cast<T>(m);
+#endif
 }
 
-#endif  // __CUDACC_RTC__
-
 namespace internal {
 /// Tag type for binary construction.
 struct binary_t {};
@@ -862,9 +952,6 @@ class alignas(2) half {
 #endif
     }
 
-#if defined(__CUDA_ARCH__)
-    AF_CONSTEXPR
-#endif
     __DH__ explicit half(double value) noexcept
         : data_(float2half<double>(value)) {}
 
@@ -876,23 +963,24 @@ class alignas(2) half {
 
     template<typename T>
     AF_CONSTEXPR __DH__ explicit half(T value) noexcept
-        : data_(int2half<T>(value)) {}
+        : data_(int2half(value)) {}
 
 #if defined(__CUDA_ARCH__)
     AF_CONSTEXPR
 #endif
     __DH__ half& operator=(const double& value) noexcept {
-        data_ = float2half<double>(value);
+        data_ = float2half(value);
         return *this;
     }
 
-#ifdef __CUDA_ARCH__
-    AF_CONSTEXPR __DH__ explicit half(__half value) noexcept : data_(value) {}
+#if defined(__CUDA_ARCH__) || defined(AF_ONEAPI)
+    AF_CONSTEXPR __DH__ explicit half(native_half_t value) noexcept
+        : data_(value) {}
 
-    AF_CONSTEXPR __DH__ half& operator=(__half value) noexcept {
-        // NOTE Assignment to ushort from __half only works with device code.
-        // using memcpy instead
-        data_ = *reinterpret_cast<native_half_t*>(&value);
+    AF_CONSTEXPR __DH__ half& operator=(native_half_t value) noexcept {
+        // NOTE Assignment to ushort from native_half_t only works with device
+        // code. using memcpy instead
+        data_ = value;
         return *this;
     }
 #endif
@@ -907,71 +995,41 @@ class alignas(2) half {
     }
 
     AF_CONSTEXPR __DH__ explicit operator short() const noexcept {
-#ifdef __CUDA_ARCH__
-        return __half2short_rn(data_);
-#else
         return half2int<std::round_indeterminate, true, short>(data_);
-#endif
     }
 
     AF_CONSTEXPR __DH__ explicit operator long long() const noexcept {
-#ifdef __CUDA_ARCH__
-        return __half2ll_rn(data_);
-#else
         return half2int<std::round_indeterminate, true, long long int>(data_);
-#endif
     }
 
     AF_CONSTEXPR __DH__ explicit operator int() const noexcept {
-#ifdef __CUDA_ARCH__
-        return __half2int_rn(data_);
-#else
         return half2int<std::round_indeterminate, true, int>(data_);
-#endif
     }
 
     AF_CONSTEXPR __DH__ explicit operator unsigned() const noexcept {
-#ifdef __CUDA_ARCH__
-        return __half2uint_rn(data_);
-#else
         return half2int<std::round_indeterminate, true, unsigned>(data_);
-#endif
     }
 
     AF_CONSTEXPR __DH__ explicit operator unsigned short() const noexcept {
-#ifdef __CUDA_ARCH__
-        return __half2ushort_rn(data_);
-#else
         return half2int<std::round_indeterminate, true, unsigned>(data_);
-#endif
     }
 
     AF_CONSTEXPR __DH__ explicit operator unsigned long long() const noexcept {
-#ifdef __CUDA_ARCH__
-        return __half2ull_rn(data_);
-#else
         return half2int<std::round_indeterminate, true, unsigned>(data_);
-#endif
     }
 
     AF_CONSTEXPR __DH__ explicit operator char() const noexcept {
-#ifdef __CUDA_ARCH__
-        return __half2short_rn(data_);
-#else
         return half2int<std::round_indeterminate, true, char>(data_);
-#endif
     }
 
     AF_CONSTEXPR __DH__ explicit operator unsigned char() const noexcept {
-#ifdef __CUDA_ARCH__
-        return __half2short_rn(data_);
-#else
         return half2int<std::round_indeterminate, true, unsigned char>(data_);
-#endif
     }
 
-#if defined(__CUDA_ARCH__)
-    AF_CONSTEXPR __DH__ operator __half() const noexcept { return data_; };
+#if defined(__CUDA_ARCH__) || defined(AF_ONEAPI)
+    AF_CONSTEXPR __DH__ operator native_half_t() const noexcept {
+        return data_;
+    };
 #endif
 
     friend AF_CONSTEXPR __DH__ bool operator==(half lhs, half rhs) noexcept;
@@ -988,6 +1046,8 @@ class alignas(2) half {
         return arrayfire::common::half(__hneg(data_));
 #elif defined(__CUDA_ARCH__)
         return arrayfire::common::half(-(__half2float(data_)));
+#elif defined(AF_ONEAPI)
+        return arrayfire::common::half(-data_);
 #else
         return arrayfire::common::half(internal::binary, data_ ^ 0x8000);
 #endif
@@ -1001,6 +1061,8 @@ class alignas(2) half {
         half out;
 #ifdef __CUDA_ARCH__
         out.data_ = __half_raw{0x7C00};
+#elif defined(AF_ONEAPI)
+        out.data_ = std::numeric_limits<sycl::half>::infinity();
 #else
         out.data_ = 0x7C00;
 #endif
@@ -1014,6 +1076,8 @@ AF_CONSTEXPR __DH__ static inline bool operator==(
     return __heq(lhs.data_, rhs.data_);
 #elif defined(__CUDA_ARCH__)
     return __half2float(lhs.data_) == __half2float(rhs.data_);
+#elif defined(AF_ONEAPI)
+    return lhs.data_ == rhs.data_;
 #else
     return (lhs.data_ == rhs.data_ || !((lhs.data_ | rhs.data_) & 0x7FFF)) &&
            !isnan(lhs);
@@ -1035,6 +1099,8 @@ __DH__ static inline bool operator<(arrayfire::common::half lhs,
     return __hlt(lhs.data_, rhs.data_);
 #elif defined(__CUDA_ARCH__)
     return __half2float(lhs.data_) < __half2float(rhs.data_);
+#elif defined(AF_ONEAPI)
+    return lhs.data_ < rhs.data_;
 #else
     int xabs = lhs.data_ & 0x7FFF, yabs = rhs.data_ & 0x7FFF;
     return xabs <= 0x7C00 && yabs <= 0x7C00 &&
@@ -1047,6 +1113,8 @@ __DH__ static inline bool operator<(arrayfire::common::half lhs,
                                     float rhs) noexcept {
 #if defined(__CUDA_ARCH__)
     return __half2float(lhs.data_) < rhs;
+#elif defined(AF_ONEAPI)
+    return lhs.data_ < rhs;
 #else
     return static_cast<float>(lhs) < rhs;
 #endif
@@ -1068,7 +1136,7 @@ static inline std::string to_string(const half&& val) {
 }  // namespace arrayfire
 
 #if !defined(__NVCC__) && !defined(__CUDACC_RTC__)
-//#endif
+// #endif
 /// Extensions to the C++ standard library.
 namespace std {
 /// Numeric limits for half-precision floats.
@@ -1230,6 +1298,8 @@ AF_CONSTEXPR __DH__ static inline bool isnan(half val) noexcept {
     return __hisnan(val.data_);
 #elif defined(__CUDA_ARCH__)
     return ::isnan(__half2float(val));
+#elif defined(AF_ONEAPI)
+    return std::isnan(val.data_);
 #else
     return (val.data_ & 0x7FFF) > 0x7C00;
 #endif
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index cca1d519f6..613e26bdb7 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -35,6 +35,12 @@ struct Param {
     // AF_DEPRECATED("Use Array<T>")
     Param(sycl::buffer<T>* data_, KParam info_) : data(data_), info(info_) {}
 
+    template<sycl::access::mode MODE>
+    sycl::accessor<data_t<T>, 1, MODE> get_accessor(sycl::handler& h) const {
+        auto o = data->template reinterpret<data_t<T>>();
+        return sycl::accessor<data_t<T>, 1, MODE>(o, h);
+    }
+
     ~Param() = default;
 };
 
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index 14e5f757c5..bb1aa99d21 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -93,9 +93,9 @@ class reduceAllKernelSMEM {
         common::Binary<compute_t<To>, op> reduce;
         common::Transform<Ti, compute_t<To>, op> transform;
 
-        const data_t<Ti> *const iptr =
-            in_.get_pointer() + wid * iInfo_.strides[3] +
-            zid * iInfo_.strides[2] + yid * iInfo_.strides[1] + iInfo_.offset;
+        auto iptr = in_.get_pointer() + wid * iInfo_.strides[3] +
+                    zid * iInfo_.strides[2] + yid * iInfo_.strides[1] +
+                    iInfo_.offset;
 
         bool cond = (yid < iInfo_.dims[1]) && (zid < iInfo_.dims[2]) &&
                     (wid < iInfo_.dims[3]);
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index 99f0452785..22b9c0f8dc 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -43,14 +43,14 @@ using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
 template<typename Ti, typename To, af_op_t op, uint dim, uint DIMY>
 class reduceDimKernelSMEM {
    public:
-    reduceDimKernelSMEM(write_accessor<To> out, KParam oInfo,
-                        read_accessor<Ti> in, KParam iInfo, uint groups_x,
+    reduceDimKernelSMEM(Param<To> out, Param<Ti> in, uint groups_x,
                         uint groups_y, uint offset_dim, bool change_nan,
-                        To nanval, local_accessor<compute_t<To>, 1> s_val)
-        : out_(out)
-        , oInfo_(oInfo)
-        , iInfo_(iInfo)
-        , in_(in)
+                        To nanval, local_accessor<compute_t<To>, 1> s_val,
+                        sycl::handler &h)
+        : out_(out.template get_accessor<sycl::access::mode::write>(h))
+        , in_(in.template get_accessor<sycl::access::mode::read>(h))
+        , oInfo_(out.info)
+        , iInfo_(in.info)
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , offset_dim_(offset_dim)
@@ -72,15 +72,16 @@ class reduceDimKernelSMEM {
         const uint yid       = groupId_y;
 
         uint ids[4] = {xid, yid, zid, wid};
+        using sycl::global_ptr;
 
-        data_t<To> *const optr =
+        global_ptr<data_t<To>> optr =
             out_.get_pointer() + ids[3] * oInfo_.strides[3] +
             ids[2] * oInfo_.strides[2] + ids[1] * oInfo_.strides[1] + ids[0];
 
         const uint groupIdx_dim = ids[dim];
         ids[dim]                = ids[dim] * g.get_local_range(1) + lidy;
 
-        const data_t<Ti> *iptr =
+        global_ptr<data_t<Ti>> iptr =
             in_.get_pointer() + ids[3] * iInfo_.strides[3] +
             ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + ids[0];
 
@@ -91,15 +92,16 @@ class reduceDimKernelSMEM {
                         (ids[2] < iInfo_.dims[2]) && (ids[3] < iInfo_.dims[3]);
 
         common::Binary<compute_t<To>, op> reduce;
-        common::Transform<Ti, compute_t<To>, op> transform;
+        common::Transform<data_t<Ti>, compute_t<To>, op> transform;
 
         compute_t<To> out_val = common::Binary<compute_t<To>, op>::init();
         for (int id = id_dim_in; is_valid && (id < iInfo_.dims[dim]);
              id += offset_dim_ * g.get_local_range(1)) {
             compute_t<To> in_val = transform(*iptr);
-            if (change_nan_)
+            if (change_nan_) {
                 in_val = !IS_NAN(in_val) ? in_val
                                          : static_cast<compute_t<To>>(nanval_);
+            }
             out_val = reduce(in_val, out_val);
             iptr += offset_dim_ * g.get_local_range(1) * istride_dim;
         }
@@ -133,9 +135,9 @@ class reduceDimKernelSMEM {
     }
 
    protected:
-    write_accessor<To> out_;
+    write_accessor<data_t<To>> out_;
+    read_accessor<data_t<Ti>> in_;
     KParam oInfo_, iInfo_;
-    read_accessor<Ti> in_;
     uint groups_x_, groups_y_, offset_dim_;
     bool change_nan_;
     To nanval_;
@@ -152,9 +154,6 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                           blocks_dim[1] * blocks_dim[3] * local[1]);
 
     getQueue().submit([=](sycl::handler &h) {
-        write_accessor<To> out_acc{*out.data, h};
-        read_accessor<Ti> in_acc{*in.data, h};
-
         auto shrdMem =
             local_accessor<compute_t<To>, 1>(creduce::THREADS_X * threads_y, h);
 
@@ -163,33 +162,29 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                 h.parallel_for(
                     sycl::nd_range<2>(global, local),
                     reduceDimKernelSMEM<Ti, To, op, dim, 8>(
-                        out_acc, out.info, in_acc, in.info, blocks_dim[0],
-                        blocks_dim[1], blocks_dim[dim], change_nan,
-                        scalar<To>(nanval), shrdMem));
+                        out, in, blocks_dim[0], blocks_dim[1], blocks_dim[dim],
+                        change_nan, scalar<To>(nanval), shrdMem, h));
                 break;
             case 4:
                 h.parallel_for(
                     sycl::nd_range<2>(global, local),
                     reduceDimKernelSMEM<Ti, To, op, dim, 4>(
-                        out_acc, out.info, in_acc, in.info, blocks_dim[0],
-                        blocks_dim[1], blocks_dim[dim], change_nan,
-                        scalar<To>(nanval), shrdMem));
+                        out, in, blocks_dim[0], blocks_dim[1], blocks_dim[dim],
+                        change_nan, scalar<To>(nanval), shrdMem, h));
                 break;
             case 2:
                 h.parallel_for(
                     sycl::nd_range<2>(global, local),
                     reduceDimKernelSMEM<Ti, To, op, dim, 2>(
-                        out_acc, out.info, in_acc, in.info, blocks_dim[0],
-                        blocks_dim[1], blocks_dim[dim], change_nan,
-                        scalar<To>(nanval), shrdMem));
+                        out, in, blocks_dim[0], blocks_dim[1], blocks_dim[dim],
+                        change_nan, scalar<To>(nanval), shrdMem, h));
                 break;
             case 1:
                 h.parallel_for(
                     sycl::nd_range<2>(global, local),
                     reduceDimKernelSMEM<Ti, To, op, dim, 1>(
-                        out_acc, out.info, in_acc, in.info, blocks_dim[0],
-                        blocks_dim[1], blocks_dim[dim], change_nan,
-                        scalar<To>(nanval), shrdMem));
+                        out, in, blocks_dim[0], blocks_dim[1], blocks_dim[dim],
+                        change_nan, scalar<To>(nanval), shrdMem, h));
                 break;
         }
     });
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index 342f6f2530..299919ae12 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -74,13 +74,12 @@ class reduceFirstKernelSMEM {
         common::Binary<compute_t<To>, op> reduce;
         common::Transform<Ti, compute_t<To>, op> transform;
 
-        const data_t<Ti> *const iptr =
-            in_.get_pointer() + wid * iInfo_.strides[3] +
-            zid * iInfo_.strides[2] + yid * iInfo_.strides[1] + iInfo_.offset;
+        Ti *const iptr = in_.get_pointer() + wid * iInfo_.strides[3] +
+                         zid * iInfo_.strides[2] + yid * iInfo_.strides[1] +
+                         iInfo_.offset;
 
-        data_t<To> *const optr = out_.get_pointer() + wid * oInfo_.strides[3] +
-                                 zid * oInfo_.strides[2] +
-                                 yid * oInfo_.strides[1];
+        auto optr = out_.get_pointer() + wid * oInfo_.strides[3] +
+                    zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
 
         bool cond = (yid < iInfo_.dims[1]) && (zid < iInfo_.dims[2]) &&
                     (wid < iInfo_.dims[3]);
diff --git a/src/backend/oneapi/types.hpp b/src/backend/oneapi/types.hpp
index 74d117a491..f4be516f3d 100644
--- a/src/backend/oneapi/types.hpp
+++ b/src/backend/oneapi/types.hpp
@@ -27,17 +27,18 @@ namespace common {
 /// are used
 template<>
 struct kernel_type<common::half> {
-    using data = common::half;
+    using data = sycl::half;
 
     // These are the types within a kernel
-    using native = float;
+    using native = sycl::half;
 
-    using compute = float;
+    using compute = sycl::half;
 };
 }  // namespace common
 }  // namespace arrayfire
 
 namespace arrayfire {
+
 namespace oneapi {
 using cdouble = std::complex<double>;
 using cfloat  = std::complex<float>;
@@ -60,7 +61,6 @@ struct ToNumStr {
     std::string operator()(CONVERSION_TYPE val);
 };
 
-namespace {
 template<typename T>
 inline const char *shortname(bool caps = false) {
     return caps ? "X" : "x";
@@ -129,7 +129,6 @@ template<>
 inline const char *getFullName<cdouble>() {
     return "double2";
 }
-}  // namespace
 
 #if 0
 template<typename... ARGS>

From d2d2bc580313406c415e5cbafb8cb71048e5cb27 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 3 Mar 2023 19:13:24 -0500
Subject: [PATCH 213/473] More header cleanup

---
 src/api/c/handle.cpp                 | 1 +
 src/api/c/hist.cpp                   | 1 +
 src/api/c/image.cpp                  | 1 +
 src/api/c/surface.cpp                | 1 +
 src/backend/common/jit/Node.hpp      | 3 ++-
 src/backend/cuda/Array.hpp           | 1 +
 src/backend/oneapi/kernel/select.hpp | 1 -
 src/backend/oneapi/wrap.cpp          | 9 +++++----
 src/backend/opencl/types.hpp         | 1 +
 9 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/api/c/handle.cpp b/src/api/c/handle.cpp
index a432d8a720..7a93847826 100644
--- a/src/api/c/handle.cpp
+++ b/src/api/c/handle.cpp
@@ -10,6 +10,7 @@
 #include <handle.hpp>
 
 #include <backend.hpp>
+#include <platform.hpp>
 #include <sparse_handle.hpp>
 
 #include <af/dim4.hpp>
diff --git a/src/api/c/hist.cpp b/src/api/c/hist.cpp
index 350d97416d..f37ba5cea1 100644
--- a/src/api/c/hist.cpp
+++ b/src/api/c/hist.cpp
@@ -15,6 +15,7 @@
 #include <copy.hpp>
 #include <handle.hpp>
 #include <hist_graphics.hpp>
+#include <platform.hpp>
 #include <reduce.hpp>
 #include <af/graphics.h>
 
diff --git a/src/api/c/image.cpp b/src/api/c/image.cpp
index 533612f45d..425530806c 100644
--- a/src/api/c/image.cpp
+++ b/src/api/c/image.cpp
@@ -21,6 +21,7 @@
 #include <handle.hpp>
 #include <image.hpp>
 #include <join.hpp>
+#include <platform.hpp>
 #include <reorder.hpp>
 #include <tile.hpp>
 
diff --git a/src/api/c/surface.cpp b/src/api/c/surface.cpp
index 62ef46e0e2..b2a6404a33 100644
--- a/src/api/c/surface.cpp
+++ b/src/api/c/surface.cpp
@@ -19,6 +19,7 @@
 #include <copy.hpp>
 #include <handle.hpp>
 #include <join.hpp>
+#include <platform.hpp>
 #include <reduce.hpp>
 #include <reorder.hpp>
 #include <surface.hpp>
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 9ed090fbaa..8a262e0734 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -11,7 +11,6 @@
 #include <backend.hpp>
 #include <common/defines.hpp>
 #include <optypes.hpp>
-#include <platform.hpp>
 #include <types.hpp>
 #include <af/defines.h>
 
@@ -38,6 +37,8 @@ class Node;
 }  // namespace arrayfire
 
 #ifdef AF_CPU
+#include <Param.hpp>
+
 namespace arrayfire {
 namespace cpu {
 namespace kernel {
diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp
index 6c00910c9d..d6774ded66 100644
--- a/src/backend/cuda/Array.hpp
+++ b/src/backend/cuda/Array.hpp
@@ -17,6 +17,7 @@
 #include <cuda_runtime_api.h>
 #include <jit/BufferNode.hpp>
 #include <memory.hpp>
+#include <platform.hpp>
 #include <traits.hpp>
 #include <types.hpp>
 #include <af/dim4.hpp>
diff --git a/src/backend/oneapi/kernel/select.hpp b/src/backend/oneapi/kernel/select.hpp
index 618cea3437..7f63f2cbea 100644
--- a/src/backend/oneapi/kernel/select.hpp
+++ b/src/backend/oneapi/kernel/select.hpp
@@ -12,7 +12,6 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
-#include <debug_oneapi.hpp>
 #include <math.hpp>
 
 #include <string>
diff --git a/src/backend/oneapi/wrap.cpp b/src/backend/oneapi/wrap.cpp
index 1400db07f0..19e8c0260e 100644
--- a/src/backend/oneapi/wrap.cpp
+++ b/src/backend/oneapi/wrap.cpp
@@ -7,15 +7,16 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <wrap.hpp>
+
+#include <kernel/wrap.hpp>
+#include <kernel/wrap_dilated.hpp>
+
 #include <Array.hpp>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
-#include <kernel/wrap.hpp>
-#include <kernel/wrap_dilated.hpp>
 #include <math.hpp>
-#include <wrap.hpp>
-#include <stdexcept>
 
 using arrayfire::common::half;
 
diff --git a/src/backend/opencl/types.hpp b/src/backend/opencl/types.hpp
index 2bc96996aa..620ab74ca9 100644
--- a/src/backend/opencl/types.hpp
+++ b/src/backend/opencl/types.hpp
@@ -13,6 +13,7 @@
 #include <common/kernel_type.hpp>
 #include <common/traits.hpp>
 #include <af/compilers.h>
+#include <af/traits.hpp>
 
 #include <algorithm>
 #include <array>

From 5544252d7eb1febf0a3e79f4056021132d92b71c Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 3 Mar 2023 19:34:13 -0500
Subject: [PATCH 214/473] Expose base KernelInterface types to derived classes

---
 src/backend/common/KernelInterface.hpp | 13 ++++++++-----
 src/backend/opencl/Kernel.hpp          |  3 ---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/backend/common/KernelInterface.hpp b/src/backend/common/KernelInterface.hpp
index 5eeb8710fd..0ead60a8cd 100644
--- a/src/backend/common/KernelInterface.hpp
+++ b/src/backend/common/KernelInterface.hpp
@@ -16,15 +16,18 @@ namespace arrayfire {
 namespace common {
 
 /// Kernel Interface that should be implemented by each backend
-template<typename ModuleType, typename KernelType, typename EnqueuerType,
-         typename DevPtrType>
+template<typename TModuleType, typename TKernelType, typename TEnqueuerType,
+         typename TDevPtrType>
 class KernelInterface {
-   private:
-    ModuleType mModuleHandle;
-    KernelType mKernelHandle;
+    TModuleType mModuleHandle;
+    TKernelType mKernelHandle;
     std::string mName;
 
    public:
+    using ModuleType   = TModuleType;
+    using KernelType   = TKernelType;
+    using EnqueuerType = TEnqueuerType;
+    using DevPtrType   = TDevPtrType;
     KernelInterface(std::string name, ModuleType mod, KernelType ker)
         : mModuleHandle(mod), mKernelHandle(ker), mName(name) {}
 
diff --git a/src/backend/opencl/Kernel.hpp b/src/backend/opencl/Kernel.hpp
index e3a05e7da8..c5582d8f1c 100644
--- a/src/backend/opencl/Kernel.hpp
+++ b/src/backend/opencl/Kernel.hpp
@@ -40,9 +40,6 @@ class Kernel
     : public common::KernelInterface<const cl::Program*, cl::Kernel, Enqueuer,
                                      cl::Buffer*> {
    public:
-    using ModuleType = const cl::Program*;
-    using KernelType = cl::Kernel;
-    using DevPtrType = cl::Buffer*;
     using BaseClass =
         common::KernelInterface<ModuleType, KernelType, Enqueuer, DevPtrType>;
 

From 5971cdcfde545b9e161cd892c6bae91f2c3df904 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 3 Mar 2023 22:26:57 -0500
Subject: [PATCH 215/473] Add flag to remove warnings on debug builds with
 oneAPI

---
 CMakeModules/InternalUtils.cmake | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index 1d1c387245..863cbaed22 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -33,6 +33,8 @@ check_cxx_compiler_flag(-fno-signed-zeros  has_cxx_no_signed_zeros)
 check_cxx_compiler_flag(-mno-ieee-fp has_cxx_no_ieee_fp)
 check_cxx_compiler_flag(-Wno-unqualified-std-cast-call has_cxx_unqualified_std_cast_call)
 check_cxx_compiler_flag(-Werror=reorder-ctor has_cxx_error_reorder_ctor)
+check_cxx_compiler_flag(-Rno-debug-disables-optimization has_cxx_debug-disables-optimization)
+
 
 function(arrayfire_set_default_cxx_flags target)
   target_compile_options(${target}
@@ -75,7 +77,10 @@ function(arrayfire_set_default_cxx_flags target)
 
               $<$<NOT:$<BOOL:${AF_WITH_FAST_MATH}>>:
                     $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>>
-                      >
+
+              $<$<BOOL:${has_cxx_debug-disables-optimization}>:
+                  $<$<CONFIG:Debug>:-Rno-debug-disables-optimization>>
+          >
     )
 
   target_compile_definitions(${target}

From ebf754353327fd17cdaf588fcac4e00df3b72a24 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 6 Mar 2023 14:40:34 -0500
Subject: [PATCH 216/473] Drop 18.04 from GitHub workflows

---
 .github/workflows/unix_cpu_build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 3c0e566d6f..01051f7e8f 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -25,7 +25,7 @@ jobs:
 
     documentation:
         name: Documentation
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-20.04
         env:
           DOXYGEN_VER: 1.8.18
         steps:
@@ -68,7 +68,7 @@ jobs:
             fail-fast: false
             matrix:
                 blas_backend: [Atlas, MKL, OpenBLAS]
-                os: [ubuntu-18.04, ubuntu-20.04, macos-latest]
+                os: [ubuntu-20.04, macos-latest]
                 compiler: [gcc, clang, icx]
                 exclude:
                     - os: macos-latest
@@ -128,7 +128,7 @@ jobs:
                   echo "CMAKE_PROGRAM=cmake" >> $GITHUB_ENV
 
             - name: Install Common Dependencies for Ubuntu
-              if: matrix.os == 'ubuntu-18.04' || matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04'
+              if: matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04'
               run: |
                   sudo add-apt-repository ppa:mhier/libboost-latest
                   sudo apt-get -qq update

From 7964d43b90a5a95d526c65c12efb429c8e354b66 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 6 Mar 2023 15:49:31 -0500
Subject: [PATCH 217/473] Remove constexpr from float2half because of the
 memcpy operation

---
 src/backend/common/half.hpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index 0de986ceb5..f427d539cf 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -77,7 +77,8 @@ AF_CONSTEXPR __DH__ native_half_t float2half_impl(double value) {
     return __float2half_rn(value);
 }
 
-AF_CONSTEXPR __DH__ inline float half2float_impl(native_half_t value) noexcept {
+AF_CONSTEXPR
+__DH__ inline float half2float_impl(native_half_t value) noexcept {
     return __half2float(value);
 }
 
@@ -135,7 +136,7 @@ AF_CONSTEXPR native_half_t float2half_impl(double value) {
     return static_cast<native_half_t>(value);
 }
 
-AF_CONSTEXPR inline float half2float_impl(native_half_t value) noexcept {
+inline float half2float_impl(native_half_t value) noexcept {
     return static_cast<float>(value);
 }
 

From 05d8b9255defc51b893a87ea668837e46fcaff71 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 8 Mar 2023 15:19:00 -0500
Subject: [PATCH 218/473] Fix half compilation on NVRTC based compilation

---
 src/backend/common/half.hpp         | 121 +++++++++++++++-------------
 src/backend/cuda/compile_module.cpp |   4 +
 2 files changed, 71 insertions(+), 54 deletions(-)

diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index f427d539cf..65a3930b15 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -39,6 +39,48 @@
 #include <backend.hpp>
 
 #ifdef __CUDACC_RTC__
+
+#if defined(__cpp_if_constexpr) || __cplusplus >= 201606L
+#define AF_IF_CONSTEXPR if constexpr
+#else
+#define AF_IF_CONSTEXPR if
+#endif
+
+namespace std {
+enum float_round_style {
+    round_indeterminate       = -1,
+    round_toward_zero         = 0,
+    round_to_nearest          = 1,
+    round_toward_infinity     = 2,
+    round_toward_neg_infinity = 3
+};
+
+template<bool B, class T = void>
+struct enable_if {};
+
+template<class T>
+struct enable_if<true, T> {
+    typedef T type;
+};
+
+template<bool B, class T = void>
+using enable_if_t = typename enable_if<B, T>::type;
+
+template<class T, class U>
+struct is_same {
+    static constexpr bool value = false;
+};
+
+template<class T>
+struct is_same<T, T> {
+    static constexpr bool value = true;
+};
+
+template<class T, class U>
+constexpr bool is_same_v = is_same<T, U>::value;
+
+}  //  namespace std
+
 using uint16_t = unsigned short;
 // we do not include the af/compilers header in nvrtc compilations so
 // we are defining the AF_CONSTEXPR expression here
@@ -140,44 +182,8 @@ inline float half2float_impl(native_half_t value) noexcept {
     return static_cast<float>(value);
 }
 
-template<typename T>
-AF_CONSTEXPR native_half_t int2half_impl(T value) noexcept;
-
-template<>
-AF_CONSTEXPR native_half_t int2half_impl(int value) noexcept {
-    return static_cast<native_half_t>(value);
-}
-
-template<>
-AF_CONSTEXPR native_half_t int2half_impl(unsigned value) noexcept {
-    return static_cast<native_half_t>(value);
-}
-
-template<>
-AF_CONSTEXPR native_half_t int2half_impl(long long value) noexcept {
-    return static_cast<native_half_t>(value);
-}
-
-template<>
-AF_CONSTEXPR native_half_t int2half_impl(unsigned long long value) noexcept {
-    return static_cast<native_half_t>(value);
-}
-
-template<>
-AF_CONSTEXPR native_half_t int2half_impl(short value) noexcept {
-    return static_cast<native_half_t>(value);
-}
-template<>
-AF_CONSTEXPR native_half_t int2half_impl(unsigned short value) noexcept {
-    return static_cast<native_half_t>(value);
-}
-
-template<>
-AF_CONSTEXPR native_half_t int2half_impl(char value) noexcept {
-    return static_cast<native_half_t>(value);
-}
-template<>
-AF_CONSTEXPR native_half_t int2half_impl(unsigned char value) noexcept {
+template<std::float_round_style R, bool S, typename T>
+AF_CONSTEXPR native_half_t int2half_impl(T value) noexcept {
     return static_cast<native_half_t>(value);
 }
 
@@ -808,24 +814,26 @@ __DH__ inline float half2float(native_half_t value) noexcept {
     return half2float_impl(value);
 }
 
+#ifndef __CUDACC_RTC__
 template<typename T, std::float_round_style R = std::round_to_nearest,
          typename std::enable_if_t<std::is_integral<T>::value &&
                                    std::is_signed<T>::value>* = nullptr>
 AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
-#if defined(__CUDACC_RTC__) || defined(AF_ONEAPI)
-    native_half_t out = int2half_impl(value);
-#else
-    uint16_t out = (value < 0) ? int2half_impl<R, true, T>(value)
-                               : int2half_impl<R, false, T>(value);
-#endif
+    native_half_t out = (value < 0) ? int2half_impl<R, true, T>(value)
+                                    : int2half_impl<R, false, T>(value);
     return out;
 }
+#endif
 
-template<typename T, std::float_round_style R = std::round_to_nearest,
+template<typename T, std::float_round_style R = std::round_to_nearest
+#ifndef __CUDACC_RTC__
+         ,
          typename std::enable_if_t<std::is_integral<T>::value &&
-                                   std::is_unsigned<T>::value>* = nullptr>
+                                   std::is_unsigned<T>::value>* = nullptr
+#endif
+         >
 AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
-#if defined(__CUDACC_RTC__) || defined(AF_ONEAPI)
+#if defined(__CUDACC_RTC__)
     return int2half_impl(value);
 #else
     return int2half_impl<R, false, T>(value);
@@ -846,18 +854,23 @@ AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
 template<std::float_round_style R, bool E, typename T>
 AF_CONSTEXPR T half2int(native_half_t value) {
 #ifdef __CUDA_ARCH__
-    if constexpr (std::is_same_v<T, short> || std::is_same_v<T, char> ||
-                  std::is_same_v<T, unsigned char>) {
+    AF_IF_CONSTEXPR(std::is_same_v<T, short> || std::is_same_v<T, char> ||
+                    std::is_same_v<T, unsigned char>) {
         return __half2short_rn(value);
-    } else if constexpr (std::is_same_v<T, unsigned short>) {
+    }
+    else AF_IF_CONSTEXPR(std::is_same_v<T, unsigned short>) {
         return __half2ushort_rn(value);
-    } else if constexpr (std::is_same_v<T, long long>) {
+    }
+    else AF_IF_CONSTEXPR(std::is_same_v<T, long long>) {
         return __half2ll_rn(value);
-    } else if constexpr (std::is_same_v<T, unsigned long long>) {
+    }
+    else AF_IF_CONSTEXPR(std::is_same_v<T, unsigned long long>) {
         return __half2ull_rn(value);
-    } else if constexpr (std::is_same_v<T, int>) {
+    }
+    else AF_IF_CONSTEXPR(std::is_same_v<T, int>) {
         return __half2int_rn(value);
-    } else if constexpr (std::is_same_v<T, unsigned>) {
+    }
+    else AF_IF_CONSTEXPR(std::is_same_v<T, unsigned>) {
         return __half2uint_rn(value);
     }
 #elif defined(AF_ONEAPI)
diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp
index 3fddb93d95..36014049a8 100644
--- a/src/backend/cuda/compile_module.cpp
+++ b/src/backend/cuda/compile_module.cpp
@@ -266,7 +266,11 @@ Module compileModule(const string &moduleKey, span<const string> sources,
              computeFlag.first, computeFlag.second);
     vector<const char *> compiler_options = {
         arch.data(),
+#if CUDA_VERSION >= 11000
+        "--std=c++17",
+#else
         "--std=c++14",
+#endif
         "--device-as-default-execution-space",
 #ifdef AF_WITH_FAST_MATH
         "--use_fast_math",

From 6736e9384478099b84e1ac49dfb2bc32fb025553 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 8 Mar 2023 15:21:14 -0500
Subject: [PATCH 219/473] Move OpenCL error functions from global namespace to
 af::ocl ns

---
 src/backend/common/err_common.hpp     |  2 +-
 src/backend/opencl/compile_module.cpp | 49 ++++++++++++++-------------
 src/backend/opencl/err_opencl.hpp     | 14 ++++++++
 3 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/src/backend/common/err_common.hpp b/src/backend/common/err_common.hpp
index 79c9d029d7..3936cee77c 100644
--- a/src/backend/common/err_common.hpp
+++ b/src/backend/common/err_common.hpp
@@ -214,5 +214,5 @@ namespace common {
 
 bool& is_stacktrace_enabled() noexcept;
 
-}
+}  // namespace common
 }  // namespace arrayfire
diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp
index 03fd41a196..832f5144a7 100644
--- a/src/backend/opencl/compile_module.cpp
+++ b/src/backend/opencl/compile_module.cpp
@@ -60,24 +60,6 @@ logger *getLogger() {
     return logger.get();
 }
 
-string getProgramBuildLog(const Program &prog) {
-    string build_error("");
-    try {
-        build_error.reserve(4096);
-        auto devices = prog.getInfo<CL_PROGRAM_DEVICES>();
-        for (auto &device : prog.getInfo<CL_PROGRAM_DEVICES>()) {
-            build_error +=
-                format("OpenCL Device: {}\n\tOptions: {}\n\tLog:\n{}\n",
-                       device.getInfo<CL_DEVICE_NAME>(),
-                       prog.getBuildInfo<CL_PROGRAM_BUILD_OPTIONS>(device),
-                       prog.getBuildInfo<CL_PROGRAM_BUILD_LOG>(device));
-        }
-    } catch (const cl::Error &e) {
-        build_error = format("Failed to fetch build log: {}", e.what());
-    }
-    return build_error;
-}
-
 #define THROW_BUILD_LOG_EXCEPTION(PROG)                              \
     do {                                                             \
         string build_error = getProgramBuildLog(PROG);               \
@@ -129,8 +111,23 @@ Program buildProgram(span<const string> kernelSources,
     return retVal;
 }
 
-}  // namespace opencl
-}  // namespace arrayfire
+string getProgramBuildLog(const Program &prog) {
+    string build_error("");
+    try {
+        build_error.reserve(4096);
+        auto devices = prog.getInfo<CL_PROGRAM_DEVICES>();
+        for (auto &device : prog.getInfo<CL_PROGRAM_DEVICES>()) {
+            build_error +=
+                format("OpenCL Device: {}\n\tOptions: {}\n\tLog:\n{}\n",
+                       device.getInfo<CL_DEVICE_NAME>(),
+                       prog.getBuildInfo<CL_PROGRAM_BUILD_OPTIONS>(device),
+                       prog.getBuildInfo<CL_PROGRAM_BUILD_LOG>(device));
+        }
+    } catch (const cl::Error &e) {
+        build_error = format("Failed to fetch build log: {}", e.what());
+    }
+    return build_error;
+}
 
 string getKernelCacheFilename(const int device, const string &key) {
     auto &dev = arrayfire::opencl::getDevice(device);
@@ -147,6 +144,9 @@ string getKernelCacheFilename(const int device, const string &key) {
            to_string(AF_API_VERSION_CURRENT) + ".bin";
 }
 
+}  // namespace opencl
+}  // namespace arrayfire
+
 namespace arrayfire {
 namespace common {
 
@@ -164,8 +164,9 @@ Module compileModule(const string &moduleKey, span<const string> sources,
     const int device             = arrayfire::opencl::getActiveDeviceId();
     const string &cacheDirectory = getCacheDirectory();
     if (!cacheDirectory.empty()) {
-        const string cacheFile = cacheDirectory + AF_PATH_SEPARATOR +
-                                 getKernelCacheFilename(device, moduleKey);
+        const string cacheFile =
+            cacheDirectory + AF_PATH_SEPARATOR +
+            opencl::getKernelCacheFilename(device, moduleKey);
         const string tempFile =
             cacheDirectory + AF_PATH_SEPARATOR + makeTempFilename();
         try {
@@ -223,7 +224,7 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
 
     auto &dev              = arrayfire::opencl::getDevice(device);
     const string cacheFile = cacheDirectory + AF_PATH_SEPARATOR +
-                             getKernelCacheFilename(device, moduleKey);
+                             opencl::getKernelCacheFilename(device, moduleKey);
     Program program;
     Module retVal{};
     try {
@@ -273,7 +274,7 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
             "{{{:<20} : Loading OpenCL binary({}) failed for {}; {}, Build "
             "Log: {}}}",
             moduleKey, cacheFile, dev.getInfo<CL_DEVICE_NAME>(), e.what(),
-            getProgramBuildLog(program));
+            opencl::getProgramBuildLog(program));
         removeFile(cacheFile);
     }
     return retVal;
diff --git a/src/backend/opencl/err_opencl.hpp b/src/backend/opencl/err_opencl.hpp
index 845db9ee02..2c1187c569 100644
--- a/src/backend/opencl/err_opencl.hpp
+++ b/src/backend/opencl/err_opencl.hpp
@@ -11,6 +11,20 @@
 
 #include <common/err_common.hpp>
 
+#include <string>
+
+namespace cl {
+class Program;
+}
+
+namespace arrayfire {
+namespace opencl {
+
+std::string getProgramBuildLog(const cl::Program &prog);
+
+}  // namespace opencl
+}  // namespace arrayfire
+
 #define OPENCL_NOT_SUPPORTED(message)                                       \
     do {                                                                    \
         throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \

From f148b178b33282d8a6b73676ac294ceb403255b1 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 9 Mar 2023 13:54:01 -0500
Subject: [PATCH 220/473] Update binary tests to use the new assert test
 functions

---
 test/binary.cpp | 102 +++++++++++++++++++++++-------------------------
 1 file changed, 48 insertions(+), 54 deletions(-)

diff --git a/test/binary.cpp b/test/binary.cpp
index f6f9a8928f..ab557f8c9a 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -44,60 +44,54 @@ af::array randgen(const int num, dtype ty) {
 
 #define MY_ASSERT_NEAR(aa, bb, cc) ASSERT_NEAR(abs(aa), abs(bb), (cc))
 
-#define BINARY_TESTS(Ta, Tb, Tc, func)                                \
-    TEST(BinaryTests, Test_##func##_##Ta##_##Tb) {                    \
-        SUPPORTED_TYPE_CHECK(Ta);                                     \
-        SUPPORTED_TYPE_CHECK(Tb);                                     \
-        SUPPORTED_TYPE_CHECK(Tc);                                     \
-                                                                      \
-        af_dtype ta = (af_dtype)dtype_traits<Ta>::af_type;            \
-        af_dtype tb = (af_dtype)dtype_traits<Tb>::af_type;            \
-        af::array a = randgen(num, ta);                               \
-        af::array b = randgen(num, tb);                               \
-        af::array c = func(a, b);                                     \
-        Ta *h_a     = a.host<Ta>();                                   \
-        Tb *h_b     = b.host<Tb>();                                   \
-        Tc *h_c     = c.host<Tc>();                                   \
-        for (int i = 0; i < num; i++)                                 \
-            ASSERT_EQ(h_c[i], func(h_a[i], h_b[i]))                   \
-                << "for values: " << h_a[i] << "," << h_b[i] << endl; \
-        af_free_host(h_a);                                            \
-        af_free_host(h_b);                                            \
-        af_free_host(h_c);                                            \
-    }                                                                 \
-                                                                      \
-    TEST(BinaryTests, Test_##func##_##Ta##_##Tb##_left) {             \
-        SUPPORTED_TYPE_CHECK(Ta);                                     \
-        SUPPORTED_TYPE_CHECK(Tb);                                     \
-                                                                      \
-        af_dtype ta = (af_dtype)dtype_traits<Ta>::af_type;            \
-        af::array a = randgen(num, ta);                               \
-        Tb h_b      = 3.0;                                            \
-        af::array c = func(a, h_b);                                   \
-        Ta *h_a     = a.host<Ta>();                                   \
-        Ta *h_c     = c.host<Ta>();                                   \
-        for (int i = 0; i < num; i++)                                 \
-            ASSERT_EQ(h_c[i], func(h_a[i], h_b))                      \
-                << "for values: " << h_a[i] << "," << h_b << endl;    \
-        af_free_host(h_a);                                            \
-        af_free_host(h_c);                                            \
-    }                                                                 \
-                                                                      \
-    TEST(BinaryTests, Test_##func##_##Ta##_##Tb##_right) {            \
-        SUPPORTED_TYPE_CHECK(Ta);                                     \
-        SUPPORTED_TYPE_CHECK(Tb);                                     \
-                                                                      \
-        af_dtype tb = (af_dtype)dtype_traits<Tb>::af_type;            \
-        Ta h_a      = 5.0;                                            \
-        af::array b = randgen(num, tb);                               \
-        af::array c = func(h_a, b);                                   \
-        Tb *h_b     = b.host<Tb>();                                   \
-        Tb *h_c     = c.host<Tb>();                                   \
-        for (int i = 0; i < num; i++)                                 \
-            ASSERT_EQ(h_c[i], func(h_a, h_b[i]))                      \
-                << "for values: " << h_a << "," << h_b[i] << endl;    \
-        af_free_host(h_b);                                            \
-        af_free_host(h_c);                                            \
+#define BINARY_TESTS(Ta, Tb, Tc, func)                                    \
+    TEST(BinaryTests, Test_##func##_##Ta##_##Tb) {                        \
+        SUPPORTED_TYPE_CHECK(Ta);                                         \
+        SUPPORTED_TYPE_CHECK(Tb);                                         \
+        SUPPORTED_TYPE_CHECK(Tc);                                         \
+                                                                          \
+        af_dtype ta = (af_dtype)dtype_traits<Ta>::af_type;                \
+        af_dtype tb = (af_dtype)dtype_traits<Tb>::af_type;                \
+        af::array a = randgen(num, ta);                                   \
+        af::array b = randgen(num, tb);                                   \
+        af::array c = func(a, b);                                         \
+        Ta *h_a     = a.host<Ta>();                                       \
+        Tb *h_b     = b.host<Tb>();                                       \
+        vector<Tc> gold(num);                                             \
+        for (int i = 0; i < num; i++) { gold[i] = func(h_a[i], h_b[i]); } \
+        ASSERT_VEC_ARRAY_EQ(gold, dim4(num), c);                          \
+        af_free_host(h_a);                                                \
+        af_free_host(h_b);                                                \
+    }                                                                     \
+                                                                          \
+    TEST(BinaryTests, Test_##func##_##Ta##_##Tb##_left) {                 \
+        SUPPORTED_TYPE_CHECK(Ta);                                         \
+        SUPPORTED_TYPE_CHECK(Tb);                                         \
+                                                                          \
+        af_dtype ta = (af_dtype)dtype_traits<Ta>::af_type;                \
+        af::array a = randgen(num, ta);                                   \
+        Tb h_b      = 3.0;                                                \
+        af::array c = func(a, h_b);                                       \
+        Ta *h_a     = a.host<Ta>();                                       \
+        vector<Tc> gold(num);                                             \
+        for (int i = 0; i < num; i++) { gold[i] = func(h_a[i], h_b); }    \
+        ASSERT_VEC_ARRAY_EQ(gold, dim4(num), c);                          \
+        af_free_host(h_a);                                                \
+    }                                                                     \
+                                                                          \
+    TEST(BinaryTests, Test_##func##_##Ta##_##Tb##_right) {                \
+        SUPPORTED_TYPE_CHECK(Ta);                                         \
+        SUPPORTED_TYPE_CHECK(Tb);                                         \
+                                                                          \
+        af_dtype tb = (af_dtype)dtype_traits<Tb>::af_type;                \
+        Ta h_a      = 5.0;                                                \
+        af::array b = randgen(num, tb);                                   \
+        af::array c = func(h_a, b);                                       \
+        Tb *h_b     = b.host<Tb>();                                       \
+        vector<Tc> gold(num);                                             \
+        for (int i = 0; i < num; i++) { gold[i] = func(h_a, h_b[i]); }    \
+        ASSERT_VEC_ARRAY_EQ(gold, dim4(num), c);                          \
+        af_free_host(h_b);                                                \
     }
 
 #define BINARY_TESTS_NEAR_GENERAL(Ta, Tb, Tc, Td, Te, func, err)      \

From 214ab0a827009d1da8ac36ee56a153c0f5df6f0f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 9 Mar 2023 13:54:34 -0500
Subject: [PATCH 221/473] Initial implementation of OpenCL based JIT for the
 oneAPI backend

This is the initial implementation of jit based on OpenCL kernels
for the oneAPI backend. This is not a feature complete implementation
and some checks for shape size have been ignored because of the lack
of the memory manager. This implementation also does not fully support
4D tensors which should be implemented later. All jit operations should
be available with this change for most JIT based tests for other
functions.
---
 src/backend/oneapi/Array.cpp                 | 145 +++--
 src/backend/oneapi/Array.hpp                 |   5 +
 src/backend/oneapi/CMakeLists.txt            |  44 +-
 src/backend/oneapi/Kernel.hpp                |  81 +--
 src/backend/oneapi/Param.hpp                 |  83 +++
 src/backend/oneapi/arith.hpp                 |   3 -
 src/backend/oneapi/device_manager.cpp        |  16 +-
 src/backend/oneapi/err_oneapi.hpp            |  28 +
 src/backend/oneapi/histogram.cpp             |   4 +-
 src/backend/oneapi/index.cpp                 |   2 +-
 src/backend/oneapi/jit.cpp                   | 568 ++++++++++++++++++-
 src/backend/oneapi/jit/BufferNode.hpp        |   4 +-
 src/backend/oneapi/jit/kernel_generators.hpp |  25 +-
 src/backend/oneapi/kernel/KParam.hpp         |   8 +-
 src/backend/oneapi/kernel/histogram.hpp      |   9 -
 src/backend/oneapi/kernel/reduce_all.hpp     |   8 +-
 src/backend/oneapi/memory.cpp                |  12 +-
 17 files changed, 833 insertions(+), 212 deletions(-)

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index a55915edb8..2db607c75c 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -214,9 +214,7 @@ void Array<T>::eval() {
 
     Param<T> res{data.get(), info};
 
-    // TODO: implement
-    ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
-    // evalNodes(res, getNode().get());
+    evalNodes(res, getNode().get());
     node.reset();
 }
 
@@ -272,9 +270,7 @@ void evalMultiple(vector<Array<T> *> arrays) {
         nodes.push_back(array->getNode().get());
     }
 
-    // TODO: implement
-    ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
-    // evalNodes(outputs, nodes);
+    evalNodes(outputs, nodes);
 
     for (Array<T> *array : output_arrays) { array->node.reset(); }
 }
@@ -283,10 +279,10 @@ template<typename T>
 Node_ptr Array<T>::getNode() {
     if (node) { return node; }
 
-    KParam kinfo   = *this;
+    AParam<T> info = *this;
     unsigned bytes = this->dims().elements() * sizeof(T);
     auto nn        = bufferNodePtr<T>();
-    nn->setData(kinfo, data, bytes, isLinear());
+    nn->setData(info, data, bytes, isLinear());
 
     return nn;
 }
@@ -318,78 +314,79 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
             return kJITHeuristics::TreeHeight;
         }
     }
-    ONEAPI_NOT_SUPPORTED("JIT NOT SUPPORTED");
 
-    // bool isBufferLimit = getMemoryPressure() >= getMemoryPressureThreshold();
+    // TODO(umar): add memory based checks for JIT kernel generation
+    bool isBufferLimit =
+        false;  // getMemoryPressure() >= getMemoryPressureThreshold();
     // auto platform      = getActivePlatform();
 
     // The Apple platform can have the nvidia card or the AMD card
     // bool isIntel = platform == AFCL_PLATFORM_INTEL;
 
-    // /// Intels param_size limit is much smaller than the other platforms
-    // /// so we need to start checking earlier with smaller trees
-    // int heightCheckLimit =
-    //     isIntel && getDeviceType() == CL_DEVICE_TYPE_GPU ? 3 : 6;
-
-    // // A lightweight check based on the height of the node. This is
-    // // an inexpensive operation and does not traverse the JIT tree.
-    // bool atHeightLimit =
-    //     std::any_of(std::begin(root_nodes), std::end(root_nodes),
-    //                 [heightCheckLimit](Node *n) {
-    //                     return (n->getHeight() + 1 >= heightCheckLimit);
-    //                 });
-
-    // if (atHeightLimit || isBufferLimit) {
-    //     // This is the base parameter size if the kernel had no
-    //     // arguments
-    //     size_t base_param_size =
-    //         (sizeof(T *) + sizeof(Param<T>)) * root_nodes.size() +
-    //         (3 * sizeof(uint));
-
-    //     const cl::Device &device = getDevice();
-    //     size_t max_param_size =
-    //     device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>();
-    //     // typical values:
-    //     //   NVIDIA     = 4096
-    //     //   AMD        = 3520  (AMD A10 iGPU = 1024)
-    //     //   Intel iGPU = 1024
-    //     max_param_size -= base_param_size;
-
-    //     struct tree_info {
-    //         size_t total_buffer_size;
-    //         size_t num_buffers;
-    //         size_t param_scalar_size;
-    //     };
-
-    //     tree_info info{0, 0, 0};
-    //     for (Node *n : root_nodes) {
-    //         NodeIterator<> it(n);
-    //         info = accumulate(
-    //             it, NodeIterator<>(), info, [](tree_info &prev, Node &n) {
-    //                 if (n.isBuffer()) {
-    //                     auto &buf_node = static_cast<BufferNode &>(n);
-    //                     // getBytes returns the size of the data Array.
-    //                     // Sub arrays will be represented by their parent
-    //                     // size.
-    //                     prev.total_buffer_size += buf_node.getBytes();
-    //                     prev.num_buffers++;
-    //                 } else {
-    //                     prev.param_scalar_size += n.getParamBytes();
-    //                 }
-    //                 return prev;
-    //             });
-    //     }
-    //     isBufferLimit = jitTreeExceedsMemoryPressure(info.total_buffer_size);
-
-    //     size_t param_size = (info.num_buffers * (sizeof(Param<T>) + sizeof(T
-    //     *)) +
-    //                          info.param_scalar_size);
-
-    //     bool isParamLimit = param_size >= max_param_size;
-
-    //     if (isParamLimit) { return kJITHeuristics::KernelParameterSize; }
-    //     if (isBufferLimit) { return kJITHeuristics::MemoryPressure; }
-    // }
+    /// Intels param_size limit is much smaller than the other platforms
+    /// so we need to start checking earlier with smaller trees
+    int heightCheckLimit = 3;
+
+    // A lightweight check based on the height of the node. This is
+    // an inexpensive operation and does not traverse the JIT tree.
+    bool atHeightLimit =
+        std::any_of(std::begin(root_nodes), std::end(root_nodes),
+                    [heightCheckLimit](Node *n) {
+                        return (n->getHeight() + 1 >= heightCheckLimit);
+                    });
+
+    if (atHeightLimit || isBufferLimit) {
+        // This is the base parameter size if the kernel had no
+        // arguments
+        size_t base_param_size =
+            (sizeof(T *) + sizeof(Param<T>)) * root_nodes.size() +
+            (3 * sizeof(uint));
+
+        const sycl::device &device = getDevice();
+        size_t max_param_size =
+            device.get_info<sycl::info::device::max_parameter_size>();
+        // typical values:
+        //   NVIDIA     = 4096
+        //   AMD        = 3520  (AMD A10 iGPU = 1024)
+        //   Intel iGPU = 1024
+        max_param_size -= base_param_size;
+
+        struct tree_info {
+            size_t total_buffer_size;
+            size_t num_buffers;
+            size_t param_scalar_size;
+        };
+
+        tree_info info{0, 0, 0};
+        for (Node *n : root_nodes) {
+            NodeIterator<> it(n);
+            info = accumulate(
+                it, NodeIterator<>(), info, [](tree_info &prev, Node &n) {
+                    if (n.isBuffer()) {
+                        auto &buf_node = static_cast<BufferNode<T> &>(n);
+                        // getBytes returns the size of the data Array.
+                        // Sub arrays will be represented by their parent
+                        // size.
+                        prev.total_buffer_size += buf_node.getBytes();
+                        prev.num_buffers++;
+                    } else {
+                        prev.param_scalar_size += n.getParamBytes();
+                    }
+                    return prev;
+                });
+        }
+        isBufferLimit = jitTreeExceedsMemoryPressure(info.total_buffer_size);
+
+        size_t param_size =
+            (info.num_buffers * (sizeof(Param<T>) + sizeof(T *)) +
+             info.param_scalar_size);
+
+        bool isParamLimit = param_size >= max_param_size;
+
+        if (isParamLimit) { return kJITHeuristics::KernelParameterSize; }
+        // TODO(umar): check buffer limit for JIT kernel generation
+        // if (isBufferLimit) { return kJITHeuristics::MemoryPressure; }
+    }
     return kJITHeuristics::Pass;
 }
 
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index d907cad92f..bc4e16c574 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -283,6 +283,11 @@ class Array {
         return out;
     }
 
+    operator AParam<T>() {
+        AParam<T> out(*getData(), dims().get(), strides().get(), getOffset());
+        return out;
+    }
+
     operator KParam() const {
         KParam kinfo = {
             {dims()[0], dims()[1], dims()[2], dims()[3]},
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 9abca35940..5b5684038d 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -7,6 +7,7 @@
 
 include(InternalUtils)
 include(build_cl2hpp)
+include(FileToString)
 
 add_library(afoneapi
   Array.cpp
@@ -93,6 +94,8 @@ add_library(afoneapi
   ireduce.cpp
   ireduce.hpp
   jit.cpp
+  jit/BufferNode.hpp
+  jit/kernel_generators.hpp
   join.cpp
   join.hpp
   logic.hpp
@@ -239,6 +242,24 @@ target_sources(afoneapi
     kernel/wrap_dilated.hpp
 )
 
+set(kernel_src
+  ${CMAKE_CURRENT_SOURCE_DIR}/../opencl/kernel/KParam.hpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/../opencl/kernel/jit.cl
+)
+
+set( kernel_headers_dir "kernel_headers")
+
+file_to_string(
+  SOURCES ${kernel_src}
+  VARNAME kernel_files
+  EXTENSION "hpp"
+  OUTPUT_DIR ${kernel_headers_dir}
+  TARGETS cl_kernel_targets
+  NAMESPACE "arrayfire oneapi opencl"
+)
+
+add_dependencies(afoneapi ${cl_kernel_targets})
+
 add_library(ArrayFire::afoneapi ALIAS afoneapi)
 
 arrayfire_set_default_cxx_flags(afoneapi)
@@ -254,23 +275,40 @@ target_include_directories(afoneapi
     $<INSTALL_INTERFACE:${AF_INSTALL_INC_DIR}>
   PRIVATE
     ${CMAKE_CURRENT_SOURCE_DIR}
-
+    ${CMAKE_CURRENT_BINARY_DIR}
   )
 
 target_compile_options(afoneapi
-  PRIVATE -fsycl)
+  PRIVATE
+    -fsycl
+    #-fsycl-targets=nvptx64-vidia-cuda
+    #-fsycl-force-target=nvptx64-nvidia-cuda-sm_86
+    #-Wno-unknown-cuda-version
+    -sycl-std=2020
+)
 
 target_compile_definitions(afoneapi
   PRIVATE
     AF_ONEAPI
+    CL_TARGET_OPENCL_VERSION=300
+    CL_HPP_TARGET_OPENCL_VERSION=300
+    CL_HPP_MINIMUM_OPENCL_VERSION=110
+    CL_HPP_ENABLE_EXCEPTIONS
   )
 
 target_link_libraries(afoneapi
   PRIVATE
+    -fsycl
+    -fno-lto
+    -fvisibility-inlines-hidden
+    #-fsycl-targets=nvptx64-nvidia-cuda-sm_86
+    #-fsycl-force-target=nvptx64-nvidia-cuda-sm_86
     c_api_interface
     cpp_api_interface
     afcommon_interface
-    -fsycl
+    OpenCL::OpenCL
+    OpenCL::cl2hpp
+    #-Wno-unknown-cuda-version
   )
 
 af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR})
diff --git a/src/backend/oneapi/Kernel.hpp b/src/backend/oneapi/Kernel.hpp
index e36e202387..ee5a2fcd02 100644
--- a/src/backend/oneapi/Kernel.hpp
+++ b/src/backend/oneapi/Kernel.hpp
@@ -26,69 +26,40 @@ inline auto getLogger() -> spdlog::logger* {
 }  // namespace kernel_logger
 
 /*
+ */
 struct Enqueuer {
     template<typename... Args>
-    void operator()(std::string name, sycl::kernel ker,
-                    const cl::EnqueueArgs& qArgs, Args&&... args) {
-        auto launchOp = cl::KernelFunctor<Args...>(ker);
+    void operator()(std::string name, sycl::kernel ker, const Enqueuer& qArgs,
+                    Args&&... args) {
+        // auto launchOp = cl::KernelFunctor<Args...>(ker);
         using namespace kernel_logger;
         AF_TRACE("Launching {}", name);
-        launchOp(qArgs, std::forward<Args>(args)...);
+        // launchOp(qArgs, std::forward<Args>(args)...);
     }
 };
 
-class Kernel
-    : public common::KernelInterface<const cl::Program*, cl::Kernel, Enqueuer,
-                                     cl::Buffer*> {
-   public:
-    using ModuleType = const sycl::program*;
-    using KernelType = sycl::kernel;
-    using DevPtrType<T> = sycl::buffer<T>*;
-    using BaseClass =
-        common::KernelInterface<ModuleType, KernelType, Enqueuer,
-DevPtrType<T>>;
-
-    Kernel() : BaseClass("", nullptr, cl::Kernel{nullptr, false}) {}
-    Kernel(std::string name, ModuleType mod, KernelType ker)
-        : BaseClass(name, mod, ker) {}
-
-    // clang-format off
-    [[deprecated("OpenCL backend doesn't need Kernel::getDevPtr method")]]
-    DevPtrType<T> getDevPtr(const char* name) final;
-    // clang-format on
-
-    void copyToReadOnly(DevPtrType<T> dst, DevPtrType<T> src, size_t bytes)
-final;
-
-    void setFlag(DevPtrType<T> dst, int* scalarValPtr,
-                 const bool syncCopy = false) final;
-
-    int getFlag(DevPtrType<T> src) final;
-};
-*/
-
 class Kernel {
-   public:
-    using ModuleType =
-        const sycl::kernel_bundle<sycl::bundle_state::executable>*;
-    using KernelType = sycl::kernel;
-    template<typename T>
-    using DevPtrType = sycl::buffer<T>*;
-    // using BaseClass =
-    // common::KernelInterface<ModuleType, KernelType, Enqueuer, DevPtrType<T>>;
-
-    Kernel() {}
-    Kernel(std::string name, ModuleType mod, KernelType ker) {}
-
-    template<typename T>
-    void copyToReadOnly(DevPtrType<T> dst, DevPtrType<T> src, size_t bytes);
-
-    template<typename T>
-    void setFlag(DevPtrType<T> dst, int* scalarValPtr,
-                 const bool syncCopy = false);
-
-    template<typename T>
-    int getFlag(DevPtrType<T> src);
+    //   public:
+    //    using BaseClass =
+    //      common::KernelInterface<ModuleType, KernelType, Enqueuer,
+    //      sycl::buffer<float>*>;
+    //
+    //  Kernel() : {}
+    //    Kernel(std::string name, ModuleType mod, KernelType ker)
+    //        : BaseClass(name, mod, ker) {}
+    //
+    //    // clang-format off
+    //    [[deprecated("OpenCL backend doesn't need Kernel::getDevPtr method")]]
+    //    DevPtrType getDevPtr(const char* name) final;
+    //    // clang-format on
+    //
+    //    void copyToReadOnly(DevPtrType dst, DevPtrType src, size_t bytes)
+    // final;
+    //
+    //    void setFlag(DevPtrType dst, int* scalarValPtr,
+    //                 const bool syncCopy = false) final;
+    //
+    //    int getFlag(DevPtrType src) final;
 };
 
 }  // namespace oneapi
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index 613e26bdb7..f6ca0ef8b1 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -14,10 +14,17 @@
 
 #include <af/dim4.hpp>
 
+/// The get_pointer function in the accessor class throws a few warnings in the
+/// 2023.0 release of the library. Review this warning in the future
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wsycl-strict"
 #include <sycl/accessor.hpp>
+#pragma clang diagnostic pop
 #include <sycl/buffer.hpp>
 #include <sycl/handler.hpp>
 
+#include <optional>
+
 namespace arrayfire {
 namespace oneapi {
 
@@ -44,9 +51,85 @@ struct Param {
     ~Param() = default;
 };
 
+template<typename T>
+struct AParam {
+    std::optional<sycl::accessor<T, 1>> data;
+    std::optional<sycl::accessor<T, 1, sycl::access::mode::read_write,
+                                 sycl::access::target::device,
+                                 sycl::access::placeholder::true_t>>
+        ph;
+    af::dim4 dims;
+    af::dim4 strides;
+    dim_t offset;
+    AParam& operator=(const AParam& other) = default;
+    AParam(const AParam& other)            = default;
+    AParam(AParam&& other)                 = default;
+
+    // AF_DEPRECATED("Use Array<T>")
+    AParam() : data(), ph(), dims{0, 0, 0, 0}, strides{0, 0, 0, 0}, offset(0) {}
+
+    AParam(sycl::buffer<T, 1>& data_, const dim_t dims_[4],
+           const dim_t strides_[4], dim_t offset_)
+        : data()
+        , ph(std::make_optional<
+              sycl::accessor<T, 1, sycl::access::mode::read_write,
+                             sycl::access::target::device,
+                             sycl::access::placeholder::true_t>>(data_))
+        , dims(4, dims_)
+        , strides(4, strides_)
+        , offset(offset_) {}
+    // AF_DEPRECATED("Use Array<T>")
+    AParam(sycl::handler& h, sycl::buffer<T, 1>& data_, const dim_t dims_[4],
+           const dim_t strides_[4], dim_t offset_)
+        : data{{data_, h}}
+        , ph(data_)
+        , dims(4, dims_)
+        , strides(4, strides_)
+        , offset(offset_) {}
+
+    template<sycl::access::mode MODE>
+    sycl::accessor<data_t<T>, 1, MODE> get_accessor(sycl::handler& h) const {
+        return *data;
+    }
+
+    void require(sycl::handler& h) {
+        if (!data) { h.require(ph.value()); }
+    }
+
+    operator KParam() const {
+        return KParam{{dims[0], dims[1], dims[2], dims[3]},
+                      {strides[0], strides[1], strides[2], strides[3]},
+                      offset};
+    }
+
+    ~AParam() = default;
+};
+
 // AF_DEPRECATED("Use Array<T>")
 template<typename T>
 Param<T> makeParam(sycl::buffer<T>& mem, int off, const int dims[4],
                    const int strides[4]);
+
+namespace opencl {
+
+template<typename T>
+struct Param {
+    cl_mem data;
+    KParam info;
+    Param& operator=(const Param& other) = default;
+    Param(const Param& other)            = default;
+    Param(Param&& other)                 = default;
+    Param(cl_mem data_, KParam info_) : data(data_), info(info_) {}
+
+    // AF_DEPRECATED("Use Array<T>")
+    Param() : data(nullptr), info{{0, 0, 0, 0}, {0, 0, 0, 0}, 0} {}
+
+    // AF_DEPRECATED("Use Array<T>")
+    Param(sycl::buffer<T>* data_, KParam info_) : data(data_), info(info_) {}
+
+    ~Param() = default;
+};
+}  // namespace opencl
+
 }  // namespace oneapi
 }  // namespace arrayfire
diff --git a/src/backend/oneapi/arith.hpp b/src/backend/oneapi/arith.hpp
index 8f31a5383e..815df91b57 100644
--- a/src/backend/oneapi/arith.hpp
+++ b/src/backend/oneapi/arith.hpp
@@ -11,7 +11,6 @@
 
 #include <Array.hpp>
 #include <common/jit/BinaryNode.hpp>
-#include <err_oneapi.hpp>
 #include <optypes.hpp>
 #include <af/dim4.hpp>
 
@@ -21,14 +20,12 @@ namespace oneapi {
 template<typename T, af_op_t op>
 Array<T> arithOp(const Array<T> &&lhs, const Array<T> &&rhs,
                  const af::dim4 &odims) {
-    ONEAPI_NOT_SUPPORTED(__FUNCTION__);
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 
 template<typename T, af_op_t op>
 Array<T> arithOp(const Array<T> &lhs, const Array<T> &rhs,
                  const af::dim4 &odims) {
-    ONEAPI_NOT_SUPPORTED(__FUNCTION__);
     return common::createBinaryNode<T, T, op>(lhs, rhs, odims);
 }
 }  // namespace oneapi
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index aea4398c66..7134109146 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -64,6 +64,16 @@ static inline bool compare_default(const unique_ptr<sycl::device>& ldev,
     return l_mem > r_mem;
 }
 
+auto arrayfire_exception_handler(sycl::exception_list exceptions) {
+    for (std::exception_ptr const& e : exceptions) {
+        try {
+            std::rethrow_exception(e);
+        } catch (sycl::exception const& ex) {
+            AF_ERROR(ex.what(), AF_ERR_INTERNAL);
+        }
+    }
+}
+
 DeviceManager::DeviceManager()
     : logger(common::loggerFactory("platform"))
     , mUserDeviceOffset(0)
@@ -115,12 +125,12 @@ DeviceManager::DeviceManager()
 
     // Create contexts and queues once the sort is done
     for (int i = 0; i < nDevices; i++) {
-        if (devices[i]->is_gpu() || devices[i]->is_cpu() ||
-            !devices[i]->is_accelerator()) {
+        if (devices[i]->is_gpu() || devices[i]->is_cpu()) {
             try {
                 mContexts.push_back(make_unique<sycl::context>(*devices[i]));
                 mQueues.push_back(
-                    make_unique<sycl::queue>(*mContexts.back(), *devices[i]));
+                    make_unique<sycl::queue>(*mContexts.back(), *devices[i],
+                                             arrayfire_exception_handler));
                 mIsGLSharingOn.push_back(false);
                 // TODO:
                 // mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
diff --git a/src/backend/oneapi/err_oneapi.hpp b/src/backend/oneapi/err_oneapi.hpp
index ff6c83d6ca..fad7d449c0 100644
--- a/src/backend/oneapi/err_oneapi.hpp
+++ b/src/backend/oneapi/err_oneapi.hpp
@@ -16,3 +16,31 @@
         throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \
                            boost::stacktrace::stacktrace());                \
     } while (0)
+
+#define CL_CHECK(call)                                                      \
+    do {                                                                    \
+        if (cl_int err = (call)) {                                          \
+            char cl_err_msg[2048];                                          \
+            const char* cl_err_call = #call;                                \
+            snprintf(cl_err_msg, sizeof(cl_err_msg),                        \
+                     "CL Error %s(%d): %d = %s\n", __FILE__, __LINE__, err, \
+                     cl_err_call);                                          \
+            AF_ERROR(cl_err_msg, AF_ERR_INTERNAL);                          \
+        }                                                                   \
+    } while (0)
+
+#define CL_CHECK_BUILD(call)                                                  \
+    do {                                                                      \
+        if (cl_int err = (call)) {                                            \
+            char log[8192];                                                   \
+            char cl_err_msg[8192];                                            \
+            const char* cl_err_call = #call;                                  \
+            size_t log_ret;                                                   \
+            clGetProgramBuildInfo(prog, dev, CL_PROGRAM_BUILD_LOG, 8192, log, \
+                                  &log_ret);                                  \
+            snprintf(cl_err_msg, sizeof(cl_err_msg),                          \
+                     "OpenCL Error building %s(%d): %d = %s\nLog:\n%s",       \
+                     __FILE__, __LINE__, err, cl_err_call, log);              \
+            AF_ERROR(cl_err_msg, AF_ERR_INTERNAL);                            \
+        }                                                                     \
+    } while (0)
diff --git a/src/backend/oneapi/histogram.cpp b/src/backend/oneapi/histogram.cpp
index 4036a5229b..4dfece0640 100644
--- a/src/backend/oneapi/histogram.cpp
+++ b/src/backend/oneapi/histogram.cpp
@@ -26,9 +26,7 @@ Array<uint> histogram(const Array<T> &in, const unsigned &nbins,
                       const bool isLinear) {
     const dim4 &dims = in.dims();
     dim4 outDims     = dim4(nbins, 1, dims[2], dims[3]);
-    // Array<uint> out  = createValueArray<uint>(outDims, uint(0));
-    // \TODO revert createEmptyArray to createValueArray once JIT functions
-    Array<uint> out = createEmptyArray<uint>(outDims);
+    Array<uint> out  = createValueArray<uint>(outDims, uint(0));
     kernel::histogram<T>(out, in, nbins, minval, maxval, isLinear);
     return out;
 }
diff --git a/src/backend/oneapi/index.cpp b/src/backend/oneapi/index.cpp
index 03a6b74c56..f0eb5e1cc4 100644
--- a/src/backend/oneapi/index.cpp
+++ b/src/backend/oneapi/index.cpp
@@ -22,7 +22,7 @@ namespace oneapi {
 
 template<typename T>
 Array<T> index(const Array<T>& in, const af_index_t idxrs[]) {
-    ONEAPI_NOT_SUPPORTED("");
+    ONEAPI_NOT_SUPPORTED("Indexing not supported");
     Array<T> out = createEmptyArray<T>(af::dim4(1));
     return out;
 }
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 3233f97430..519d4efeea 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -7,21 +7,32 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <CL/cl.h>
+#include <jit/kernel_generators.hpp>
+
+#include <kernel_headers/KParam.hpp>
+#include <kernel_headers/jit.hpp>
+
 #include <Array.hpp>
-#include <common/compile_module.hpp>
+#include <common/debug.hpp>
 #include <common/dispatch.hpp>
+#include <common/half.hpp>
 #include <common/jit/ModdimNode.hpp>
 #include <common/jit/Node.hpp>
 #include <common/jit/NodeIterator.hpp>
-#include <common/kernel_cache.hpp>
 #include <common/util.hpp>
 #include <copy.hpp>
 #include <device_manager.hpp>
 #include <err_oneapi.hpp>
+#include <jit/BufferNode.hpp>
+#include <platform.hpp>
+#include <type_util.hpp>
 #include <af/dim4.hpp>
 
-//#include <jit/BufferNode.hpp>
+#include <sycl/backend.hpp>
+#include <sycl/interop_handle.hpp>
 
+#include <array>
 #include <cstdio>
 #include <functional>
 #include <sstream>
@@ -30,44 +41,553 @@
 #include <vector>
 
 using arrayfire::common::getFuncName;
+using arrayfire::common::half;
+using arrayfire::common::ModdimNode;
 using arrayfire::common::Node;
 using arrayfire::common::Node_ids;
 using arrayfire::common::Node_map_t;
+using arrayfire::common::Node_ptr;
+using arrayfire::common::NodeIterator;
+using arrayfire::oneapi::getActiveDeviceBaseBuildFlags;
+using arrayfire::oneapi::jit::BufferNode;
 
+using std::array;
+using std::find_if;
 using std::string;
 using std::stringstream;
 using std::to_string;
 using std::vector;
 
+using sycl::backend;
+
 namespace arrayfire {
-namespace oneapi {
 
-string getKernelString(const string &funcName, const vector<Node *> &full_nodes,
-                       const vector<Node_ids> &full_ids,
-                       const vector<int> &output_ids, bool is_linear) {
-    ONEAPI_NOT_SUPPORTED("");
-    return "";
-}
+namespace opencl {
+string getKernelString(const string& funcName, const vector<Node*>& full_nodes,
+                       const vector<Node_ids>& full_ids,
+                       const vector<int>& output_ids, const bool is_linear,
+                       const bool loop0, const bool loop1, const bool loop3) {
+    // Common OpenCL code
+    // This part of the code does not change with the kernel.
+
+    static const char* kernelVoid = R"JIT(
+__kernel void )JIT";
+    static const char* dimParams  = "KParam oInfo";
+    static const char* blockStart = "{";
+    static const char* blockEnd   = "\n}\n";
+
+    static const char* linearInit = R"JIT(
+   int idx = get_global_id(0);
+   const int idxEnd = oInfo.dims[0];
+   if (idx < idxEnd) {
+)JIT";
+    static const char* linearEnd  = R"JIT(
+   })JIT";
+
+    static const char* linearLoop0Start = R"JIT(
+        const int idxID0Inc = get_global_size(0);
+        do {)JIT";
+    static const char* linearLoop0End   = R"JIT(
+            idx += idxID0Inc;
+            if (idx >= idxEnd) break;
+        } while (true);)JIT";
+
+    // ///////////////////////////////////////////////
+    // oInfo = output optimized information (dims, strides, offset).
+    //         oInfo has removed dimensions, to optimized block scheduling
+    // iInfo = input internal information (dims, strides, offset)
+    //         iInfo has the original dimensions, auto generated code
+    //
+    // Loop3 is fastest and becomes inside loop, since
+    //      - #of loops is known upfront
+    // Loop1 is used for extra dynamic looping (writing into cache)
+    // All loops are conditional and idependent
+    // Format Loop1 & Loop3
+    // ////////////////////////////
+    //  *stridedLoopNInit               // Always
+    //  *stridedLoop1Init               // Conditional
+    //  *stridedLoop2Init               // Conditional
+    //  *stridedLoop3Init               // Conditional
+    //  *stridedLoop1Start              // Conditional
+    //      *stridedLoop3Start          // Conditional
+    //          auto generated code     // Always
+    //      *stridedLoop3End            // Conditional
+    //  *stridedLoop1End                // Conditional
+    //  *StridedEnd                     // Always
+    //
+    // format loop0 (Vector only)
+    // //////////////////////////
+    // *stridedLoop0Init                // Always
+    // *stridedLoop0Start               // Always
+    //      auto generated code         // Always
+    // *stridedLoop0End                 // Always
+    // *stridedEnd                      // Always
+
+    static const char* stridedLoop0Init  = R"JIT(
+    int id0 = get_global_id(0);
+    const int id0End = oInfo.dims[0];
+    if (id0 < id0End) {
+#define id1 0
+#define id2 0
+#define id3 0
+        const int ostrides0 = oInfo.strides[0];
+        int idx = ostrides0*id0;)JIT";
+    static const char* stridedLoop0Start = R"JIT(
+        const int id0Inc = get_global_size(0);
+        const int idxID0Inc = ostrides0*id0Inc;
+        do {)JIT";
+    static const char* stridedLoop0End   = R"JIT(
+            id0 += id0Inc;
+            if (id0 >= id0End) break;
+            idx += idxID0Inc;
+        } while (true);)JIT";
+
+    // -------------
+    static const char* stridedLoopNInit = R"JIT(
+    int id0 = get_global_id(0);
+    int id1 = get_global_id(1);
+    const int id0End = oInfo.dims[0];
+    const int id1End = oInfo.dims[1];
+    //printf("id0: %d  id1: %d id0End: %d, id1End: %d\n")
+    if ((id0 < id0End) & (id1 < id1End)) {
+        const int id2 = get_global_id(2);
+#define id3 0
+        const int ostrides1 = oInfo.strides[1];
+        int idx = (int)oInfo.strides[0]*id0 + ostrides1*id1 + (int)oInfo.strides[2]*id2;)JIT";
+    static const char* stridedEnd       = R"JIT(
+    })JIT";
+
+    static const char* stridedLoop3Init  = R"JIT(
+#undef id3
+        int id3 = 0;
+        const int id3End = oInfo.dims[3];
+        const int idxID3Inc = oInfo.strides[3];)JIT";
+    static const char* stridedLoop3Start = R"JIT(
+                const int idxBaseID3 = idx;
+                do {)JIT";
+    static const char* stridedLoop3End   = R"JIT(
+                    ++id3;
+                    if (id3 == id3End) break;
+                    idx += idxID3Inc;
+                } while (true);
+                id3 = 0;
+                idx = idxBaseID3;)JIT";
 
-/*
-cl::Kernel getKernel(const vector<Node *> &output_nodes,
-                     const vector<int> &output_ids,
-                     const vector<Node *> &full_nodes,
-                     const vector<Node_ids> &full_ids, const bool is_linear) {
-    ONEAPI_NOT_SUPPORTED("");
-    return common::getKernel("", "", true).get();
+    static const char* stridedLoop1Init  = R"JIT(
+        const int id1Inc = get_global_size(1);
+        const int idxID1Inc = id1Inc * ostrides1;)JIT";
+    static const char* stridedLoop1Start = R"JIT(
+        do {)JIT";
+    static const char* stridedLoop1End   = R"JIT(
+            id1 += id1Inc;
+            if (id1 >= id1End) break;
+            idx += idxID1Inc;
+        } while (true);)JIT";
+
+    // Reuse stringstreams, because they are very costly during initilization
+    thread_local stringstream inParamStream;
+    thread_local stringstream outParamStream;
+    thread_local stringstream outOffsetStream;
+    thread_local stringstream inOffsetsStream;
+    thread_local stringstream opsStream;
+
+    int oid{0};
+    for (size_t i{0}; i < full_nodes.size(); i++) {
+        const auto& node{full_nodes[i]};
+        const auto& ids_curr{full_ids[i]};
+        // Generate input parameters, only needs current id
+        node->genParams(inParamStream, ids_curr.id, is_linear);
+        // Generate input offsets, only needs current id
+        node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
+        // Generate the core function body, needs children ids as well
+        node->genFuncs(opsStream, ids_curr);
+        for (auto outIt{begin(output_ids)}, endIt{end(output_ids)};
+             (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) {
+            // Generate also output parameters
+            outParamStream << "__global "
+                           << full_nodes[ids_curr.id]->getTypeStr() << " *out"
+                           << oid << ", int offset" << oid << ",\n";
+            // Apply output offset
+            outOffsetStream << "\nout" << oid << " += offset" << oid << ';';
+            // Generate code to write the output
+            opsStream << "out" << oid << "[idx] = val" << ids_curr.id << ";\n";
+            ++oid;
+        }
+    }
+
+    thread_local stringstream kerStream;
+    kerStream << kernelVoid << funcName << "(\n"
+              << inParamStream.str() << outParamStream.str() << dimParams << ")"
+              << blockStart;
+    if (is_linear) {
+        kerStream << linearInit << inOffsetsStream.str()
+                  << outOffsetStream.str() << '\n';
+        if (loop0) kerStream << linearLoop0Start;
+        kerStream << "\n\n" << opsStream.str();
+        if (loop0) kerStream << linearLoop0End;
+        kerStream << linearEnd;
+    } else {
+        if (loop0) {
+            kerStream << stridedLoop0Init << outOffsetStream.str() << '\n'
+                      << stridedLoop0Start;
+        } else {
+            kerStream << stridedLoopNInit << outOffsetStream.str() << '\n';
+            if (loop3) kerStream << stridedLoop3Init;
+            if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
+            if (loop3) kerStream << stridedLoop3Start;
+        }
+        kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
+        if (loop3) kerStream << stridedLoop3End;
+        if (loop1) kerStream << stridedLoop1End;
+        if (loop0) kerStream << stridedLoop0End;
+        kerStream << stridedEnd;
+    }
+    kerStream << blockEnd;
+    const string ret{kerStream.str()};
+
+    // Prepare for next round, limit memory
+    inParamStream.str("");
+    outParamStream.str("");
+    inOffsetsStream.str("");
+    outOffsetStream.str("");
+    opsStream.str("");
+    kerStream.str("");
+
+    return ret;
 }
-*/
 
-/*
-void evalNodes(vector<Param> &outputs, const vector<Node *> &output_nodes) {
-    ONEAPI_NOT_SUPPORTED("");
+// cl::Kernel getKernel(const vector<Node*>& output_nodes,
+//                      const vector<int>& output_ids,
+//                      const vector<Node*>& full_nodes,
+//                      const vector<Node_ids>& full_ids, const bool is_linear)
+//                      {
+//     ONEAPI_NOT_SUPPORTED("");
+//     return common::getKernel("", "", true).get();
+// }
+
+}  // namespace opencl
+
+namespace oneapi {
+
+template<typename T>
+void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
+    if (outputs.empty()) return;
+    Node_map_t nodes;
+    vector<Node*> full_nodes;
+    vector<Node_ids> full_ids;
+    vector<int> output_ids;
+    vector<Node_ptr> node_clones;
+
+    bool is_linear{true};
+    dim_t numOutElems{1};
+    KParam& out_info{outputs[0].info};
+    dim_t* outDims{out_info.dims};
+    dim_t* outStrides{out_info.strides};
+
+    dim_t ndims{outDims[3] > 1   ? 4
+                : outDims[2] > 1 ? 3
+                : outDims[1] > 1 ? 2
+                : outDims[0] > 0 ? 1
+                                 : 0};
+    for (dim_t dim{0}; dim < ndims; ++dim) {
+        is_linear &= (numOutElems == outStrides[dim]);
+        numOutElems *= outDims[dim];
+    }
+    if (numOutElems == 0) { return; }
+
+    const af::dtype outputType{output_nodes[0]->getType()};
+    for (Node* node : output_nodes) {
+        assert(node->getType() == outputType);
+        const int id{node->getNodesMap(nodes, full_nodes, full_ids)};
+        output_ids.push_back(id);
+    }
+
+    bool moddimsFound{false};
+    for (const Node* node : full_nodes) {
+        is_linear &= node->isLinear(outDims);
+        moddimsFound |= (node->getOp() == af_moddims_t);
+    }
+
+    bool emptyColumnsFound{false};
+    if (is_linear) {
+        outDims[0]    = numOutElems;
+        outDims[1]    = 1;
+        outDims[2]    = 1;
+        outDims[3]    = 1;
+        outStrides[0] = 1;
+        outStrides[1] = numOutElems;
+        outStrides[2] = numOutElems;
+        outStrides[3] = numOutElems;
+        ndims         = 1;
+    } else {
+        emptyColumnsFound = ndims > (outDims[0] == 1   ? 1
+                                     : outDims[1] == 1 ? 2
+                                     : outDims[2] == 1 ? 3
+                                                       : 4);
+    }
+
+    // for (auto* node : full_nodes) SHOW(*node);
+    //  Keep in global scope, so that the nodes remain active for later
+    //  referral in case moddims operations or column elimination have to
+    //  take place
+    //  Avoid all cloning/copying when no moddims node is present (high
+    //  chance)
+    if (moddimsFound || emptyColumnsFound) {
+        node_clones.clear();
+        node_clones.reserve(full_nodes.size());
+        for (Node* node : full_nodes) {
+            node_clones.emplace_back(node->clone());
+        }
+
+        for (const Node_ids& ids : full_ids) {
+            auto& children{node_clones[ids.id]->m_children};
+            for (int i{0}; i < Node::kMaxChildren && children[i] != nullptr;
+                 i++) {
+                children[i] = node_clones[ids.child_ids[i]];
+            }
+        }
+
+        if (moddimsFound) {
+            const auto isModdim{[](const Node_ptr& ptr) {
+                return ptr->getOp() == af_moddims_t;
+            }};
+            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
+                 (nodeIt = find_if(nodeIt, endIt, isModdim)) != endIt;
+                 ++nodeIt) {
+                const ModdimNode* mn{static_cast<ModdimNode*>(nodeIt->get())};
+
+                const auto new_strides{calcStrides(mn->m_new_shape)};
+                const auto isBuffer{
+                    [](const Node& node) { return node.isBuffer(); }};
+                for (NodeIterator<> it{nodeIt->get()}, end{NodeIterator<>()};
+                     (it = find_if(it, end, isBuffer)) != end; ++it) {
+                    jit::BufferNode<T>* buf{
+                        static_cast<jit::BufferNode<T>*>(&(*it))};
+                    buf->m_param.dims[0]    = mn->m_new_shape[0];
+                    buf->m_param.dims[1]    = mn->m_new_shape[1];
+                    buf->m_param.dims[2]    = mn->m_new_shape[2];
+                    buf->m_param.dims[3]    = mn->m_new_shape[3];
+                    buf->m_param.strides[0] = new_strides[0];
+                    buf->m_param.strides[1] = new_strides[1];
+                    buf->m_param.strides[2] = new_strides[2];
+                    buf->m_param.strides[3] = new_strides[3];
+                }
+            }
+        }
+        if (emptyColumnsFound) {
+            const auto isBuffer{
+                [](const Node_ptr& ptr) { return ptr->isBuffer(); }};
+            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
+                 (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt;
+                 ++nodeIt) {
+                BufferNode<T>* buf{static_cast<BufferNode<T>*>(nodeIt->get())};
+                removeEmptyColumns(outDims, ndims, buf->m_param.dims.get(),
+                                   buf->m_param.strides.get());
+            }
+            for_each(++begin(outputs), end(outputs),
+                     [outDims, ndims](Param<T>& output) {
+                         removeEmptyColumns(outDims, ndims, output.info.dims,
+                                            output.info.strides);
+                     });
+            ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
+        }
+
+        full_nodes.clear();
+        for (Node_ptr& node : node_clones) { full_nodes.push_back(node.get()); }
+    }
+
+    const string funcName{getFuncName(output_nodes, full_nodes, full_ids,
+                                      is_linear, false, false, false,
+                                      outputs[0].info.dims[2] > 1)};
+
+    getQueue()
+        .submit([&](sycl::handler& h) {
+            for (Node* node : full_nodes) {
+                if (node->isBuffer()) {
+                    BufferNode<T>* n = static_cast<BufferNode<T>*>(node);
+                    n->m_param.require(h);
+                }
+            }
+            vector<AParam<T>> ap;
+            transform(begin(outputs), end(outputs), back_inserter(ap),
+                      [&](const Param<T>& p) {
+                          return AParam<T>(h, *p.data, p.info.dims,
+                                           p.info.strides, p.info.offset);
+                      });
+
+            h.host_task([ap, full_nodes, output_ids, full_ids, is_linear,
+                         funcName](sycl::interop_handle hh) {
+                switch (hh.get_backend()) {
+                    case backend::opencl: {
+                        string jitstr = arrayfire::opencl::getKernelString(
+                            funcName, full_nodes, full_ids, output_ids,
+                            is_linear, false, false, ap[0].dims[2] > 1);
+
+                        cl_command_queue q =
+                            hh.get_native_queue<backend::opencl>();
+                        cl_context ctx =
+                            hh.get_native_context<backend::opencl>();
+                        cl_device_id dev =
+                            hh.get_native_device<backend::opencl>();
+
+                        cl_int err;
+                        vector<const char*> jitsources = {
+                            {arrayfire::oneapi::opencl::KParam_hpp,
+                             arrayfire::oneapi::opencl::jit_cl,
+                             jitstr.c_str()}};
+                        vector<size_t> jitsizes = {
+                            arrayfire::oneapi::opencl::KParam_hpp_len,
+                            arrayfire::oneapi::opencl::jit_cl_len,
+                            jitstr.size()};
+
+                        cl_program prog = clCreateProgramWithSource(
+                            ctx, jitsources.size(), jitsources.data(),
+                            jitsizes.data(), &err);
+
+                        std::string options = getActiveDeviceBaseBuildFlags();
+
+                        CL_CHECK_BUILD(clBuildProgram(
+                            prog, 1, &dev, options.c_str(), nullptr, nullptr));
+
+                        vector<cl_kernel> kernels(10);
+                        cl_uint ret_kernels = 0;
+                        CL_CHECK(clCreateKernelsInProgram(
+                            prog, 1, kernels.data(), &ret_kernels));
+                        int nargs{0};
+                        for (Node* node : full_nodes) {
+                            if (node->isBuffer()) {
+                                nargs = node->setArgs(
+                                    nargs, is_linear,
+                                    [&kernels, &hh, &is_linear](
+                                        int id, const void* ptr,
+                                        size_t arg_size) {
+                                        AParam<T>* info =
+                                            static_cast<AParam<T>*>(
+                                                const_cast<void*>(ptr));
+                                        vector<cl_mem> mem =
+                                            hh.get_native_mem<backend::opencl>(
+                                                info->ph.value());
+                                        if (is_linear) {
+                                            CL_CHECK(clSetKernelArg(
+                                                kernels[0], id++,
+                                                sizeof(cl_mem), &mem[0]));
+                                            CL_CHECK(clSetKernelArg(
+                                                kernels[0], id++, sizeof(dim_t),
+                                                &info->offset));
+                                        } else {
+                                            CL_CHECK(clSetKernelArg(
+                                                kernels[0], id++,
+                                                sizeof(cl_mem), &mem[0]));
+                                            KParam ooo = *info;
+                                            CL_CHECK(clSetKernelArg(
+                                                kernels[0], id++,
+                                                sizeof(KParam), &ooo));
+                                        }
+                                    });
+                            } else {
+                                nargs = node->setArgs(
+                                    nargs, is_linear,
+                                    [&kernels](int id, const void* ptr,
+                                               size_t arg_size) {
+                                        CL_CHECK(clSetKernelArg(kernels[0], id,
+                                                                arg_size, ptr));
+                                    });
+                            }
+                        }
+
+                        // Set output parameters
+                        vector<cl_mem> mem;
+                        for (const auto& output : ap) {
+                            mem = hh.get_native_mem<backend::opencl>(
+                                output.data.value());
+                            cl_mem mmm = mem[0];
+                            CL_CHECK(clSetKernelArg(kernels[0], nargs++,
+                                                    sizeof(cl_mem), &mmm));
+                            int off = output.offset;
+                            CL_CHECK(clSetKernelArg(kernels[0], nargs++,
+                                                    sizeof(int), &off));
+                        }
+                        const KParam ooo = ap[0];
+                        CL_CHECK(clSetKernelArg(kernels[0], nargs++,
+                                                sizeof(KParam), &ooo));
+                        array<size_t, 3> offset{0, 0, 0};
+                        array<size_t, 3> global;
+                        int ndims = 0;
+                        if (is_linear) {
+                            global = {(size_t)ap[0].dims.elements(), 0, 0};
+                            ndims  = 1;
+                        } else {
+                            global = {(size_t)ap[0].dims[0],
+                                      (size_t)ap[0].dims[1],
+                                      (size_t)ap[0].dims[2]};
+                            ndims  = 3;
+                        }
+                        // SHOW(global);
+                        CL_CHECK(clEnqueueNDRangeKernel(
+                            q, kernels[0], ndims, offset.data(), global.data(),
+                            nullptr, 0, nullptr, nullptr));
+
+                        CL_CHECK(clReleaseKernel(kernels[0]));
+                        CL_CHECK(clReleaseProgram(prog));
+                        CL_CHECK(clReleaseDevice(dev));
+                        CL_CHECK(clReleaseContext(ctx));
+                        CL_CHECK(clReleaseCommandQueue(q));
+
+                    } break;
+                    default: ONEAPI_NOT_SUPPORTED("Backend not supported");
+                }
+            });
+        })
+        .wait();
 }
 
-void evalNodes(Param &out, Node *node) {
-    ONEAPI_NOT_SUPPORTED("");
+template<typename T>
+void evalNodes(Param<T>& out, Node* node) {
+    vector<Param<T>> outputs{out};
+    vector<Node*> nodes{node};
+    oneapi::evalNodes(outputs, nodes);
 }
-*/
+
+template void evalNodes<float>(Param<float>& out, Node* node);
+template void evalNodes<double>(Param<double>& out, Node* node);
+template void evalNodes<cfloat>(Param<cfloat>& out, Node* node);
+template void evalNodes<cdouble>(Param<cdouble>& out, Node* node);
+template void evalNodes<int>(Param<int>& out, Node* node);
+template void evalNodes<uint>(Param<uint>& out, Node* node);
+template void evalNodes<char>(Param<char>& out, Node* node);
+template void evalNodes<uchar>(Param<uchar>& out, Node* node);
+template void evalNodes<intl>(Param<intl>& out, Node* node);
+template void evalNodes<uintl>(Param<uintl>& out, Node* node);
+template void evalNodes<short>(Param<short>& out, Node* node);
+template void evalNodes<ushort>(Param<ushort>& out, Node* node);
+template void evalNodes<half>(Param<half>& out, Node* node);
+
+template void evalNodes<float>(vector<Param<float>>& out,
+                               const vector<Node*>& node);
+template void evalNodes<double>(vector<Param<double>>& out,
+                                const vector<Node*>& node);
+template void evalNodes<cfloat>(vector<Param<cfloat>>& out,
+                                const vector<Node*>& node);
+template void evalNodes<cdouble>(vector<Param<cdouble>>& out,
+                                 const vector<Node*>& node);
+template void evalNodes<int>(vector<Param<int>>& out,
+                             const vector<Node*>& node);
+template void evalNodes<uint>(vector<Param<uint>>& out,
+                              const vector<Node*>& node);
+template void evalNodes<char>(vector<Param<char>>& out,
+                              const vector<Node*>& node);
+template void evalNodes<uchar>(vector<Param<uchar>>& out,
+                               const vector<Node*>& node);
+template void evalNodes<intl>(vector<Param<intl>>& out,
+                              const vector<Node*>& node);
+template void evalNodes<uintl>(vector<Param<uintl>>& out,
+                               const vector<Node*>& node);
+template void evalNodes<short>(vector<Param<short>>& out,
+                               const vector<Node*>& node);
+template void evalNodes<ushort>(vector<Param<ushort>>& out,
+                                const vector<Node*>& node);
+template void evalNodes<half>(vector<Param<half>>& out,
+                              const vector<Node*>& node);
 
 }  // namespace oneapi
 }  // namespace arrayfire
diff --git a/src/backend/oneapi/jit/BufferNode.hpp b/src/backend/oneapi/jit/BufferNode.hpp
index 5f8ead77e0..b6bedc5baf 100644
--- a/src/backend/oneapi/jit/BufferNode.hpp
+++ b/src/backend/oneapi/jit/BufferNode.hpp
@@ -8,7 +8,9 @@
  ********************************************************/
 
 #pragma once
+#include <Param.hpp>
 #include <common/jit/BufferNodeBase.hpp>
+#include <jit/kernel_generators.hpp>
 
 #include <memory>
 
@@ -17,7 +19,7 @@ namespace oneapi {
 namespace jit {
 template<typename T>
 using BufferNode =
-    common::BufferNodeBase<std::shared_ptr<sycl::buffer<T>>, KParam>;
+    common::BufferNodeBase<std::shared_ptr<sycl::buffer<T>>, AParam<T>>;
 }
 }  // namespace oneapi
 
diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
index b3753955b9..3a15f78e8e 100644
--- a/src/backend/oneapi/jit/kernel_generators.hpp
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -8,11 +8,16 @@
  ********************************************************/
 
 #pragma once
+#include <Param.hpp>
+#include <err_oneapi.hpp>
+
+#include <sycl/buffer.hpp>
+
+#include <functional>
+#include <memory>
 #include <sstream>
 #include <string>
 
-#include <err_oneapi.hpp>
-
 namespace arrayfire {
 namespace oneapi {
 
@@ -27,7 +32,7 @@ inline void generateParamDeclaration(std::stringstream& kerStream, int id,
                   << ", dim_t iInfo" << id << "_offset, \n";
     } else {
         kerStream << "__global " << m_type_str << " *in" << id
-                  << ", Param iInfo" << id << ", \n";
+                  << ", KParam iInfo" << id << ", \n";
     }
 }
 
@@ -36,18 +41,8 @@ template<typename T>
 inline int setKernelArguments(
     int start_id, bool is_linear,
     std::function<void(int id, const void* ptr, size_t arg_size)>& setArg,
-    const std::shared_ptr<sycl::buffer<T>>& ptr, const KParam& info) {
-    // TODO(oneapi)
-    ONEAPI_NOT_SUPPORTED("ERROR");
-    // setArg(start_id + 0, static_cast<const void*>(&ptr.get()->operator()()),
-    // sizeof(cl_mem));
-    if (is_linear) {
-        // setArg(start_id + 1, static_cast<const void*>(&info.offset),
-        // sizeof(dim_t));
-    } else {
-        // setArg(start_id + 1, static_cast<const void*>(&info),
-        // sizeof(KParam));
-    }
+    const std::shared_ptr<sycl::buffer<T>>& ptr, const AParam<T>& info) {
+    setArg(start_id + 0, static_cast<const void*>(&info), sizeof(Param<T>));
     return start_id + 2;
 }
 
diff --git a/src/backend/oneapi/kernel/KParam.hpp b/src/backend/oneapi/kernel/KParam.hpp
index b5bb98e850..c1cf30be4b 100644
--- a/src/backend/oneapi/kernel/KParam.hpp
+++ b/src/backend/oneapi/kernel/KParam.hpp
@@ -10,11 +10,11 @@
 #ifndef __KPARAM_H
 #define __KPARAM_H
 
-//#ifndef __OPENCL_VERSION__
-// Only define dim_t in host code. dim_t is defined when setting the program
-// options in program.cpp
+// #ifndef __OPENCL_VERSION__
+//  Only define dim_t in host code. dim_t is defined when setting the program
+//  options in program.cpp
 #include <af/defines.h>
-//#endif
+// #endif
 
 // Defines the size and shape of the data in the OpenCL buffer
 typedef struct {
diff --git a/src/backend/oneapi/kernel/histogram.hpp b/src/backend/oneapi/kernel/histogram.hpp
index ea6c4c229a..3d53930bd4 100644
--- a/src/backend/oneapi/kernel/histogram.hpp
+++ b/src/backend/oneapi/kernel/histogram.hpp
@@ -142,15 +142,6 @@ void histogram(Param<uint> out, const Param<T> in, int nbins, float minval,
     const size_t global1 = in.info.dims[3];
     auto global          = sycl::range{global0, global1};
 
-    // \TODO drop this first memset once createEmptyArray is reverted back to
-    //       createValueArray in ../histogram.cpp
-    getQueue()
-        .submit([&](sycl::handler &h) {
-            auto outAcc = out.data->get_access(h);
-            h.parallel_for(sycl::range<1>{(size_t)nbins},
-                           [=](sycl::id<1> idx) { outAcc[idx[0]] = 0; });
-        })
-        .wait();
     getQueue().submit([&](sycl::handler &h) {
         auto inAcc  = in.data->get_access(h);
         auto outAcc = out.data->get_access(h);
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index bb1aa99d21..ee27b706d9 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -262,14 +262,8 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
             AF_ERR_RUNTIME);
     }
     Array<To> tmp = createEmptyArray<To>(tmp_elements);
-    // TODO: JIT dependency
-    // Array<unsigned> retirementCount = createValueArray<unsigned>(1, 0);
-    Array<unsigned> retirementCount = createEmptyArray<unsigned>(1);
-    getQueue().submit([=](sycl::handler &h) {
-        auto acc = retirementCount.getData()->get_access(h);
-        h.single_task([=] { acc[0] = 0; });
-    });
 
+    Array<unsigned> retirementCount = createValueArray<unsigned>(1, 0);
     getQueue().submit([=](sycl::handler &h) {
         write_accessor<To> out_acc{*out.data, h};
         auto retCount_acc = retirementCount.getData()->get_access(h);
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index ee47082295..aa620e8e2c 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -205,18 +205,10 @@ void Allocator::shutdown() {
     // }
 }
 
-int Allocator::getActiveDeviceId() {
-    ONEAPI_NOT_SUPPORTED("Allocator::getActiveDeviceId Not supported");
-
-    return 0;
-    // return opencl::getActiveDeviceId();
-}
+int Allocator::getActiveDeviceId() { return oneapi::getActiveDeviceId(); }
 
 size_t Allocator::getMaxMemorySize(int id) {
-    ONEAPI_NOT_SUPPORTED("Allocator::getMaxMemorySize Not supported");
-
-    return 0;
-    // return opencl::getDeviceMemorySize(id);
+    return oneapi::getDeviceMemorySize(id);
 }
 
 void *Allocator::nativeAlloc(const size_t bytes) {

From c167e9f16eb4b7bdff1018e7a0630dd722b3f57c Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 7 Dec 2022 17:57:11 -0500
Subject: [PATCH 222/473] Add device-code-split flags to oneapi backend to
 avoid double failures

---
 src/backend/oneapi/CMakeLists.txt | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 5b5684038d..f7d1033b5a 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -281,9 +281,6 @@ target_include_directories(afoneapi
 target_compile_options(afoneapi
   PRIVATE
     -fsycl
-    #-fsycl-targets=nvptx64-vidia-cuda
-    #-fsycl-force-target=nvptx64-nvidia-cuda-sm_86
-    #-Wno-unknown-cuda-version
     -sycl-std=2020
 )
 
@@ -301,14 +298,14 @@ target_link_libraries(afoneapi
     -fsycl
     -fno-lto
     -fvisibility-inlines-hidden
-    #-fsycl-targets=nvptx64-nvidia-cuda-sm_86
-    #-fsycl-force-target=nvptx64-nvidia-cuda-sm_86
     c_api_interface
     cpp_api_interface
     afcommon_interface
     OpenCL::OpenCL
     OpenCL::cl2hpp
-    #-Wno-unknown-cuda-version
+    -fsycl
+    -fsycl-device-code-split=per_kernel
+    -fsycl-link-huge-device-code
   )
 
 af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR})

From 1e4b1a02623d4240d1d7aaaaf14f5d9e467c199d Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Tue, 14 Mar 2023 19:26:16 +0100
Subject: [PATCH 223/473] multithreaded OPENBLAS & FFTW

---
 vcpkg.json | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/vcpkg.json b/vcpkg.json
index 72625d8fa9..5cf6972ce0 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -46,8 +46,14 @@
         "openblasfftw": {
             "description": "Build with OpenBLAS/FFTW",
             "dependencies": [
-                "fftw3",
-                "openblas",
+                {
+                    "name": "fftw3",
+                    "features": [ "threads" ]
+                },
+                {
+                    "name": "openblas",
+                    "features": [ "threads" ]
+                },
                 "lapack"
             ]
         },

From 97d4e61cc5941c0b63ca700b0398d19d42c162e4 Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Fri, 17 Mar 2023 13:39:40 -0400
Subject: [PATCH 224/473] Implementation of FFT for oneAPI (#3379)

FFT implementation for the oneAPI backend

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
---
 src/backend/oneapi/CMakeLists.txt     |   3 +
 src/backend/oneapi/fft.cpp            | 184 ++++++++++++++++++--------
 src/backend/oneapi/kernel/memcopy.hpp |  41 +++---
 src/backend/oneapi/reshape.cpp        |  13 +-
 4 files changed, 162 insertions(+), 79 deletions(-)

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index f7d1033b5a..7e61118811 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -281,6 +281,7 @@ target_include_directories(afoneapi
 target_compile_options(afoneapi
   PRIVATE
     -fsycl
+    -qopenmp -qmkl=parallel
     -sycl-std=2020
 )
 
@@ -306,6 +307,8 @@ target_link_libraries(afoneapi
     -fsycl
     -fsycl-device-code-split=per_kernel
     -fsycl-link-huge-device-code
+    -qopenmp
+    -qmkl=parallel
   )
 
 af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR})
diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
index 9ccdcfcb86..8ac2cd410c 100644
--- a/src/backend/oneapi/fft.cpp
+++ b/src/backend/oneapi/fft.cpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2022, ArrayFire
+ * Copyright (c) 2023, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -7,6 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <common/dispatch.hpp>
+
 #include <fft.hpp>
 
 #include <copy.hpp>
@@ -15,78 +17,156 @@
 #include <memory.hpp>
 #include <af/dim4.hpp>
 
+#include <array>
+using std::array;
+
 using af::dim4;
 
+#include <oneapi/mkl/dfti.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 
 void setFFTPlanCacheSize(size_t numPlans) {}
 
-/*
-template<typename T>
-struct Precision;
-template<>
-struct Precision<cfloat> {
-    enum { type = CLFFT_SINGLE };
-};
-template<>
-struct Precision<cdouble> {
-    enum { type = CLFFT_DOUBLE };
-};
-*/
-
-void computeDims(size_t rdims[AF_MAX_DIMS], const dim4 &idims) {
-    for (int i = 0; i < AF_MAX_DIMS; i++) {
-        rdims[i] = static_cast<size_t>(idims[i]);
-    }
-}
-
-//(currently) true is in clFFT if length is a power of 2,3,5
-inline bool isSupLen(dim_t length) {
-    while (length > 1) {
-        if (length % 2 == 0) {
-            length /= 2;
-        } else if (length % 3 == 0) {
-            length /= 3;
-        } else if (length % 5 == 0) {
-            length /= 5;
-        } else if (length % 7 == 0) {
-            length /= 7;
-        } else if (length % 11 == 0) {
-            length /= 11;
-        } else if (length % 13 == 0) {
-            length /= 13;
-        } else {
-            return false;
-        }
-    }
-    return true;
-}
-
-void verifySupported(const int rank, const dim4 &dims) {
-    for (int i = 0; i < rank; i++) { ARG_ASSERT(1, isSupLen(dims[i])); }
+inline array<int, AF_MAX_DIMS> computeDims(const int rank, const dim4 &idims) {
+    array<int, AF_MAX_DIMS> retVal = {};
+    for (int i = 0; i < rank; i++) { retVal[i] = idims[(rank - 1) - i]; }
+    return retVal;
 }
 
 template<typename T>
 void fft_inplace(Array<T> &in, const int rank, const bool direction) {
-    ONEAPI_NOT_SUPPORTED("");
+    const dim4 idims    = in.dims();
+    const dim4 istrides = in.strides();
+
+    constexpr bool is_single = std::is_same_v<T, cfloat>;
+    constexpr auto precision = (is_single)
+                                   ? ::oneapi::mkl::dft::precision::SINGLE
+                                   : ::oneapi::mkl::dft::precision::DOUBLE;
+    using desc_ty =
+        ::oneapi::mkl::dft::descriptor<precision,
+                                       ::oneapi::mkl::dft::domain::COMPLEX>;
+
+    auto desc = [rank, &idims]() {
+        if (rank == 1) return desc_ty(idims[0]);
+        if (rank == 2) return desc_ty({idims[0], idims[1]});
+        if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]});
+        return desc_ty({idims[0], idims[1], idims[2], idims[3]});
+    }();
+
+    desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT, DFTI_INPLACE);
+
+    int batch = 1;
+    for (int i = rank; i < 4; i++) { batch *= idims[i]; }
+    desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
+                   (int64_t)batch);
+
+    desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE,
+                   istrides[rank]);
+    desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE,
+                   istrides[rank]);
+
+    desc.commit(getQueue());
+    if (direction)
+        ::oneapi::mkl::dft::compute_forward(desc, *(in.getData()));
+    else
+        ::oneapi::mkl::dft::compute_backward(desc, *(in.getData()));
 }
 
 template<typename Tc, typename Tr>
 Array<Tc> fft_r2c(const Array<Tr> &in, const int rank) {
-    ONEAPI_NOT_SUPPORTED("");
-    dim4 odims = in.dims();
-
-    odims[0] = odims[0] / 2 + 1;
+    const dim4 idims    = in.dims();
+    const dim4 istrides = in.strides();
+    Array<Tc> out       = createEmptyArray<Tc>(
+        dim4({idims[0] / 2 + 1, idims[1], idims[2], idims[3]}));
+    const dim4 ostrides = out.strides();
+
+    constexpr bool is_single = std::is_same_v<Tr, float>;
+    constexpr auto precision = (is_single)
+                                   ? ::oneapi::mkl::dft::precision::SINGLE
+                                   : ::oneapi::mkl::dft::precision::DOUBLE;
+    using desc_ty =
+        ::oneapi::mkl::dft::descriptor<precision,
+                                       ::oneapi::mkl::dft::domain::REAL>;
+
+    auto desc = [rank, &idims]() {
+        if (rank == 1) return desc_ty(idims[0]);
+        if (rank == 2) return desc_ty({idims[0], idims[1]});
+        if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]});
+        return desc_ty({idims[0], idims[1], idims[2], idims[3]});
+    }();
+
+    desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
+                   DFTI_NOT_INPLACE);
+
+    int batch = 1;
+    for (int i = rank; i < 4; i++) { batch *= idims[i]; }
+    desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
+                   (int64_t)batch);
+
+    desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE,
+                   ostrides[rank]);
+    desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE,
+                   istrides[rank]);
+
+    const std::int64_t fft_output_strides[5] = {
+        0, ostrides[(rank == 2) ? 1 : 0], ostrides[(rank == 2) ? 0 : 1],
+        ostrides[2], ostrides[3]};
+    desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
+                   fft_output_strides, rank);
+
+    desc.commit(getQueue());
+    ::oneapi::mkl::dft::compute_forward(desc, *(in.getData()),
+                                        *(out.getData()));
 
-    Array<Tc> out = createEmptyArray<Tc>(odims);
     return out;
 }
 
 template<typename Tr, typename Tc>
 Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<Tr> out = createEmptyArray<Tr>(odims);
+    const dim4 idims    = in.dims();
+    const dim4 istrides = in.strides();
+    Array<Tr> out       = createEmptyArray<Tr>(odims);
+    const dim4 ostrides = out.strides();
+
+    constexpr bool is_single = std::is_same_v<Tr, float>;
+    constexpr auto precision = (is_single)
+                                   ? ::oneapi::mkl::dft::precision::SINGLE
+                                   : ::oneapi::mkl::dft::precision::DOUBLE;
+    using desc_ty =
+        ::oneapi::mkl::dft::descriptor<precision,
+                                       ::oneapi::mkl::dft::domain::REAL>;
+
+    auto desc = [rank, &odims]() {
+        if (rank == 1) return desc_ty(odims[0]);
+        if (rank == 2) return desc_ty({odims[0], odims[1]});
+        if (rank == 3) return desc_ty({odims[0], odims[1], odims[2]});
+        return desc_ty({odims[0], odims[1], odims[2], odims[3]});
+    }();
+
+    desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
+                   DFTI_NOT_INPLACE);
+
+    int batch = 1;
+    for (int i = rank; i < 4; i++) { batch *= idims[i]; }
+    desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
+                   (int64_t)batch);
+
+    desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE,
+                   istrides[rank]);
+    desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE,
+                   ostrides[rank]);
+
+    const std::int64_t fft_output_strides[5] = {
+        0, ostrides[(rank == 2) ? 1 : 0], ostrides[(rank == 2) ? 0 : 1],
+        ostrides[2], ostrides[3]};
+    desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
+                   fft_output_strides, rank);
+
+    desc.commit(getQueue());
+    ::oneapi::mkl::dft::compute_backward(desc, *(in.getData()),
+                                         *(out.getData()));
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 294573b1bf..9d5f966dc2 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2022, ArrayFire
+ * Copyright (c) 2023, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -9,6 +9,7 @@
 
 #pragma once
 
+#include <CL/sycl.hpp>
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
@@ -131,54 +132,54 @@ cdouble scale<cdouble>(cdouble value, double factor) {
 }
 
 template<typename inType, typename outType>
-outType convertType(inType value) {
+static outType convertType(inType value) {
     return static_cast<outType>(value);
 }
 
 template<>
-char convertType<compute_t<arrayfire::common::half>, char>(
+static char convertType<compute_t<arrayfire::common::half>, char>(
     compute_t<arrayfire::common::half> value) {
     return (char)((short)value);
 }
 
 template<>
-compute_t<arrayfire::common::half>
-convertType<char, compute_t<arrayfire::common::half>>(char value) {
+compute_t<arrayfire::common::half> static convertType<
+    char, compute_t<arrayfire::common::half>>(char value) {
     return compute_t<arrayfire::common::half>(value);
 }
 
 template<>
-unsigned char convertType<compute_t<arrayfire::common::half>, unsigned char>(
+static unsigned char
+convertType<compute_t<arrayfire::common::half>, unsigned char>(
     compute_t<arrayfire::common::half> value) {
     return (unsigned char)((short)value);
 }
 
 template<>
-compute_t<arrayfire::common::half>
-convertType<unsigned char, compute_t<arrayfire::common::half>>(
-    unsigned char value) {
+compute_t<arrayfire::common::half> static convertType<
+    unsigned char, compute_t<arrayfire::common::half>>(unsigned char value) {
     return compute_t<arrayfire::common::half>(value);
 }
 
 template<>
-cdouble convertType<cfloat, cdouble>(cfloat value) {
+static cdouble convertType<cfloat, cdouble>(cfloat value) {
     return cdouble(value.real(), value.imag());
 }
 
 template<>
-cfloat convertType<cdouble, cfloat>(cdouble value) {
+static cfloat convertType<cdouble, cfloat>(cdouble value) {
     return cfloat(value.real(), value.imag());
 }
 
-#define OTHER_SPECIALIZATIONS(IN_T)                      \
-    template<>                                           \
-    cfloat convertType<IN_T, cfloat>(IN_T value) {       \
-        return cfloat(static_cast<float>(value), 0.0f);  \
-    }                                                    \
-                                                         \
-    template<>                                           \
-    cdouble convertType<IN_T, cdouble>(IN_T value) {     \
-        return cdouble(static_cast<double>(value), 0.0); \
+#define OTHER_SPECIALIZATIONS(IN_T)                         \
+    template<>                                              \
+    static cfloat convertType<IN_T, cfloat>(IN_T value) {   \
+        return cfloat(static_cast<float>(value), 0.0f);     \
+    }                                                       \
+                                                            \
+    template<>                                              \
+    static cdouble convertType<IN_T, cdouble>(IN_T value) { \
+        return cdouble(static_cast<double>(value), 0.0);    \
     }
 
 OTHER_SPECIALIZATIONS(float)
diff --git a/src/backend/oneapi/reshape.cpp b/src/backend/oneapi/reshape.cpp
index 768a167480..8f1b6f0ecb 100644
--- a/src/backend/oneapi/reshape.cpp
+++ b/src/backend/oneapi/reshape.cpp
@@ -1,6 +1,5 @@
-
 /*******************************************************
- * Copyright (c) 2020, ArrayFire
+ * Copyright (c) 2023, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -12,7 +11,7 @@
 #include <err_oneapi.hpp>
 
 #include <common/half.hpp>
-// #include <kernel/memcopy.hpp>
+#include <kernel/memcopy.hpp>
 
 using arrayfire::common::half;
 
@@ -22,11 +21,11 @@ namespace oneapi {
 template<typename inType, typename outType>
 Array<outType> reshape(const Array<inType> &in, const dim4 &outDims,
                        outType defaultValue, double scale) {
-    ONEAPI_NOT_SUPPORTED("reshape Not supported");
-
     Array<outType> out = createEmptyArray<outType>(outDims);
-    // kernel::copy<inType, outType>(out, in, in.ndims(), defaultValue, scale,
-    //                               in.dims() == outDims);
+    if (out.elements() > 0) {
+        kernel::copy<inType, outType>(out, in, in.ndims(), defaultValue, scale,
+                                      in.dims() == outDims);
+    }
     return out;
 }
 

From 8fc5ee650fdbeb2078746b5d6fd33789f55ab347 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 22 Mar 2023 15:04:58 -0400
Subject: [PATCH 225/473] Fix warnings in oneAPI backend and remove
 sycl::streams

---
 src/backend/oneapi/Kernel.hpp                 |  1 -
 src/backend/oneapi/Module.hpp                 |  3 +-
 src/backend/oneapi/compile_module.cpp         |  1 -
 src/backend/oneapi/copy.cpp                   | 10 ++--
 src/backend/oneapi/kernel/bilateral.hpp       |  4 +-
 src/backend/oneapi/kernel/convolve2.hpp       |  3 --
 src/backend/oneapi/kernel/convolve3.hpp       | 18 +++-----
 src/backend/oneapi/kernel/memcopy.hpp         | 38 ++++++---------
 src/backend/oneapi/kernel/random_engine.hpp   | 46 ++++++++-----------
 .../oneapi/kernel/random_engine_mersenne.hpp  | 28 ++++-------
 .../oneapi/kernel/random_engine_philox.hpp    | 18 ++------
 .../oneapi/kernel/random_engine_threefry.hpp  | 14 ++----
 .../oneapi/kernel/random_engine_write.hpp     |  3 +-
 src/backend/oneapi/kernel/range.hpp           | 33 ++++---------
 src/backend/oneapi/kernel/rotate.hpp          |  3 --
 src/backend/oneapi/kernel/transform.hpp       |  6 ---
 .../oneapi/kernel/transpose_inplace.hpp       | 25 ++++------
 17 files changed, 85 insertions(+), 169 deletions(-)

diff --git a/src/backend/oneapi/Kernel.hpp b/src/backend/oneapi/Kernel.hpp
index ee5a2fcd02..3fcf7b66b8 100644
--- a/src/backend/oneapi/Kernel.hpp
+++ b/src/backend/oneapi/Kernel.hpp
@@ -12,7 +12,6 @@
 #include <common/KernelInterface.hpp>
 #include <common/Logger.hpp>
 
-#include <CL/sycl.hpp>
 #include <backend.hpp>
 #include <string>
 
diff --git a/src/backend/oneapi/Module.hpp b/src/backend/oneapi/Module.hpp
index c4de202761..cb4c4e130c 100644
--- a/src/backend/oneapi/Module.hpp
+++ b/src/backend/oneapi/Module.hpp
@@ -9,9 +9,10 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <common/ModuleInterface.hpp>
 
+#include <sycl/kernel_bundle.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 
diff --git a/src/backend/oneapi/compile_module.cpp b/src/backend/oneapi/compile_module.cpp
index 39783a3c53..640fcc797c 100644
--- a/src/backend/oneapi/compile_module.cpp
+++ b/src/backend/oneapi/compile_module.cpp
@@ -10,7 +10,6 @@
 #include <common/compile_module.hpp>  //compileModule & loadModuleFromDisk
 #include <common/kernel_cache.hpp>    //getKernel(Module&, ...)
 
-#include <CL/sycl.hpp>
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
 #include <common/util.hpp>
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index 23106f7dd1..4059bd27f0 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -219,10 +219,12 @@ T getScalar(const Array<T> &in) {
 
     getQueue()
         .submit([&](sycl::handler &h) {
-            auto acc_in = in.getData()->get_access(
-                h, sycl::range{1},
-                sycl::id{static_cast<uintl>(in.getOffset())});
-            auto acc_out = retBuffer.get_access();
+            auto acc_in =
+                in.get()->template get_access<sycl::access::mode::read>(
+                    h, sycl::range{1},
+                    sycl::id{static_cast<uintl>(in.getOffset())});
+            auto acc_out =
+                retBuffer.template get_access<sycl::access::mode::write>(h);
             h.copy(acc_in, acc_out);
         })
         .wait();
diff --git a/src/backend/oneapi/kernel/bilateral.hpp b/src/backend/oneapi/kernel/bilateral.hpp
index cb3d323f07..c01ee4a4a5 100644
--- a/src/backend/oneapi/kernel/bilateral.hpp
+++ b/src/backend/oneapi/kernel/bilateral.hpp
@@ -148,8 +148,8 @@ class bilateralKernel {
     void load2LocalMem(local_accessor<outType, 1> shrd, const inType* in,
                        int lx, int ly, int shrdStride, int dim0, int dim1,
                        int gx, int gy, int inStride1, int inStride0) const {
-        int gx_ = std::clamp(gx, 0, dim0 - 1);
-        int gy_ = std::clamp(gy, 0, dim1 - 1);
+        int gx_ = sycl::clamp(gx, 0, dim0 - 1);
+        int gy_ = sycl::clamp(gy, 0, dim1 - 1);
         shrd[lIdx(lx, ly, shrdStride, 1)] =
             (outType)in[lIdx(gx_, gy_, inStride1, inStride0)];
     }
diff --git a/src/backend/oneapi/kernel/convolve2.hpp b/src/backend/oneapi/kernel/convolve2.hpp
index 173405bdb8..5de34a2023 100644
--- a/src/backend/oneapi/kernel/convolve2.hpp
+++ b/src/backend/oneapi/kernel/convolve2.hpp
@@ -121,9 +121,6 @@ template<typename T, typename aT>
 void conv2Helper(const conv_kparam_t<aT> &param, Param<T> out,
                  const Param<T> signal, const Param<aT> filter,
                  const bool expand) {
-    constexpr bool IsComplex =
-        std::is_same<T, cfloat>::value || std::is_same<T, cdouble>::value;
-
     const int f0 = filter.info.dims[0];
     const int f1 = filter.info.dims[1];
     const size_t LOC_SIZE =
diff --git a/src/backend/oneapi/kernel/convolve3.hpp b/src/backend/oneapi/kernel/convolve3.hpp
index 57f1538ddc..0e2dee72fe 100644
--- a/src/backend/oneapi/kernel/convolve3.hpp
+++ b/src/backend/oneapi/kernel/convolve3.hpp
@@ -55,18 +55,12 @@ class conv3HelperCreateKernel {
              sstep3_ *
                  sInfo_.strides[3]); /* activated with batched input filter */
 
-        int lx  = it.get_local_id(0);
-        int ly  = it.get_local_id(1);
-        int lz  = it.get_local_id(2);
-        int gx  = g.get_local_range(0) * (g.get_group_id(0) - b2 * nBBS0_) + lx;
-        int gy  = g.get_local_range(1) * g.get_group_id(1) + ly;
-        int gz  = g.get_local_range(2) * g.get_group_id(2) + lz;
-        int lx2 = lx + g.get_local_range(0);
-        int ly2 = ly + g.get_local_range(1);
-        int lz2 = lz + g.get_local_range(2);
-        int gx2 = gx + g.get_local_range(0);
-        int gy2 = gy + g.get_local_range(1);
-        int gz2 = gz + g.get_local_range(2);
+        int lx = it.get_local_id(0);
+        int ly = it.get_local_id(1);
+        int lz = it.get_local_id(2);
+        int gx = g.get_local_range(0) * (g.get_group_id(0) - b2 * nBBS0_) + lx;
+        int gy = g.get_local_range(1) * g.get_group_id(1) + ly;
+        int gz = g.get_local_range(2) * g.get_group_id(2) + lz;
 
         int s0 = sInfo_.strides[0];
         int s1 = sInfo_.strides[1];
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 9d5f966dc2..adabe3b29d 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -9,7 +9,6 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
@@ -116,69 +115,60 @@ void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
 }
 
 template<typename T>
-static T scale(T value, double factor) {
+inline T scale(T value, double factor) {
     return (T)(double(value) * factor);
 }
 
 template<>
-cfloat scale<cfloat>(cfloat value, double factor) {
+inline cfloat scale<cfloat>(cfloat value, double factor) {
     return cfloat{static_cast<float>(value.real() * factor),
                   static_cast<float>(value.imag() * factor)};
 }
 
 template<>
-cdouble scale<cdouble>(cdouble value, double factor) {
+inline cdouble scale<cdouble>(cdouble value, double factor) {
     return cdouble{value.real() * factor, value.imag() * factor};
 }
 
 template<typename inType, typename outType>
-static outType convertType(inType value) {
+inline outType convertType(inType value) {
     return static_cast<outType>(value);
 }
 
 template<>
-static char convertType<compute_t<arrayfire::common::half>, char>(
+inline char convertType<compute_t<arrayfire::common::half>, char>(
     compute_t<arrayfire::common::half> value) {
     return (char)((short)value);
 }
 
 template<>
-compute_t<arrayfire::common::half> static convertType<
-    char, compute_t<arrayfire::common::half>>(char value) {
+inline compute_t<arrayfire::common::half>
+convertType<char, compute_t<arrayfire::common::half>>(char value) {
     return compute_t<arrayfire::common::half>(value);
 }
 
 template<>
-static unsigned char
-convertType<compute_t<arrayfire::common::half>, unsigned char>(
+unsigned char inline convertType<compute_t<arrayfire::common::half>,
+                                 unsigned char>(
     compute_t<arrayfire::common::half> value) {
     return (unsigned char)((short)value);
 }
 
 template<>
-compute_t<arrayfire::common::half> static convertType<
-    unsigned char, compute_t<arrayfire::common::half>>(unsigned char value) {
+inline compute_t<arrayfire::common::half>
+convertType<unsigned char, compute_t<arrayfire::common::half>>(
+    unsigned char value) {
     return compute_t<arrayfire::common::half>(value);
 }
 
-template<>
-static cdouble convertType<cfloat, cdouble>(cfloat value) {
-    return cdouble(value.real(), value.imag());
-}
-
-template<>
-static cfloat convertType<cdouble, cfloat>(cdouble value) {
-    return cfloat(value.real(), value.imag());
-}
-
 #define OTHER_SPECIALIZATIONS(IN_T)                         \
     template<>                                              \
-    static cfloat convertType<IN_T, cfloat>(IN_T value) {   \
+    inline cfloat convertType<IN_T, cfloat>(IN_T value) {   \
         return cfloat(static_cast<float>(value), 0.0f);     \
     }                                                       \
                                                             \
     template<>                                              \
-    static cdouble convertType<IN_T, cdouble>(IN_T value) { \
+    inline cdouble convertType<IN_T, cdouble>(IN_T value) { \
         return cdouble(static_cast<double>(value), 0.0);    \
     }
 
diff --git a/src/backend/oneapi/kernel/random_engine.hpp b/src/backend/oneapi/kernel/random_engine.hpp
index 66e286fea9..329387eef5 100644
--- a/src/backend/oneapi/kernel/random_engine.hpp
+++ b/src/backend/oneapi/kernel/random_engine.hpp
@@ -58,11 +58,9 @@ void uniformDistributionCBRNG(Param<T> out, const size_t elements,
             getQueue().submit([=](sycl::handler &h) {
                 auto out_acc = out.data->get_access(h);
 
-                sycl::stream debug_stream(2048, 128, h);
-                h.parallel_for(
-                    ndrange,
-                    uniformPhilox<T>(out_acc, hi, lo, hic, loc,
-                                     elementsPerBlock, elements, debug_stream));
+                h.parallel_for(ndrange,
+                               uniformPhilox<T>(out_acc, hi, lo, hic, loc,
+                                                elementsPerBlock, elements));
             });
             ONEAPI_DEBUG_FINISH(getQueue());
             break;
@@ -70,11 +68,9 @@ void uniformDistributionCBRNG(Param<T> out, const size_t elements,
             getQueue().submit([=](sycl::handler &h) {
                 auto out_acc = out.data->get_access(h);
 
-                sycl::stream debug_stream(2048, 128, h);
                 h.parallel_for(ndrange,
                                uniformThreefry<T>(out_acc, hi, lo, hic, loc,
-                                                  elementsPerBlock, elements,
-                                                  debug_stream));
+                                                  elementsPerBlock, elements));
             });
             ONEAPI_DEBUG_FINISH(getQueue());
             break;
@@ -102,22 +98,18 @@ void normalDistributionCBRNG(Param<T> out, const size_t elements,
             getQueue().submit([=](sycl::handler &h) {
                 auto out_acc = out.data->get_access(h);
 
-                sycl::stream debug_stream(2048, 128, h);
-                h.parallel_for(
-                    ndrange,
-                    normalPhilox<T>(out_acc, hi, lo, hic, loc, elementsPerBlock,
-                                    elements, debug_stream));
+                h.parallel_for(ndrange,
+                               normalPhilox<T>(out_acc, hi, lo, hic, loc,
+                                               elementsPerBlock, elements));
             });
             break;
         case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
             getQueue().submit([=](sycl::handler &h) {
                 auto out_acc = out.data->get_access(h);
 
-                sycl::stream debug_stream(2048, 128, h);
                 h.parallel_for(ndrange,
                                normalThreefry<T>(out_acc, hi, lo, hic, loc,
-                                                 elementsPerBlock, elements,
-                                                 debug_stream));
+                                                 elementsPerBlock, elements));
             });
             break;
         default:
@@ -154,12 +146,11 @@ void uniformDistributionMT(Param<T> out, const size_t elements,
         auto lrecursion_acc = local_accessor<uint, 1>(TABLE_SIZE, h);
         auto ltemper_acc    = local_accessor<uint, 1>(TABLE_SIZE, h);
 
-        sycl::stream debug_stream(2048, 128, h);
-        h.parallel_for(ndrange, uniformMersenne<T>(
-                                    out_acc, state_acc, pos_acc, sh1_acc,
-                                    sh2_acc, mask, recursion_acc, temper_acc,
-                                    lstate_acc, lrecursion_acc, ltemper_acc,
-                                    elementsPerBlock, elements, debug_stream));
+        h.parallel_for(
+            ndrange, uniformMersenne<T>(
+                         out_acc, state_acc, pos_acc, sh1_acc, sh2_acc, mask,
+                         recursion_acc, temper_acc, lstate_acc, lrecursion_acc,
+                         ltemper_acc, elementsPerBlock, elements));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
@@ -191,12 +182,11 @@ void normalDistributionMT(Param<T> out, const size_t elements,
         auto lrecursion_acc = local_accessor<uint, 1>(TABLE_SIZE, h);
         auto ltemper_acc    = local_accessor<uint, 1>(TABLE_SIZE, h);
 
-        sycl::stream debug_stream(2048, 128, h);
-        h.parallel_for(ndrange, normalMersenne<T>(
-                                    out_acc, state_acc, pos_acc, sh1_acc,
-                                    sh2_acc, mask, recursion_acc, temper_acc,
-                                    lstate_acc, lrecursion_acc, ltemper_acc,
-                                    elementsPerBlock, elements, debug_stream));
+        h.parallel_for(
+            ndrange, normalMersenne<T>(out_acc, state_acc, pos_acc, sh1_acc,
+                                       sh2_acc, mask, recursion_acc, temper_acc,
+                                       lstate_acc, lrecursion_acc, ltemper_acc,
+                                       elementsPerBlock, elements));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
diff --git a/src/backend/oneapi/kernel/random_engine_mersenne.hpp b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
index e0a0f57c8d..bbf5dae3e0 100644
--- a/src/backend/oneapi/kernel/random_engine_mersenne.hpp
+++ b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
@@ -107,13 +107,8 @@ static inline uint temper(const uint *const temper_table, const uint v,
 class initMersenneKernel {
    public:
     initMersenneKernel(sycl::accessor<uint> state, sycl::accessor<uint> tbl,
-                       local_accessor<uint, 1> lstate, uintl seed,
-                       sycl::stream debug_stream)
-        : state_(state)
-        , tbl_(tbl)
-        , lstate_(lstate)
-        , seed_(seed)
-        , debug_(debug_stream) {}
+                       local_accessor<uint, 1> lstate, uintl seed)
+        : state_(state), tbl_(tbl), lstate_(lstate), seed_(seed) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
@@ -147,7 +142,6 @@ class initMersenneKernel {
     sycl::accessor<uint> state_, tbl_;
     local_accessor<uint, 1> lstate_;
     uintl seed_;
-    sycl::stream debug_;
 };
 
 void initMersenneState(Param<uint> state, const Param<uint> tbl, uintl seed) {
@@ -157,10 +151,8 @@ void initMersenneState(Param<uint> state, const Param<uint> tbl, uintl seed) {
         auto tbl_acc    = tbl.data->get_access(h);
         auto lstate_acc = local_accessor<uint, 1>(N, h);
 
-        sycl::stream debug_stream(2048, 128, h);
-        h.parallel_for(ndrange,
-                       initMersenneKernel(state_acc, tbl_acc, lstate_acc, seed,
-                                          debug_stream));
+        h.parallel_for(
+            ndrange, initMersenneKernel(state_acc, tbl_acc, lstate_acc, seed));
     });
     // TODO: do we need to sync before using Mersenne generators?
     // force wait() here?
@@ -179,7 +171,7 @@ class uniformMersenne {
                     local_accessor<uint, 1> state,
                     local_accessor<uint, 1> recursion_table,
                     local_accessor<uint, 1> temper_table, uint elementsPerBlock,
-                    size_t elements, sycl::stream debug)
+                    size_t elements)
         : out_(out)
         , gState_(gState)
         , pos_tbl_(pos_tbl)
@@ -192,8 +184,7 @@ class uniformMersenne {
         , recursion_table_(recursion_table)
         , temper_table_(temper_table)
         , elementsPerBlock_(elementsPerBlock)
-        , elements_(elements)
-        , debug_(debug) {}
+        , elements_(elements) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
@@ -263,7 +254,6 @@ class uniformMersenne {
     local_accessor<uint, 1> state_, recursion_table_, temper_table_;
     uint elementsPerBlock_;
     size_t elements_;
-    sycl::stream debug_;
 };
 
 template<typename T>
@@ -278,7 +268,7 @@ class normalMersenne {
                    local_accessor<uint, 1> state,
                    local_accessor<uint, 1> recursion_table,
                    local_accessor<uint, 1> temper_table, uint elementsPerBlock,
-                   size_t elements, sycl::stream debug)
+                   size_t elements)
         : out_(out)
         , gState_(gState)
         , pos_tbl_(pos_tbl)
@@ -291,8 +281,7 @@ class normalMersenne {
         , recursion_table_(recursion_table)
         , temper_table_(temper_table)
         , elementsPerBlock_(elementsPerBlock)
-        , elements_(elements)
-        , debug_(debug) {}
+        , elements_(elements) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
@@ -363,7 +352,6 @@ class normalMersenne {
     local_accessor<uint, 1> state_, recursion_table_, temper_table_;
     uint elementsPerBlock_;
     size_t elements_;
-    sycl::stream debug_;
 };
 
 }  // namespace kernel
diff --git a/src/backend/oneapi/kernel/random_engine_philox.hpp b/src/backend/oneapi/kernel/random_engine_philox.hpp
index b5887aa16e..3bfe44251d 100644
--- a/src/backend/oneapi/kernel/random_engine_philox.hpp
+++ b/src/backend/oneapi/kernel/random_engine_philox.hpp
@@ -107,22 +107,18 @@ template<typename T>
 class uniformPhilox {
    public:
     uniformPhilox(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
-                  uint elementsPerBlock, uint elements,
-                  sycl::stream debug_stream)
+                  uint elementsPerBlock, uint elements)
         : out_(out)
         , hi_(hi)
         , lo_(lo)
         , hic_(hic)
         , loc_(loc)
         , elementsPerBlock_(elementsPerBlock)
-        , elements_(elements)
-        , debug_(debug_stream) {}
+        , elements_(elements) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
 
-        // debug_ << "<" << g.get_group_id(0)  << ":" << it.get_local_id(0) <<
-        // "/" << g.get_group_range(0) << sycl::stream_manipulator::endl;
         uint index = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
         uint key[2] = {lo_, hi_};
         uint ctr[4] = {loc_, hic_, 0, 0};
@@ -145,28 +141,23 @@ class uniformPhilox {
     sycl::accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
-    sycl::stream debug_;
 };
 
 template<typename T>
 class normalPhilox {
    public:
     normalPhilox(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
-                 uint elementsPerBlock, uint elements,
-                 sycl::stream debug_stream)
+                 uint elementsPerBlock, uint elements)
         : out_(out)
         , hi_(hi)
         , lo_(lo)
         , hic_(hic)
         , loc_(loc)
         , elementsPerBlock_(elementsPerBlock)
-        , elements_(elements)
-        , debug_(debug_stream) {}
+        , elements_(elements) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
-        // debug_ << "<" << g.get_group_id(0)  << ":" << it.get_local_id(0) <<
-        // "/" << g.get_group_range(0) << sycl::stream_manipulator::endl;
 
         uint index = g.get_group_id(0) * elementsPerBlock_ + it.get_local_id(0);
         uint key[2] = {lo_, hi_};
@@ -192,7 +183,6 @@ class normalPhilox {
     sycl::accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
-    sycl::stream debug_;
 };
 
 }  // namespace kernel
diff --git a/src/backend/oneapi/kernel/random_engine_threefry.hpp b/src/backend/oneapi/kernel/random_engine_threefry.hpp
index 2e8b6e0d16..919f04d010 100644
--- a/src/backend/oneapi/kernel/random_engine_threefry.hpp
+++ b/src/backend/oneapi/kernel/random_engine_threefry.hpp
@@ -162,16 +162,14 @@ template<typename T>
 class uniformThreefry {
    public:
     uniformThreefry(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
-                    uint elementsPerBlock, uint elements,
-                    sycl::stream debug_stream)
+                    uint elementsPerBlock, uint elements)
         : out_(out)
         , hi_(hi)
         , lo_(lo)
         , hic_(hic)
         , loc_(loc)
         , elementsPerBlock_(elementsPerBlock)
-        , elements_(elements)
-        , debug_(debug_stream) {}
+        , elements_(elements) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
@@ -203,23 +201,20 @@ class uniformThreefry {
     sycl::accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
-    sycl::stream debug_;
 };
 
 template<typename T>
 class normalThreefry {
    public:
     normalThreefry(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
-                   uint elementsPerBlock, uint elements,
-                   sycl::stream debug_stream)
+                   uint elementsPerBlock, uint elements)
         : out_(out)
         , hi_(hi)
         , lo_(lo)
         , hic_(hic)
         , loc_(loc)
         , elementsPerBlock_(elementsPerBlock)
-        , elements_(elements)
-        , debug_(debug_stream) {}
+        , elements_(elements) {}
 
     void operator()(sycl::nd_item<1> it) const {
         sycl::group g = it.get_group();
@@ -251,7 +246,6 @@ class normalThreefry {
     sycl::accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
-    sycl::stream debug_;
 };
 
 }  // namespace kernel
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index 9769285d2f..b3a4d60ed7 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -7,7 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 #pragma once
-#include <CL/sycl.hpp>
+
+#include <sycl/builtins.hpp>
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
index cce47881f2..fb7e5ea449 100644
--- a/src/backend/oneapi/kernel/range.hpp
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -29,20 +29,14 @@ template<typename T>
 class rangeOp {
    public:
     rangeOp(sycl::accessor<T> out, KParam oinfo, const int dim,
-            const int blocksPerMatX, const int blocksPerMatY,
-            sycl::stream debug)
+            const int blocksPerMatX, const int blocksPerMatY)
         : out_(out)
         , oinfo_(oinfo)
         , dim_(dim)
         , blocksPerMatX_(blocksPerMatX)
-        , blocksPerMatY_(blocksPerMatY)
-        , debug_(debug) {}
+        , blocksPerMatY_(blocksPerMatY) {}
 
     void operator()(sycl::nd_item<2> it) const {
-        // printf("[%d,%d]\n", it.get_global_id(0), it.get_global_id(1));
-        // debug_ << "[" << it.get_global_id(0) << "," << it.get_global_id(1) <<
-        // "]" << sycl::stream_manipulator::endl;
-
         const int mul0 = (dim_ == 0);
         const int mul1 = (dim_ == 1);
         const int mul2 = (dim_ == 2);
@@ -67,15 +61,15 @@ class rangeOp {
         const int incy = blocksPerMatY_ * g.get_local_range(1);
         const int incx = blocksPerMatX_ * g.get_local_range(0);
 
-        T valZW = (mul3 * ow) + (mul2 * oz);
+        compute_t<T> valZW = (mul3 * ow) + (mul2 * oz);
 
         T* optr = out_.get_pointer();
         for (int oy = yy; oy < oinfo_.dims[1]; oy += incy) {
-            T valYZW = valZW + (mul1 * oy);
-            int oyzw = ozw + oy * oinfo_.strides[1];
+            compute_t<T> valYZW = valZW + (mul1 * oy);
+            int oyzw            = ozw + oy * oinfo_.strides[1];
             for (int ox = xx; ox < oinfo_.dims[0]; ox += incx) {
-                int oidx = oyzw + ox;
-                T val    = valYZW + (mul0 * ox);
+                int oidx         = oyzw + ox;
+                compute_t<T> val = valYZW + (mul0 * ox);
 
                 optr[oidx] = val;
             }
@@ -87,7 +81,6 @@ class rangeOp {
     KParam oinfo_;
     int dim_;
     int blocksPerMatX_, blocksPerMatY_;
-    sycl::stream debug_;
 };
 
 template<typename T>
@@ -108,20 +101,12 @@ void range(Param<T> out, const int dim) {
     getQueue().submit([=](sycl::handler& h) {
         auto out_acc = out.data->get_access(h);
 
-        sycl::stream debug_stream(2048, 128, h);
-
-        h.parallel_for(ndrange,
-                       rangeOp<T>(out_acc, out.info, dim, blocksPerMatX,
-                                  blocksPerMatY, debug_stream));
+        h.parallel_for(ndrange, rangeOp<T>(out_acc, out.info, dim,
+                                           blocksPerMatX, blocksPerMatY));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 
-template<>
-void range(Param<arrayfire::common::half> out, const int dim) {
-    ONEAPI_NOT_SUPPORTED("TODO: fix arrayfire::common::half support");
-}
-
 }  // namespace kernel
 }  // namespace oneapi
 }  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/rotate.hpp b/src/backend/oneapi/kernel/rotate.hpp
index 61d736763a..b8c8357e79 100644
--- a/src/backend/oneapi/kernel/rotate.hpp
+++ b/src/backend/oneapi/kernel/rotate.hpp
@@ -136,9 +136,6 @@ void rotate(Param<T> out, const Param<T> in, const float theta,
 
     // Used for batching images
     constexpr int TI = 4;
-    constexpr bool isComplex =
-        static_cast<af_dtype>(dtype_traits<T>::af_type) == c32 ||
-        static_cast<af_dtype>(dtype_traits<T>::af_type) == c64;
 
     const float c = cos(-theta), s = sin(-theta);
     float tx, ty;
diff --git a/src/backend/oneapi/kernel/transform.hpp b/src/backend/oneapi/kernel/transform.hpp
index b67a11c660..c18ac6c827 100644
--- a/src/backend/oneapi/kernel/transform.hpp
+++ b/src/backend/oneapi/kernel/transform.hpp
@@ -126,7 +126,6 @@ class transformCreateKernel {
 
         // Index of transform
         const int eTfs2 = sycl::max((nTfs2_ / nImg2_), 1);
-        const int eTfs3 = sycl::max((nTfs3_ / nImg3_), 1);
 
         int t_idx3        = -1;  // init
         int t_idx2        = -1;  // init
@@ -243,8 +242,6 @@ template<typename T>
 void transform(Param<T> out, const Param<T> in, const Param<float> tf,
                bool isInverse, bool isPerspective, af_interp_type method,
                int order) {
-    static int counter = 0;
-
     using std::string;
 
     using BT = typename dtype_traits<T>::base_type;
@@ -253,9 +250,6 @@ void transform(Param<T> out, const Param<T> in, const Param<float> tf,
     constexpr int TY = 16;
     // Used for batching images
     constexpr int TI = 4;
-    constexpr bool isComplex =
-        static_cast<af_dtype>(dtype_traits<T>::af_type) == c32 ||
-        static_cast<af_dtype>(dtype_traits<T>::af_type) == c64;
 
     const int nImg2 = in.info.dims[2];
     const int nImg3 = in.info.dims[3];
diff --git a/src/backend/oneapi/kernel/transpose_inplace.hpp b/src/backend/oneapi/kernel/transpose_inplace.hpp
index d397436dfc..3dda946ced 100644
--- a/src/backend/oneapi/kernel/transpose_inplace.hpp
+++ b/src/backend/oneapi/kernel/transpose_inplace.hpp
@@ -41,9 +41,9 @@ cdouble getConjugate(const cdouble &in) {
 
 #define doOp(v) (conjugate_ ? getConjugate((v)) : (v))
 
-constexpr int TILE_DIM  = 16;
-constexpr int THREADS_X = TILE_DIM;
-constexpr int THREADS_Y = 256 / TILE_DIM;
+constexpr dim_t TILE_DIM  = 16;
+constexpr dim_t THREADS_X = TILE_DIM;
+constexpr dim_t THREADS_Y = 256 / TILE_DIM;
 
 template<typename T, int dimensions>
 using local_accessor =
@@ -57,8 +57,7 @@ class transposeInPlaceKernel {
                            const int blocksPerMatX, const int blocksPerMatY,
                            const bool conjugate, const bool IS32MULTIPLE,
                            local_accessor<T, 1> shrdMem_s,
-                           local_accessor<T, 1> shrdMem_d,
-                           sycl::stream debugStream)
+                           local_accessor<T, 1> shrdMem_d)
         : iData_(iData)
         , in_(in)
         , blocksPerMatX_(blocksPerMatX)
@@ -66,8 +65,7 @@ class transposeInPlaceKernel {
         , conjugate_(conjugate)
         , IS32MULTIPLE_(IS32MULTIPLE)
         , shrdMem_s_(shrdMem_s)
-        , shrdMem_d_(shrdMem_d)
-        , debugStream_(debugStream) {}
+        , shrdMem_d_(shrdMem_d) {}
     void operator()(sycl::nd_item<2> it) const {
         const int shrdStride = TILE_DIM + 1;
 
@@ -165,7 +163,6 @@ class transposeInPlaceKernel {
     bool IS32MULTIPLE_;
     local_accessor<T, 1> shrdMem_s_;
     local_accessor<T, 1> shrdMem_d_;
-    sycl::stream debugStream_;
 };
 
 template<typename T>
@@ -180,16 +177,14 @@ void transpose_inplace(Param<T> in, const bool conjugate,
                               blk_y * local[1] * in.info.dims[3]};
 
     getQueue().submit([&](sycl::handler &h) {
-        auto r = in.data->get_access(h);
-        sycl::stream debugStream(128, 128, h);
-
+        auto r         = in.data->get_access(h);
         auto shrdMem_s = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
         auto shrdMem_d = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
 
-        h.parallel_for(sycl::nd_range{global, local},
-                       transposeInPlaceKernel<T>(
-                           r, in.info, blk_x, blk_y, conjugate, IS32MULTIPLE,
-                           shrdMem_s, shrdMem_d, debugStream));
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            transposeInPlaceKernel<T>(r, in.info, blk_x, blk_y, conjugate,
+                                      IS32MULTIPLE, shrdMem_s, shrdMem_d));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }

From 9ceaa06eeebc51ef9f8c0a9acd8e0fd0b0fe4864 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 23 Mar 2023 19:26:10 -0400
Subject: [PATCH 226/473] Remove getData usage in the oneAPI backend and make
 it private

The getData function should not be used to get the buffer pointer. The get
function should be called instead because the Array could be a node array and in
that case it would be a null pointer. The get function will evaluate the buffer
and then return the resulting array. This was causing crashes after the JIT
changes
---
 src/api/c/memory.cpp                     | 16 ++++------------
 src/backend/cuda/Array.hpp               |  3 ++-
 src/backend/oneapi/Array.hpp             | 13 +++----------
 src/backend/oneapi/fft.cpp               | 10 ++++------
 src/backend/oneapi/kernel/mean.hpp       | 10 ++++------
 src/backend/oneapi/kernel/reduce_all.hpp |  6 +++---
 src/backend/opencl/Array.hpp             |  4 ++--
 7 files changed, 22 insertions(+), 40 deletions(-)

diff --git a/src/api/c/memory.cpp b/src/api/c/memory.cpp
index 17ea0a4d73..fbff61720e 100644
--- a/src/api/c/memory.cpp
+++ b/src/api/c/memory.cpp
@@ -144,10 +144,7 @@ af_err af_get_device_ptr(void **data, const af_array arr) {
 
 template<typename T>
 inline void lockArray(const af_array arr) {
-    // Ideally we need to use .get(false), i.e. get ptr without offset
-    // This is however not supported in opencl
-    // Use getData().get() as alternative
-    memLock(getArray<T>(arr).getData().get());
+    memLock(getArray<T>(arr).get());
 }
 
 af_err af_lock_device_ptr(const af_array arr) { return af_lock_array(arr); }
@@ -180,10 +177,8 @@ af_err af_lock_array(const af_array arr) {
 
 template<typename T>
 inline bool checkUserLock(const af_array arr) {
-    // Ideally we need to use .get(false), i.e. get ptr without offset
-    // This is however not supported in opencl
-    // Use getData().get() as alternative
-    return isLocked(static_cast<void *>(getArray<T>(arr).getData().get()));
+    detail::Array<T> &out = const_cast<detail::Array<T> &>(getArray<T>(arr));
+    return isLocked(static_cast<void *>(out.get()));
 }
 
 af_err af_is_locked_array(bool *res, const af_array arr) {
@@ -214,10 +209,7 @@ af_err af_is_locked_array(bool *res, const af_array arr) {
 
 template<typename T>
 inline void unlockArray(const af_array arr) {
-    // Ideally we need to use .get(false), i.e. get ptr without offset
-    // This is however not supported in opencl
-    // Use getData().get() as alternative
-    memUnlock(getArray<T>(arr).getData().get());
+    memUnlock(getArray<T>(arr).get());
 }
 
 af_err af_unlock_device_ptr(const af_array arr) { return af_unlock_array(arr); }
diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp
index d6774ded66..7e1324d016 100644
--- a/src/backend/cuda/Array.hpp
+++ b/src/backend/cuda/Array.hpp
@@ -156,6 +156,8 @@ class Array {
     Array(Param<T> &tmp, bool owner);
     Array(const af::dim4 &dims, common::Node_ptr n);
 
+    std::shared_ptr<T> getData() const { return data; }
+
    public:
     Array(const Array<T> &other) = default;
 
@@ -227,7 +229,6 @@ class Array {
     void eval() const;
 
     dim_t getOffset() const { return info.getOffset(); }
-    std::shared_ptr<T> getData() const { return data; }
 
     dim4 getDataDims() const { return data_dims; }
 
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index bc4e16c574..9a4de1285c 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -177,6 +177,8 @@ class Array {
     explicit Array(const af::dim4 &dims, sycl::buffer<T> *const mem,
                    size_t offset, bool copy);
 
+    std::shared_ptr<sycl::buffer<T>> getData() const { return data; }
+
    public:
     Array(const Array<T> &other) = default;
 
@@ -250,14 +252,7 @@ class Array {
         return const_cast<Array<T> *>(this)->device();
     }
 
-    // FIXME: This should do a copy if it is not owner. You do not want to
-    // overwrite parents data
-    sycl::buffer<T> *get() {
-        if (!isReady()) eval();
-        return data.get();
-    }
-
-    const sycl::buffer<T> *get() const {
+    sycl::buffer<T> *get() const {
         if (!isReady()) eval();
         return data.get();
     }
@@ -266,8 +261,6 @@ class Array {
 
     dim_t getOffset() const { return info.getOffset(); }
 
-    std::shared_ptr<sycl::buffer<T>> getData() const { return data; }
-
     dim4 getDataDims() const { return data_dims; }
 
     void setDataDims(const dim4 &new_dims);
diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
index 8ac2cd410c..eff8770bfc 100644
--- a/src/backend/oneapi/fft.cpp
+++ b/src/backend/oneapi/fft.cpp
@@ -69,9 +69,9 @@ void fft_inplace(Array<T> &in, const int rank, const bool direction) {
 
     desc.commit(getQueue());
     if (direction)
-        ::oneapi::mkl::dft::compute_forward(desc, *(in.getData()));
+        ::oneapi::mkl::dft::compute_forward(desc, *in.get());
     else
-        ::oneapi::mkl::dft::compute_backward(desc, *(in.getData()));
+        ::oneapi::mkl::dft::compute_backward(desc, *in.get());
 }
 
 template<typename Tc, typename Tr>
@@ -117,8 +117,7 @@ Array<Tc> fft_r2c(const Array<Tr> &in, const int rank) {
                    fft_output_strides, rank);
 
     desc.commit(getQueue());
-    ::oneapi::mkl::dft::compute_forward(desc, *(in.getData()),
-                                        *(out.getData()));
+    ::oneapi::mkl::dft::compute_forward(desc, *in.get(), *out.get());
 
     return out;
 }
@@ -165,8 +164,7 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank) {
                    fft_output_strides, rank);
 
     desc.commit(getQueue());
-    ::oneapi::mkl::dft::compute_backward(desc, *(in.getData()),
-                                         *(out.getData()));
+    ::oneapi::mkl::dft::compute_backward(desc, *in.get(), *out.get());
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 3f3dbc378b..4353bfff26 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -629,13 +629,12 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
 
         auto e1 = getQueue().submit([&](sycl::handler &h) {
             auto acc_in =
-                tmpOut.getData()->get_access(h, sycl::range{tmp_elements});
+                tmpOut.get()->get_access(h, sycl::range{tmp_elements});
             auto acc_out = hBuffer.get_access();
             h.copy(acc_in, acc_out);
         });
         auto e2 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in =
-                tmpWt.getData()->get_access(h, sycl::range{tmp_elements});
+            auto acc_in = tmpWt.get()->get_access(h, sycl::range{tmp_elements});
             auto acc_out = hwBuffer.get_access();
             h.copy(acc_in, acc_out);
         });
@@ -733,13 +732,12 @@ To mean_all(Param<Ti> in) {
 
         auto e1 = getQueue().submit([&](sycl::handler &h) {
             auto acc_in =
-                tmpOut.getData()->get_access(h, sycl::range{tmp_elements});
+                tmpOut.get()->get_access(h, sycl::range{tmp_elements});
             auto acc_out = hBuffer.get_access();
             h.copy(acc_in, acc_out);
         });
         auto e2 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in =
-                tmpCt.getData()->get_access(h, sycl::range{tmp_elements});
+            auto acc_in = tmpCt.get()->get_access(h, sycl::range{tmp_elements});
             auto acc_out = hcBuffer.get_access();
             h.copy(acc_in, acc_out);
         });
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index ee27b706d9..2089b60175 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -264,10 +264,10 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
     Array<To> tmp = createEmptyArray<To>(tmp_elements);
 
     Array<unsigned> retirementCount = createValueArray<unsigned>(1, 0);
-    getQueue().submit([=](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         write_accessor<To> out_acc{*out.data, h};
-        auto retCount_acc = retirementCount.getData()->get_access(h);
-        auto tmp_acc      = tmp.getData()->get_access(h);
+        auto retCount_acc = retirementCount.get()->get_access(h);
+        auto tmp_acc      = tmp.get()->get_access(h);
         read_accessor<Ti> in_acc{*in.data, h};
 
         auto shrdMem =
diff --git a/src/backend/opencl/Array.hpp b/src/backend/opencl/Array.hpp
index 6951021f19..3a672d00f6 100644
--- a/src/backend/opencl/Array.hpp
+++ b/src/backend/opencl/Array.hpp
@@ -161,6 +161,8 @@ class Array {
     explicit Array(const af::dim4 &dims, const T *const in_data);
     explicit Array(const af::dim4 &dims, cl_mem mem, size_t offset, bool copy);
 
+    std::shared_ptr<cl::Buffer> getData() const { return data; }
+
    public:
     Array(const Array<T> &other) = default;
 
@@ -250,8 +252,6 @@ class Array {
 
     dim_t getOffset() const { return info.getOffset(); }
 
-    std::shared_ptr<cl::Buffer> getData() const { return data; }
-
     dim4 getDataDims() const { return data_dims; }
 
     void setDataDims(const dim4 &new_dims);

From e5d2dda8b91d0ae8ef1aabb5af590aa23f4deae9 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 24 Mar 2023 18:00:29 -0400
Subject: [PATCH 227/473] adds ireduce kernels

---
 src/backend/oneapi/CMakeLists.txt     |   4 +-
 src/backend/oneapi/ireduce.cpp        |   9 +-
 src/backend/oneapi/kernel/ireduce.hpp | 698 ++++++++++++++++++++++++++
 src/backend/oneapi/minmax_op.hpp      |  87 ++++
 4 files changed, 793 insertions(+), 5 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/ireduce.hpp
 create mode 100644 src/backend/oneapi/minmax_op.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 7e61118811..64a2b34715 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -117,6 +117,7 @@ add_library(afoneapi
   memory.cpp
   memory.hpp
   min.cpp
+  minmax_op.hpp
   moments.cpp
   moments.hpp
   morph.cpp
@@ -217,6 +218,7 @@ target_sources(afoneapi
     kernel/diff.hpp
     kernel/interp.hpp
     kernel/iota.hpp
+    kernel/ireduce.hpp
     kernel/histogram.hpp
     kernel/memcopy.hpp
     kernel/mean.hpp
@@ -281,7 +283,7 @@ target_include_directories(afoneapi
 target_compile_options(afoneapi
   PRIVATE
     -fsycl
-    -qopenmp -qmkl=parallel
+    -openmp -Qmkl=parallel
     -sycl-std=2020
 )
 
diff --git a/src/backend/oneapi/ireduce.cpp b/src/backend/oneapi/ireduce.cpp
index 6cca678b20..c7b4d263ab 100644
--- a/src/backend/oneapi/ireduce.cpp
+++ b/src/backend/oneapi/ireduce.cpp
@@ -11,6 +11,7 @@
 #include <Array.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/ireduce.hpp>
 #include <optypes.hpp>
 #include <af/dim4.hpp>
 #include <complex>
@@ -24,19 +25,19 @@ namespace oneapi {
 template<af_op_t op, typename T>
 void ireduce(Array<T> &out, Array<uint> &loc, const Array<T> &in,
              const int dim) {
-    ONEAPI_NOT_SUPPORTED("");
+    Array<uint> rlen = createEmptyArray<uint>(af::dim4(0));
+    kernel::ireduce<T, op>(out, loc, in, dim, rlen);
 }
 
 template<af_op_t op, typename T>
 void rreduce(Array<T> &out, Array<uint> &loc, const Array<T> &in, const int dim,
              const Array<uint> &rlen) {
-    ONEAPI_NOT_SUPPORTED("");
+    kernel::ireduce<T, op>(out, loc, in, dim, rlen);
 }
 
 template<af_op_t op, typename T>
 T ireduce_all(unsigned *loc, const Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("");
-    return T(0);
+    return kernel::ireduce_all<T, op>(loc, in);
 }
 
 #define INSTANTIATE(ROp, T)                                           \
diff --git a/src/backend/oneapi/kernel/ireduce.hpp b/src/backend/oneapi/kernel/ireduce.hpp
new file mode 100644
index 0000000000..9e4e35c51d
--- /dev/null
+++ b/src/backend/oneapi/kernel/ireduce.hpp
@@ -0,0 +1,698 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/reduce_config.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+#include <minmax_op.hpp>
+
+#include <sycl/sycl.hpp>  //TODO: exact headers
+
+#include <algorithm>
+#include <climits>
+#include <complex>
+#include <iostream>
+#include <memory>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T, int dimensions>
+using local_accessor =
+    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
+                   sycl::access::target::local>;
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T, af_op_t op, uint dim, bool is_first, uint DIMY>
+class ireduceDimKernelSMEM {
+   public:
+    ireduceDimKernelSMEM(write_accessor<T> out, KParam oInfo,
+                         write_accessor<uint> oloc, KParam olocInfo,
+                         read_accessor<T> in, KParam iInfo,
+                         read_accessor<uint> iloc, KParam ilocInfo,
+                         uint groups_x, uint groups_y, uint groups_dim,
+                         read_accessor<uint> rlen, KParam rlenInfo,
+                         local_accessor<compute_t<T>, 1> s_val,
+                         local_accessor<uint, 1> s_idx)
+        : out_(out)
+        , oInfo_(oInfo)
+        , oloc_(oloc)
+        , olocInfo_(olocInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , iloc_(iloc)
+        , ilocInfo_(ilocInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , groups_dim_(groups_dim)
+        , rlen_(rlen)
+        , rlenInfo_(rlenInfo)
+        , s_val_(s_val)
+        , s_idx_(s_idx) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid       = groupId_x * g.get_local_range(0) + lidx;
+        const uint yid       = groupId_y;
+
+        uint ids[4] = {xid, yid, zid, wid};
+        T *optr     = out_.get_pointer() + ids[3] * oInfo_.strides[3] +
+                  ids[2] * oInfo_.strides[2] + ids[1] * oInfo_.strides[1] +
+                  ids[0] + oInfo_.offset;
+
+        uint *olptr = oloc_.get_pointer() + ids[3] * oInfo_.strides[3] +
+                      ids[2] * oInfo_.strides[2] + ids[1] * oInfo_.strides[1] +
+                      ids[0] + oInfo_.offset;
+
+        // There is only one element per block for out
+        // There are blockDim.y elements per block for in
+        // Hence increment ids[dim] just after offseting out and before
+        // offsetting in
+        const bool rlen_valid =
+            (ids[0] < rlenInfo_.dims[0]) && (ids[1] < rlenInfo_.dims[1]) &&
+            (ids[2] < rlenInfo_.dims[2]) && (ids[3] < rlenInfo_.dims[3]);
+        const bool rlen_nonnull = (rlenInfo_.dims[0] * rlenInfo_.dims[1] *
+                                   rlenInfo_.dims[2] * rlenInfo_.dims[3]) > 0;
+        uint *const rlenptr =
+            (rlen_nonnull && rlen_valid)
+                ? rlen_.get_pointer() + ids[3] * rlenInfo_.strides[3] +
+                      ids[2] * rlenInfo_.strides[2] +
+                      ids[1] * rlenInfo_.strides[1] + ids[0] + rlenInfo_.offset
+                : nullptr;
+
+        const uint groupIdx_dim = ids[dim];
+
+        // add thread offset for reduced dim for inputs
+        ids[dim] = ids[dim] * g.get_local_range(1) + lidy;
+
+        T *iptr = in_.get_pointer() + ids[3] * iInfo_.strides[3] +
+                  ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] +
+                  ids[0] + iInfo_.offset;
+        uint *ilptr;
+        if (!is_first) {
+            ilptr = iloc_.get_pointer() + ids[3] * iInfo_.strides[3] +
+                    ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] +
+                    ids[0] + iInfo_.offset;
+        }
+
+        const uint id_dim_in   = ids[dim];
+        const uint istride_dim = iInfo_.strides[dim];
+
+        size_t xlim   = iInfo_.dims[0];
+        size_t ylim   = iInfo_.dims[1];
+        size_t zlim   = iInfo_.dims[2];
+        size_t wlim   = iInfo_.dims[3];
+        bool is_valid = (ids[0] < xlim) && (ids[1] < ylim) && (ids[2] < zlim) &&
+                        (ids[3] < wlim);
+
+        compute_t<T> out_val = common::Binary<compute_t<T>, op>::init();
+        uint out_idx         = id_dim_in;
+
+        uint lim = rlenptr ? *rlenptr : iInfo_.dims[0];
+        lim      = is_first ? sycl::min((uint)iInfo_.dims[dim], lim) : lim;
+
+        bool within_ragged_bounds =
+            (is_first) ? (out_idx < lim)
+                       : ((rlenptr) ? ((is_valid) && (*ilptr < lim)) : true);
+        if (is_valid && id_dim_in < iInfo_.dims[dim] && within_ragged_bounds) {
+            out_val = *iptr;
+            if (!is_first) out_idx = *ilptr;
+        }
+
+        MinMaxOp<op, compute_t<T>> Op(out_val, out_idx);
+
+        const uint id_dim_in_start =
+            id_dim_in + groups_dim_ * g.get_local_range(1);
+        for (int id = id_dim_in_start; is_valid && (id < lim);
+             id += groups_dim_ * g.get_local_range(1)) {
+            iptr = iptr + groups_dim_ * g.get_local_range(1) * istride_dim;
+            if (!is_first) {
+                ilptr =
+                    ilptr + groups_dim_ * g.get_local_range(1) * istride_dim;
+                Op(*iptr, *ilptr);
+            } else {
+                Op(*iptr, id);
+            }
+        }
+
+        s_val_[lid] = Op.m_val;
+        s_idx_[lid] = Op.m_idx;
+        it.barrier();
+
+        compute_t<T> *s_vptr = s_val_.get_pointer() + lid;
+        uint *s_iptr         = s_idx_.get_pointer() + lid;
+
+        if (DIMY == 8) {
+            if (lidy < 4) {
+                Op(s_vptr[g.get_local_range(0) * 4],
+                   s_iptr[g.get_local_range(0) * 4]);
+                *s_vptr = Op.m_val;
+                *s_iptr = Op.m_idx;
+            }
+            it.barrier();
+        }
+        if (DIMY >= 4) {
+            if (lidy < 2) {
+                Op(s_vptr[g.get_local_range(0) * 2],
+                   s_iptr[g.get_local_range(0) * 2]);
+                *s_vptr = Op.m_val;
+                *s_iptr = Op.m_idx;
+            }
+            it.barrier();
+        }
+        if (DIMY >= 2) {
+            if (lidy < 1) {
+                Op(s_vptr[g.get_local_range(0) * 1],
+                   s_iptr[g.get_local_range(0) * 1]);
+                *s_vptr = Op.m_val;
+                *s_iptr = Op.m_idx;
+            }
+            it.barrier();
+        }
+        if (is_valid && lidy == 0 && (groupIdx_dim < oInfo_.dims[dim])) {
+            *optr  = data_t<T>(s_vptr[0]);
+            *olptr = s_iptr[0];
+        }
+    }
+
+   protected:
+    write_accessor<T> out_;
+    KParam oInfo_;
+    write_accessor<uint> oloc_;
+    KParam olocInfo_;
+    read_accessor<T> in_;
+    KParam iInfo_;
+    read_accessor<uint> iloc_;
+    KParam ilocInfo_;
+    uint groups_x_, groups_y_, groups_dim_;
+    read_accessor<uint> rlen_;
+    KParam rlenInfo_;
+    local_accessor<compute_t<T>, 1> s_val_;
+    local_accessor<uint, 1> s_idx_;
+};
+
+template<typename T, af_op_t op, int dim, bool is_first>
+void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
+                          Param<uint> iloc, const uint threads_y,
+                          const dim_t groups_dim[4], Param<uint> rlen) {
+    sycl::range<2> local(creduce::THREADS_X, threads_y);
+    sycl::range<2> global(groups_dim[0] * groups_dim[2] * local[0],
+                          groups_dim[1] * groups_dim[3] * local[1]);
+
+    sycl::buffer<uint, 1> empty{sycl::range<1>(1)};
+    try {
+        getQueue().submit([&](sycl::handler &h) {
+            write_accessor<T> out_acc{*out.data, h};
+            write_accessor<uint> oloc_acc{*oloc.data, h};
+            read_accessor<T> in_acc{*in.data, h};
+
+            read_accessor<uint> iloc_acc{empty, h};
+            if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] *
+                    iloc.info.dims[3] >
+                0) {
+                iloc_acc = read_accessor<uint>{*iloc.data, h};
+            }
+
+            read_accessor<uint> rlen_acc{empty, h};
+            if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] *
+                    rlen.info.dims[3] >
+                0) {
+                rlen_acc = read_accessor<uint>{*rlen.data, h};
+            }
+
+            auto shrdVal =
+                local_accessor<compute_t<T>, 1>(creduce::THREADS_PER_BLOCK, h);
+            auto shrdLoc =
+                local_accessor<uint, 1>(creduce::THREADS_PER_BLOCK, h);
+
+            switch (threads_y) {
+                case 8:
+                    h.parallel_for(
+                        sycl::nd_range<2>(global, local),
+                        ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
+                            in.info, iloc_acc, iloc.info, groups_dim[0],
+                            groups_dim[1], groups_dim[dim], rlen_acc, rlen.info,
+                            shrdVal, shrdLoc));
+                    break;
+                case 4:
+                    h.parallel_for(
+                        sycl::nd_range<2>(global, local),
+                        ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
+                            in.info, iloc_acc, iloc.info, groups_dim[0],
+                            groups_dim[1], groups_dim[dim], rlen_acc, rlen.info,
+                            shrdVal, shrdLoc));
+                    break;
+                case 2:
+                    h.parallel_for(
+                        sycl::nd_range<2>(global, local),
+                        ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
+                            in.info, iloc_acc, iloc.info, groups_dim[0],
+                            groups_dim[1], groups_dim[dim], rlen_acc, rlen.info,
+                            shrdVal, shrdLoc));
+                    break;
+                case 1:
+                    h.parallel_for(
+                        sycl::nd_range<2>(global, local),
+                        ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
+                            in.info, iloc_acc, iloc.info, groups_dim[0],
+                            groups_dim[1], groups_dim[dim], rlen_acc, rlen.info,
+                            shrdVal, shrdLoc));
+                    break;
+            }
+        });
+        getQueue().wait_and_throw();
+        ONEAPI_DEBUG_FINISH(getQueue());
+    } catch (sycl::exception &e) { std::cout << e.what() << std::endl; }
+}
+
+template<typename T, af_op_t op, int dim>
+void ireduce_dim(Param<T> out, Param<uint> oloc, Param<T> in,
+                 Param<uint> rlen) {
+    uint threads_y = std::min(creduce::THREADS_Y, nextpow2(in.info.dims[dim]));
+    uint threads_x = creduce::THREADS_X;
+
+    dim_t blocks_dim[] = {divup(in.info.dims[0], threads_x), in.info.dims[1],
+                          in.info.dims[2], in.info.dims[3]};
+
+    blocks_dim[dim] = divup(in.info.dims[dim], threads_y * creduce::REPEAT);
+
+    Param<T> tmp      = out;
+    Param<uint> tlptr = oloc;
+    bufptr<T> tmp_alloc;
+    bufptr<uint> tlptr_alloc;
+
+    if (blocks_dim[dim] > 1) {
+        int tmp_elements   = 1;
+        tmp.info.dims[dim] = blocks_dim[dim];
+
+        for (int k = 0; k < 4; k++) tmp_elements *= tmp.info.dims[k];
+        tmp_alloc   = memAlloc<T>(tmp_elements);
+        tlptr_alloc = memAlloc<uint>(tmp_elements);
+        tmp.data    = tmp_alloc.get();
+        tlptr.data  = tlptr_alloc.get();
+
+        for (int k = dim + 1; k < 4; k++)
+            tmp.info.strides[k] *= blocks_dim[dim];
+    }
+
+    Param<uint> nullparam;
+    ireduce_dim_launcher<T, op, dim, true>(tmp, tlptr, in, nullparam, threads_y,
+                                           blocks_dim, rlen);
+
+    if (blocks_dim[dim] > 1) {
+        blocks_dim[dim] = 1;
+
+        ireduce_dim_launcher<T, op, dim, false>(out, oloc, tmp, tlptr,
+                                                threads_y, blocks_dim, rlen);
+    }
+}
+
+template<typename T, af_op_t op, bool is_first, uint DIMX>
+class ireduceFirstKernelSMEM {
+   public:
+    ireduceFirstKernelSMEM(write_accessor<T> out, KParam oInfo,
+                           write_accessor<uint> oloc, KParam olocInfo,
+                           read_accessor<T> in, KParam iInfo,
+                           read_accessor<uint> iloc, KParam ilocInfo,
+                           uint groups_x, uint groups_y, uint repeat,
+                           read_accessor<uint> rlen, KParam rlenInfo,
+                           local_accessor<compute_t<T>, 1> s_val,
+                           local_accessor<uint, 1> s_idx)
+        : out_(out)
+        , oInfo_(oInfo)
+        , oloc_(oloc)
+        , olocInfo_(olocInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , iloc_(iloc)
+        , ilocInfo_(ilocInfo)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y)
+        , repeat_(repeat)
+        , rlen_(rlen)
+        , rlenInfo_(rlenInfo)
+        , s_val_(s_val)
+        , s_idx_(s_idx) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g   = it.get_group();
+        const uint lidx = it.get_local_id(0);
+        const uint lidy = it.get_local_id(1);
+        const uint lid  = lidy * g.get_local_range(0) + lidx;
+
+        const uint zid       = g.get_group_id(0) / groups_x_;
+        const uint wid       = g.get_group_id(1) / groups_y_;
+        const uint groupId_x = g.get_group_id(0) - (groups_x_)*zid;
+        const uint groupId_y = g.get_group_id(1) - (groups_y_)*wid;
+        const uint xid = groupId_x * g.get_local_range(0) * repeat_ + lidx;
+        const uint yid = groupId_y * g.get_local_range(1) + lidy;
+
+        T *const iptr = in_.get_pointer() + wid * iInfo_.strides[3] +
+                        zid * iInfo_.strides[2] + yid * iInfo_.strides[1] +
+                        iInfo_.offset;
+
+        T *optr = out_.get_pointer() + wid * oInfo_.strides[3] +
+                  zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
+
+        const bool rlenvalid = (rlenInfo_.dims[0] * rlenInfo_.dims[1] *
+                                rlenInfo_.dims[2] * rlenInfo_.dims[3]) > 0;
+        uint *const rlenptr =
+            (rlenvalid)
+                ? rlen_.get_pointer() + wid * rlenInfo_.strides[3] +
+                      zid * rlenInfo_.strides[2] + yid * rlenInfo_.strides[1]
+                : nullptr;
+
+        uint *ilptr;
+        if (!is_first) {
+            ilptr = iloc_.get_pointer() + wid * iInfo_.strides[3] +
+                    zid * iInfo_.strides[2] + yid * iInfo_.strides[1];
+        }
+        uint *olptr = oloc_.get_pointer() + wid * oInfo_.strides[3] +
+                      zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
+
+        size_t ylim   = iInfo_.dims[1];
+        size_t zlim   = iInfo_.dims[2];
+        size_t wlim   = iInfo_.dims[3];
+        bool is_valid = (yid < ylim) && (zid < zlim) && (wid < wlim);
+        // bool is_valid = (yid < iInfo_.dims[1]) && (zid < iInfo_.dims[2]) &&
+        //(wid < iInfo_.dims[3]);
+
+        int minlen = rlenptr ? sycl::min(*rlenptr, (uint)iInfo_.dims[0])
+                             : iInfo_.dims[0];
+        int lim    = sycl::min((int)(xid + repeat_ * DIMX), minlen);
+
+        compute_t<T> out_val = common::Binary<compute_t<T>, op>::init();
+        uint idx             = xid;
+
+        if (xid < lim) {
+            out_val = static_cast<compute_t<T>>(iptr[xid]);
+            if (!is_first) idx = ilptr[xid];
+        }
+
+        MinMaxOp<op, compute_t<T>> Op(out_val, idx);
+        for (int id = xid; is_valid && id < lim; id += DIMX) {
+            Op(static_cast<compute_t<T>>(iptr[id]),
+               (!is_first) ? ilptr[id] : id);
+        }
+
+        s_val_[lid] = Op.m_val;
+        s_idx_[lid] = Op.m_idx;
+        it.barrier();
+
+        compute_t<T> *s_vptr = s_val_.get_pointer() + lidy * DIMX;
+        uint *s_iptr         = s_idx_.get_pointer() + lidy * DIMX;
+
+        if (DIMX == 256) {
+            if (lidx < 128) {
+                Op(s_vptr[lidx + 128], s_iptr[lidx + 128]);
+                s_vptr[lidx] = Op.m_val;
+                s_iptr[lidx] = Op.m_idx;
+            }
+            it.barrier();
+        }
+
+        if (DIMX >= 128) {
+            if (lidx < 64) {
+                Op(s_vptr[lidx + 64], s_iptr[lidx + 64]);
+                s_vptr[lidx] = Op.m_val;
+                s_iptr[lidx] = Op.m_idx;
+            }
+            it.barrier();
+        }
+
+        if (DIMX >= 64) {
+            if (lidx < 32) {
+                Op(s_vptr[lidx + 32], s_iptr[lidx + 32]);
+                s_vptr[lidx] = Op.m_val;
+                s_iptr[lidx] = Op.m_idx;
+            }
+            it.barrier();
+        }
+
+        // TODO: replace with subgroup operations in optimized kernels
+        if (lidx < 16) {
+            Op(s_vptr[lidx + 16], s_iptr[lidx + 16]);
+            s_vptr[lidx] = Op.m_val;
+            s_iptr[lidx] = Op.m_idx;
+        }
+        it.barrier();
+
+        if (lidx < 8) {
+            Op(s_vptr[lidx + 8], s_iptr[lidx + 8]);
+            s_vptr[lidx] = Op.m_val;
+            s_iptr[lidx] = Op.m_idx;
+        }
+        it.barrier();
+
+        if (lidx < 4) {
+            Op(s_vptr[lidx + 4], s_iptr[lidx + 4]);
+            s_vptr[lidx] = Op.m_val;
+            s_iptr[lidx] = Op.m_idx;
+        }
+        it.barrier();
+
+        if (lidx < 2) {
+            Op(s_vptr[lidx + 2], s_iptr[lidx + 2]);
+            s_vptr[lidx] = Op.m_val;
+            s_iptr[lidx] = Op.m_idx;
+        }
+        it.barrier();
+
+        if (lidx < 1) {
+            Op(s_vptr[lidx + 1], s_iptr[lidx + 1]);
+            s_vptr[lidx] = Op.m_val;
+            s_iptr[lidx] = Op.m_idx;
+        }
+        it.barrier();
+
+        if (is_valid && lidx == 0) {
+            optr[groupId_x]  = data_t<T>(s_vptr[0]);
+            olptr[groupId_x] = s_iptr[0];
+        }
+    }
+
+   protected:
+    write_accessor<T> out_;
+    KParam oInfo_;
+    write_accessor<uint> oloc_;
+    KParam olocInfo_;
+    read_accessor<T> in_;
+    KParam iInfo_;
+    read_accessor<uint> iloc_;
+    KParam ilocInfo_;
+    uint groups_x_, groups_y_, repeat_;
+    read_accessor<uint> rlen_;
+    KParam rlenInfo_;
+    local_accessor<compute_t<T>, 1> s_val_;
+    local_accessor<uint, 1> s_idx_;
+};
+
+template<typename T, af_op_t op, bool is_first>
+void ireduce_first_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
+                            Param<uint> iloc, const uint groups_x,
+                            const uint groups_y, const uint threads_x,
+                            Param<uint> rlen) {
+    sycl::range<2> local(threads_x, creduce::THREADS_PER_BLOCK / threads_x);
+    sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
+                          groups_y * in.info.dims[3] * local[1]);
+
+    uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
+
+    sycl::buffer<uint, 1> empty{sycl::range<1>(1)};
+    try {
+        getQueue().submit([&](sycl::handler &h) {
+            write_accessor<T> out_acc{*out.data, h};
+            write_accessor<uint> oloc_acc{*oloc.data, h};
+            read_accessor<T> in_acc{*in.data, h};
+
+            read_accessor<uint> iloc_acc{empty, h};
+            if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] *
+                    iloc.info.dims[3] >
+                0) {
+                iloc_acc = read_accessor<uint>{*iloc.data, h};
+            }
+
+            read_accessor<uint> rlen_acc{empty, h};
+            if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] *
+                    rlen.info.dims[3] >
+                0) {
+                rlen_acc = read_accessor<uint>{*rlen.data, h};
+            }
+
+            auto shrdVal =
+                local_accessor<compute_t<T>, 1>(creduce::THREADS_PER_BLOCK, h);
+            auto shrdLoc =
+                local_accessor<uint, 1>(creduce::THREADS_PER_BLOCK, h);
+
+            switch (threads_x) {
+                case 32:
+                    h.parallel_for(
+                        sycl::nd_range<2>(global, local),
+                        ireduceFirstKernelSMEM<T, op, is_first, 32>(
+                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
+                            in.info, iloc_acc, iloc.info, groups_x, groups_y,
+                            repeat, rlen_acc, rlen.info, shrdVal, shrdLoc));
+                    break;
+                case 64:
+                    h.parallel_for(
+                        sycl::nd_range<2>(global, local),
+                        ireduceFirstKernelSMEM<T, op, is_first, 64>(
+                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
+                            in.info, iloc_acc, iloc.info, groups_x, groups_y,
+                            repeat, rlen_acc, rlen.info, shrdVal, shrdLoc));
+                    break;
+                case 128:
+                    h.parallel_for(
+                        sycl::nd_range<2>(global, local),
+                        ireduceFirstKernelSMEM<T, op, is_first, 128>(
+                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
+                            in.info, iloc_acc, iloc.info, groups_x, groups_y,
+                            repeat, rlen_acc, rlen.info, shrdVal, shrdLoc));
+                    break;
+                case 256:
+                    h.parallel_for(
+                        sycl::nd_range<2>(global, local),
+                        ireduceFirstKernelSMEM<T, op, is_first, 256>(
+                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
+                            in.info, iloc_acc, iloc.info, groups_x, groups_y,
+                            repeat, rlen_acc, rlen.info, shrdVal, shrdLoc));
+                    break;
+            }
+        });
+        getQueue().wait_and_throw();
+        ONEAPI_DEBUG_FINISH(getQueue());
+    } catch (sycl::exception &e) { std::cout << e.what() << std::endl; }
+}
+
+template<typename T, af_op_t op>
+void ireduce_first(Param<T> out, Param<uint> oloc, Param<T> in,
+                   Param<uint> rlen) {
+    uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+    threads_x      = std::min(threads_x, creduce::THREADS_PER_BLOCK);
+    uint threads_y = creduce::THREADS_PER_BLOCK / threads_x;
+
+    uint blocks_x = divup(in.info.dims[0], threads_x * creduce::REPEAT);
+    uint blocks_y = divup(in.info.dims[1], threads_y);
+
+    Param<T> tmp      = out;
+    Param<uint> tlptr = oloc;
+    bufptr<T> tmp_alloc;
+    bufptr<uint> tlptr_alloc;
+    if (blocks_x > 1) {
+        auto elements =
+            blocks_x * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
+        tmp_alloc   = memAlloc<T>(elements);
+        tlptr_alloc = memAlloc<uint>(elements);
+        tmp.data    = tmp_alloc.get();
+        tlptr.data  = tlptr_alloc.get();
+
+        tmp.info.dims[0] = blocks_x;
+        for (int k = 1; k < 4; k++) tmp.info.strides[k] *= blocks_x;
+    }
+
+    Param<uint> nullparam;
+    ireduce_first_launcher<T, op, true>(tmp, tlptr, in, nullparam, blocks_x,
+                                        blocks_y, threads_x, rlen);
+
+    if (blocks_x > 1) {
+        ireduce_first_launcher<T, op, false>(out, oloc, tmp, tlptr, 1, blocks_y,
+                                             threads_x, rlen);
+    }
+}
+
+template<typename T, af_op_t op>
+void ireduce(Param<T> out, Param<uint> oloc, Param<T> in, int dim,
+             Param<uint> rlen) {
+    switch (dim) {
+        case 0: return ireduce_first<T, op>(out, oloc, in, rlen);
+        case 1: return ireduce_dim<T, op, 1>(out, oloc, in, rlen);
+        case 2: return ireduce_dim<T, op, 2>(out, oloc, in, rlen);
+        case 3: return ireduce_dim<T, op, 3>(out, oloc, in, rlen);
+    }
+}
+
+template<typename T, af_op_t op>
+T ireduce_all(uint *idx, Param<T> in) {
+    int in_elements =
+        in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
+
+    bool is_linear = (in.info.strides[0] == 1);
+    for (int k = 1; k < 4; k++) {
+        is_linear &= (in.info.strides[k] ==
+                      (in.info.strides[k - 1] * in.info.dims[k - 1]));
+    }
+
+    if (is_linear) {
+        in.info.dims[0] = in_elements;
+        for (int k = 1; k < 4; k++) {
+            in.info.dims[k]    = 1;
+            in.info.strides[k] = in_elements;
+        }
+    }
+
+    uint threads_x = nextpow2(std::max(32u, (uint)in.info.dims[0]));
+    threads_x      = std::min(threads_x, creduce::THREADS_PER_BLOCK);
+    uint threads_y = creduce::THREADS_PER_BLOCK / threads_x;
+
+    // TODO: perf REPEAT, consider removing or runtime eval
+    // max problem size < SM resident threads, don't use REPEAT
+    uint groups_x = divup(in.info.dims[0], threads_x * creduce::REPEAT);
+    uint groups_y = divup(in.info.dims[1], threads_y);
+
+    Array<T> tmp = createEmptyArray<T>(
+        {groups_x, in.info.dims[1], in.info.dims[2], in.info.dims[3]});
+
+    int tmp_elements  = tmp.elements();
+    Array<uint> tlptr = createEmptyArray<uint>({tmp_elements, 1, 1, 1});
+
+    Param<uint> nullparam;
+    Array<uint> rlen = createEmptyArray<uint>(af::dim4(0));
+    ireduce_first_launcher<T, op, true>(tmp, tlptr, in, nullparam, groups_x,
+                                        groups_y, threads_x, rlen);
+
+    sycl::host_accessor h_ptr_raw{*tmp.get()};
+    sycl::host_accessor h_lptr_raw{*tlptr.get()};
+
+    MinMaxOp<op, T> Op(h_ptr_raw[0], h_lptr_raw[0]);
+
+    for (int i = 1; i < tmp_elements; i++) { Op(h_ptr_raw[i], h_lptr_raw[i]); }
+
+    *idx = Op.m_idx;
+    return Op.m_val;
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/minmax_op.hpp b/src/backend/oneapi/minmax_op.hpp
new file mode 100644
index 0000000000..f006ff419c
--- /dev/null
+++ b/src/backend/oneapi/minmax_op.hpp
@@ -0,0 +1,87 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <common/Binary.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+
+template<typename T>
+static double cabs(const T &in) {
+    return (double)in;
+}
+
+template<>
+double cabs<char>(const char &in) {
+    return (double)(in > 0);
+}
+
+template<>
+double cabs<cfloat>(const cfloat &in) {
+    return (double)abs(in);
+}
+
+template<>
+double cabs<cdouble>(const cdouble &in) {
+    return (double)abs(in);
+}
+
+template<typename T>
+static bool is_nan(const T &in) {
+    return in != in;
+}
+
+template<>
+bool is_nan<cfloat>(const cfloat &in) {
+    return in.real() != in.real() || in.imag() != in.imag();
+}
+
+template<>
+bool is_nan<cdouble>(const cdouble &in) {
+    return in.real() != in.real() || in.imag() != in.imag();
+}
+
+template<af_op_t op, typename T>
+struct MinMaxOp {
+    T m_val;
+    uint m_idx;
+    MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) {
+        if (is_nan(val)) { m_val = common::Binary<compute_t<T>, op>::init(); }
+    }
+
+    void operator()(T val, uint idx) {
+        if ((cabs(val) < cabs(m_val) ||
+             (cabs(val) == cabs(m_val) && idx > m_idx))) {
+            m_val = val;
+            m_idx = idx;
+        }
+    }
+};
+
+template<typename T>
+struct MinMaxOp<af_max_t, T> {
+    T m_val;
+    uint m_idx;
+    MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) {
+        if (is_nan(val)) { m_val = common::Binary<T, af_max_t>::init(); }
+    }
+
+    void operator()(T val, uint idx) {
+        if ((cabs(val) > cabs(m_val) ||
+             (cabs(val) == cabs(m_val) && idx <= m_idx))) {
+            m_val = val;
+            m_idx = idx;
+        }
+    }
+};
+
+}  // namespace oneapi
+}  // namespace arrayfire

From 115a942544e3723bda4c650e051b364cb72e337d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 27 Mar 2023 10:51:24 -0400
Subject: [PATCH 228/473] Workaround compiler bug in range for oneAPI

---
 src/backend/oneapi/kernel/range.hpp | 33 ++++++++++++++++-------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
index fb7e5ea449..9cfea27964 100644
--- a/src/backend/oneapi/kernel/range.hpp
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -52,26 +52,29 @@ class rangeOp {
         const int xx = it.get_local_id(0) + blockIdx_x * it.get_local_range(0);
         const int yy = it.get_local_id(1) + blockIdx_y * it.get_local_range(1);
 
-        if (xx >= oinfo_.dims[0] || yy >= oinfo_.dims[1] ||
-            oz >= oinfo_.dims[2] || ow >= oinfo_.dims[3])
-            return;
+        const size_t odx = oinfo_.dims[0];
+        const size_t ody = oinfo_.dims[1];
+        const size_t odz = oinfo_.dims[2];
+        const size_t odw = oinfo_.dims[3];
 
-        const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
+        if (xx < odx && yy < ody && oz < odz && ow < odw) {
+            const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
 
-        const int incy = blocksPerMatY_ * g.get_local_range(1);
-        const int incx = blocksPerMatX_ * g.get_local_range(0);
+            const int incy = blocksPerMatY_ * g.get_local_range(1);
+            const int incx = blocksPerMatX_ * g.get_local_range(0);
 
-        compute_t<T> valZW = (mul3 * ow) + (mul2 * oz);
+            compute_t<T> valZW = (mul3 * ow) + (mul2 * oz);
 
-        T* optr = out_.get_pointer();
-        for (int oy = yy; oy < oinfo_.dims[1]; oy += incy) {
-            compute_t<T> valYZW = valZW + (mul1 * oy);
-            int oyzw            = ozw + oy * oinfo_.strides[1];
-            for (int ox = xx; ox < oinfo_.dims[0]; ox += incx) {
-                int oidx         = oyzw + ox;
-                compute_t<T> val = valYZW + (mul0 * ox);
+            T* optr = out_.get_pointer();
+            for (int oy = yy; oy < oinfo_.dims[1]; oy += incy) {
+                compute_t<T> valYZW = valZW + (mul1 * oy);
+                int oyzw            = ozw + oy * oinfo_.strides[1];
+                for (int ox = xx; ox < oinfo_.dims[0]; ox += incx) {
+                    int oidx         = oyzw + ox;
+                    compute_t<T> val = valYZW + (mul0 * ox);
 
-                optr[oidx] = val;
+                    optr[oidx] = val;
+                }
             }
         }
     }

From 40f9896c97b394150e03e63f8736cd394b4bd931 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 27 Mar 2023 10:52:03 -0400
Subject: [PATCH 229/473] Narrow access modes for transpose

---
 src/backend/oneapi/kernel/transpose.hpp | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index d22a6f4475..43b741ca32 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -50,11 +50,12 @@ using local_accessor =
 template<typename T>
 class transposeKernel {
    public:
-    transposeKernel(sycl::accessor<T> oData, const KParam out,
-                    const sycl::accessor<T> iData, const KParam in,
-                    const int blocksPerMatX, const int blocksPerMatY,
-                    const bool conjugate, const bool IS32MULTIPLE,
-                    local_accessor<T, 1> shrdMem)
+    transposeKernel(sycl::accessor<T, 1, sycl::access::mode::write> oData,
+                    const KParam out,
+                    const sycl::accessor<T, 1, sycl::access::mode::read> iData,
+                    const KParam in, const int blocksPerMatX,
+                    const int blocksPerMatY, const bool conjugate,
+                    const bool IS32MULTIPLE, local_accessor<T, 1> shrdMem)
         : oData_(oData)
         , out_(out)
         , iData_(iData)
@@ -124,9 +125,9 @@ class transposeKernel {
     }
 
    private:
-    sycl::accessor<T> oData_;
+    sycl::accessor<T, 1, sycl::access::mode::write> oData_;
     KParam out_;
-    sycl::accessor<T> iData_;
+    sycl::accessor<T, 1, sycl::access::mode::read> iData_;
     KParam in_;
     int blocksPerMatX_;
     int blocksPerMatY_;
@@ -147,8 +148,8 @@ void transpose(Param<T> out, const Param<T> in, const bool conjugate,
                               blk_y * local[1] * in.info.dims[3]};
 
     getQueue().submit([&](sycl::handler &h) {
-        auto r = in.data->get_access(h);
-        auto q = out.data->get_access(h);
+        auto r = in.data->template get_access<sycl::access::mode::read>(h);
+        auto q = out.data->template get_access<sycl::access::mode::write>(h);
 
         auto shrdMem = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
 

From 56be86b6a01c372a2da7e3f2e5ca608f3edaf154 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 27 Mar 2023 17:41:53 -0400
Subject: [PATCH 230/473] Pass values into JIT lambda by value

---
 src/backend/oneapi/jit.cpp              |  2 +-
 src/backend/oneapi/jit/BufferNode.hpp   |  2 +-
 src/backend/oneapi/kernel/bilateral.hpp |  2 --
 src/backend/oneapi/platform.cpp         | 12 ++++--------
 4 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 519d4efeea..bd4a5f2d43 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -401,7 +401,7 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                                       outputs[0].info.dims[2] > 1)};
 
     getQueue()
-        .submit([&](sycl::handler& h) {
+        .submit([=](sycl::handler& h) {
             for (Node* node : full_nodes) {
                 if (node->isBuffer()) {
                     BufferNode<T>* n = static_cast<BufferNode<T>*>(node);
diff --git a/src/backend/oneapi/jit/BufferNode.hpp b/src/backend/oneapi/jit/BufferNode.hpp
index b6bedc5baf..8c8d61abf2 100644
--- a/src/backend/oneapi/jit/BufferNode.hpp
+++ b/src/backend/oneapi/jit/BufferNode.hpp
@@ -20,7 +20,7 @@ namespace jit {
 template<typename T>
 using BufferNode =
     common::BufferNodeBase<std::shared_ptr<sycl::buffer<T>>, AParam<T>>;
-}
+}  // namespace jit
 }  // namespace oneapi
 
 namespace common {
diff --git a/src/backend/oneapi/kernel/bilateral.hpp b/src/backend/oneapi/kernel/bilateral.hpp
index c01ee4a4a5..2a5cf59fb1 100644
--- a/src/backend/oneapi/kernel/bilateral.hpp
+++ b/src/backend/oneapi/kernel/bilateral.hpp
@@ -121,9 +121,7 @@ class bilateralKernel {
             int joff = (ly - radius) * shrdLen + (lx - radius);
             int goff = 0;
 
-#pragma unroll
             for (int wj = 0; wj < window_size; ++wj) {
-#pragma unroll
                 for (int wi = 0; wi < window_size; ++wi) {
                     outType tmp_color = localMem_[joff + wi];
                     const outType c   = center_color - tmp_color;
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index b95b5326bc..918b4666f4 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -320,14 +320,10 @@ const std::string& getActiveDeviceBaseBuildFlags() {
 size_t getDeviceMemorySize(int device) {
     DeviceManager& devMngr = DeviceManager::getInstance();
 
-    sycl::device dev;
-    {
-        common::lock_guard_t lock(devMngr.deviceMutex);
-        // Assuming devices don't deallocate or are invalidated during execution
-        dev = *devMngr.mDevices[device];
-    }
-    size_t msize = dev.get_info<sycl::info::device::global_mem_size>();
-    return msize;
+    common::lock_guard_t lock(devMngr.deviceMutex);
+    // Assuming devices don't deallocate or are invalidated during execution
+    sycl::device& dev = *devMngr.mDevices[device];
+    return dev.get_info<sycl::info::device::global_mem_size>();
 }
 
 size_t getHostMemorySize() { return common::getHostMemorySize(); }

From 13cbcf1d15ccf7f5a4a5fafe1390a798d808b9d6 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 27 Mar 2023 17:43:22 -0400
Subject: [PATCH 231/473] Use SKIP_TEST rather than returning for disabled
 feature checks

---
 test/anisotropic_diffusion.cpp |  2 +-
 test/arrayfire_test.cpp        | 12 ------------
 test/bilateral.cpp             |  2 +-
 test/canny.cpp                 |  4 ++--
 test/cholesky_dense.cpp        |  2 +-
 test/confidence_connected.cpp  |  2 +-
 test/fast.cpp                  |  4 ++--
 test/gloh.cpp                  |  4 ++--
 test/harris.cpp                |  4 ++--
 test/homography.cpp            |  4 ++--
 test/imageio.cpp               | 26 +++++++++++++-------------
 test/inverse_deconv.cpp        |  2 +-
 test/inverse_dense.cpp         |  2 +-
 test/iterative_deconv.cpp      |  2 +-
 test/lu_dense.cpp              | 14 +++++++-------
 test/meanshift.cpp             |  4 ++--
 test/medfilt.cpp               |  2 +-
 test/moments.cpp               |  2 +-
 test/morph.cpp                 |  4 ++--
 test/orb.cpp                   |  4 ++--
 test/qr_dense.cpp              |  6 +++---
 test/rank_dense.cpp            | 10 +++++-----
 test/sift.cpp                  |  4 ++--
 test/solve_common.hpp          |  6 +++---
 test/solve_dense.cpp           |  6 +++---
 test/susan.cpp                 |  2 +-
 test/svd_dense.cpp             |  6 +++---
 test/testHelpers.hpp           | 12 +++++++-----
 test/threading.cpp             |  2 +-
 test/transform.cpp             |  8 ++++----
 30 files changed, 77 insertions(+), 87 deletions(-)

diff --git a/test/anisotropic_diffusion.cpp b/test/anisotropic_diffusion.cpp
index f4d78382f3..afeda45d52 100644
--- a/test/anisotropic_diffusion.cpp
+++ b/test/anisotropic_diffusion.cpp
@@ -50,7 +50,7 @@ void imageTest(string pTestFile, const float dt, const float K,
             OutType;
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     using af::dim4;
 
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index 2128f7fbd3..4c6e966220 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -822,18 +822,6 @@ void cleanSlate() {
     ASSERT_EQ(af::getMemStepSize(), step_bytes);
 }
 
-bool noImageIOTests() {
-    bool ret = !af::isImageIOAvailable();
-    if (ret) printf("Image IO Not Configured. Test will exit\n");
-    return ret;
-}
-
-bool noLAPACKTests() {
-    bool ret = !af::isLAPACKAvailable();
-    if (ret) printf("LAPACK Not Configured. Test will exit\n");
-    return ret;
-}
-
 template<typename inType, typename outType>
 void readTestsFromFile(const std::string &FileName,
                        std::vector<af::dim4> &inputDims,
diff --git a/test/bilateral.cpp b/test/bilateral.cpp
index 8d83d2798b..f4ff949b55 100644
--- a/test/bilateral.cpp
+++ b/test/bilateral.cpp
@@ -25,7 +25,7 @@ using std::vector;
 template<typename T, bool isColor>
 void bilateralTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/canny.cpp b/test/canny.cpp
index b34a4923b4..7f2fa2918c 100644
--- a/test/canny.cpp
+++ b/test/canny.cpp
@@ -93,7 +93,7 @@ TEST(Canny, DISABLED_Exact) {
 template<typename T>
 void cannyImageOtsuTest(string pTestFile, bool isColor) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     using af::dim4;
 
@@ -220,7 +220,7 @@ TEST(CannyEdgeDetector, Sobel5x5_Invalid) {
 template<typename T>
 void cannyImageOtsuBatchTest(string pTestFile, const dim_t targetBatchCount) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     using af::array;
     using af::canny;
diff --git a/test/cholesky_dense.cpp b/test/cholesky_dense.cpp
index 0631ec2bad..dea036eca1 100644
--- a/test/cholesky_dense.cpp
+++ b/test/cholesky_dense.cpp
@@ -34,7 +34,7 @@ using std::vector;
 template<typename T>
 void choleskyTester(const int n, double eps, bool is_upper) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     dtype ty = (dtype)dtype_traits<T>::af_type;
 
diff --git a/test/confidence_connected.cpp b/test/confidence_connected.cpp
index 9d081f068d..ac5b0bf2bc 100644
--- a/test/confidence_connected.cpp
+++ b/test/confidence_connected.cpp
@@ -58,7 +58,7 @@ void testImage(const std::string pTestFile, const size_t numSeeds,
                const int multiplier, const unsigned neighborhood_radius,
                const int iter) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<af::dim4> inDims;
     vector<string> inFiles;
diff --git a/test/fast.cpp b/test/fast.cpp
index 316fe57ad6..1d494641ff 100644
--- a/test/fast.cpp
+++ b/test/fast.cpp
@@ -69,7 +69,7 @@ TYPED_TEST_SUITE(FixedFAST, FixedTestTypes);
 template<typename T>
 void fastTest(string pTestFile, bool nonmax) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
@@ -180,7 +180,7 @@ using af::features;
 using af::loadImage;
 
 TEST(FloatFAST, CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/gloh.cpp b/test/gloh.cpp
index e370984fbf..b360ac6a18 100644
--- a/test/gloh.cpp
+++ b/test/gloh.cpp
@@ -137,7 +137,7 @@ TYPED_TEST_SUITE(GLOH, TestTypes);
 template<typename T>
 void glohTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
@@ -261,7 +261,7 @@ GLOH_INIT(man, man);
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(GLOH, CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/harris.cpp b/test/harris.cpp
index ec6a1fa626..43c0bb6433 100644
--- a/test/harris.cpp
+++ b/test/harris.cpp
@@ -61,7 +61,7 @@ TYPED_TEST_SUITE(Harris, TestTypes);
 template<typename T>
 void harrisTest(string pTestFile, float sigma, unsigned block_size) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
@@ -167,7 +167,7 @@ using af::harris;
 using af::loadImage;
 
 TEST(FloatHarris, CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/homography.cpp b/test/homography.cpp
index c6a6e43450..f4c1c75259 100644
--- a/test/homography.cpp
+++ b/test/homography.cpp
@@ -49,7 +49,7 @@ void homographyTest(string pTestFile, const af_homography_type htype,
     using af::Pi;
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
@@ -220,7 +220,7 @@ using af::features;
 using af::loadImage;
 
 TEST(Homography, CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/imageio.cpp b/test/imageio.cpp
index 4869e50e15..00834fb693 100644
--- a/test/imageio.cpp
+++ b/test/imageio.cpp
@@ -36,7 +36,7 @@ typedef ::testing::Types<float> TestTypes;
 TYPED_TEST_SUITE(ImageIO, TestTypes);
 
 void loadImageTest(string pTestFile, string pImageFile, const bool isColor) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> numDims;
 
@@ -93,7 +93,7 @@ TYPED_TEST(ImageIO, ColorSeq) {
 }
 
 void loadimageArgsTest(string pImageFile, const bool isColor, af_err err) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     af_array imgArray = 0;
 
@@ -122,7 +122,7 @@ using af::saveImageMem;
 using af::span;
 
 TEST(ImageIO, CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> numDims;
 
@@ -150,7 +150,7 @@ TEST(ImageIO, CPP) {
 }
 
 TEST(ImageIO, SavePNGCPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     array input(10, 10, 3, f32);
 
@@ -170,7 +170,7 @@ TEST(ImageIO, SavePNGCPP) {
 }
 
 TEST(ImageIO, SaveBMPCPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     array input(10, 10, 3, f32);
 
@@ -190,7 +190,7 @@ TEST(ImageIO, SaveBMPCPP) {
 }
 
 TEST(ImageMem, SaveMemPNG) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     array img =
         loadImage(string(TEST_DIR "/imageio/color_seq.png").c_str(), true);
@@ -205,7 +205,7 @@ TEST(ImageMem, SaveMemPNG) {
 }
 
 TEST(ImageMem, SaveMemJPG1) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     array img =
         loadImage(string(TEST_DIR "/imageio/color_seq.png").c_str(), false);
@@ -222,7 +222,7 @@ TEST(ImageMem, SaveMemJPG1) {
 }
 
 TEST(ImageMem, SaveMemJPG3) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     array img =
         loadImage(string(TEST_DIR "/imageio/color_seq.png").c_str(), true);
@@ -239,7 +239,7 @@ TEST(ImageMem, SaveMemJPG3) {
 }
 
 TEST(ImageMem, SaveMemBMP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     array img =
         loadImage(string(TEST_DIR "/imageio/color_rand.png").c_str(), true);
@@ -254,7 +254,7 @@ TEST(ImageMem, SaveMemBMP) {
 }
 
 TEST(ImageIO, LoadImage16CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> numDims;
 
@@ -284,7 +284,7 @@ TEST(ImageIO, LoadImage16CPP) {
 }
 
 TEST(ImageIO, SaveImage16CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     dim4 dims(16, 24, 3);
 
@@ -312,7 +312,7 @@ using af::saveImageNative;
 
 template<typename T>
 void loadImageNativeCPPTest(string pTestFile, string pImageFile) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> numDims;
 
@@ -362,7 +362,7 @@ TEST(ImageIONative, LoadImageNative16GrayCPP) {
 
 template<typename T>
 void saveLoadImageNativeCPPTest(dim4 dims) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     array input = randu(dims, (af_dtype)dtype_traits<T>::af_type);
 
diff --git a/test/inverse_deconv.cpp b/test/inverse_deconv.cpp
index 9cce59ea62..b6db793f4b 100644
--- a/test/inverse_deconv.cpp
+++ b/test/inverse_deconv.cpp
@@ -38,7 +38,7 @@ void invDeconvImageTest(string pTestFile, const float gamma,
             OutType;
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     using af::dim4;
 
diff --git a/test/inverse_dense.cpp b/test/inverse_dense.cpp
index a0bb6145d9..0d502389b8 100644
--- a/test/inverse_dense.cpp
+++ b/test/inverse_dense.cpp
@@ -34,7 +34,7 @@ using std::abs;
 template<typename T>
 void inverseTester(const int m, const int n, double eps) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 #if 1
     array A = cpu_randu<T>(dim4(m, n));
 #else
diff --git a/test/iterative_deconv.cpp b/test/iterative_deconv.cpp
index 59e6b4598b..e59440b977 100644
--- a/test/iterative_deconv.cpp
+++ b/test/iterative_deconv.cpp
@@ -38,7 +38,7 @@ void iterDeconvImageTest(string pTestFile, const unsigned iters, const float rf,
             OutType;
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     using af::dim4;
 
diff --git a/test/lu_dense.cpp b/test/lu_dense.cpp
index ec69e1ccd9..35c925ab57 100644
--- a/test/lu_dense.cpp
+++ b/test/lu_dense.cpp
@@ -37,7 +37,7 @@ using std::string;
 using std::vector;
 
 TEST(LU, InPlaceSmall) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     int resultIdx = 0;
 
@@ -75,7 +75,7 @@ TEST(LU, InPlaceSmall) {
 }
 
 TEST(LU, SplitSmall) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     int resultIdx = 0;
 
@@ -128,7 +128,7 @@ TEST(LU, SplitSmall) {
 template<typename T>
 void luTester(const int m, const int n, double eps) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
 #if 1
     array a_orig = cpu_randu<T>(dim4(m, n));
@@ -237,7 +237,7 @@ TYPED_TEST(LU, RectangularMultipleOfTwoLarge1) {
 }
 
 TEST(LU, NullLowerOutput) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
     dim4 dims(3, 3);
     af_array in = 0;
     ASSERT_SUCCESS(af_randu(&in, dims.ndims(), dims.get(), f32));
@@ -248,7 +248,7 @@ TEST(LU, NullLowerOutput) {
 }
 
 TEST(LU, NullUpperOutput) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
     dim4 dims(3, 3);
     af_array in = 0;
     ASSERT_SUCCESS(af_randu(&in, dims.ndims(), dims.get(), f32));
@@ -259,7 +259,7 @@ TEST(LU, NullUpperOutput) {
 }
 
 TEST(LU, NullPivotOutput) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
     dim4 dims(3, 3);
     af_array in = 0;
     ASSERT_SUCCESS(af_randu(&in, dims.ndims(), dims.get(), f32));
@@ -270,7 +270,7 @@ TEST(LU, NullPivotOutput) {
 }
 
 TEST(LU, InPlaceNullOutput) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
     dim4 dims(3, 3);
     af_array in = 0;
     ASSERT_SUCCESS(af_randu(&in, dims.ndims(), dims.get(), f32));
diff --git a/test/meanshift.cpp b/test/meanshift.cpp
index 59f6bd2ee7..1f0aa697b3 100644
--- a/test/meanshift.cpp
+++ b/test/meanshift.cpp
@@ -54,7 +54,7 @@ TYPED_TEST(Meanshift, InvalidArgs) {
 template<typename T, bool isColor>
 void meanshiftTest(string pTestFile, const float ss) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
@@ -131,7 +131,7 @@ using af::seq;
 using af::span;
 
 TEST(Meanshift, Color_CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/medfilt.cpp b/test/medfilt.cpp
index 2120da8e4c..1939379974 100644
--- a/test/medfilt.cpp
+++ b/test/medfilt.cpp
@@ -166,7 +166,7 @@ TYPED_TEST(MedianFilter1d, BATCH_SYMMETRIC_PAD_3) {
 template<typename T, bool isColor>
 void medfiltImageTest(string pTestFile, dim_t w_len, dim_t w_wid) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/moments.cpp b/test/moments.cpp
index d7a396ea95..6b02cb614a 100644
--- a/test/moments.cpp
+++ b/test/moments.cpp
@@ -98,7 +98,7 @@ void momentsTest(string pTestFile) {
 }
 
 void momentsOnImageTest(string pTestFile, string pImageFile, bool isColor) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
     vector<dim4> numDims;
 
     vector<vector<float>> in;
diff --git a/test/morph.cpp b/test/morph.cpp
index b24106b88b..9cc2255fb5 100644
--- a/test/morph.cpp
+++ b/test/morph.cpp
@@ -136,7 +136,7 @@ TYPED_TEST(Morph, Erode4x4x4) {
 template<typename T, bool isDilation, bool isColor>
 void morphImageTest(string pTestFile, dim_t seLen) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
@@ -390,7 +390,7 @@ using af::span;
 template<typename T, bool isDilation, bool isColor>
 void cppMorphImageTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/orb.cpp b/test/orb.cpp
index b29c7021ba..e519fd91dc 100644
--- a/test/orb.cpp
+++ b/test/orb.cpp
@@ -129,7 +129,7 @@ TYPED_TEST_SUITE(ORB, TestTypes);
 template<typename T>
 void orbTest(string pTestFile) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
@@ -246,7 +246,7 @@ TYPED_TEST(ORB, Lena) { orbTest<TypeParam>(string(TEST_DIR "/orb/lena.test")); }
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(ORB, CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/qr_dense.cpp b/test/qr_dense.cpp
index 9d5f3f1c78..d87cb7b565 100644
--- a/test/qr_dense.cpp
+++ b/test/qr_dense.cpp
@@ -34,7 +34,7 @@ using std::vector;
 
 ///////////////////////////////// CPP ////////////////////////////////////
 TEST(QRFactorized, CPP) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     int resultIdx = 0;
 
@@ -90,7 +90,7 @@ template<typename T>
 void qrTester(const int m, const int n, double eps) {
     try {
         SUPPORTED_TYPE_CHECK(T);
-        if (noLAPACKTests()) return;
+        LAPACK_ENABLED_CHECK();
 
 #if 1
         array in = cpu_randu<T>(dim4(m, n));
@@ -181,7 +181,7 @@ TYPED_TEST(QR, RectangularMultipleOfTwoLarge1) {
 }
 
 TEST(QR, InPlaceNullOutput) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
     dim4 dims(3, 3);
     af_array in = 0;
     ASSERT_SUCCESS(af_randu(&in, dims.ndims(), dims.get(), f32));
diff --git a/test/rank_dense.cpp b/test/rank_dense.cpp
index bb838686f5..7625ab82d2 100644
--- a/test/rank_dense.cpp
+++ b/test/rank_dense.cpp
@@ -46,7 +46,7 @@ TYPED_TEST_SUITE(Det, TestTypes);
 template<typename T>
 void rankSmall() {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     T ha[] = {1, 4, 7, 2, 5, 8, 3, 6, 20};
     array a(3, 3, ha);
@@ -57,7 +57,7 @@ void rankSmall() {
 template<typename T>
 void rankBig(const int num) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     dtype dt = (dtype)dtype_traits<T>::af_type;
     array a  = randu(num, num, dt);
@@ -71,7 +71,7 @@ void rankBig(const int num) {
 template<typename T>
 void rankLow(const int num) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     dtype dt = (dtype)dtype_traits<T>::af_type;
 
@@ -93,7 +93,7 @@ TYPED_TEST(Rank, low) { rankBig<TypeParam>(512); }
 template<typename T>
 void detTest() {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     dtype dt = (dtype)dtype_traits<T>::af_type;
 
@@ -114,7 +114,7 @@ void detTest() {
 TYPED_TEST(Det, Small) { detTest<TypeParam>(); }
 
 TEST(Rank, NullOutput) {
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
     dim4 dims(3, 3);
     af_array in = 0;
     af_randu(&in, dims.ndims(), dims.get(), f32);
diff --git a/test/sift.cpp b/test/sift.cpp
index 2410472b53..621659e259 100644
--- a/test/sift.cpp
+++ b/test/sift.cpp
@@ -138,7 +138,7 @@ template<typename T>
 void siftTest(string pTestFile, unsigned nLayers, float contrastThr,
               float edgeThr, float initSigma, bool doubleInput) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
@@ -272,7 +272,7 @@ SIFT_INIT(Man_NoDoubleInput, man_nodoubleinput, 3, 0.04f, 10.0f, 1.6f, false);
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(SIFT, CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/solve_common.hpp b/test/solve_common.hpp
index c464bfdc47..0eee3d7029 100644
--- a/test/solve_common.hpp
+++ b/test/solve_common.hpp
@@ -35,7 +35,7 @@ void solveTester(const int m, const int n, const int k, double eps,
     af::deviceGC();
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
 #if 1
     af::array A  = cpu_randu<T>(af::dim4(m, n));
@@ -65,7 +65,7 @@ void solveLUTester(const int n, const int k, double eps,
     af::deviceGC();
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
 #if 1
     af::array A  = cpu_randu<T>(af::dim4(n, n));
@@ -95,7 +95,7 @@ void solveTriangleTester(const int n, const int k, bool is_upper, double eps,
     af::deviceGC();
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
 #if 1
     af::array A  = cpu_randu<T>(af::dim4(n, n));
diff --git a/test/solve_dense.cpp b/test/solve_dense.cpp
index b09c77645c..161aa7a212 100644
--- a/test/solve_dense.cpp
+++ b/test/solve_dense.cpp
@@ -51,7 +51,7 @@ void solveTester(const int m, const int n, const int k, const int b, double eps,
     deviceGC();
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
 #if 1
     array A  = cpu_randu<T>(dim4(m, n, b));
@@ -88,7 +88,7 @@ void solveLUTester(const int n, const int k, double eps,
     deviceGC();
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
 #if 1
     array A  = cpu_randu<T>(dim4(n, n));
@@ -125,7 +125,7 @@ void solveTriangleTester(const int n, const int k, bool is_upper, double eps,
     deviceGC();
 
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
 #if 1
     array A  = cpu_randu<T>(dim4(n, n));
diff --git a/test/susan.cpp b/test/susan.cpp
index 9bdc16d3d9..34929c22c0 100644
--- a/test/susan.cpp
+++ b/test/susan.cpp
@@ -67,7 +67,7 @@ TYPED_TEST_SUITE(Susan, TestTypes);
 template<typename T>
 void susanTest(string pTestFile, float t, float g) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;
diff --git a/test/svd_dense.cpp b/test/svd_dense.cpp
index e31603a84b..f0da346ce4 100644
--- a/test/svd_dense.cpp
+++ b/test/svd_dense.cpp
@@ -58,7 +58,7 @@ double get_val<cdouble>(cdouble val) {
 template<typename T>
 void svdTest(const int M, const int N) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     dtype ty = (dtype)dtype_traits<T>::af_type;
 
@@ -87,7 +87,7 @@ void svdTest(const int M, const int N) {
 template<typename T>
 void svdInPlaceTest(const int M, const int N) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     dtype ty = (dtype)dtype_traits<T>::af_type;
 
@@ -115,7 +115,7 @@ void svdInPlaceTest(const int M, const int N) {
 template<typename T>
 void checkInPlaceSameResults(const int M, const int N) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noLAPACKTests()) return;
+    LAPACK_ENABLED_CHECK();
 
     dtype ty = (dtype)dtype_traits<T>::af_type;
 
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 2382060ebf..3f1beb55bb 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -227,7 +227,13 @@ bool noHalfTests(af::dtype ty);
     if (noDoubleTests((af_dtype)af::dtype_traits<type>::af_type)) \
         GTEST_SKIP() << "Device doesn't support Doubles";         \
     if (noHalfTests((af_dtype)af::dtype_traits<type>::af_type))   \
-        GTEST_SKIP() << "Device doesn't support Half";
+    GTEST_SKIP() << "Device doesn't support Half"
+
+#define LAPACK_ENABLED_CHECK() \
+    if (!af::isLAPACKAvailable()) GTEST_SKIP() << "LAPACK Not Configured."
+
+#define IMAGEIO_ENABLED_CHECK() \
+    if (!af::isImageIOAvailable()) GTEST_SKIP() << "Image IO Not Configured"
 
 #ifdef AF_WITH_FAST_MATH
 #define SKIP_IF_FAST_MATH_ENABLED() \
@@ -236,10 +242,6 @@ bool noHalfTests(af::dtype ty);
 #define SKIP_IF_FAST_MATH_ENABLED()
 #endif
 
-bool noImageIOTests();
-
-bool noLAPACKTests();
-
 template<typename TO, typename FROM>
 TO convert_to(FROM in) {
     return TO(in);
diff --git a/test/threading.cpp b/test/threading.cpp
index 96dd894e4f..41c4ebb723 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -141,7 +141,7 @@ void morphTest(const array input, const array mask, const bool isDilation,
 }
 
 TEST(Threading, SetPerThreadActiveDevice) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<bool> isDilationFlags;
     vector<bool> isColorFlags;
diff --git a/test/transform.cpp b/test/transform.cpp
index b7719d46fc..e3e0efe640 100644
--- a/test/transform.cpp
+++ b/test/transform.cpp
@@ -97,7 +97,7 @@ template<typename T>
 void transformTest(string pTestFile, string pHomographyFile,
                    const af_interp_type method, const bool invert) {
     SUPPORTED_TYPE_CHECK(T);
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     af_array sceneArray = 0;
     af_array goldArray  = 0;
@@ -304,7 +304,7 @@ class TransformV2 : public Transform<T> {
     }
 
     void setTestData(string pTestFile, string pHomographyFile) {
-        if (noImageIOTests()) return;
+        IMAGEIO_ENABLED_CHECK();
         releaseArrays();
 
         genTestData<T>(&gold, &in, &transform, &odim0, &odim1, pTestFile,
@@ -390,7 +390,7 @@ class TransformV2 : public Transform<T> {
 
     void testSpclOutArray(TestOutputArrayType out_array_type) {
         SUPPORTED_TYPE_CHECK(T);
-        if (noImageIOTests()) return;
+        IMAGEIO_ENABLED_CHECK();
 
         af_array out = 0;
         TestOutputArrayInfo metadata(out_array_type);
@@ -481,7 +481,7 @@ TEST_F(TransformNullArgs, V2NullTransformArray) {
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(Transform, CPP) {
-    if (noImageIOTests()) return;
+    IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
     vector<string> inFiles;

From be1ef85ea620219140c32a66575d5ba498f9f130 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 29 Mar 2023 09:39:29 -0400
Subject: [PATCH 232/473] Add setting of default device using environment
 variable for oneAPI

---
 src/backend/oneapi/device_manager.cpp | 45 ++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index 7134109146..13a314b2aa 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -163,13 +163,50 @@ DeviceManager::DeviceManager()
 
     bool default_device_set = false;
     string deviceENV        = getEnvVar("AF_ONEAPI_DEFAULT_DEVICE");
+
     if (!deviceENV.empty()) {
-        // TODO: handle default device from env variable
+        stringstream s(deviceENV);
+        int def_device = -1;
+        s >> def_device;
+        if (def_device >= static_cast<int>(mQueues.size()) ||
+            def_device >= static_cast<int>(DeviceManager::MAX_DEVICES)) {
+            AF_TRACE(
+                "AF_ONEAPI_DEFAULT_DEVICE ({}) \
+                   is out of range, Setting default device to 0",
+                def_device);
+            def_device = 0;
+        } else {
+            setActiveContext(def_device);
+            default_device_set = true;
+        }
     }
 
-    deviceENV = getEnvVar("AF_OPENCL_DEFAULT_DEVICE_TYPE");
+    deviceENV = getEnvVar("AF_ONEAPI_DEFAULT_DEVICE_TYPE");
     if (!default_device_set && !deviceENV.empty()) {
-        // TODO: handle default device by type env variable
+        sycl::info::device_type default_device_type =
+            sycl::info::device_type::gpu;
+        if (deviceENV == "CPU") {
+            default_device_type = sycl::info::device_type::cpu;
+        } else if (deviceENV == "ACC") {
+            default_device_type = sycl::info::device_type::accelerator;
+        }
+
+        bool default_device_set = false;
+        for (int i = 0; i < nDevices; i++) {
+            if (mDevices[i]->get_info<sycl::info::device::device_type>() ==
+                default_device_type) {
+                default_device_set = true;
+                AF_TRACE("Setting to first available {}({})", deviceENV, i);
+                setActiveContext(i);
+                break;
+            }
+        }
+        if (!default_device_set) {
+            AF_TRACE(
+                "AF_ONEAPI_DEFAULT_DEVICE_TYPE={} \
+                   is not available, Using default device as 0",
+                deviceENV);
+        }
     }
 
     // Define AF_DISABLE_GRAPHICS with any value to disable initialization
@@ -182,7 +219,7 @@ DeviceManager::DeviceManager()
 
     // TODO: init other needed libraries?
     // blas? program cache?
-    // AF_TRACE("Default device: {}", getActiveDeviceId());
+    AF_TRACE("Default device: {}", getActiveDeviceId());
 }
 
 spdlog::logger* DeviceManager::getLogger() { return logger.get(); }

From 40a71025c206a69f9637facda3c3966a2a55504d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 29 Mar 2023 09:40:34 -0400
Subject: [PATCH 233/473] Updating device sorting compare function to prefer
 GPUS for oneAPI

---
 src/backend/oneapi/device_manager.cpp | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index 13a314b2aa..05d6cb454d 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -57,11 +57,26 @@ namespace oneapi {
 
 static inline bool compare_default(const unique_ptr<sycl::device>& ldev,
                                    const unique_ptr<sycl::device>& rdev) {
-    // TODO: update sorting criteria
-    // select according to something applicable to oneapi backend
-    auto l_mem = ldev->get_info<sycl::info::device::global_mem_size>();
-    auto r_mem = rdev->get_info<sycl::info::device::global_mem_size>();
-    return l_mem > r_mem;
+    using sycl::info::device_type;
+
+    auto ldt = ldev->get_info<sycl::info::device::device_type>();
+    auto rdt = rdev->get_info<sycl::info::device::device_type>();
+
+    if (ldt == rdt) {
+        auto l_mem = ldev->get_info<sycl::info::device::global_mem_size>();
+        auto r_mem = rdev->get_info<sycl::info::device::global_mem_size>();
+        return l_mem > r_mem;
+    } else {
+        if (ldt == device_type::gpu)
+            return true;
+        else if (rdt == device_type::gpu)
+            return false;
+        else if (ldt == device_type::cpu)
+            return true;
+        else if (rdt == device_type::cpu)
+            return false;
+    }
+    return false;
 }
 
 auto arrayfire_exception_handler(sycl::exception_list exceptions) {

From c872f5686385f41e4e724a505058efe8caceb066 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 29 Mar 2023 09:56:28 -0400
Subject: [PATCH 234/473] Update oneAPI af_info and print floating point
 aspects for devices

---
 src/backend/oneapi/platform.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index 918b4666f4..6d4c7df84c 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -41,6 +41,7 @@
 #include <utility>
 #include <vector>
 
+using sycl::aspect;
 using sycl::context;
 using sycl::device;
 using sycl::platform;
@@ -139,7 +140,7 @@ af_oneapi_platform getPlatformEnum(sycl::device dev) {
 
 string getDeviceInfo() noexcept {
     ostringstream info;
-    info << "ArrayFire v" << AF_VERSION << " (OpenCL, " << get_system()
+    info << "ArrayFire v" << AF_VERSION << " (oneAPI, " << get_system()
          << ", build " << AF_REVISION << ")\n";
 
     try {
@@ -156,11 +157,14 @@ string getDeviceInfo() noexcept {
 
             string id = (show_braces ? string("[") : "-") +
                         to_string(nDevices) + (show_braces ? string("]") : "-");
-
             size_t msize =
                 device->get_info<sycl::info::device::global_mem_size>();
             info << id << " " << getPlatformName(*device) << ": " << ltrim(dstr)
                  << ", " << msize / 1048576 << " MB";
+            info << " (";
+            if (device->has(aspect::fp64)) { info << "fp64 "; }
+            if (device->has(aspect::fp16)) { info << "fp16 "; }
+            info << "\b)";
 #ifndef NDEBUG
             info << " -- ";
             string devVersion = device->get_info<sycl::info::device::version>();
@@ -168,11 +172,7 @@ string getDeviceInfo() noexcept {
                 device->get_info<sycl::info::device::driver_version>();
             info << devVersion;
             info << " -- Device driver " << driVersion;
-            info << " -- FP64 Support: "
-                 << (device->get_info<sycl::info::device::
-                                          preferred_vector_width_double>() > 0
-                         ? "True"
-                         : "False");
+
             info << " -- Unified Memory ("
                  << (isHostUnifiedMemory(*device) ? "True" : "False") << ")";
 #endif

From 2b8e5eae49a0d3ce6a900ccce6b51c1d02ad719d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 29 Mar 2023 09:57:27 -0400
Subject: [PATCH 235/473] Update info test print the default device before
 setting device

---
 test/info.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/info.cpp b/test/info.cpp
index f1519d3380..5cd82a6201 100644
--- a/test/info.cpp
+++ b/test/info.cpp
@@ -48,6 +48,7 @@ void infoTest() {
         testFunction<float>();
     } else {
         int oldDevice = getDevice();
+        testFunction<float>();
         for (int d = 0; d < nDevices; d++) {
             setDevice(d);
             testFunction<float>();

From be3334ef5ae9a401194c3f12b83b43e8e83f3dbd Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 29 Mar 2023 12:28:01 -0400
Subject: [PATCH 236/473] Use Intel's MKLConfig.cmake instead of FindMKL for
 oneAPI.

---
 CMakeLists.txt                                | 27 +++++++++++++++++--
 .../{FindMKL.cmake => FindAF_MKL.cmake}       |  0
 src/backend/oneapi/CMakeLists.txt             |  5 +---
 3 files changed, 26 insertions(+), 6 deletions(-)
 rename CMakeModules/{FindMKL.cmake => FindAF_MKL.cmake} (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b049258552..708ef7f390 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -46,7 +46,15 @@ if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
 endif()
 
 #Set Intel OpenMP as default MKL thread layer
-set(MKL_THREAD_LAYER "Intel OpenMP" CACHE STRING "The thread layer to choose for MKL")
+if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+  set(MKL_THREAD_LAYER "TBB" CACHE STRING "The thread layer to choose for MKL")
+  set(MKL_INTERFACE "ilp64")
+  set(MKL_INTERFACE_INTEGER_SIZE 8)
+else()
+  set(MKL_THREAD_LAYER "Intel OpenMP" CACHE STRING "The thread layer to choose for MKL")
+  set(MKL_INTERFACE "lp64")
+  set(MKL_INTERFACE_INTEGER_SIZE 4)
+endif()
 
 find_package(CUDA 10.2)
 find_package(cuDNN 4.0)
@@ -59,7 +67,7 @@ find_package(FFTW)
 find_package(CBLAS)
 find_package(LAPACKE)
 find_package(Doxygen)
-find_package(MKL)
+find_package(AF_MKL)
 find_package(spdlog QUIET ${AF_REQUIRED} NO_CMAKE_PACKAGE_REGISTRY)
 find_package(fmt QUIET ${AF_REQUIRED})
 find_package(span-lite QUIET)
@@ -103,6 +111,21 @@ if(MKL_FOUND)
   set(default_compute_library "Intel-MKL")
 endif()
 
+if(AF_WITH_STATIC_MKL)
+  set(MKL_LINK static)
+endif()
+if(MKL_THREAD_LAYER STREQUAL "Sequential")
+  set(MKL_THREADING "sequential")
+elseif(MKL_THREAD_LAYER STREQUAL "GNU OpenMP")
+  set(MKL_THREADING "gnu_thread")
+elseif(MKL_THREAD_LAYER STREQUAL "Intel OpenMP")
+  set(MKL_THREADING "intel_thread")
+elseif(MKL_THREAD_LAYER STREQUAL "TBB")
+  set(MKL_THREADING "tbb_thread")
+else()
+endif()
+find_package(MKL)
+
 af_multiple_option(NAME        AF_COMPUTE_LIBRARY
                    DEFAULT     ${default_compute_library}
                    DESCRIPTION "Compute library for signal processing and linear algebra routines"
diff --git a/CMakeModules/FindMKL.cmake b/CMakeModules/FindAF_MKL.cmake
similarity index 100%
rename from CMakeModules/FindMKL.cmake
rename to CMakeModules/FindAF_MKL.cmake
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 64a2b34715..c003f72152 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -283,7 +283,6 @@ target_include_directories(afoneapi
 target_compile_options(afoneapi
   PRIVATE
     -fsycl
-    -openmp -Qmkl=parallel
     -sycl-std=2020
 )
 
@@ -299,7 +298,6 @@ target_compile_definitions(afoneapi
 target_link_libraries(afoneapi
   PRIVATE
     -fsycl
-    -fno-lto
     -fvisibility-inlines-hidden
     c_api_interface
     cpp_api_interface
@@ -309,8 +307,7 @@ target_link_libraries(afoneapi
     -fsycl
     -fsycl-device-code-split=per_kernel
     -fsycl-link-huge-device-code
-    -qopenmp
-    -qmkl=parallel
+    MKL::MKL_DPCPP
   )
 
 af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR})

From fe15ffeee5695fac7a735a50666e537119b5939e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 30 Mar 2023 13:19:48 -0400
Subject: [PATCH 237/473] Fix the scalar funciton for the GPU in the oneAPI
 backend

---
 src/backend/oneapi/copy.cpp | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index 4059bd27f0..d9d2fba2c5 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -214,18 +214,13 @@ template<typename T>
 T getScalar(const Array<T> &in) {
     T retVal{};
 
-    sycl::buffer retBuffer(&retVal, {1},
-                           {sycl::property::buffer::use_host_ptr()});
-
     getQueue()
         .submit([&](sycl::handler &h) {
             auto acc_in =
                 in.get()->template get_access<sycl::access::mode::read>(
                     h, sycl::range{1},
                     sycl::id{static_cast<uintl>(in.getOffset())});
-            auto acc_out =
-                retBuffer.template get_access<sycl::access::mode::write>(h);
-            h.copy(acc_in, acc_out);
+            h.copy(acc_in, &retVal);
         })
         .wait();
 

From 5bb5c167b7ab7732fbab9823ddf6778c97261cbd Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 30 Mar 2023 13:22:22 -0400
Subject: [PATCH 238/473] Refactor array test assert calls

---
 test/array.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/array.cpp b/test/array.cpp
index eeb7f2952b..5962797083 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -502,12 +502,12 @@ TEST(DeviceId, Different) {
 
 TEST(Device, empty) {
     array a = array();
-    ASSERT_EQ(a.device<float>() == NULL, 1);
+    ASSERT_EQ(a.device<float>(), nullptr);
 }
 
 TEST(Device, JIT) {
     array a = constant(1, 5, 5);
-    ASSERT_EQ(a.device<float>() != NULL, 1);
+    ASSERT_NE(a.device<float>(), nullptr);
 }
 
 TYPED_TEST(Array, Scalar) {
@@ -520,7 +520,7 @@ TYPED_TEST(Array, Scalar) {
 
     a.host((void *)gold.data());
 
-    EXPECT_EQ(true, gold[0] == a.scalar<TypeParam>());
+    EXPECT_EQ(gold[0], a.scalar<TypeParam>());
 }
 
 TEST(Array, ScalarTypeMismatch) {

From e6bc701a8b42658908c98df7feeab9051d0cb0d7 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 30 Mar 2023 13:23:03 -0400
Subject: [PATCH 239/473] Return a pointer to the buffer<T> object when calling
 device in oneAPI

---
 src/backend/oneapi/Array.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 2db607c75c..93c9e0df7e 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -397,10 +397,6 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
 template<typename T>
 void *getDevicePtr(const Array<T> &arr) {
     const buffer<T> *buf = arr.device();
-    // if (!buf) { return NULL; }
-    // memLock(buf);
-    // cl_mem mem = (*buf)();
-    ONEAPI_NOT_SUPPORTED("pointer to sycl::buffer should be accessor");
     return (void *)buf;
 }
 

From 6cb7924c7026d9f9968b2cdaf864a4bf8052485a Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 30 Mar 2023 14:19:08 -0400
Subject: [PATCH 240/473] Add the ability to run individual gtests separately
 in ctest

---
 CMakeLists.txt      |  1 +
 test/CMakeLists.txt | 37 +++++++++++++++++++++++++++++--------
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 708ef7f390..29e2880949 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -101,6 +101,7 @@ option(AF_WITH_STATIC_CUDA_NUMERIC_LIBS "Link libafcuda with static numeric libr
 option(AF_WITH_SPDLOG_HEADER_ONLY "Build ArrayFire with header only version of spdlog" OFF)
 option(AF_WITH_FMT_HEADER_ONLY "Build ArrayFire with header only version of fmt" OFF)
 option(AF_WITH_FAST_MATH "Use lower precision but high performance numeric optimizations" OFF)
+option(AF_CTEST_SEPARATED "Run tests separately when called from ctest(increases test times)" OFF)
 
 if(AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   option(AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS "Prune CUDA static libraries to reduce binary size.(WARNING: May break some libs on older CUDA toolkits for some compute arch)" OFF)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index dbd81ea6e7..6f385f666a 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -11,6 +11,10 @@ set(AF_TEST_WITH_MTX_FILES
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
 
+if(AF_CTEST_SEPARATED)
+  include(GoogleTest)
+endif()
+
 if(AF_TEST_WITH_MTX_FILES)
   include(download_sparse_datasets)
 endif()
@@ -55,6 +59,28 @@ if(NOT TARGET mmio)
   add_subdirectory(mmio)
 endif()
 
+
+# Registers test with ctest
+#
+# Parameters
+#  target: The target associated with this test
+#  backend: The backend associated with this test
+#  is_serial: If true the test will be serialized
+function(af_add_test target backend is_serial)
+  if(AF_CTEST_SEPARATED)
+    gtest_discover_tests(${target}
+      TEST_PREFIX $<UPPER_CASE:${backend}>.
+      DISCOVERY_TIMEOUT 40)
+  else()
+    add_test(NAME ${target} COMMAND ${target})
+    if(${is_serial})
+      set_tests_properties(${target}
+        PROPERTIES
+        RUN_SERIAL ON)
+    endif(${is_serial})
+  endif()
+endfunction()
+
 # Reset the CXX flags for tests
 set(CMAKE_CXX_STANDARD 11)
 
@@ -238,12 +264,7 @@ function(make_test)
 
     # TODO(umar): Create this executable separately
     if(NOT ${backend} STREQUAL "unified" OR ${target} STREQUAL "backend_unified")
-      add_test(NAME ${target} COMMAND ${target})
-	    if(${mt_args_SERIAL})
-        set_tests_properties(${target}
-          PROPERTIES
-            RUN_SERIAL ON)
-	    endif(${mt_args_SERIAL})
+      af_add_test(${target} ${backend} ${mt_args_SERIAL})
     endif()
 
   endforeach()
@@ -387,7 +408,7 @@ if(CUDA_FOUND)
           OUTPUT_NAME "cuda_${backend}")
 
       if(NOT ${backend} STREQUAL "unified")
-        add_test(NAME ${target} COMMAND ${target})
+        af_add_test(${target} ${backend} ON)
       endif()
     endif()
   endforeach()
@@ -457,7 +478,7 @@ foreach(backend ${enabled_backends})
       PRIVATE
       ArrayFire::af${backend})
   endif()
-  add_test(NAME test_${target} COMMAND ${target})
+  af_add_test(${target} ${backend} ON)
 endforeach()
 
 if(AF_TEST_WITH_MTX_FILES)

From 7e7c250e5cc0373b57adc49aef29fc8296be62c4 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Fri, 31 Mar 2023 21:57:13 +0200
Subject: [PATCH 241/473] Corrected availability check of
 work_group_collection_functions for OCL2.X

---
 .../opencl/kernel/reduce_blocks_by_key_dim.cl | 29 +++++++++++--------
 .../kernel/reduce_blocks_by_key_first.cl      | 23 ++++++++++-----
 2 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl b/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl
index 1fbd594e0a..66bbb3e6d2 100644
--- a/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl
+++ b/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl
@@ -9,7 +9,12 @@
 
 // Starting from OpenCL 2.0, core profile includes work group level
 // inclusive scan operations, hence skip defining custom one
-#if __OPENCL_VERSION__ < 200
+#if __OPENCL_C_VERSION__ == 200 || __OPENCL_C_VERSION__ == 210 || \
+    __OPENCL_C_VERSION__ == 220 || __opencl_c_work_group_collective_functions
+#define BUILTIN_WORK_GROUP_COLLECTIVE_FUNCTIONS
+#endif
+
+#ifndef BUILTIN_WORK_GROUP_COLLECTIVE_FUNCTIONS
 int work_group_scan_inclusive_add(local int *wg_temp, __local int *arr) {
     local int *active_buf;
 
@@ -29,15 +34,15 @@ int work_group_scan_inclusive_add(local int *wg_temp, __local int *arr) {
     int res = active_buf[lid];
     return res;
 }
-#endif  // __OPENCL_VERSION__ < 200
+#endif
 
 kernel void reduce_blocks_by_key_dim(global int *reduced_block_sizes,
-                                       global Tk *oKeys, KParam oKInfo,
-                                       global To *oVals, KParam oVInfo,
-                                       const global Tk *iKeys, KParam iKInfo,
-                                       const global Ti *iVals, KParam iVInfo,
-                                       int change_nan, To nanval, int n,
-                                       const int nBlocksZ) {
+                                     global Tk *oKeys, KParam oKInfo,
+                                     global To *oVals, KParam oVInfo,
+                                     const global Tk *iKeys, KParam iKInfo,
+                                     const global Ti *iVals, KParam iVInfo,
+                                     int change_nan, To nanval, int n,
+                                     const int nBlocksZ) {
     const uint lid  = get_local_id(0);
     const uint gidx = get_global_id(0);
 
@@ -50,7 +55,7 @@ kernel void reduce_blocks_by_key_dim(global int *reduced_block_sizes,
     local Tk reduced_keys[DIMX];
     local To reduced_vals[DIMX];
     local int unique_ids[DIMX];
-#if __OPENCL_VERSION__ < 200
+#ifndef BUILTIN_WORK_GROUP_COLLECTIVE_FUNCTIONS
     local int wg_temp[DIMX];
     local int unique_flags[DIMX];
 #endif
@@ -98,11 +103,11 @@ kernel void reduce_blocks_by_key_dim(global int *reduced_block_sizes,
     int eq_check    = (lid > 0) ? (k != reduced_keys[lid - 1]) : 0;
     int unique_flag = (eq_check || (lid == 0)) && (gidx < n);
 
-#if __OPENCL_VERSION__ < 200
+#ifdef BUILTIN_WORK_GROUP_COLLECTIVE_FUNCTIONS
+    int unique_id = work_group_scan_inclusive_add(unique_flag);
+#else
     unique_flags[lid] = unique_flag;
     int unique_id     = work_group_scan_inclusive_add(wg_temp, unique_flags);
-#else
-    int unique_id = work_group_scan_inclusive_add(unique_flag);
 #endif
     unique_ids[lid] = unique_id;
 
diff --git a/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl b/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
index e473244152..f184e94818 100644
--- a/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
+++ b/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
@@ -9,7 +9,12 @@
 
 // Starting from OpenCL 2.0, core profile includes work group level
 // inclusive scan operations, hence skip defining custom one
-#if !__opencl_c_work_group_collective_functions
+#if __OPENCL_C_VERSION__ == 200 || __OPENCL_C_VERSION__ == 210 || \
+    __OPENCL_C_VERSION__ == 220 || __opencl_c_work_group_collective_functions
+#define BUILTIN_WORK_GROUP_COLLECTIVE_FUNCTIONS
+#endif
+
+#ifndef BUILTIN_WORK_GROUP_COLLECTIVE_FUNCTIONS
 int work_group_scan_inclusive_add(local int *wg_temp, __local int *arr) {
     local int *active_buf;
 
@@ -31,11 +36,13 @@ int work_group_scan_inclusive_add(local int *wg_temp, __local int *arr) {
 }
 #endif
 
-kernel void reduce_blocks_by_key_first(
-    global int *reduced_block_sizes, __global Tk *oKeys, KParam oKInfo,
-    global To *oVals, KParam oVInfo, const __global Tk *iKeys, KParam iKInfo,
-    const global Ti *iVals, KParam iVInfo, int change_nan, To nanval, int n,
-    const int nBlocksZ) {
+kernel void reduce_blocks_by_key_first(global int *reduced_block_sizes,
+                                       __global Tk *oKeys, KParam oKInfo,
+                                       global To *oVals, KParam oVInfo,
+                                       const __global Tk *iKeys, KParam iKInfo,
+                                       const global Ti *iVals, KParam iVInfo,
+                                       int change_nan, To nanval, int n,
+                                       const int nBlocksZ) {
     const uint lid = get_local_id(0);
     const uint gid = get_global_id(0);
 
@@ -48,7 +55,7 @@ kernel void reduce_blocks_by_key_first(
     local Tk reduced_keys[DIMX];
     local To reduced_vals[DIMX];
     local int unique_ids[DIMX];
-#if !__opencl_c_work_group_collective_functions
+#ifndef BUILTIN_WORK_GROUP_COLLECTIVE_FUNCTIONS
     local int wg_temp[DIMX];
     local int unique_flags[DIMX];
 #endif
@@ -84,7 +91,7 @@ kernel void reduce_blocks_by_key_first(
     int eq_check    = (lid > 0) ? (k != reduced_keys[lid - 1]) : 0;
     int unique_flag = (eq_check || (lid == 0)) && (gid < n);
 
-#if __opencl_c_work_group_collective_functions
+#ifdef BUILTIN_WORK_GROUP_COLLECTIVE_FUNCTIONS
     int unique_id = work_group_scan_inclusive_add(unique_flag);
 #else
     unique_flags[lid] = unique_flag;

From afac0eaa884ff9524aa62d3cbfc52aa66a554323 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 31 Mar 2023 16:52:41 -0400
Subject: [PATCH 242/473] Fix oneAPI find_package command to look at the
 MKLROOT env var

---
 CMakeLists.txt | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 29e2880949..e7bf293ce4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -45,8 +45,8 @@ if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
   set(AF_REQUIRED REQUIRED)
 endif()
 
-#Set Intel OpenMP as default MKL thread layer
-if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM" OR CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+if(CXX_COMPILER_NAME STREQUAL "dpcpp" OR CXX_COMPILER_NAME STREQUAL "dpcpp.exe"
+   OR CXX_COMPILER_NAME STREQUAL "icpx" OR CXX_COMPILER_NAME STREQUAL "icx.exe")
   set(MKL_THREAD_LAYER "TBB" CACHE STRING "The thread layer to choose for MKL")
   set(MKL_INTERFACE "ilp64")
   set(MKL_INTERFACE_INTEGER_SIZE 8)
@@ -125,7 +125,14 @@ elseif(MKL_THREAD_LAYER STREQUAL "TBB")
   set(MKL_THREADING "tbb_thread")
 else()
 endif()
-find_package(MKL)
+
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
+  # VCPKG overrides the find_package command and the PATH parameter is currently
+  # broken with the current version of VCPKG so we are setting the MKL_ROOT
+  # directory to the MKLROOT environment variable.
+  set(MKL_ROOT "$ENV{MKLROOT}")
+  find_package(MKL)
+endif()
 
 af_multiple_option(NAME        AF_COMPUTE_LIBRARY
                    DEFAULT     ${default_compute_library}
@@ -218,7 +225,7 @@ if(${AF_BUILD_CPU} OR ${AF_BUILD_OPENCL})
   if("${AF_COMPUTE_LIBRARY}" STREQUAL "Intel-MKL"
       OR "${AF_COMPUTE_LIBRARY}" STREQUAL "MKL")
     af_mkl_batch_check()
-    dependency_check(MKL_FOUND "Please ensure Intel-MKL / oneAPI-oneMKL is installed")
+    dependency_check(MKL_Shared_FOUND "Please ensure Intel-MKL / oneAPI-oneMKL is installed")
     set(BUILD_WITH_MKL ON)
   elseif("${AF_COMPUTE_LIBRARY}" STREQUAL "FFTW/LAPACK/BLAS")
     dependency_check(FFTW_FOUND "FFTW not found")

From 6503b97d686d0b158e684e400cb683c7aa3f3826 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sun, 2 Apr 2023 08:39:03 -0400
Subject: [PATCH 243/473] Increase test timeout to avoid failures with
 jit_opencl on CI

---
 test/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 6f385f666a..5b7c869eba 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -76,7 +76,9 @@ function(af_add_test target backend is_serial)
     if(${is_serial})
       set_tests_properties(${target}
         PROPERTIES
-        RUN_SERIAL ON)
+          ENVIRONMENT AF_PRINT_ERRORS=1
+          TIMEOUT 900
+          RUN_SERIAL ON)
     endif(${is_serial})
   endif()
 endfunction()

From de0aba43640d6c6e39aa3d090aeb72c962b6cd7c Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sun, 2 Apr 2023 14:54:15 -0400
Subject: [PATCH 244/473] Remove exceptions thrown on OpenCL kernel cache miss.

---
 src/backend/opencl/compile_module.cpp | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp
index 832f5144a7..89d382c9c0 100644
--- a/src/backend/opencl/compile_module.cpp
+++ b/src/backend/opencl/compile_module.cpp
@@ -230,7 +230,10 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
     try {
         std::ifstream in(cacheFile, std::ios::binary);
         if (!in.is_open()) {
-            AF_ERROR("Unable to open binary cache file", AF_ERR_INTERNAL);
+            AF_TRACE("{{{:<20} : Unable to open {} for {}}}", moduleKey,
+                     cacheFile, dev.getInfo<CL_DEVICE_NAME>());
+            removeFile(cacheFile);
+            return retVal;
         }
         in.exceptions(std::ios::failbit | std::ios::badbit);
 
@@ -247,7 +250,11 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
         const size_t recomputedHash =
             deterministicHash(clbin.data(), clbinSize);
         if (recomputedHash != clbinHash) {
-            AF_ERROR("Binary on disk seems to be corrupted", AF_ERR_LOAD_SYM);
+            AF_TRACE(
+                "{{{:<20} : Corrupt binary({}) found on disk for {}, removed}}",
+                moduleKey, cacheFile, dev.getInfo<CL_DEVICE_NAME>());
+            removeFile(cacheFile);
+            return retVal;
         }
         program = Program(arrayfire::opencl::getContext(), {dev}, {clbin});
         program.build();
@@ -255,16 +262,6 @@ Module loadModuleFromDisk(const int device, const string &moduleKey,
         AF_TRACE("{{{:<20} : loaded from {} for {} }}", moduleKey, cacheFile,
                  dev.getInfo<CL_DEVICE_NAME>());
         retVal.set(program);
-    } catch (const AfError &e) {
-        if (e.getError() == AF_ERR_LOAD_SYM) {
-            AF_TRACE(
-                "{{{:<20} : Corrupt binary({}) found on disk for {}, removed}}",
-                moduleKey, cacheFile, dev.getInfo<CL_DEVICE_NAME>());
-        } else {
-            AF_TRACE("{{{:<20} : Unable to open {} for {}}}", moduleKey,
-                     cacheFile, dev.getInfo<CL_DEVICE_NAME>());
-        }
-        removeFile(cacheFile);
     } catch (const std::ios_base::failure &e) {
         AF_TRACE("{{{:<20} : IO failure while loading {} for {}; {}}}",
                  moduleKey, cacheFile, dev.getInfo<CL_DEVICE_NAME>(), e.what());

From d70a59b4004e8afa4b2537d6fb065b14776d4a02 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sun, 2 Apr 2023 15:16:02 -0400
Subject: [PATCH 245/473] Limit the maximum kernel size to 5kb for OpenCL to
 manage compile times

---
 src/backend/opencl/Array.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 810666b9a6..311ec715b9 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -334,11 +334,15 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
             (3 * sizeof(uint));
 
         const cl::Device &device = getDevice();
-        size_t max_param_size = device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>();
         // typical values:
         //   NVIDIA     = 4096
         //   AMD        = 3520  (AMD A10 iGPU = 1024)
         //   Intel iGPU = 1024
+        //
+        // Setting the maximum to 5120 bytes to keep the compile times
+        // resonable. This still results in large kernels but its not excessive.
+        size_t max_param_size =
+            min(5120UL, device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>());
         max_param_size -= base_param_size;
 
         struct tree_info {

From aaca3e6f81669d033fafc5bec7be5e9f0a8e7f05 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 3 Apr 2023 15:22:23 -0400
Subject: [PATCH 246/473] set CXX_COMPILER_NAME based on the current compiler
 to detect sycl

---
 CMakeLists.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e7bf293ce4..eed62e23a6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -45,6 +45,7 @@ if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
   set(AF_REQUIRED REQUIRED)
 endif()
 
+get_filename_component(CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
 if(CXX_COMPILER_NAME STREQUAL "dpcpp" OR CXX_COMPILER_NAME STREQUAL "dpcpp.exe"
    OR CXX_COMPILER_NAME STREQUAL "icpx" OR CXX_COMPILER_NAME STREQUAL "icx.exe")
   set(MKL_THREAD_LAYER "TBB" CACHE STRING "The thread layer to choose for MKL")
@@ -130,7 +131,9 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
   # VCPKG overrides the find_package command and the PATH parameter is currently
   # broken with the current version of VCPKG so we are setting the MKL_ROOT
   # directory to the MKLROOT environment variable.
-  set(MKL_ROOT "$ENV{MKLROOT}")
+  if(DEFINED ENV{MKLROOT} AND NOT DEFINED MKL_ROOT)
+    set(MKL_ROOT "$ENV{MKLROOT}")
+  endif()
   find_package(MKL)
 endif()
 

From 1e15e1f926a51468f094044b64f5d81f05912957 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 6 Apr 2023 15:22:16 -0400
Subject: [PATCH 247/473] Implement BLAS functions in oneAPI (#3396)

* initial blas function implementation
---
 src/backend/oneapi/Array.hpp          |   1 +
 src/backend/oneapi/Kernel.hpp         |   1 +
 src/backend/oneapi/blas.cpp           | 133 ++++++++++++++++++++++----
 src/backend/oneapi/blas.hpp           |   5 +-
 src/backend/oneapi/compile_module.cpp |   1 +
 src/backend/oneapi/jit.cpp            |   1 -
 src/backend/oneapi/kernel/memcopy.hpp |   1 +
 7 files changed, 121 insertions(+), 22 deletions(-)

diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index 9a4de1285c..d3173d7fb8 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 
+#include <Param.hpp>
 #include <backend.hpp>
 #include <common/ArrayInfo.hpp>
 #include <kernel/KParam.hpp>
diff --git a/src/backend/oneapi/Kernel.hpp b/src/backend/oneapi/Kernel.hpp
index 3fcf7b66b8..c0f15356f8 100644
--- a/src/backend/oneapi/Kernel.hpp
+++ b/src/backend/oneapi/Kernel.hpp
@@ -13,6 +13,7 @@
 #include <common/Logger.hpp>
 
 #include <backend.hpp>
+#include <sycl/sycl.hpp>
 #include <string>
 
 namespace arrayfire {
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
index 4a3b5e180d..964dcb6cde 100644
--- a/src/backend/oneapi/blas.cpp
+++ b/src/backend/oneapi/blas.cpp
@@ -12,53 +12,142 @@
 #include <Array.hpp>
 #include <arith.hpp>
 #include <common/half.hpp>
+#include <common/kernel_type.hpp>
 #include <common/traits.hpp>
 #include <complex.hpp>
+#include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
 #include <math.hpp>
+#include <platform.hpp>
 #include <reduce.hpp>
 #include <transpose.hpp>
+#include <types.hpp>
+
+#include <sycl/sycl.hpp>
+#include "oneapi/mkl/blas.hpp"
 
 #include <complex>
 #include <vector>
 
 using arrayfire::common::half;
 
-namespace arrayfire {
-namespace oneapi {
-
-void initBlas() { /*gpu_blas_init();*/
+// Converts an af_mat_prop options to a transpose type for mkl
+static oneapi::mkl::transpose toBlasTranspose(af_mat_prop opt) {
+    switch (opt) {
+        case AF_MAT_NONE: return oneapi::mkl::transpose::nontrans;
+        case AF_MAT_TRANS: return oneapi::mkl::transpose::trans;
+        case AF_MAT_CTRANS: return oneapi::mkl::transpose::conjtrans;
+        default: AF_ERROR("INVALID af_mat_prop", AF_ERR_ARG);
+    }
 }
 
-void deInitBlas() { /*gpu_blas_deinit();*/
+template<typename T>
+static void gemvDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts, int M,
+                         int N, const T *alpha,
+                         const arrayfire::oneapi::Array<T> &lhs, dim_t lStride,
+                         const arrayfire::oneapi::Array<T> &x, dim_t incx,
+                         const T *beta, arrayfire::oneapi::Array<T> &out,
+                         dim_t oInc) {
+    using Dt                   = arrayfire::oneapi::data_t<T>;
+    sycl::buffer<Dt, 1> lhsBuf = lhs.get()->template reinterpret<Dt, 1>();
+    sycl::buffer<Dt, 1> xBuf   = x.get()->template reinterpret<Dt, 1>();
+    sycl::buffer<Dt, 1> outBuf = out.get()->template reinterpret<Dt, 1>();
+    ::oneapi::mkl::blas::gemv(queue, lOpts, (int64_t)M, (int64_t)N, (T)*alpha,
+                              lhsBuf, (int64_t)lStride, xBuf, (int64_t)incx,
+                              (T)*beta, outBuf, (int64_t)oInc);
 }
 
 template<typename T>
-void gemm_fallback(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs,
-                   const T *alpha, const Array<T> &lhs, const Array<T> &rhs,
-                   const T *beta) {
-    ONEAPI_NOT_SUPPORTED("");
+static void gemmDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts,
+                         oneapi::mkl::transpose rOpts, int M, int N, int K,
+                         const T *alpha, const arrayfire::oneapi::Array<T> &lhs,
+                         dim_t lStride, const arrayfire::oneapi::Array<T> &rhs,
+                         dim_t rStride, const T *beta,
+                         arrayfire::oneapi::Array<T> &out, dim_t oleading) {
+    using Dt                   = arrayfire::oneapi::data_t<T>;
+    sycl::buffer<Dt, 1> lhsBuf = lhs.get()->template reinterpret<Dt, 1>();
+    sycl::buffer<Dt, 1> rhsBuf = rhs.get()->template reinterpret<Dt, 1>();
+    sycl::buffer<Dt, 1> outBuf = out.get()->template reinterpret<Dt, 1>();
+    ::oneapi::mkl::blas::gemm(queue, lOpts, rOpts, M, N, K, *alpha, lhsBuf,
+                              lStride, rhsBuf, rStride, *beta, outBuf,
+                              oleading);
 }
 
-template<>
-void gemm_fallback<half>(Array<half> & /*out*/, af_mat_prop /*optLhs*/,
-                         af_mat_prop /*optRhs*/, const half * /*alpha*/,
-                         const Array<half> & /*lhs*/,
-                         const Array<half> & /*rhs*/, const half * /*beta*/) {
-    ONEAPI_NOT_SUPPORTED("");
-    assert(false && "CPU fallback not implemented for f16");
+namespace arrayfire {
+namespace oneapi {
+
+void initBlas() { /*gpu_blas_init();*/
+}
+
+void deInitBlas() { /*gpu_blas_deinit();*/
 }
 
 template<typename T>
 void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
           const Array<T> &lhs, const Array<T> &rhs, const T *beta) {
-    ONEAPI_NOT_SUPPORTED("");
+    const auto lOpts = toBlasTranspose(optLhs);
+    const auto rOpts = toBlasTranspose(optRhs);
+
+    const auto aRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1;
+    const auto aColDim = (optLhs == AF_MAT_NONE) ? 1 : 0;
+    const auto bColDim = (optRhs == AF_MAT_NONE) ? 1 : 0;
+
+    const dim4 &lDims = lhs.dims();
+    const dim4 &rDims = rhs.dims();
+    const int M       = lDims[aRowDim];
+    const int N       = rDims[bColDim];
+    const int K       = lDims[aColDim];
+    const dim4 oDims  = out.dims();
+
+    const dim4 &lStrides = lhs.strides();
+    const dim4 &rStrides = rhs.strides();
+    const dim4 oStrides  = out.strides();
+
+    if (oDims.ndims() <= 2) {  // if non-batched
+        if (rhs.dims()[bColDim] == 1) {
+            dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
+            gemvDispatch<T>(getQueue(), lOpts, lDims[0], lDims[1], alpha, lhs,
+                            lStrides[1], rhs, incr, beta, out, oStrides[0]);
+        } else {
+            gemmDispatch<T>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
+                            lStrides[1], rhs, rStrides[1], beta, out,
+                            oStrides[1]);
+        }
+    } else {  // if batched
+        using Dt = arrayfire::oneapi::data_t<T>;
+
+        sycl::buffer<Dt, 1> lhsBuf = lhs.get()->template reinterpret<Dt, 1>();
+        sycl::buffer<Dt, 1> rhsBuf = rhs.get()->template reinterpret<Dt, 1>();
+        sycl::buffer<Dt, 1> outBuf = out.get()->template reinterpret<Dt, 1>();
+
+        const int64_t lda = lStrides[1];
+        const int64_t ldb = rStrides[1];
+        const int64_t ldc = oStrides[1];
+
+        int64_t batchSize = static_cast<int64_t>(oDims[2] * oDims[3]);
+
+        const bool not_l_batched =
+            (oDims[2] != lDims[2] && oDims[3] != lDims[3]);
+        const bool not_r_batched =
+            (oDims[2] != rDims[2] && oDims[3] != rDims[3]);
+
+        ::oneapi::mkl::blas::gemm_batch(
+            getQueue(), lOpts, rOpts, M, N, K, *alpha, lhsBuf, lda,
+            not_l_batched ? 0 : lStrides[2], rhsBuf, ldb,
+            not_r_batched ? 0 : rStrides[2], *beta, outBuf, ldc, oStrides[2],
+            batchSize);
+    }
+
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 template<typename T>
 Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
              af_mat_prop optRhs) {
-    ONEAPI_NOT_SUPPORTED("");
+    auto lhs_ = (optLhs == AF_MAT_NONE ? lhs : conj<T>(lhs));
+    auto rhs_ = (optRhs == AF_MAT_NONE ? rhs : conj<T>(rhs));
+    auto temp = arithOp<T, af_mul_t>(lhs_, rhs_, lhs_.dims());
+    return reduce<af_add_t, T, T>(temp, 0, false, 0);
 }
 
 #define INSTANTIATE_GEMM(TYPE)                                               \
@@ -71,7 +160,13 @@ INSTANTIATE_GEMM(float)
 INSTANTIATE_GEMM(cfloat)
 INSTANTIATE_GEMM(double)
 INSTANTIATE_GEMM(cdouble)
-INSTANTIATE_GEMM(half)
+// INSTANTIATE_GEMM(half)
+template<>
+void gemm(Array<half> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+          const half *alpha, const Array<half> &lhs, const Array<half> &rhs,
+          const half *beta) {
+    ONEAPI_NOT_SUPPORTED("");
+}
 
 #define INSTANTIATE_DOT(TYPE)                                                  \
     template Array<TYPE> dot<TYPE>(const Array<TYPE> &lhs,                     \
diff --git a/src/backend/oneapi/blas.hpp b/src/backend/oneapi/blas.hpp
index 605b3f6d6c..194fc4e6fb 100644
--- a/src/backend/oneapi/blas.hpp
+++ b/src/backend/oneapi/blas.hpp
@@ -30,8 +30,8 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
     int Ndim     = optRhs == AF_MAT_NONE ? 1 : 0;
     Array<T> res = createEmptyArray<T>(
         dim4(lhs.dims()[Mdim], rhs.dims()[Ndim], lhs.dims()[2], lhs.dims()[3]));
-    static const T alpha = T(1.0);
-    static const T beta  = T(0.0);
+    static constexpr T alpha = 1.0;
+    static constexpr T beta  = 0.0;
     gemm(res, optLhs, optRhs, &alpha, lhs, rhs, &beta);
     return res;
 }
@@ -39,5 +39,6 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
 template<typename T>
 Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
              af_mat_prop optRhs);
+
 }  // namespace oneapi
 }  // namespace arrayfire
diff --git a/src/backend/oneapi/compile_module.cpp b/src/backend/oneapi/compile_module.cpp
index 640fcc797c..4731d7dd87 100644
--- a/src/backend/oneapi/compile_module.cpp
+++ b/src/backend/oneapi/compile_module.cpp
@@ -13,6 +13,7 @@
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
 #include <common/util.hpp>
+#include <sycl/sycl.hpp>
 //#include <debug_opencl.hpp> TODO: remove?
 #include <err_oneapi.hpp>
 //#include <kernel_headers/KParam.hpp>
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index bd4a5f2d43..57c299a3f2 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -14,7 +14,6 @@
 #include <kernel_headers/jit.hpp>
 
 #include <Array.hpp>
-#include <common/debug.hpp>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <common/jit/ModdimNode.hpp>
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index adabe3b29d..59990dea39 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -14,6 +14,7 @@
 #include <common/half.hpp>
 #include <common/traits.hpp>
 #include <debug_oneapi.hpp>
+#include <sycl/sycl.hpp>
 #include <traits.hpp>
 
 #include <algorithm>

From ded9b338aa49e9ef79a5590fa0f5dcb548a37fdd Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 6 Apr 2023 16:01:54 -0400
Subject: [PATCH 248/473] Add linear algebra support to oneAPI (#3389)

This commit adds Linear Algebra support to oneAPI
---
 src/backend/oneapi/CMakeLists.txt             |   8 +-
 src/backend/oneapi/Module.hpp                 |   5 +-
 src/backend/oneapi/cholesky.cpp               |  46 +++-
 src/backend/oneapi/compile_module.cpp         |   9 +-
 src/backend/oneapi/copy.hpp                   |   2 +-
 src/backend/oneapi/identity.cpp               |   3 +-
 src/backend/oneapi/iir.cpp                    |   2 +-
 src/backend/oneapi/kernel/identity.hpp        |  86 +++++++
 src/backend/oneapi/kernel/lu_split.hpp        | 143 ++++++++++++
 .../oneapi/kernel/random_engine_write.hpp     |   3 +-
 src/backend/oneapi/lu.cpp                     |  67 +++++-
 src/backend/oneapi/platform.cpp               |   1 -
 src/backend/oneapi/reduce_impl.hpp            |   2 +-
 src/backend/oneapi/sort_by_key.cpp            |   2 +-
 src/backend/oneapi/sparse_blas.cpp            |  88 +++----
 src/backend/oneapi/svd.cpp                    | 221 +++---------------
 16 files changed, 416 insertions(+), 272 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/identity.hpp
 create mode 100644 src/backend/oneapi/kernel/lu_split.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index c003f72152..60c5aa9379 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -216,10 +216,12 @@ target_sources(afoneapi
     kernel/bilateral.hpp
     kernel/diagonal.hpp
     kernel/diff.hpp
+    kernel/histogram.hpp
+    kernel/identity.hpp
     kernel/interp.hpp
     kernel/iota.hpp
     kernel/ireduce.hpp
-    kernel/histogram.hpp
+    kernel/lu_split.hpp
     kernel/memcopy.hpp
     kernel/mean.hpp
     kernel/random_engine.hpp
@@ -268,7 +270,8 @@ arrayfire_set_default_cxx_flags(afoneapi)
 
 target_include_directories(afoneapi
   SYSTEM PRIVATE
-    ${SYCL_INCLUDE_DIR})
+    ${SYCL_INCLUDE_DIR}
+)
 
 target_include_directories(afoneapi
   PUBLIC
@@ -289,6 +292,7 @@ target_compile_options(afoneapi
 target_compile_definitions(afoneapi
   PRIVATE
     AF_ONEAPI
+    WITH_LINEAR_ALGEBRA
     CL_TARGET_OPENCL_VERSION=300
     CL_HPP_TARGET_OPENCL_VERSION=300
     CL_HPP_MINIMUM_OPENCL_VERSION=110
diff --git a/src/backend/oneapi/Module.hpp b/src/backend/oneapi/Module.hpp
index cb4c4e130c..6a5ce71985 100644
--- a/src/backend/oneapi/Module.hpp
+++ b/src/backend/oneapi/Module.hpp
@@ -10,6 +10,7 @@
 #pragma once
 
 #include <common/ModuleInterface.hpp>
+#include <sycl/sycl.hpp>
 
 #include <sycl/kernel_bundle.hpp>
 
@@ -19,9 +20,9 @@ namespace oneapi {
 /// oneapi backend wrapper for cl::Program object
 class Module
     : public common::ModuleInterface<
-          sycl::kernel_bundle<sycl::bundle_state::executable>*> {
+          sycl::kernel_bundle<sycl::bundle_state::executable> *> {
    public:
-    using ModuleType = sycl::kernel_bundle<sycl::bundle_state::executable>*;
+    using ModuleType = sycl::kernel_bundle<sycl::bundle_state::executable> *;
     using BaseClass  = common::ModuleInterface<ModuleType>;
 
     /// \brief Create an uninitialized Module
diff --git a/src/backend/oneapi/cholesky.cpp b/src/backend/oneapi/cholesky.cpp
index 905a3208c5..4fb0e08c58 100644
--- a/src/backend/oneapi/cholesky.cpp
+++ b/src/backend/oneapi/cholesky.cpp
@@ -11,23 +11,55 @@
 #include <cholesky.hpp>
 #include <copy.hpp>
 #include <err_oneapi.hpp>
+#include <platform.hpp>
 
 #if defined(WITH_LINEAR_ALGEBRA)
-//#include <triangle.hpp>
+#include <triangle.hpp>
+#include "oneapi/mkl/lapack.hpp"
 
 namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 int cholesky_inplace(Array<T> &in, const bool is_upper) {
-    ONEAPI_NOT_SUPPORTED("");
+    dim4 iDims    = in.dims();
+    dim4 iStrides = in.strides();
+    int64_t N     = iDims[0];
+    int64_t LDA   = iStrides[1];
+
+    int64_t lwork = 0;
+
+    ::oneapi::mkl::uplo uplo = ::oneapi::mkl::uplo::lower;
+    if (is_upper) { uplo = ::oneapi::mkl::uplo::upper; }
+
+    lwork = ::oneapi::mkl::lapack::potrf_scratchpad_size<T>(getQueue(), uplo, N,
+                                                            LDA);
+
+    Array<T> workspace = createEmptyArray<T>(af::dim4(lwork));
+    Array<int> d_info  = createEmptyArray<int>(af::dim4(1));
+
+    try {
+        ::oneapi::mkl::lapack::potrf(getQueue(), uplo, N, *in.get(), LDA,
+                                     *workspace.get(), lwork);
+    } catch (::oneapi::mkl::lapack::exception const &e) {
+        AF_ERROR(
+            "Unexpected exception caught during synchronous\
+                call to LAPACK API",
+            AF_ERR_RUNTIME);
+        return e.info();
+    }
+
     return 0;
 }
 
 template<typename T>
 Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper) {
-    ONEAPI_NOT_SUPPORTED("");
-    return 0;
+    Array<T> out = copyArray<T>(in);
+    *info        = cholesky_inplace(out, is_upper);
+
+    triangle<T>(out, out, is_upper, false);
+
+    return out;
 }
 
 #define INSTANTIATE_CH(T)                                                 \
@@ -50,12 +82,14 @@ namespace oneapi {
 
 template<typename T>
 Array<T> cholesky(int *info, const Array<T> &in, const bool is_upper) {
-    AF_ERROR("Linear Algebra is disabled on OpenCL", AF_ERR_NOT_CONFIGURED);
+    AF_ERROR("Linear Algebra is disabled on OneAPI backend",
+             AF_ERR_NOT_CONFIGURED);
 }
 
 template<typename T>
 int cholesky_inplace(Array<T> &in, const bool is_upper) {
-    AF_ERROR("Linear Algebra is disabled on OpenCL", AF_ERR_NOT_CONFIGURED);
+    AF_ERROR("Linear Algebra is disabled on OneAPI backend",
+             AF_ERR_NOT_CONFIGURED);
 }
 
 #define INSTANTIATE_CH(T)                                                 \
diff --git a/src/backend/oneapi/compile_module.cpp b/src/backend/oneapi/compile_module.cpp
index 4731d7dd87..7fce4b70c0 100644
--- a/src/backend/oneapi/compile_module.cpp
+++ b/src/backend/oneapi/compile_module.cpp
@@ -13,10 +13,9 @@
 #include <common/Logger.hpp>
 #include <common/defines.hpp>
 #include <common/util.hpp>
-#include <sycl/sycl.hpp>
-//#include <debug_opencl.hpp> TODO: remove?
 #include <err_oneapi.hpp>
-//#include <kernel_headers/KParam.hpp>
+#include <sycl/sycl.hpp>
+// #include <kernel_headers/KParam.hpp>
 #include <platform.hpp>
 #include <traits.hpp>
 
@@ -73,7 +72,7 @@ namespace arrayfire {
 namespace oneapi {
 
 // const static string DEFAULT_MACROS_STR(
-//"\n\
+// "\n\
                                            //#ifdef USE_DOUBLE\n\
                                            //#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n\
                                            //#endif\n                     \
@@ -85,7 +84,7 @@ namespace oneapi {
                                            //#ifndef M_PI\n               \
                                            //#define
 // M_PI 3.1415926535897932384626433832795028841971693993751058209749445923078164\n
-//\
+// \
                                            //#endif\n                     \
                                            //");
 
diff --git a/src/backend/oneapi/copy.hpp b/src/backend/oneapi/copy.hpp
index 048c89260a..4b05151dbd 100644
--- a/src/backend/oneapi/copy.hpp
+++ b/src/backend/oneapi/copy.hpp
@@ -9,7 +9,7 @@
 #pragma once
 
 #include <Array.hpp>
-//#include <kernel/pad_array_borders.hpp>
+// #include <kernel/pad_array_borders.hpp>
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/identity.cpp b/src/backend/oneapi/identity.cpp
index c7db8e7d44..5a838a4cf0 100644
--- a/src/backend/oneapi/identity.cpp
+++ b/src/backend/oneapi/identity.cpp
@@ -11,6 +11,7 @@
 #include <Array.hpp>
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/identity.hpp>
 #include <af/dim4.hpp>
 
 using arrayfire::common::half;
@@ -19,8 +20,8 @@ namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> identity(const dim4& dims) {
-    ONEAPI_NOT_SUPPORTED("");
     Array<T> out = createEmptyArray<T>(dims);
+    kernel::identity<T>(out);
     return out;
 }
 
diff --git a/src/backend/oneapi/iir.cpp b/src/backend/oneapi/iir.cpp
index e0223ca6f1..e38a70294f 100644
--- a/src/backend/oneapi/iir.cpp
+++ b/src/backend/oneapi/iir.cpp
@@ -12,7 +12,7 @@
 #include <convolve.hpp>
 #include <err_oneapi.hpp>
 #include <iir.hpp>
-//#include <kernel/iir.hpp>
+// #include <kernel/iir.hpp>
 #include <math.hpp>
 #include <af/dim4.hpp>
 
diff --git a/src/backend/oneapi/kernel/identity.hpp b/src/backend/oneapi/kernel/identity.hpp
new file mode 100644
index 0000000000..20553a2149
--- /dev/null
+++ b/src/backend/oneapi/kernel/identity.hpp
@@ -0,0 +1,86 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <math.hpp>
+#include <types.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T>
+class identityKernel {
+   public:
+    identityKernel(write_accessor<T> out, KParam oInfo, const int groups_x,
+                   const int groups_y)
+        : out_(out), oInfo_(oInfo), groups_x_(groups_x), groups_y_(groups_y) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        size_t idz = g.get_group_id(0) / groups_x_;
+        size_t idw = g.get_group_id(1) / groups_y_;
+
+        size_t groupId_x = g.get_group_id(0) - idz * groups_x_;
+        size_t groupId_y = g.get_group_id(1) - idw * groups_y_;
+
+        size_t idx = it.get_local_id(0) + groupId_x * g.get_local_range(0);
+        size_t idy = it.get_local_id(1) + groupId_y * g.get_local_range(1);
+
+        size_t xlim = oInfo_.dims[0];
+        size_t ylim = oInfo_.dims[1];
+        size_t zlim = oInfo_.dims[2];
+        size_t wlim = oInfo_.dims[3];
+        if (idx < xlim && idy < ylim && idz < zlim && idw < wlim) {
+            const T one  = scalar<T>(1);
+            const T zero = scalar<T>(0);
+
+            T *ptr = out_.get_pointer() + idz * oInfo_.strides[2] +
+                     idw * oInfo_.strides[3];
+            T val                              = (idx == idy) ? one : zero;
+            ptr[idx + idy * oInfo_.strides[1]] = val;
+        }
+    }
+
+   protected:
+    write_accessor<T> out_;
+    KParam oInfo_;
+    int groups_x_;
+    int groups_y_;
+};
+
+template<typename T>
+void identity(Param<T> out) {
+    sycl::range<2> local{32, 8};
+
+    int groups_x = divup(out.info.dims[0], local[0]);
+    int groups_y = divup(out.info.dims[1], local[1]);
+    sycl::range<2> global{groups_x * out.info.dims[2] * local[0],
+                          groups_y * out.info.dims[3] * local[1]};
+
+    getQueue().submit([&](sycl::handler &h) {
+        write_accessor<T> oData{*out.data, h};
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       identityKernel<T>(oData, out.info, groups_x, groups_y));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/lu_split.hpp b/src/backend/oneapi/kernel/lu_split.hpp
new file mode 100644
index 0000000000..f42cf8644c
--- /dev/null
+++ b/src/backend/oneapi/kernel/lu_split.hpp
@@ -0,0 +1,143 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+
+#include <math.hpp>
+#include <array>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T, bool same_dims>
+class luSplitKernel {
+   public:
+    luSplitKernel(write_accessor<T> lower, KParam lInfo,
+                  write_accessor<T> upper, KParam uInfo, read_accessor<T> in,
+                  KParam iInfo, const int groupsPerMatX,
+                  const int groupsPerMatY)
+        : lower_(lower)
+        , lInfo_(lInfo)
+        , upper_(upper)
+        , uInfo_(uInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , groupsPerMatX_(groupsPerMatX)
+        , groupsPerMatY_(groupsPerMatY) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const int oz  = g.get_group_id(0) / groupsPerMatX_;
+        const int ow  = g.get_group_id(1) / groupsPerMatY_;
+
+        const int blockIdx_x = g.get_group_id(0) - oz * groupsPerMatX_;
+        const int blockIdx_y = g.get_group_id(1) - ow * groupsPerMatY_;
+
+        const int xx = it.get_local_id(0) + blockIdx_x * g.get_local_range(0);
+        const int yy = it.get_local_id(1) + blockIdx_y * g.get_local_range(1);
+
+        const int incy = groupsPerMatY_ * g.get_local_range(1);
+        const int incx = groupsPerMatX_ * g.get_local_range(0);
+
+        T *d_l = lower_.get_pointer();
+        T *d_u = upper_.get_pointer();
+        T *d_i = in_.get_pointer();
+
+        if (oz < iInfo_.dims[2] && ow < iInfo_.dims[3]) {
+            d_i = d_i + oz * iInfo_.strides[2] + ow * iInfo_.strides[3];
+            d_l = d_l + oz * lInfo_.strides[2] + ow * lInfo_.strides[3];
+            d_u = d_u + oz * uInfo_.strides[2] + ow * uInfo_.strides[3];
+
+            for (int oy = yy; oy < iInfo_.dims[1]; oy += incy) {
+                T *Yd_i = d_i + oy * iInfo_.strides[1];
+                T *Yd_l = d_l + oy * lInfo_.strides[1];
+                T *Yd_u = d_u + oy * uInfo_.strides[1];
+                for (int ox = xx; ox < iInfo_.dims[0]; ox += incx) {
+                    if (ox > oy) {
+                        if (same_dims || oy < lInfo_.dims[1])
+                            Yd_l[ox] = Yd_i[ox];
+                        if (!same_dims || ox < uInfo_.dims[0])
+                            Yd_u[ox] = scalar<T>(0);
+                    } else if (oy > ox) {
+                        if (same_dims || oy < lInfo_.dims[1])
+                            Yd_l[ox] = scalar<T>(0);
+                        if (!same_dims || ox < uInfo_.dims[0])
+                            Yd_u[ox] = Yd_i[ox];
+                    } else if (ox == oy) {
+                        if (same_dims || oy < lInfo_.dims[1])
+                            Yd_l[ox] = scalar<T>(1.0);
+                        if (!same_dims || ox < uInfo_.dims[0])
+                            Yd_u[ox] = Yd_i[ox];
+                    }
+                }
+            }
+        }
+    }
+
+   protected:
+    write_accessor<T> lower_;
+    KParam lInfo_;
+    write_accessor<T> upper_;
+    KParam uInfo_;
+    read_accessor<T> in_;
+    KParam iInfo_;
+    int groupsPerMatX_;
+    int groupsPerMatY_;
+};
+
+template<typename T>
+void lu_split(Param<T> lower, Param<T> upper, Param<T> in) {
+    constexpr unsigned TX    = 32;
+    constexpr unsigned TY    = 8;
+    constexpr unsigned TILEX = 128;
+    constexpr unsigned TILEY = 32;
+
+    const bool sameDims = lower.info.dims[0] == in.info.dims[0] &&
+                          lower.info.dims[1] == in.info.dims[1];
+
+    sycl::range<2> local(TX, TY);
+
+    int groupsPerMatX = divup(in.info.dims[0], TILEX);
+    int groupsPerMatY = divup(in.info.dims[1], TILEY);
+    sycl::range<2> global(groupsPerMatX * in.info.dims[2] * local[0],
+                          groupsPerMatY * in.info.dims[3] * local[1]);
+
+    getQueue().submit([&](sycl::handler &h) {
+        read_accessor<T> iData{*in.data, h};
+        write_accessor<T> lData{*lower.data, h};
+        write_accessor<T> uData{*upper.data, h};
+
+        if (sameDims) {
+            h.parallel_for(sycl::nd_range{global, local},
+                           luSplitKernel<T, true>(
+                               lData, lower.info, uData, upper.info, iData,
+                               in.info, groupsPerMatX, groupsPerMatY));
+        } else {
+            h.parallel_for(sycl::nd_range{global, local},
+                           luSplitKernel<T, false>(
+                               lData, lower.info, uData, upper.info, iData,
+                               in.info, groupsPerMatX, groupsPerMatY));
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index b3a4d60ed7..dcd20dec13 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -7,8 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 #pragma once
-
-#include <sycl/builtins.hpp>
+#include <sycl/sycl.hpp>
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/lu.cpp b/src/backend/oneapi/lu.cpp
index b1d0b4b746..200b85d23b 100644
--- a/src/backend/oneapi/lu.cpp
+++ b/src/backend/oneapi/lu.cpp
@@ -13,28 +13,77 @@
 #if defined(WITH_LINEAR_ALGEBRA)
 #include <blas.hpp>
 #include <copy.hpp>
+#include <kernel/lu_split.hpp>
 #include <platform.hpp>
+#include "oneapi/mkl/lapack.hpp"
 
 namespace arrayfire {
 namespace oneapi {
 
-Array<int> convertPivot(int *ipiv, int in_sz, int out_sz) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<int> out = createEmptyArray<in_t>(af::dim4(1));
-    return out;
+Array<int> convertPivot(sycl::buffer<int64_t> &pivot, int out_sz,
+                        bool convert_pivot) {
+    dim_t d0 = pivot.get_range()[0];
+
+    std::vector<int> d_po(out_sz);
+    for (int i = 0; i < out_sz; i++) { d_po[i] = i; }
+
+    auto d_pi = pivot.get_host_access();
+
+    if (convert_pivot) {
+        for (int j = 0; j < d0; j++) {
+            // 1 indexed in pivot
+            std::swap(d_po[j], d_po[d_pi[j] - 1]);
+        }
+
+        Array<int> res = createHostDataArray(dim4(out_sz), &d_po[0]);
+        return res;
+    } else {
+        d_po.resize(d0);
+        for (int j = 0; j < d0; j++) { d_po[j] = static_cast<int>(d_pi[j]); }
+    }
+    Array<int> res = createHostDataArray(dim4(d0), &d_po[0]);
+    return res;
 }
 
 template<typename T>
 void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
         const Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("");
+    dim4 iDims = in.dims();
+    int M      = iDims[0];
+    int N      = iDims[1];
+    int MN     = std::min(M, N);
+
+    Array<T> in_copy = copyArray<T>(in);
+    pivot            = lu_inplace(in_copy);
+
+    // SPLIT into lower and upper
+    dim4 ldims(M, MN);
+    dim4 udims(MN, N);
+    lower = createEmptyArray<T>(ldims);
+    upper = createEmptyArray<T>(udims);
+    kernel::lu_split<T>(lower, upper, in_copy);
 }
 
 template<typename T>
 Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<int> out = createEmptyArray<in_t>(af::dim4(1));
-    return out;
+    dim4 iDims    = in.dims();
+    dim4 iStrides = in.strides();
+    int64_t M     = iDims[0];
+    int64_t N     = iDims[1];
+    int64_t MN    = std::min(M, N);
+    int64_t LDA   = iStrides[1];
+
+    std::int64_t scratchpad_size =
+        ::oneapi::mkl::lapack::getrf_scratchpad_size<T>(getQueue(), M, N, LDA);
+
+    sycl::buffer<int64_t, 1> ipiv{sycl::range<1>(MN)};
+    Array<T> scratch = createEmptyArray<T>(af::dim4(scratchpad_size));
+
+    ::oneapi::mkl::lapack::getrf(getQueue(), M, N, *in.get(), LDA, ipiv,
+                                 *scratch.get(), scratchpad_size);
+
+    Array<int> pivot = convertPivot(ipiv, M, convert_pivot);
+    return pivot;
 }
 
 bool isLAPACKAvailable() { return true; }
@@ -61,14 +110,12 @@ namespace oneapi {
 template<typename T>
 void lu(Array<T> &lower, Array<T> &upper, Array<int> &pivot,
         const Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("");
     AF_ERROR("Linear Algebra is disabled on OneAPI backend",
              AF_ERR_NOT_CONFIGURED);
 }
 
 template<typename T>
 Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
-    ONEAPI_NOT_SUPPORTED("");
     AF_ERROR("Linear Algebra is disabled on OneAPI backend",
              AF_ERR_NOT_CONFIGURED);
 }
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index 6d4c7df84c..e0959a9390 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -172,7 +172,6 @@ string getDeviceInfo() noexcept {
                 device->get_info<sycl::info::device::driver_version>();
             info << devVersion;
             info << " -- Device driver " << driVersion;
-
             info << " -- Unified Memory ("
                  << (isHostUnifiedMemory(*device) ? "True" : "False") << ")";
 #endif
diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp
index 898f77d006..14b5a9e269 100644
--- a/src/backend/oneapi/reduce_impl.hpp
+++ b/src/backend/oneapi/reduce_impl.hpp
@@ -10,7 +10,7 @@
 #include <Array.hpp>
 #include <err_oneapi.hpp>
 #include <kernel/reduce.hpp>
-//#include <kernel/reduce_by_key.hpp>
+// #include <kernel/reduce_by_key.hpp>
 #include <reduce.hpp>
 #include <af/dim4.hpp>
 #include <complex>
diff --git a/src/backend/oneapi/sort_by_key.cpp b/src/backend/oneapi/sort_by_key.cpp
index f7b5beca91..00a5bb55fa 100644
--- a/src/backend/oneapi/sort_by_key.cpp
+++ b/src/backend/oneapi/sort_by_key.cpp
@@ -10,7 +10,7 @@
 #include <Array.hpp>
 #include <copy.hpp>
 #include <err_oneapi.hpp>
-//#include <kernel/sort_by_key.hpp>
+// #include <kernel/sort_by_key.hpp>
 #include <math.hpp>
 #include <reorder.hpp>
 #include <sort_by_key.hpp>
diff --git a/src/backend/oneapi/sparse_blas.cpp b/src/backend/oneapi/sparse_blas.cpp
index 6d414c8ee0..67d7cb8352 100644
--- a/src/backend/oneapi/sparse_blas.cpp
+++ b/src/backend/oneapi/sparse_blas.cpp
@@ -27,7 +27,7 @@
 #include <af/dim4.hpp>
 
 #if defined(WITH_LINEAR_ALGEBRA)
-#include <cpu/cpu_sparse_blas.hpp>
+// #include <cpu/cpu_sparse_blas.hpp>
 #endif  // WITH_LINEAR_ALGEBRA
 
 namespace arrayfire {
@@ -39,55 +39,55 @@ template<typename T>
 Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhsIn,
                 af_mat_prop optLhs, af_mat_prop optRhs) {
     ONEAPI_NOT_SUPPORTED("sparse matmul Not supported");
-    //#if defined(WITH_LINEAR_ALGEBRA)
-    //    if (OpenCLCPUOffload(
-    //            false)) {  // Do not force offload gemm on OSX Intel devices
-    //        return cpu::matmul(lhs, rhsIn, optLhs, optRhs);
-    //    }
-    //#endif
+    // #if defined(WITH_LINEAR_ALGEBRA)
+    //     if (OpenCLCPUOffload(
+    //             false)) {  // Do not force offload gemm on OSX Intel devices
+    //         return cpu::matmul(lhs, rhsIn, optLhs, optRhs);
+    //     }
+    // #endif
     //
-    //    int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1;
-    //    // int lColDim = (optLhs == AF_MAT_NONE) ? 1 : 0;
-    //    static const int rColDim =
-    //        1;  // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0;
+    //     int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1;
+    //     // int lColDim = (optLhs == AF_MAT_NONE) ? 1 : 0;
+    //     static const int rColDim =
+    //         1;  // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0;
     //
-    //    dim4 lDims = lhs.dims();
-    //    dim4 rDims = rhsIn.dims();
-    //    int M      = lDims[lRowDim];
-    //    int N      = rDims[rColDim];
-    //    // int K = lDims[lColDim];
+    //     dim4 lDims = lhs.dims();
+    //     dim4 rDims = rhsIn.dims();
+    //     int M      = lDims[lRowDim];
+    //     int N      = rDims[rColDim];
+    //     // int K = lDims[lColDim];
     //
-    //    const Array<T> rhs =
-    //        (N != 1 && optLhs == AF_MAT_NONE) ? transpose(rhsIn, false) :
-    //        rhsIn;
-    //    Array<T> out = createEmptyArray<T>(af::dim4(M, N, 1, 1));
+    //     const Array<T> rhs =
+    //         (N != 1 && optLhs == AF_MAT_NONE) ? transpose(rhsIn, false) :
+    //         rhsIn;
+    //     Array<T> out = createEmptyArray<T>(af::dim4(M, N, 1, 1));
     //
-    //    static const T alpha = scalar<T>(1.0);
-    //    static const T beta  = scalar<T>(0.0);
+    //     static const T alpha = scalar<T>(1.0);
+    //     static const T beta  = scalar<T>(0.0);
     //
-    //    const Array<T>& values   = lhs.getValues();
-    //    const Array<int>& rowIdx = lhs.getRowIdx();
-    //    const Array<int>& colIdx = lhs.getColIdx();
+    //     const Array<T>& values   = lhs.getValues();
+    //     const Array<int>& rowIdx = lhs.getRowIdx();
+    //     const Array<int>& colIdx = lhs.getColIdx();
     //
-    //    if (optLhs == AF_MAT_NONE) {
-    //        if (N == 1) {
-    //            kernel::csrmv(out, values, rowIdx, colIdx, rhs, alpha, beta);
-    //        } else {
-    //            kernel::csrmm_nt(out, values, rowIdx, colIdx, rhs, alpha,
-    //            beta);
-    //        }
-    //    } else {
-    //        // CSR transpose is a CSC matrix
-    //        if (N == 1) {
-    //            kernel::cscmv(out, values, rowIdx, colIdx, rhs, alpha, beta,
-    //                          optLhs == AF_MAT_CTRANS);
-    //        } else {
-    //            kernel::cscmm_nn(out, values, rowIdx, colIdx, rhs, alpha,
-    //            beta,
-    //                             optLhs == AF_MAT_CTRANS);
-    //        }
-    //    }
-    //    return out;
+    //     if (optLhs == AF_MAT_NONE) {
+    //         if (N == 1) {
+    //             kernel::csrmv(out, values, rowIdx, colIdx, rhs, alpha, beta);
+    //         } else {
+    //             kernel::csrmm_nt(out, values, rowIdx, colIdx, rhs, alpha,
+    //             beta);
+    //         }
+    //     } else {
+    //         // CSR transpose is a CSC matrix
+    //         if (N == 1) {
+    //             kernel::cscmv(out, values, rowIdx, colIdx, rhs, alpha, beta,
+    //                           optLhs == AF_MAT_CTRANS);
+    //         } else {
+    //             kernel::cscmm_nn(out, values, rowIdx, colIdx, rhs, alpha,
+    //             beta,
+    //                              optLhs == AF_MAT_CTRANS);
+    //         }
+    //     }
+    //     return out;
 }
 
 #define INSTANTIATE_SPARSE(T)                                            \
diff --git a/src/backend/oneapi/svd.cpp b/src/backend/oneapi/svd.cpp
index fad4c2f35b..2c9b751d15 100644
--- a/src/backend/oneapi/svd.cpp
+++ b/src/backend/oneapi/svd.cpp
@@ -9,219 +9,50 @@
 
 #include <Array.hpp>
 #include <blas.hpp>
+#include <common/err_common.hpp>
 #include <copy.hpp>
 #include <err_oneapi.hpp>  // error check functions and Macros
 #include <math.hpp>
+#include <memory.hpp>
+#include <platform.hpp>
 #include <reduce.hpp>
 #include <svd.hpp>  // oneapi backend function header
 #include <transpose.hpp>
 
 #if defined(WITH_LINEAR_ALGEBRA)
-
-#include <cpu/cpu_svd.hpp>
-#include <magma/magma.h>
-#include <magma/magma_cpu_lapack.h>
-#include <magma/magma_helper.h>
-#include <platform.hpp>
+#include "oneapi/mkl/lapack.hpp"
 
 namespace arrayfire {
 namespace oneapi {
 
-template<typename Tr>
-Tr calc_scale(Tr From, Tr To) {
-    // FIXME: I am not sure this is correct, removing this for now
-#if 0
-    //http://www.netlib.org/lapack/explore-3.1.1-html/dlascl.f.html
-    cpu_lapack_lamch_func<Tr> cpu_lapack_lamch;
-
-    Tr S = cpu_lapack_lamch('S');
-    Tr B = 1.0 / S;
-
-    Tr FromCopy = From, ToCopy = To;
-
-    Tr Mul = 1;
-
-    while (true) {
-        Tr From1 = FromCopy * S, To1 = ToCopy / B;
-        if (std::abs(From1) > std::abs(ToCopy) && ToCopy != 0) {
-            Mul *= S;
-            FromCopy = From1;
-        } else if (std::abs(To1) > std::abs(FromCopy)) {
-            Mul *= B;
-            ToCopy = To1;
-        } else {
-            Mul *= (ToCopy) / (FromCopy);
-            break;
-        }
-    }
-
-    return Mul;
-#else
-    return To / From;
-#endif
-}
-
-template<typename T, typename Tr>
-void svd(Array<T> &arrU, Array<Tr> &arrS, Array<T> &arrVT, Array<T> &arrA,
-         bool want_vectors = true) {
-    ONEAPI_NOT_SUPPORTED("");
-    dim4 idims    = arrA.dims();
-    dim4 istrides = arrA.strides();
-
-    const int m      = static_cast<int>(idims[0]);
-    const int n      = static_cast<int>(idims[1]);
-    const int ldda   = static_cast<int>(istrides[1]);
-    const int lda    = m;
-    const int min_mn = std::min(m, n);
-    const int ldu    = m;
-    const int ldvt   = n;
-
-    const int nb    = magma_get_gebrd_nb<T>(n);
-    const int lwork = (m + n) * nb;
-
-    cpu_lapack_lacpy_func<T> cpu_lapack_lacpy;
-    cpu_lapack_bdsqr_work_func<T> cpu_lapack_bdsqr_work;
-    cpu_lapack_ungbr_work_func<T> cpu_lapack_ungbr_work;
-    cpu_lapack_lamch_func<Tr> cpu_lapack_lamch;
-
-    // Get machine constants
-    static const double eps    = cpu_lapack_lamch('P');
-    static const double smlnum = std::sqrt(cpu_lapack_lamch('S')) / eps;
-    static const double bignum = 1. / smlnum;
-
-    Tr anrm = abs(getScalar<T>(reduce_all<af_max_t, T, T>(arrA)));
-
-    T scale                = scalar<T>(1);
-    static const int ione  = 1;
-    static const int izero = 0;
-
-    bool iscl = false;
-    if (anrm > 0. && anrm < smlnum) {
-        iscl  = true;
-        scale = scalar<T>(calc_scale<Tr>(anrm, smlnum));
-    } else if (anrm > bignum) {
-        iscl  = true;
-        scale = scalar<T>(calc_scale<Tr>(anrm, bignum));
-    }
-
-    if (iscl == 1) { multiply_inplace(arrA, abs(scale)); }
-
-    int nru  = 0;
-    int ncvt = 0;
-
-    // Instead of copying U, S, VT, and A to the host and copying the results
-    // back to the device, create a pointer that's mapped to device memory where
-    // the computation can directly happen
-    T *mappedA = static_cast<T *>(getQueue().enqueueMapBuffer(
-        *arrA.get(), CL_FALSE, CL_MAP_READ, sizeof(T) * arrA.getOffset(),
-        sizeof(T) * arrA.elements()));
-    std::vector<T> tauq(min_mn), taup(min_mn);
-    std::vector<T> work(lwork);
-    Tr *mappedS0 = (Tr *)getQueue().enqueueMapBuffer(
-        *arrS.get(), CL_TRUE, CL_MAP_WRITE, sizeof(Tr) * arrS.getOffset(),
-        sizeof(Tr) * arrS.elements());
-    std::vector<Tr> s1(min_mn - 1);
-    std::vector<Tr> rwork(5 * min_mn);
-
-    int info = 0;
-
-    // Bidiagonalize A
-    // (CWorkspace: need 2*N + M, prefer 2*N + (M + N)*NB)
-    // (RWorkspace: need N)
-    magma_gebrd_hybrid<T>(m, n, mappedA, lda, (*arrA.get())(), arrA.getOffset(),
-                          ldda, (void *)mappedS0, static_cast<void *>(&s1[0]),
-                          &tauq[0], &taup[0], &work[0], lwork, getQueue()(),
-                          &info, false);
-
-    T *mappedU = nullptr, *mappedVT = nullptr;
-    std::vector<T> cdummy(1);
-
-    if (want_vectors) {
-        mappedU  = static_cast<T *>(getQueue().enqueueMapBuffer(
-             *arrU.get(), CL_FALSE, CL_MAP_WRITE, sizeof(T) * arrU.getOffset(),
-             sizeof(T) * arrU.elements()));
-        mappedVT = static_cast<T *>(getQueue().enqueueMapBuffer(
-            *arrVT.get(), CL_TRUE, CL_MAP_WRITE, sizeof(T) * arrVT.getOffset(),
-            sizeof(T) * arrVT.elements()));
-
-        // If left singular vectors desired in U, copy result to U
-        // and generate left bidiagonalizing vectors in U
-        // (CWorkspace: need 2*N + NCU, prefer 2*N + NCU*NB)
-        // (RWorkspace: 0)
-        LAPACKE_CHECK(cpu_lapack_lacpy('L', m, n, mappedA, lda, mappedU, ldu));
-
-        int ncu = m;
-        LAPACKE_CHECK(cpu_lapack_ungbr_work('Q', m, ncu, n, mappedU, ldu,
-                                            &tauq[0], &work[0], lwork));
-
-        // If right singular vectors desired in VT, copy result to
-        // VT and generate right bidiagonalizing vectors in VT
-        // (CWorkspace: need 3*N-1, prefer 2*N + (N-1)*NB)
-        // (RWorkspace: 0)
-        LAPACKE_CHECK(
-            cpu_lapack_lacpy('U', n, n, mappedA, lda, mappedVT, ldvt));
-        LAPACKE_CHECK(cpu_lapack_ungbr_work('P', n, n, n, mappedVT, ldvt,
-                                            &taup[0], &work[0], lwork));
-
-        nru  = m;
-        ncvt = n;
-    }
-    getQueue().enqueueUnmapMemObject(*arrA.get(), mappedA);
-
-    // Perform bidiagonal QR iteration, if desired, computing
-    // left singular vectors in U and computing right singular
-    // vectors in VT
-    // (CWorkspace: need 0)
-    // (RWorkspace: need BDSPAC)
-    LAPACKE_CHECK(cpu_lapack_bdsqr_work('U', n, ncvt, nru, izero, mappedS0,
-                                        &s1[0], mappedVT, ldvt, mappedU, ldu,
-                                        &cdummy[0], ione, &rwork[0]));
-
-    if (want_vectors) {
-        getQueue().enqueueUnmapMemObject(*arrU.get(), mappedU);
-        getQueue().enqueueUnmapMemObject(*arrVT.get(), mappedVT);
-    }
-
-    getQueue().enqueueUnmapMemObject(*arrS.get(), mappedS0);
-
-    if (iscl == 1) {
-        Tr rscale = scalar<Tr>(1);
-        if (anrm > bignum) {
-            rscale = calc_scale<Tr>(bignum, anrm);
-        } else if (anrm < smlnum) {
-            rscale = calc_scale<Tr>(smlnum, anrm);
-        }
-        multiply_inplace(arrS, rscale);
-    }
-}
-
 template<typename T, typename Tr>
 void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("");
-    // if (OpenCLCPUOffload()) { return cpu::svdInPlace(s, u, vt, in); }
-
-    // svd<T, Tr>(u, s, vt, in, true);
+    dim4 iDims = in.dims();
+    int64_t M  = iDims[0];
+    int64_t N  = iDims[1];
+
+    dim4 iStrides = in.strides();
+    dim4 uStrides = u.strides();
+    dim4 vStrides = vt.strides();
+    int64_t LDA   = iStrides[1];
+    int64_t LDU   = uStrides[1];
+    int64_t LDVt  = vStrides[1];
+
+    int64_t scratch_size = ::oneapi::mkl::lapack::gesvd_scratchpad_size<T>(
+        getQueue(), ::oneapi::mkl::jobsvd::vectors,
+        ::oneapi::mkl::jobsvd::vectors, M, N, LDA, LDU, LDVt);
+    Array<T> scratchpad = createEmptyArray<T>(af::dim4(scratch_size));
+
+    ::oneapi::mkl::lapack::gesvd(
+        getQueue(), ::oneapi::mkl::jobsvd::vectors,
+        ::oneapi::mkl::jobsvd::vectors, M, N, *in.get(), LDA, *s.get(),
+        *u.get(), LDU, *vt.get(), LDVt, *scratchpad.get(), scratch_size);
 }
 
 template<typename T, typename Tr>
 void svd(Array<Tr> &s, Array<T> &u, Array<T> &vt, const Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("");
-
-    // if (OpenCLCPUOffload()) { return cpu::svd(s, u, vt, in); }
-
-    // dim4 iDims = in.dims();
-    // int M      = iDims[0];
-    // int N      = iDims[1];
-
-    // if (M >= N) {
-    //     Array<T> in_copy = copyArray(in);
-    //     svdInPlace(s, u, vt, in_copy);
-    // } else {
-    //     Array<T> in_trans = transpose(in, true);
-    //     svdInPlace(s, vt, u, in_trans);
-    //     transpose_inplace(u, true);
-    //     transpose_inplace(vt, true);
-    // }
+    Array<T> in_copy = copyArray<T>(in);
+    svdInPlace(s, u, vt, in_copy);
 }
 
 #define INSTANTIATE(T, Tr)                                               \

From 1ab90475d1cae36224c7c6fa3552279225ce0a52 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 4 Apr 2023 16:26:19 -0400
Subject: [PATCH 249/473] Updates to require oneAPI 2023.1

---
 CMakeLists.txt                                       |  2 +-
 src/backend/common/half.hpp                          |  2 +-
 src/backend/oneapi/Array.hpp                         |  2 +-
 src/backend/oneapi/Event.hpp                         |  4 +---
 src/backend/oneapi/Module.hpp                        |  3 +--
 src/backend/oneapi/Param.hpp                         | 10 +---------
 src/backend/oneapi/blas.cpp                          |  3 ++-
 src/backend/oneapi/compile_module.cpp                |  4 ++--
 src/backend/oneapi/device_manager.cpp                |  7 +------
 src/backend/oneapi/device_manager.hpp                |  4 +---
 src/backend/oneapi/jit.cpp                           |  2 +-
 src/backend/oneapi/jit/kernel_generators.hpp         |  2 +-
 src/backend/oneapi/kernel/approx1.hpp                |  5 +++--
 src/backend/oneapi/kernel/approx2.hpp                |  5 +++--
 src/backend/oneapi/kernel/assign.hpp                 |  2 ++
 src/backend/oneapi/kernel/bilateral.hpp              |  2 +-
 src/backend/oneapi/kernel/convolve.hpp               |  5 ++++-
 src/backend/oneapi/kernel/convolve1.hpp              | 10 ++++++++++
 src/backend/oneapi/kernel/diagonal.hpp               |  2 ++
 src/backend/oneapi/kernel/diff.hpp                   |  2 ++
 src/backend/oneapi/kernel/gradient.hpp               |  2 ++
 src/backend/oneapi/kernel/histogram.hpp              |  2 +-
 src/backend/oneapi/kernel/interp.hpp                 |  2 +-
 src/backend/oneapi/kernel/iota.hpp                   |  2 ++
 src/backend/oneapi/kernel/ireduce.hpp                |  2 +-
 src/backend/oneapi/kernel/lookup.hpp                 |  2 ++
 src/backend/oneapi/kernel/mean.hpp                   |  3 +--
 src/backend/oneapi/kernel/meanshift.hpp              |  2 ++
 src/backend/oneapi/kernel/memcopy.hpp                |  2 ++
 src/backend/oneapi/kernel/random_engine_mersenne.hpp |  2 ++
 src/backend/oneapi/kernel/range.hpp                  |  2 ++
 src/backend/oneapi/kernel/reduce_all.hpp             |  5 +----
 src/backend/oneapi/kernel/reduce_first.hpp           |  2 ++
 src/backend/oneapi/kernel/reorder.hpp                |  2 ++
 src/backend/oneapi/kernel/resize.hpp                 |  2 +-
 src/backend/oneapi/kernel/rotate.hpp                 |  2 ++
 src/backend/oneapi/kernel/scan_dim.hpp               |  3 +--
 src/backend/oneapi/kernel/scan_first.hpp             |  3 +--
 src/backend/oneapi/kernel/select.hpp                 |  2 ++
 src/backend/oneapi/kernel/tile.hpp                   |  2 ++
 src/backend/oneapi/kernel/transform.hpp              |  4 ++--
 src/backend/oneapi/kernel/transpose.hpp              |  2 ++
 src/backend/oneapi/kernel/transpose_inplace.hpp      |  2 ++
 src/backend/oneapi/kernel/triangle.hpp               |  2 ++
 src/backend/oneapi/kernel/unwrap.hpp                 |  2 ++
 src/backend/oneapi/kernel/where.hpp                  |  2 ++
 src/backend/oneapi/kernel/wrap.hpp                   |  2 ++
 src/backend/oneapi/memory.cpp                        |  3 +--
 src/backend/oneapi/memory.hpp                        |  2 +-
 src/backend/oneapi/platform.cpp                      |  2 +-
 src/backend/oneapi/platform.hpp                      |  4 +---
 src/backend/oneapi/types.hpp                         |  2 +-
 52 files changed, 92 insertions(+), 61 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index eed62e23a6..a4c3eef645 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -134,7 +134,7 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
   if(DEFINED ENV{MKLROOT} AND NOT DEFINED MKL_ROOT)
     set(MKL_ROOT "$ENV{MKLROOT}")
   endif()
-  find_package(MKL)
+  find_package(MKL 2023.1)
 endif()
 
 af_multiple_option(NAME        AF_COMPUTE_LIBRARY
diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index 65a3930b15..515c301079 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -33,7 +33,7 @@
 #endif
 
 #ifdef AF_ONEAPI
-#include <sycl/half_type.hpp>
+#include <sycl/sycl.hpp>
 #endif
 
 #include <backend.hpp>
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index d3173d7fb8..d3f81bff2c 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -17,7 +17,7 @@
 #include <types.hpp>
 #include <af/dim4.hpp>
 
-#include <sycl/buffer.hpp>
+#include <sycl/sycl.hpp>
 
 #include <nonstd/span.hpp>
 #include <algorithm>
diff --git a/src/backend/oneapi/Event.hpp b/src/backend/oneapi/Event.hpp
index 90aaf1b2ca..ae7fdd8c29 100644
--- a/src/backend/oneapi/Event.hpp
+++ b/src/backend/oneapi/Event.hpp
@@ -9,11 +9,9 @@
 #pragma once
 
 #include <common/EventBase.hpp>
-
 #include <af/event.h>
 
-#include <sycl/event.hpp>
-#include <sycl/queue.hpp>
+#include <sycl/sycl.hpp>
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/Module.hpp b/src/backend/oneapi/Module.hpp
index 6a5ce71985..dc2afe676d 100644
--- a/src/backend/oneapi/Module.hpp
+++ b/src/backend/oneapi/Module.hpp
@@ -10,9 +10,8 @@
 #pragma once
 
 #include <common/ModuleInterface.hpp>
-#include <sycl/sycl.hpp>
 
-#include <sycl/kernel_bundle.hpp>
+#include <sycl/sycl.hpp>
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index f6ca0ef8b1..4a95dff6ec 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -11,17 +11,9 @@
 
 #include <kernel/KParam.hpp>
 #include <types.hpp>
-
 #include <af/dim4.hpp>
 
-/// The get_pointer function in the accessor class throws a few warnings in the
-/// 2023.0 release of the library. Review this warning in the future
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wsycl-strict"
-#include <sycl/accessor.hpp>
-#pragma clang diagnostic pop
-#include <sycl/buffer.hpp>
-#include <sycl/handler.hpp>
+#include <sycl/sycl.hpp>
 
 #include <optional>
 
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
index 964dcb6cde..73dbadfcfd 100644
--- a/src/backend/oneapi/blas.cpp
+++ b/src/backend/oneapi/blas.cpp
@@ -24,7 +24,8 @@
 #include <types.hpp>
 
 #include <sycl/sycl.hpp>
-#include "oneapi/mkl/blas.hpp"
+
+#include <oneapi/mkl/blas.hpp>
 
 #include <complex>
 #include <vector>
diff --git a/src/backend/oneapi/compile_module.cpp b/src/backend/oneapi/compile_module.cpp
index 7fce4b70c0..2737909208 100644
--- a/src/backend/oneapi/compile_module.cpp
+++ b/src/backend/oneapi/compile_module.cpp
@@ -14,11 +14,11 @@
 #include <common/defines.hpp>
 #include <common/util.hpp>
 #include <err_oneapi.hpp>
-#include <sycl/sycl.hpp>
-// #include <kernel_headers/KParam.hpp>
 #include <platform.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <algorithm>
 #include <cctype>
 #include <cstdio>
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index 05d6cb454d..ac06d5768c 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -23,12 +23,7 @@
 #include <af/oneapi.h>
 #include <af/version.h>
 
-#include <sycl/context.hpp>
-#include <sycl/device.hpp>
-#include <sycl/exception.hpp>
-#include <sycl/exception_list.hpp>
-#include <sycl/platform.hpp>
-#include <sycl/queue.hpp>
+#include <sycl/sycl.hpp>
 
 #include <algorithm>
 #include <iterator>
diff --git a/src/backend/oneapi/device_manager.hpp b/src/backend/oneapi/device_manager.hpp
index 198ddd07e0..28be51631b 100644
--- a/src/backend/oneapi/device_manager.hpp
+++ b/src/backend/oneapi/device_manager.hpp
@@ -9,9 +9,7 @@
 
 #pragma once
 
-#include <sycl/context.hpp>
-#include <sycl/device.hpp>
-#include <sycl/queue.hpp>
+#include <sycl/sycl.hpp>
 
 #include <memory>
 #include <mutex>
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 57c299a3f2..562a0ed1a2 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -29,7 +29,7 @@
 #include <af/dim4.hpp>
 
 #include <sycl/backend.hpp>
-#include <sycl/interop_handle.hpp>
+#include <sycl/sycl.hpp>
 
 #include <array>
 #include <cstdio>
diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
index 3a15f78e8e..a69553acd3 100644
--- a/src/backend/oneapi/jit/kernel_generators.hpp
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -11,7 +11,7 @@
 #include <Param.hpp>
 #include <err_oneapi.hpp>
 
-#include <sycl/buffer.hpp>
+#include <sycl/sycl.hpp>
 
 #include <functional>
 #include <memory>
diff --git a/src/backend/oneapi/kernel/approx1.hpp b/src/backend/oneapi/kernel/approx1.hpp
index 4d9d039f1b..3f0e2cfbe5 100644
--- a/src/backend/oneapi/kernel/approx1.hpp
+++ b/src/backend/oneapi/kernel/approx1.hpp
@@ -14,9 +14,10 @@
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
 #include <kernel/interp.hpp>
-#include <af/constants.h>
-// #include <kernel/default_config.hpp>
 #include <traits.hpp>
+#include <af/constants.h>
+
+#include <sycl/sycl.hpp>
 
 #include <string>
 #include <vector>
diff --git a/src/backend/oneapi/kernel/approx2.hpp b/src/backend/oneapi/kernel/approx2.hpp
index 5b7e509f9b..8713d87d20 100644
--- a/src/backend/oneapi/kernel/approx2.hpp
+++ b/src/backend/oneapi/kernel/approx2.hpp
@@ -14,9 +14,10 @@
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
 #include <kernel/interp.hpp>
-#include <af/constants.h>
-// #include <kernel/default_config.hpp>
 #include <traits.hpp>
+#include <af/constants.h>
+
+#include <sycl/sycl.hpp>
 
 #include <string>
 #include <vector>
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 0876b9e16c..1ab8c42732 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -14,6 +14,8 @@
 #include <debug_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/bilateral.hpp b/src/backend/oneapi/kernel/bilateral.hpp
index 2a5cf59fb1..0fb213999a 100644
--- a/src/backend/oneapi/kernel/bilateral.hpp
+++ b/src/backend/oneapi/kernel/bilateral.hpp
@@ -15,7 +15,7 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
-#include <sycl/builtins.hpp>
+#include <sycl/sycl.hpp>
 
 #include <string>
 #include <vector>
diff --git a/src/backend/oneapi/kernel/convolve.hpp b/src/backend/oneapi/kernel/convolve.hpp
index 9f868ce729..ba1bda6b7c 100644
--- a/src/backend/oneapi/kernel/convolve.hpp
+++ b/src/backend/oneapi/kernel/convolve.hpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2023, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -9,11 +9,14 @@
 
 #pragma once
 #include <Param.hpp>
+#include <accessor.hpp>
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
 #include <af/defines.h>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/convolve1.hpp b/src/backend/oneapi/kernel/convolve1.hpp
index 1d3df7ef3b..ca20b7a89e 100644
--- a/src/backend/oneapi/kernel/convolve1.hpp
+++ b/src/backend/oneapi/kernel/convolve1.hpp
@@ -1,3 +1,13 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
 template<typename T, typename aT>
 class conv1HelperCreateKernel {
    public:
diff --git a/src/backend/oneapi/kernel/diagonal.hpp b/src/backend/oneapi/kernel/diagonal.hpp
index c49d9871e3..8da78dba70 100644
--- a/src/backend/oneapi/kernel/diagonal.hpp
+++ b/src/backend/oneapi/kernel/diagonal.hpp
@@ -15,6 +15,8 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/diff.hpp b/src/backend/oneapi/kernel/diff.hpp
index f5a73c8c40..478da588c0 100644
--- a/src/backend/oneapi/kernel/diff.hpp
+++ b/src/backend/oneapi/kernel/diff.hpp
@@ -15,6 +15,8 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/gradient.hpp b/src/backend/oneapi/kernel/gradient.hpp
index fbaae20b51..7f29b4cec3 100644
--- a/src/backend/oneapi/kernel/gradient.hpp
+++ b/src/backend/oneapi/kernel/gradient.hpp
@@ -15,6 +15,8 @@
 #include <debug_oneapi.hpp>
 #include <kernel/default_config.hpp>
 
+#include <sycl/sycl.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 namespace kernel {
diff --git a/src/backend/oneapi/kernel/histogram.hpp b/src/backend/oneapi/kernel/histogram.hpp
index 3d53930bd4..606bbebc35 100644
--- a/src/backend/oneapi/kernel/histogram.hpp
+++ b/src/backend/oneapi/kernel/histogram.hpp
@@ -15,7 +15,7 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
-#include <sycl/atomic_ref.hpp>
+#include <sycl/sycl.hpp>
 
 #include <string>
 #include <vector>
diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
index d6bb62b177..f1e74d6c87 100644
--- a/src/backend/oneapi/kernel/interp.hpp
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -12,7 +12,7 @@
 #include <types.hpp>
 #include <af/constants.h>
 
-#include <sycl/builtins.hpp>
+#include <sycl/sycl.hpp>
 
 #include <algorithm>
 
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index e326ff9416..8f102ed87f 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -16,6 +16,8 @@
 #include <traits.hpp>
 #include <af/dim4.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/ireduce.hpp b/src/backend/oneapi/kernel/ireduce.hpp
index 9e4e35c51d..e047826b08 100644
--- a/src/backend/oneapi/kernel/ireduce.hpp
+++ b/src/backend/oneapi/kernel/ireduce.hpp
@@ -19,7 +19,7 @@
 #include <memory.hpp>
 #include <minmax_op.hpp>
 
-#include <sycl/sycl.hpp>  //TODO: exact headers
+#include <sycl/sycl.hpp>
 
 #include <algorithm>
 #include <climits>
diff --git a/src/backend/oneapi/kernel/lookup.hpp b/src/backend/oneapi/kernel/lookup.hpp
index 8baf14ad21..a5d29fea09 100644
--- a/src/backend/oneapi/kernel/lookup.hpp
+++ b/src/backend/oneapi/kernel/lookup.hpp
@@ -14,6 +14,8 @@
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 4353bfff26..1d58458e46 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -20,8 +20,7 @@
 #include <math.hpp>
 #include <memory.hpp>
 
-#include <sycl/builtins.hpp>
-#include <sycl/group_algorithm.hpp>
+#include <sycl/sycl.hpp>
 
 #include <memory>
 #include <vector>
diff --git a/src/backend/oneapi/kernel/meanshift.hpp b/src/backend/oneapi/kernel/meanshift.hpp
index 8dfb96a3b7..2211d81b73 100644
--- a/src/backend/oneapi/kernel/meanshift.hpp
+++ b/src/backend/oneapi/kernel/meanshift.hpp
@@ -14,6 +14,8 @@
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <algorithm>
 #include <string>
 #include <vector>
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 59990dea39..87b46a4c22 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -17,6 +17,8 @@
 #include <sycl/sycl.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <algorithm>
 #include <string>
 #include <vector>
diff --git a/src/backend/oneapi/kernel/random_engine_mersenne.hpp b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
index bbf5dae3e0..f78bc8d732 100644
--- a/src/backend/oneapi/kernel/random_engine_mersenne.hpp
+++ b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
@@ -44,6 +44,8 @@
 #pragma once
 #include <kernel/random_engine_write.hpp>
 
+#include <sycl/sycl.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 namespace kernel {
diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
index 9cfea27964..1c8512be0b 100644
--- a/src/backend/oneapi/kernel/range.hpp
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -18,6 +18,8 @@
 #include <traits.hpp>
 #include <af/dim4.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index 2089b60175..0878f33329 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -19,10 +19,7 @@
 #include <math.hpp>
 #include <memory.hpp>
 
-#include <sycl/atomic_fence.hpp>
-#include <sycl/atomic_ref.hpp>
-#include <sycl/builtins.hpp>
-#include <sycl/group_algorithm.hpp>
+#include <sycl/sycl.hpp>
 
 #include <algorithm>
 #include <climits>
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index 299919ae12..42ffb9199d 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -19,6 +19,8 @@
 #include <math.hpp>
 #include <memory.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <algorithm>
 #include <climits>
 #include <complex>
diff --git a/src/backend/oneapi/kernel/reorder.hpp b/src/backend/oneapi/kernel/reorder.hpp
index b643bb6fc8..f3ee445fe7 100644
--- a/src/backend/oneapi/kernel/reorder.hpp
+++ b/src/backend/oneapi/kernel/reorder.hpp
@@ -14,6 +14,8 @@
 #include <debug_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/resize.hpp b/src/backend/oneapi/kernel/resize.hpp
index 5443815b75..b14ceafe14 100644
--- a/src/backend/oneapi/kernel/resize.hpp
+++ b/src/backend/oneapi/kernel/resize.hpp
@@ -15,7 +15,7 @@
 #include <debug_oneapi.hpp>
 #include <traits.hpp>
 
-#include <sycl/builtins.hpp>
+#include <sycl/sycl.hpp>
 
 #include <string>
 #include <vector>
diff --git a/src/backend/oneapi/kernel/rotate.hpp b/src/backend/oneapi/kernel/rotate.hpp
index b8c8357e79..84641a3f76 100644
--- a/src/backend/oneapi/kernel/rotate.hpp
+++ b/src/backend/oneapi/kernel/rotate.hpp
@@ -16,6 +16,8 @@
 #include <math.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 namespace kernel {
diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp
index b4a2678dac..a9ce3d7838 100644
--- a/src/backend/oneapi/kernel/scan_dim.hpp
+++ b/src/backend/oneapi/kernel/scan_dim.hpp
@@ -17,8 +17,7 @@
 #include <kernel/default_config.hpp>
 #include <memory.hpp>
 
-#include <sycl/builtins.hpp>
-#include <sycl/group_algorithm.hpp>
+#include <sycl/sycl.hpp>
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index 777f8f205e..8660494657 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -17,8 +17,7 @@
 #include <kernel/default_config.hpp>
 #include <memory.hpp>
 
-#include <sycl/builtins.hpp>
-#include <sycl/group_algorithm.hpp>
+#include <sycl/sycl.hpp>
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/kernel/select.hpp b/src/backend/oneapi/kernel/select.hpp
index 7f63f2cbea..abba384f80 100644
--- a/src/backend/oneapi/kernel/select.hpp
+++ b/src/backend/oneapi/kernel/select.hpp
@@ -14,6 +14,8 @@
 #include <common/kernel_cache.hpp>
 #include <math.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/tile.hpp b/src/backend/oneapi/kernel/tile.hpp
index 24112442a9..2c44594a34 100644
--- a/src/backend/oneapi/kernel/tile.hpp
+++ b/src/backend/oneapi/kernel/tile.hpp
@@ -14,6 +14,8 @@
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/transform.hpp b/src/backend/oneapi/kernel/transform.hpp
index c18ac6c827..6760e1a489 100644
--- a/src/backend/oneapi/kernel/transform.hpp
+++ b/src/backend/oneapi/kernel/transform.hpp
@@ -12,13 +12,13 @@
 #include <Param.hpp>
 #include <common/complex.hpp>
 #include <common/dispatch.hpp>
-// #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
-// #include <kernel/config.hpp>
 #include <kernel/interp.hpp>
 #include <math.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index 43b741ca32..eeb9387145 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -15,6 +15,8 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/transpose_inplace.hpp b/src/backend/oneapi/kernel/transpose_inplace.hpp
index 3dda946ced..23f04c6559 100644
--- a/src/backend/oneapi/kernel/transpose_inplace.hpp
+++ b/src/backend/oneapi/kernel/transpose_inplace.hpp
@@ -16,6 +16,8 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/triangle.hpp b/src/backend/oneapi/kernel/triangle.hpp
index 2f65abe20c..f4705035b3 100644
--- a/src/backend/oneapi/kernel/triangle.hpp
+++ b/src/backend/oneapi/kernel/triangle.hpp
@@ -15,6 +15,8 @@
 #include <err_oneapi.hpp>
 #include <traits.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/kernel/unwrap.hpp b/src/backend/oneapi/kernel/unwrap.hpp
index a6fa8ee64e..0c88bd4348 100644
--- a/src/backend/oneapi/kernel/unwrap.hpp
+++ b/src/backend/oneapi/kernel/unwrap.hpp
@@ -15,6 +15,8 @@
 #include <debug_oneapi.hpp>
 #include <kernel/default_config.hpp>
 
+#include <sycl/sycl.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 namespace kernel {
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index 3d8fe3324f..64b25ec211 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -16,6 +16,8 @@
 #include <kernel/scan_first.hpp>
 #include <memory.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <Param.hpp>
 #include <backend.hpp>
 #include <math.hpp>
diff --git a/src/backend/oneapi/kernel/wrap.hpp b/src/backend/oneapi/kernel/wrap.hpp
index e574b4a127..ba503a1f56 100644
--- a/src/backend/oneapi/kernel/wrap.hpp
+++ b/src/backend/oneapi/kernel/wrap.hpp
@@ -16,6 +16,8 @@
 #include <kernel/default_config.hpp>
 #include <math.hpp>
 
+#include <sycl/sycl.hpp>
+
 #include <string>
 #include <vector>
 
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index aa620e8e2c..971fa05b64 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -18,8 +18,7 @@
 #include <types.hpp>
 #include <af/dim4.hpp>
 
-#include <sycl/builtins.hpp>
-#include <sycl/usm.hpp>
+#include <sycl/sycl.hpp>
 
 #include <utility>
 
diff --git a/src/backend/oneapi/memory.hpp b/src/backend/oneapi/memory.hpp
index 462c1498f1..dea5e62f5a 100644
--- a/src/backend/oneapi/memory.hpp
+++ b/src/backend/oneapi/memory.hpp
@@ -10,7 +10,7 @@
 
 #include <common/AllocatorInterface.hpp>
 
-#include <sycl/buffer.hpp>
+#include <sycl/sycl.hpp>
 
 #include <cstdlib>
 #include <functional>
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index e0959a9390..edd62e0d6a 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -28,7 +28,7 @@
 #include <OpenGL/CGLCurrent.h>
 #endif
 
-#include <sycl/platform.hpp>
+#include <sycl/sycl.hpp>
 
 #include <cctype>
 #include <cstdlib>
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index de6ae498dc..86439a685c 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -11,9 +11,7 @@
 
 #include <af/oneapi.h>
 
-#include <sycl/context.hpp>
-#include <sycl/device.hpp>
-#include <sycl/queue.hpp>
+#include <sycl/sycl.hpp>
 
 #include <memory>
 #include <string>
diff --git a/src/backend/oneapi/types.hpp b/src/backend/oneapi/types.hpp
index f4be516f3d..4537f27987 100644
--- a/src/backend/oneapi/types.hpp
+++ b/src/backend/oneapi/types.hpp
@@ -14,7 +14,7 @@
 #include <af/compilers.h>
 #include <af/traits.hpp>
 
-#include <sycl/aliases.hpp>
+#include <sycl/sycl.hpp>
 
 #include <algorithm>
 #include <array>

From e9fe5d3e2904e0a8e4202ef199c318c45c545669 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 5 Apr 2023 10:02:10 -0400
Subject: [PATCH 250/473] Use accessor directly in the AParam object.

---
 src/backend/oneapi/Param.hpp | 27 ++++++++++-----------------
 src/backend/oneapi/jit.cpp   |  6 +++---
 2 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index 4a95dff6ec..447e8fb117 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -45,11 +45,9 @@ struct Param {
 
 template<typename T>
 struct AParam {
-    std::optional<sycl::accessor<T, 1>> data;
-    std::optional<sycl::accessor<T, 1, sycl::access::mode::read_write,
-                                 sycl::access::target::device,
-                                 sycl::access::placeholder::true_t>>
-        ph;
+    sycl::accessor<T, 1, sycl::access_mode::read_write, sycl::target::device,
+                   sycl::access::placeholder::true_t>
+        data;
     af::dim4 dims;
     af::dim4 strides;
     dim_t offset;
@@ -58,35 +56,30 @@ struct AParam {
     AParam(AParam&& other)                 = default;
 
     // AF_DEPRECATED("Use Array<T>")
-    AParam() : data(), ph(), dims{0, 0, 0, 0}, strides{0, 0, 0, 0}, offset(0) {}
+    AParam() : data(), dims{0, 0, 0, 0}, strides{0, 0, 0, 0}, offset(0) {}
 
     AParam(sycl::buffer<T, 1>& data_, const dim_t dims_[4],
            const dim_t strides_[4], dim_t offset_)
-        : data()
-        , ph(std::make_optional<
-              sycl::accessor<T, 1, sycl::access::mode::read_write,
-                             sycl::access::target::device,
-                             sycl::access::placeholder::true_t>>(data_))
+        : data(data_.get_access())
         , dims(4, dims_)
         , strides(4, strides_)
         , offset(offset_) {}
     // AF_DEPRECATED("Use Array<T>")
     AParam(sycl::handler& h, sycl::buffer<T, 1>& data_, const dim_t dims_[4],
            const dim_t strides_[4], dim_t offset_)
-        : data{{data_, h}}
-        , ph(data_)
+        : data(data_.get_access())
         , dims(4, dims_)
         , strides(4, strides_)
-        , offset(offset_) {}
+        , offset(offset_) {
+        require(h);
+    }
 
     template<sycl::access::mode MODE>
     sycl::accessor<data_t<T>, 1, MODE> get_accessor(sycl::handler& h) const {
         return *data;
     }
 
-    void require(sycl::handler& h) {
-        if (!data) { h.require(ph.value()); }
-    }
+    void require(sycl::handler& h) { h.require(data); }
 
     operator KParam() const {
         return KParam{{dims[0], dims[1], dims[2], dims[3]},
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 562a0ed1a2..2190dd8070 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -465,7 +465,7 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                                                 const_cast<void*>(ptr));
                                         vector<cl_mem> mem =
                                             hh.get_native_mem<backend::opencl>(
-                                                info->ph.value());
+                                                info->data);
                                         if (is_linear) {
                                             CL_CHECK(clSetKernelArg(
                                                 kernels[0], id++,
@@ -497,8 +497,8 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                         // Set output parameters
                         vector<cl_mem> mem;
                         for (const auto& output : ap) {
-                            mem = hh.get_native_mem<backend::opencl>(
-                                output.data.value());
+                            mem =
+                                hh.get_native_mem<backend::opencl>(output.data);
                             cl_mem mmm = mem[0];
                             CL_CHECK(clSetKernelArg(kernels[0], nargs++,
                                                     sizeof(cl_mem), &mmm));

From 82cf75f1de7b3728b2c32c32096f22e5199a507f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 6 Apr 2023 10:26:17 -0400
Subject: [PATCH 251/473] Workaround for the long long compiler bug for iota

---
 src/backend/oneapi/kernel/iota.hpp | 39 ++++++++++++++++--------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index 8f102ed87f..1ec05f31b0 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -51,24 +51,27 @@ class iotaKernel {
         const int xx = it.get_local_id(0) + blockIdx_x * gg.get_local_range(0);
         const int yy = it.get_local_id(1) + blockIdx_y * gg.get_local_range(1);
 
-        if (xx >= oinfo_.dims[0] || yy >= oinfo_.dims[1] ||
-            oz >= oinfo_.dims[2] || ow >= oinfo_.dims[3])
-            return;
-
-        const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
-
-        T val = static_cast<T>((ow % s3_) * s2_ * s1_ * s0_);
-        val += static_cast<T>((oz % s2_) * s1_ * s0_);
-
-        const int incy = blocksPerMatY_ * gg.get_local_range(1);
-        const int incx = blocksPerMatX_ * gg.get_local_range(0);
-
-        for (int oy = yy; oy < oinfo_.dims[1]; oy += incy) {
-            T valY   = val + (oy % s1_) * s0_;
-            int oyzw = ozw + oy * oinfo_.strides[1];
-            for (int ox = xx; ox < oinfo_.dims[0]; ox += incx) {
-                int oidx   = oyzw + ox;
-                out_[oidx] = valY + (ox % s0_);
+        size_t odims0 = oinfo_.dims[0];
+        size_t odims1 = oinfo_.dims[1];
+        size_t odims2 = oinfo_.dims[2];
+        size_t odims3 = oinfo_.dims[3];
+
+        if (xx < odims0 && yy < odims1 && oz < odims2 && ow < odims3) {
+            const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
+
+            T val = static_cast<T>((ow % s3_) * s2_ * s1_ * s0_);
+            val += static_cast<T>((oz % s2_) * s1_ * s0_);
+
+            const int incy = blocksPerMatY_ * gg.get_local_range(1);
+            const int incx = blocksPerMatX_ * gg.get_local_range(0);
+
+            for (int oy = yy; oy < odims1; oy += incy) {
+                T valY   = val + (oy % s1_) * s0_;
+                int oyzw = ozw + oy * oinfo_.strides[1];
+                for (int ox = xx; ox < odims0; ox += incx) {
+                    int oidx   = oyzw + ox;
+                    out_[oidx] = valY + (ox % s0_);
+                }
             }
         }
     }

From 4d139af32e2e2a4bff602a283d37d39301ab9845 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 6 Apr 2023 10:26:46 -0400
Subject: [PATCH 252/473] Workaround for the long long compiler bug in memcopy

---
 src/backend/oneapi/kernel/memcopy.hpp | 30 ++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 87b46a4c22..c3b317ef17 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -69,14 +69,19 @@ class memCopy {
                 id1 * istrides_.dim[1];
 
         int istride0 = istrides_.dim[0];
-        if (id0 < idims_.dim[0] && id1 < idims_.dim[1] && id2 < idims_.dim[2] &&
-            id3 < idims_.dim[3]) {
+        size_t idd0  = idims_.dim[0];
+        size_t idd1  = idims_.dim[1];
+        size_t idd2  = idims_.dim[2];
+        size_t idd3  = idims_.dim[3];
+
+        if (id0 < idd0 && id1 < idd1 && id2 < idd2 && id3 < idd3) {
             optr[id0] = iptr[id0 * istride0];
         }
     }
 
    protected:
-    sycl::accessor<T> out_, in_;
+    sycl::accessor<T> out_;
+    sycl::accessor<T> in_;
     dims_t ostrides_, idims_, istrides_;
     int offset_, groups_0_, groups_1_;
 };
@@ -228,13 +233,22 @@ class reshapeCopy {
         uint istride0 = iInfo_.strides[0];
         uint ostride0 = oInfo_.strides[0];
 
-        if (gy < oInfo_.dims[1] && gz < oInfo_.dims[2] && gw < oInfo_.dims[3]) {
+        size_t odims0 = oInfo_.dims[0];
+        size_t odims1 = oInfo_.dims[1];
+        size_t odims2 = oInfo_.dims[2];
+        size_t odims3 = oInfo_.dims[3];
+
+        size_t tdims0 = trgt_.dim[0];
+        size_t tdims1 = trgt_.dim[1];
+        size_t tdims2 = trgt_.dim[2];
+        size_t tdims3 = trgt_.dim[3];
+
+        if (gy < odims1 && gz < odims2 && gw < odims3) {
             int loop_offset = gg.get_local_range(0) * blk_x_;
-            bool cond =
-                gy < trgt_.dim[1] && gz < trgt_.dim[2] && gw < trgt_.dim[3];
-            for (int rep = gx; rep < oInfo_.dims[0]; rep += loop_offset) {
+            bool cond       = gy < tdims1 && gz < tdims2 && gw < tdims3;
+            for (int rep = gx; rep < odims0; rep += loop_offset) {
                 outType temp = default_value_;
-                if (SAMEDIMS || (rep < trgt_.dim[0] && cond)) {
+                if (SAMEDIMS || (rep < tdims0 && cond)) {
                     temp = convertType<inType, outType>(
                         scale<inType>(in[rep * istride0], factor_));
                 }

From 7ed1972285b4519d93141d0392504e29eed68aec Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 6 Apr 2023 10:27:37 -0400
Subject: [PATCH 253/473] Update common/debug.hpp to handle up to 6 variables

---
 src/backend/common/debug.hpp | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/backend/common/debug.hpp b/src/backend/common/debug.hpp
index e91c903d53..54e74a2953 100644
--- a/src/backend/common/debug.hpp
+++ b/src/backend/common/debug.hpp
@@ -43,15 +43,18 @@ void print(const char *F, const first &FF, ARGS... args) {
 #define SHOW5(val1, val2, val3, val4, val5)                              \
     debugging::print(#val1, val1, #val2, val2, #val3, val3, #val4, val4, \
                      #val5, val5)
+#define SHOW6(val1, val2, val3, val4, val5, val6)                        \
+    debugging::print(#val1, val1, #val2, val2, #val3, val3, #val4, val4, \
+                     #val5, val5, #val6, val6)
 
-#define GET_MACRO(_1, _2, _3, _4, _5, NAME, ...) NAME
+#define GET_MACRO(_1, _2, _3, _4, _5, _6, NAME, ...) NAME
 
-#define SHOW(...)                                                 \
-    do {                                                          \
-        fmt::print(std::cout, "{}:({}): ", __FILE__, __LINE__);   \
-        GET_MACRO(__VA_ARGS__, SHOW5, SHOW4, SHOW3, SHOW2, SHOW1) \
-        (__VA_ARGS__);                                            \
-        fmt::print(std::cout, "\n");                              \
+#define SHOW(...)                                                        \
+    do {                                                                 \
+        fmt::print(std::cout, "{}:({}): ", __FILE__, __LINE__);          \
+        GET_MACRO(__VA_ARGS__, SHOW6, SHOW5, SHOW4, SHOW3, SHOW2, SHOW1) \
+        (__VA_ARGS__);                                                   \
+        fmt::print(std::cout, "\n");                                     \
     } while (0)
 
 #define PRINTVEC(val)                                                        \

From 3e95f2bcd118597a42508fa232e6aef83c0988c8 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 7 Apr 2023 14:31:23 -0400
Subject: [PATCH 254/473] Add half support for iota in oneAPI

---
 src/backend/oneapi/iota.cpp        | 10 +---------
 src/backend/oneapi/kernel/iota.hpp | 10 ++++++----
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/src/backend/oneapi/iota.cpp b/src/backend/oneapi/iota.cpp
index 84bf693f1b..6d511df23f 100644
--- a/src/backend/oneapi/iota.cpp
+++ b/src/backend/oneapi/iota.cpp
@@ -29,15 +29,6 @@ Array<T> iota(const dim4 &dims, const dim4 &tile_dims) {
     return out;
 }
 
-template<>
-Array<half> iota(const dim4 &dims, const dim4 &tile_dims) {
-    ONEAPI_NOT_SUPPORTED("");
-    // dim4 outdims = dims * tile_dims;
-
-    // Array<half> out = createEmptyArray<half>(outdims);
-    // return out;
-}
-
 #define INSTANTIATE(T) \
     template Array<T> iota<T>(const af::dim4 &dims, const af::dim4 &tile_dims);
 
@@ -50,5 +41,6 @@ INSTANTIATE(uintl)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(half)
 }  // namespace oneapi
 }  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index 1ec05f31b0..87dbfc923c 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -14,6 +14,7 @@
 #include <common/half.hpp>
 #include <debug_oneapi.hpp>
 #include <traits.hpp>
+#include <types.hpp>
 #include <af/dim4.hpp>
 
 #include <sycl/sycl.hpp>
@@ -59,15 +60,16 @@ class iotaKernel {
         if (xx < odims0 && yy < odims1 && oz < odims2 && ow < odims3) {
             const int ozw = ow * oinfo_.strides[3] + oz * oinfo_.strides[2];
 
-            T val = static_cast<T>((ow % s3_) * s2_ * s1_ * s0_);
-            val += static_cast<T>((oz % s2_) * s1_ * s0_);
+            compute_t<T> val =
+                static_cast<compute_t<T>>((ow % s3_) * s2_ * s1_ * s0_);
+            val += static_cast<compute_t<T>>((oz % s2_) * s1_ * s0_);
 
             const int incy = blocksPerMatY_ * gg.get_local_range(1);
             const int incx = blocksPerMatX_ * gg.get_local_range(0);
 
             for (int oy = yy; oy < odims1; oy += incy) {
-                T valY   = val + (oy % s1_) * s0_;
-                int oyzw = ozw + oy * oinfo_.strides[1];
+                compute_t<T> valY = val + (oy % s1_) * s0_;
+                int oyzw          = ozw + oy * oinfo_.strides[1];
                 for (int ox = xx; ox < odims0; ox += incx) {
                     int oidx   = oyzw + ox;
                     out_[oidx] = valY + (ox % s0_);

From 31d5a368f00e14621125577fdaf0a384f43dd020 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 7 Apr 2023 14:31:58 -0400
Subject: [PATCH 255/473] Fix CMake warning in FindAF_MKL

---
 CMakeModules/FindAF_MKL.cmake | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CMakeModules/FindAF_MKL.cmake b/CMakeModules/FindAF_MKL.cmake
index 7c9baefecb..662f0046da 100644
--- a/CMakeModules/FindAF_MKL.cmake
+++ b/CMakeModules/FindAF_MKL.cmake
@@ -73,7 +73,6 @@
 
 include(CheckTypeSize)
 include(FindPackageHandleStandardArgs)
-find_package(OpenMP QUIET)
 
 check_type_size("int" INT_SIZE
   BUILTIN_TYPES_ONLY LANGUAGE C)

From ef1d3a51e99d155567f91c5d49bf77c065f9f71e Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 10 Apr 2023 19:09:45 -0400
Subject: [PATCH 256/473] Add sort and sort_by_key support for oneAPI (#3390)

Add sort and sort_by_key support in the oneAPI backend.

---------

Co-authored-by: Umar Arshad <umar@arrayfire.com>
---
 src/backend/common/half.hpp                   |   3 +
 src/backend/oneapi/CMakeLists.txt             |   5 +
 src/backend/oneapi/kernel/bilateral.hpp       |  21 +-
 src/backend/oneapi/kernel/convolve.hpp        |   3 -
 src/backend/oneapi/kernel/convolve1.hpp       |   6 +-
 src/backend/oneapi/kernel/convolve2.hpp       |   6 +-
 src/backend/oneapi/kernel/convolve3.hpp       |   6 +-
 src/backend/oneapi/kernel/histogram.hpp       |   7 +-
 src/backend/oneapi/kernel/interp.hpp          |   6 +-
 src/backend/oneapi/kernel/reorder.hpp         |   6 +-
 src/backend/oneapi/kernel/sort.hpp            | 119 ++++++++++
 src/backend/oneapi/kernel/sort_by_key.hpp     |  29 +++
 .../oneapi/kernel/sort_by_key/CMakeLists.txt  |  53 +++++
 .../kernel/sort_by_key/sort_by_key_impl.cpp   |  20 ++
 .../oneapi/kernel/sort_by_key_impl.hpp        | 206 ++++++++++++++++++
 src/backend/oneapi/kernel/wrap.hpp            |   5 +-
 src/backend/oneapi/kernel/wrap_dilated.hpp    |   5 +-
 src/backend/oneapi/sort.cpp                   |  20 +-
 src/backend/oneapi/sort_by_key.cpp            |  32 ++-
 src/backend/oneapi/sort_index.cpp             |  24 +-
 src/backend/oneapi/topk.cpp                   | 128 +----------
 21 files changed, 531 insertions(+), 179 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/sort.hpp
 create mode 100644 src/backend/oneapi/kernel/sort_by_key.hpp
 create mode 100644 src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
 create mode 100644 src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp
 create mode 100644 src/backend/oneapi/kernel/sort_by_key_impl.hpp

diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index 515c301079..67bd47829f 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -919,10 +919,12 @@ AF_CONSTEXPR __DH__ static inline bool operator==(
     arrayfire::common::half lhs, arrayfire::common::half rhs) noexcept;
 AF_CONSTEXPR __DH__ static inline bool operator!=(
     arrayfire::common::half lhs, arrayfire::common::half rhs) noexcept;
+
 __DH__ static inline bool operator<(arrayfire::common::half lhs,
                                     arrayfire::common::half rhs) noexcept;
 __DH__ static inline bool operator<(arrayfire::common::half lhs,
                                     float rhs) noexcept;
+
 AF_CONSTEXPR __DH__ static inline bool isinf(half val) noexcept;
 
 /// Classification implementation.
@@ -1052,6 +1054,7 @@ class alignas(2) half {
                                  arrayfire::common::half rhs) noexcept;
     friend __DH__ bool operator<(arrayfire::common::half lhs,
                                  float rhs) noexcept;
+
     friend AF_CONSTEXPR __DH__ bool isinf(half val) noexcept;
     friend AF_CONSTEXPR __DH__ inline bool isnan(half val) noexcept;
 
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 60c5aa9379..7a58966711 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -237,6 +237,8 @@ target_sources(afoneapi
     kernel/reorder.hpp
     kernel/scan_first.hpp
     kernel/scan_dim.hpp
+    kernel/sort.hpp
+    kernel/sort_by_key.hpp
     kernel/transpose.hpp
     kernel/transpose_inplace.hpp
     kernel/triangle.hpp
@@ -268,6 +270,8 @@ add_library(ArrayFire::afoneapi ALIAS afoneapi)
 
 arrayfire_set_default_cxx_flags(afoneapi)
 
+include("${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/CMakeLists.txt")
+
 target_include_directories(afoneapi
   SYSTEM PRIVATE
     ${SYCL_INCLUDE_DIR}
@@ -305,6 +309,7 @@ target_link_libraries(afoneapi
     -fvisibility-inlines-hidden
     c_api_interface
     cpp_api_interface
+    oneapi_sort_by_key
     afcommon_interface
     OpenCL::OpenCL
     OpenCL::cl2hpp
diff --git a/src/backend/oneapi/kernel/bilateral.hpp b/src/backend/oneapi/kernel/bilateral.hpp
index 0fb213999a..8c340ccb81 100644
--- a/src/backend/oneapi/kernel/bilateral.hpp
+++ b/src/backend/oneapi/kernel/bilateral.hpp
@@ -57,14 +57,14 @@ class bilateralKernel {
         , nBBS0_(nBBS0)
         , nBBS1_(nBBS1) {}
     void operator()(sycl::nd_item<2> it) const {
-        sycl::group g                   = it.get_group();
-        const int radius                = fmax((int)(sigma_space_ * 1.5f), 1);
-        const int padding               = 2 * radius;
-        const int window_size           = padding + 1;
-        const int shrdLen               = g.get_local_range(0) + padding;
-        const float variance_range      = sigma_color_ * sigma_color_;
-        const float variance_space      = sigma_space_ * sigma_space_;
-        const float variance_space_neg2 = -2.0 * variance_space;
+        sycl::group g              = it.get_group();
+        const int radius           = sycl::max((int)(sigma_space_ * 1.5f), 1);
+        const int padding          = 2 * radius;
+        const int window_size      = padding + 1;
+        const int shrdLen          = g.get_local_range(0) + padding;
+        const float variance_range = sigma_color_ * sigma_color_;
+        const float variance_space = sigma_space_ * sigma_space_;
+        const float variance_space_neg2     = -2.0 * variance_space;
         const float inv_variance_range_neg2 = -0.5 / (variance_range);
 
         // gfor batch offsets
@@ -143,6 +143,11 @@ class bilateralKernel {
         return (y * stride1 + x * stride0);
     }
 
+    template<class T>
+    constexpr const T& clamp0(const T& v, const T& lo, const T& hi) const {
+        return (v < lo) ? lo : (hi < v) ? hi : v;
+    }
+
     void load2LocalMem(local_accessor<outType, 1> shrd, const inType* in,
                        int lx, int ly, int shrdStride, int dim0, int dim1,
                        int gx, int gy, int inStride1, int inStride0) const {
diff --git a/src/backend/oneapi/kernel/convolve.hpp b/src/backend/oneapi/kernel/convolve.hpp
index ba1bda6b7c..276c84c3af 100644
--- a/src/backend/oneapi/kernel/convolve.hpp
+++ b/src/backend/oneapi/kernel/convolve.hpp
@@ -109,9 +109,6 @@ void memcpyBuffer(sycl::buffer<T, 1> &dest, sycl::buffer<T, 1> &src,
     });
 }
 
-template<typename T>
-using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
-                                      sycl::access::target::local>;
 template<typename T>
 using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
 template<typename T>
diff --git a/src/backend/oneapi/kernel/convolve1.hpp b/src/backend/oneapi/kernel/convolve1.hpp
index ca20b7a89e..e156308b34 100644
--- a/src/backend/oneapi/kernel/convolve1.hpp
+++ b/src/backend/oneapi/kernel/convolve1.hpp
@@ -13,7 +13,7 @@ class conv1HelperCreateKernel {
    public:
     conv1HelperCreateKernel(write_accessor<T> out, KParam oInfo,
                             read_accessor<T> signal, KParam sInfo,
-                            local_accessor<aT> localMem,
+                            sycl::local_accessor<aT> localMem,
                             read_accessor<aT> impulse, KParam fInfo, int nBBS0,
                             int nBBS1, int ostep1, int ostep2, int ostep3,
                             int sstep1, int sstep2, int sstep3,
@@ -97,7 +97,7 @@ class conv1HelperCreateKernel {
     KParam oInfo_;
     read_accessor<T> signal_;
     KParam sInfo_;
-    local_accessor<aT> localMem_;
+    sycl::local_accessor<aT> localMem_;
     read_accessor<aT> impulse_;
     KParam fInfo_;
     int nBBS0_;
@@ -117,7 +117,7 @@ void conv1Helper(const conv_kparam_t<aT> &param, Param<T> &out,
                  const int rank, const bool expand) {
     auto Q = getQueue();
     Q.submit([&](auto &h) {
-        local_accessor<aT> localMem(param.loc_size, h);
+        sycl::local_accessor<aT> localMem(param.loc_size, h);
         write_accessor<T> outAcc{*out.data, h};
         read_accessor<T> signalAcc{*signal.data, h};
         read_accessor<aT> impulseAcc{*param.impulse, h};
diff --git a/src/backend/oneapi/kernel/convolve2.hpp b/src/backend/oneapi/kernel/convolve2.hpp
index 5de34a2023..fc5db9c06a 100644
--- a/src/backend/oneapi/kernel/convolve2.hpp
+++ b/src/backend/oneapi/kernel/convolve2.hpp
@@ -5,7 +5,7 @@ class conv2HelperCreateKernel {
                             read_accessor<T> signal, KParam sInfo,
                             read_accessor<aT> impulse, KParam fInfo, int nBBS0,
                             int nBBS1, int ostep2, int ostep3, int sstep2,
-                            int sstep3, local_accessor<aT> localMem,
+                            int sstep3, sycl::local_accessor<aT> localMem,
                             const int f0, const int f1, const bool expand)
         : out_(out)
         , oInfo_(oInfo)
@@ -111,7 +111,7 @@ class conv2HelperCreateKernel {
     int ostep3_;
     int sstep2_;
     int sstep3_;
-    local_accessor<aT> localMem_;
+    sycl::local_accessor<aT> localMem_;
     const int f0_;
     const int f1_;
     const bool expand_;
@@ -128,7 +128,7 @@ void conv2Helper(const conv_kparam_t<aT> &param, Param<T> out,
 
     auto Q = getQueue();
     Q.submit([&](auto &h) {
-        local_accessor<aT> localMem(LOC_SIZE, h);
+        sycl::local_accessor<aT> localMem(LOC_SIZE, h);
         write_accessor<T> outAcc{*out.data, h};
         read_accessor<T> signalAcc{*signal.data, h};
         read_accessor<aT> impulseAcc{*param.impulse, h};
diff --git a/src/backend/oneapi/kernel/convolve3.hpp b/src/backend/oneapi/kernel/convolve3.hpp
index 0e2dee72fe..30861a2a63 100644
--- a/src/backend/oneapi/kernel/convolve3.hpp
+++ b/src/backend/oneapi/kernel/convolve3.hpp
@@ -7,7 +7,7 @@ class conv3HelperCreateKernel {
    public:
     conv3HelperCreateKernel(write_accessor<T> out, KParam oInfo,
                             read_accessor<T> signal, KParam sInfo,
-                            local_accessor<aT> localMem,
+                            sycl::local_accessor<aT> localMem,
                             read_accessor<aT> impulse, KParam fInfo, int nBBS0,
                             int nBBS1, int ostep1, int ostep2, int ostep3,
                             int sstep1, int sstep2, int sstep3,
@@ -117,7 +117,7 @@ class conv3HelperCreateKernel {
     KParam oInfo_;
     read_accessor<T> signal_;
     KParam sInfo_;
-    local_accessor<aT> localMem_;
+    sycl::local_accessor<aT> localMem_;
     read_accessor<aT> impulse_;
     KParam fInfo_;
     int nBBS0_;
@@ -137,7 +137,7 @@ void conv3Helper(const conv_kparam_t<aT> &param, Param<T> &out,
                  const int rank, const bool EXPAND) {
     auto Q = getQueue();
     Q.submit([&](auto &h) {
-        local_accessor<aT> localMem(param.loc_size, h);
+        sycl::local_accessor<aT> localMem(param.loc_size, h);
         write_accessor<T> outAcc{*out.data, h};
         read_accessor<T> signalAcc{*signal.data, h};
         read_accessor<aT> impulseAcc{*param.impulse, h};
diff --git a/src/backend/oneapi/kernel/histogram.hpp b/src/backend/oneapi/kernel/histogram.hpp
index 606bbebc35..35f21fc9b6 100644
--- a/src/backend/oneapi/kernel/histogram.hpp
+++ b/src/backend/oneapi/kernel/histogram.hpp
@@ -71,7 +71,8 @@ class histogramKernel {
         int start     = (g.get_group_id(0) - b2 * nBBS_) * THRD_LOAD *
                         g.get_local_range(0) +
                     it.get_local_id(0);
-        int end = fmin((int)(start + THRD_LOAD * g.get_local_range(0)), len_);
+        int end =
+            sycl::min((int)(start + THRD_LOAD * g.get_local_range(0)), len_);
 
         // offset input and output to account for batch ops
         const T *in = d_src_.get_pointer() + b2 * iInfo_.strides[2] +
@@ -96,8 +97,8 @@ class histogramKernel {
             const int idx = isLinear_ ? row : i0 + i1 * iInfo_.strides[1];
 
             int bin = (int)(((float)in[idx] - minval_) / dx);
-            bin     = fmax(bin, 0);
-            bin     = fmin(bin, (int)nbins_ - 1);
+            bin     = sycl::max(bin, 0);
+            bin     = sycl::min(bin, (int)nbins_ - 1);
 
             if (use_global) {
                 global_atomic_ref<uint>(d_dst_[outOffset + bin])++;
diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
index f1e74d6c87..516acea466 100644
--- a/src/backend/oneapi/kernel/interp.hpp
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -115,7 +115,7 @@ struct Interp1<Ty, Tp, 1> {
 
         int xid = (method == AF_INTERP_LOWER ? sycl::floor(x) : sycl::round(x));
         bool cond = xid >= 0 && xid < x_lim;
-        if (clamp) xid = std::max((int)0, std::min(xid, x_lim));
+        if (clamp) xid = sycl::max((int)0, sycl::min(xid, x_lim));
 
         const int idx = ioff + xid * x_stride;
 
@@ -218,8 +218,8 @@ struct Interp2<Ty, Tp, 1> {
         const int y_stride = iInfo.strides[ydim];
 
         if (clamp) {
-            xid = std::max(0, std::min(xid, (int)iInfo.dims[xdim]));
-            yid = std::max(0, std::min(yid, (int)iInfo.dims[ydim]));
+            xid = sycl::max(0, sycl::min(xid, (int)iInfo.dims[xdim]));
+            yid = sycl::max(0, sycl::min(yid, (int)iInfo.dims[ydim]));
         }
 
         const int idx = ioff + yid * y_stride + xid * x_stride;
diff --git a/src/backend/oneapi/kernel/reorder.hpp b/src/backend/oneapi/kernel/reorder.hpp
index f3ee445fe7..1064047f77 100644
--- a/src/backend/oneapi/kernel/reorder.hpp
+++ b/src/backend/oneapi/kernel/reorder.hpp
@@ -63,9 +63,9 @@ class reorderCreateKernel {
         const int incy = blocksPerMatY_ * g.get_local_range(1);
         const int incx = blocksPerMatX_ * g.get_local_range(0);
 
-        const int o_off   = ow * op_.strides[3] + oz * op_.strides[2];
-        const int rdims[] = {d0_, d1_, d2_, d3_};
-        int ids[4]        = {0};
+        const int o_off    = ow * op_.strides[3] + oz * op_.strides[2];
+        const int rdims[4] = {d0_, d1_, d2_, d3_};
+        int ids[4]         = {0};
 
         ids[rdims[3]] = ow;
         ids[rdims[2]] = oz;
diff --git a/src/backend/oneapi/kernel/sort.hpp b/src/backend/oneapi/kernel/sort.hpp
new file mode 100644
index 0000000000..1789887b82
--- /dev/null
+++ b/src/backend/oneapi/kernel/sort.hpp
@@ -0,0 +1,119 @@
+/*******************************************************
+ * Copyright (c) 2022, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+// oneDPL headers should be included before standard headers
+#define ONEDPL_USE_PREDEFINED_POLICIES 0
+#include <oneapi/dpl/algorithm>
+#include <oneapi/dpl/execution>
+#include <oneapi/dpl/iterator>
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/half.hpp>
+#include <debug_oneapi.hpp>
+#include <iota.hpp>
+#include <traits.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+void sort0Iterative(Param<T> val, bool isAscending) {
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
+    for (int w = 0; w < val.info.dims[3]; w++) {
+        int valW = w * val.info.strides[3];
+        for (int z = 0; z < val.info.dims[2]; z++) {
+            int valWZ = valW + z * val.info.strides[2];
+            for (int y = 0; y < val.info.dims[1]; y++) {
+                int valOffset = valWZ + y * val.info.strides[1];
+
+                auto buf_begin = ::oneapi::dpl::begin(*val.data) + valOffset;
+                auto buf_end   = buf_begin + val.info.dims[0];
+                if (isAscending) {
+                    std::sort(dpl_policy, buf_begin, buf_end,
+                              [](auto lhs, auto rhs) { return lhs < rhs; });
+                    // std::less<T>()); // mangled name errors in icx for now
+                } else {
+                    std::sort(dpl_policy, buf_begin, buf_end,
+                              [](auto lhs, auto rhs) { return lhs > rhs; });
+                    // std::greater<T>()); // mangled name errors in icx for now
+                }
+            }
+        }
+    }
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+void sortBatched(Param<T> pVal, int dim, bool isAscending) {
+    af::dim4 inDims;
+    for (int i = 0; i < 4; i++) inDims[i] = pVal.info.dims[i];
+
+    // Sort dimension
+    af::dim4 tileDims(1);
+    af::dim4 seqDims = inDims;
+    tileDims[dim]    = inDims[dim];
+    seqDims[dim]     = 1;
+
+    // Create/call iota
+    Array<uint> pKey = iota<uint>(seqDims, tileDims);
+
+    pKey.setDataDims(inDims.elements());
+
+    // Flat
+    pVal.info.dims[0]    = inDims.elements();
+    pVal.info.strides[0] = 1;
+    for (int i = 1; i < 4; i++) {
+        pVal.info.dims[i]    = 1;
+        pVal.info.strides[i] = pVal.info.strides[i - 1] * pVal.info.dims[i - 1];
+    }
+
+    // Sort indices
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
+
+    auto key_begin    = ::oneapi::dpl::begin(*pKey.get());
+    auto key_end      = ::oneapi::dpl::end(*pKey.get());
+    auto val_begin    = ::oneapi::dpl::begin(*pVal.data);
+    auto val_end      = ::oneapi::dpl::end(*pVal.data);
+    auto zipped_begin = dpl::make_zip_iterator(key_begin, val_begin);
+    auto zipped_end   = dpl::make_zip_iterator(key_end, val_end);
+
+    // sort values first
+    if (isAscending) {
+        std::sort(dpl_policy, zipped_begin, zipped_end, [](auto lhs, auto rhs) {
+            return std::get<1>(lhs) < std::get<1>(rhs);
+        });
+    } else {
+        std::sort(dpl_policy, zipped_begin, zipped_end, [](auto lhs, auto rhs) {
+            return std::get<1>(lhs) > std::get<1>(rhs);
+        });
+    }
+    // sort according to keys second
+    std::sort(dpl_policy, zipped_begin, zipped_end, [](auto lhs, auto rhs) {
+        return std::get<0>(lhs) < std::get<0>(rhs);
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+void sort0(Param<T> val, bool isAscending) {
+    int higherDims = val.info.dims[1] * val.info.dims[2] * val.info.dims[3];
+    // TODO Make a better heurisitic
+    if (higherDims > 10)
+        sortBatched<T>(val, 0, isAscending);
+    else
+        sort0Iterative<T>(val, isAscending);
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/sort_by_key.hpp b/src/backend/oneapi/kernel/sort_by_key.hpp
new file mode 100644
index 0000000000..3a1d7d38a8
--- /dev/null
+++ b/src/backend/oneapi/kernel/sort_by_key.hpp
@@ -0,0 +1,29 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <Param.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename Tk, typename Tv>
+void sort0ByKeyIterative(Param<Tk> pKey, Param<Tv> pVal, bool isAscending);
+
+template<typename Tk, typename Tv>
+void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
+                      bool isAscending);
+
+template<typename Tk, typename Tv>
+void sort0ByKey(Param<Tk> pKey, Param<Tv> pVal, bool isAscending);
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt b/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
new file mode 100644
index 0000000000..ce184639eb
--- /dev/null
+++ b/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
@@ -0,0 +1,53 @@
+# Copyright (c) 2017, ArrayFire
+# All rights reserved.
+#
+# This file is distributed under 3-clause BSD license.
+# The complete license agreement can be obtained at:
+# http://arrayfire.com/licenses/BSD-3-Clause
+
+file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/sort_by_key_impl.cpp" FILESTRINGS)
+
+foreach(STR ${FILESTRINGS})
+    if(${STR} MATCHES "// SBK_TYPES")
+        string(REPLACE "// SBK_TYPES:" "" TEMP ${STR})
+        string(REPLACE " " ";" SBK_TYPES ${TEMP})
+    endif()
+endforeach()
+
+add_library(oneapi_sort_by_key INTERFACE)
+foreach(SBK_TYPE ${SBK_TYPES})
+  add_library(oneapi_sort_by_key_${SBK_TYPE} OBJECT
+        "${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/sort_by_key_impl.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key_impl.hpp"
+    )
+  set_target_properties(oneapi_sort_by_key_${SBK_TYPE}
+    PROPERTIES
+      COMPILE_DEFINITIONS "TYPE=${SBK_TYPE};AFDLL;$<TARGET_PROPERTY:Boost::boost,INTERFACE_COMPILE_DEFINITIONS>"
+      CXX_STANDARD 17
+      CXX_EXTENSIONS OFF
+      CXX_VISIBILITY_PRESET hidden
+      FOLDER "Generated Targets")
+
+  arrayfire_set_default_cxx_flags(oneapi_sort_by_key_${SBK_TYPE})
+
+  target_include_directories(oneapi_sort_by_key_${SBK_TYPE}
+    PUBLIC
+      .
+      ../../api/c
+      ${ArrayFire_SOURCE_DIR}/include
+      ${ArrayFire_BINARY_DIR}/include
+    PRIVATE
+      ../common
+      ..
+      )
+
+  target_include_directories(oneapi_sort_by_key_${SBK_TYPE}
+    SYSTEM PRIVATE
+      ${span-lite_SOURCE_DIR}/include
+      $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>)
+
+  target_compile_options(oneapi_sort_by_key_${SBK_TYPE} PUBLIC -fsycl)
+  set_target_properties(oneapi_sort_by_key_${SBK_TYPE} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+  target_sources(oneapi_sort_by_key
+    INTERFACE $<TARGET_OBJECTS:oneapi_sort_by_key_${SBK_TYPE}>)
+endforeach(SBK_TYPE ${SBK_TYPES})
diff --git a/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp
new file mode 100644
index 0000000000..9b04402904
--- /dev/null
+++ b/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp
@@ -0,0 +1,20 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <kernel/sort_by_key_impl.hpp>
+
+// SBK_TYPES:float double int uint intl uintl short ushort char uchar half
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+INSTANTIATE1(TYPE);
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/sort_by_key_impl.hpp b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
new file mode 100644
index 0000000000..c0c57d8eff
--- /dev/null
+++ b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
@@ -0,0 +1,206 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+// oneDPL headers should be included before standard headers
+#define ONEDPL_USE_PREDEFINED_POLICIES 0
+#include <oneapi/dpl/algorithm>
+#include <oneapi/dpl/execution>
+#include <oneapi/dpl/iterator>
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/half.hpp>
+#include <debug_oneapi.hpp>
+#include <iota.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+#include <traits.hpp>
+#include <types.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+using arrayfire::common::half;
+
+template<typename Tk, typename Tv>
+void sort0ByKeyIterative(Param<Tk> pKey, Param<Tv> pVal, bool isAscending) {
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
+
+    for (int w = 0; w < pKey.info.dims[3]; w++) {
+        int pKeyW = w * pKey.info.strides[3];
+        int pValW = w * pVal.info.strides[3];
+        for (int z = 0; z < pKey.info.dims[2]; z++) {
+            int pKeyWZ = pKeyW + z * pKey.info.strides[2];
+            int pValWZ = pValW + z * pVal.info.strides[2];
+            for (int y = 0; y < pKey.info.dims[1]; y++) {
+                int pKeyOffset = pKeyWZ + y * pKey.info.strides[1];
+                int pValOffset = pValWZ + y * pVal.info.strides[1];
+
+                auto key_begin =
+                    ::oneapi::dpl::begin(
+                        pKey.data->template reinterpret<compute_t<Tk>>()) +
+                    pKeyOffset;
+                auto key_end   = key_begin + pKey.info.dims[0];
+                auto val_begin = ::oneapi::dpl::begin(*pVal.data) + pValOffset;
+                auto val_end   = val_begin + pVal.info.dims[0];
+
+                auto zipped_begin =
+                    ::oneapi::dpl::make_zip_iterator(key_begin, val_begin);
+                auto zipped_end =
+                    ::oneapi::dpl::make_zip_iterator(key_end, val_end);
+
+                // sort by key
+                if (isAscending) {
+                    std::sort(dpl_policy, zipped_begin, zipped_end,
+                              [](auto lhs, auto rhs) {
+                                  return std::get<0>(lhs) < std::get<0>(rhs);
+                              });
+                } else {
+                    std::sort(dpl_policy, zipped_begin, zipped_end,
+                              [](auto lhs, auto rhs) {
+                                  return std::get<0>(lhs) > std::get<0>(rhs);
+                              });
+                }
+            }
+        }
+    }
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Tk, typename Tv>
+void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
+                      bool isAscending) {
+    af::dim4 inDims;
+    for (int i = 0; i < 4; i++) inDims[i] = pKey.info.dims[i];
+
+    const dim_t elements = inDims.elements();
+
+    // Sort dimension
+    // tileDims * seqDims = inDims
+    af::dim4 tileDims(1);
+    af::dim4 seqDims = inDims;
+    tileDims[dim]    = inDims[dim];
+    seqDims[dim]     = 1;
+
+    // Create/call iota
+    Array<uint> Seq = iota<uint>(seqDims, tileDims);
+
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
+
+    // set up iterators for seq, key, val, and new cKey
+    auto seq_begin = ::oneapi::dpl::begin(*Seq.get());
+    auto seq_end   = ::oneapi::dpl::end(*Seq.get());
+    auto key_begin =
+        ::oneapi::dpl::begin(pKey.data->template reinterpret<compute_t<Tk>>());
+    auto key_end =
+        ::oneapi::dpl::end(pKey.data->template reinterpret<compute_t<Tk>>());
+    auto val_begin = ::oneapi::dpl::begin(*pVal.data);
+    auto val_end   = ::oneapi::dpl::end(*pVal.data);
+
+    auto cKey = memAlloc<Tk>(elements);
+    getQueue().submit([&](sycl::handler &h) {
+        h.copy(pKey.data->template reinterpret<compute_t<Tk>>().get_access(),
+               cKey.get()->template reinterpret<compute_t<Tk>>().get_access());
+    });
+    auto ckey_begin =
+        ::oneapi::dpl::begin(cKey.get()->template reinterpret<compute_t<Tk>>());
+    auto ckey_end =
+        ::oneapi::dpl::end(cKey.get()->template reinterpret<compute_t<Tk>>());
+
+    {
+        auto zipped_begin_KV  = dpl::make_zip_iterator(key_begin, val_begin);
+        auto zipped_end_KV    = dpl::make_zip_iterator(key_end, val_end);
+        auto zipped_begin_cKS = dpl::make_zip_iterator(ckey_begin, seq_begin);
+        auto zipped_end_cKS   = dpl::make_zip_iterator(ckey_end, seq_end);
+        if (isAscending) {
+            std::sort(dpl_policy, zipped_begin_KV, zipped_end_KV,
+                      [](auto lhs, auto rhs) {
+                          return std::get<0>(lhs) < std::get<0>(rhs);
+                      });
+            std::sort(dpl_policy, zipped_begin_cKS, zipped_end_cKS,
+                      [](auto lhs, auto rhs) {
+                          return std::get<0>(lhs) < std::get<0>(rhs);
+                      });
+        } else {
+            std::sort(dpl_policy, zipped_begin_KV, zipped_end_KV,
+                      [](auto lhs, auto rhs) {
+                          return std::get<0>(lhs) > std::get<0>(rhs);
+                      });
+            std::sort(dpl_policy, zipped_begin_cKS, zipped_end_cKS,
+                      [](auto lhs, auto rhs) {
+                          return std::get<0>(lhs) > std::get<0>(rhs);
+                      });
+        }
+    }
+
+    auto cSeq = memAlloc<uint>(elements);
+    getQueue().submit([&](sycl::handler &h) {
+        h.copy(Seq.get()->get_access(), cSeq.get()->get_access());
+    });
+    auto cseq_begin = ::oneapi::dpl::begin(*cSeq.get());
+    auto cseq_end   = ::oneapi::dpl::end(*cSeq.get());
+
+    {
+        auto zipped_begin_SV  = dpl::make_zip_iterator(seq_begin, val_begin);
+        auto zipped_end_SV    = dpl::make_zip_iterator(seq_end, val_end);
+        auto zipped_begin_cSK = dpl::make_zip_iterator(cseq_begin, key_begin);
+        auto zipped_end_cSK   = dpl::make_zip_iterator(cseq_end, key_end);
+        std::sort(dpl_policy, zipped_begin_SV, zipped_end_SV,
+                  [](auto lhs, auto rhs) {
+                      return std::get<0>(lhs) < std::get<0>(rhs);
+                  });
+        std::sort(dpl_policy, zipped_begin_cSK, zipped_end_cSK,
+                  [](auto lhs, auto rhs) {
+                      return std::get<0>(lhs) < std::get<0>(rhs);
+                  });
+    }
+}
+
+template<typename Tk, typename Tv>
+void sort0ByKey(Param<Tk> pKey, Param<Tv> pVal, bool isAscending) {
+    int higherDims = pKey.info.dims[1] * pKey.info.dims[2] * pKey.info.dims[3];
+    // Batched sort performs 4x sort by keys
+    // But this is only useful before GPU is saturated
+    // The GPU is saturated at around 1000,000 integers
+    // Call batched sort only if both conditions are met
+    if (higherDims > 4 && pKey.info.dims[0] < 1000000) {
+        kernel::sortByKeyBatched<Tk, Tv>(pKey, pVal, 0, isAscending);
+    } else {
+        kernel::sort0ByKeyIterative<Tk, Tv>(pKey, pVal, isAscending);
+    }
+}
+
+#define INSTANTIATE(Tk, Tv)                                                   \
+    template void sort0ByKey<Tk, Tv>(Param<Tk> okey, Param<Tv> oval,          \
+                                     bool isAscending);                       \
+    template void sort0ByKeyIterative<Tk, Tv>(Param<Tk> okey, Param<Tv> oval, \
+                                              bool isAscending);              \
+    template void sortByKeyBatched<Tk, Tv>(Param<Tk> okey, Param<Tv> oval,    \
+                                           const int dim, bool isAscending);
+
+#define INSTANTIATE1(Tk)     \
+    INSTANTIATE(Tk, float)   \
+    INSTANTIATE(Tk, double)  \
+    INSTANTIATE(Tk, cfloat)  \
+    INSTANTIATE(Tk, cdouble) \
+    INSTANTIATE(Tk, int)     \
+    INSTANTIATE(Tk, uint)    \
+    INSTANTIATE(Tk, short)   \
+    INSTANTIATE(Tk, ushort)  \
+    INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, uchar)   \
+    INSTANTIATE(Tk, intl)    \
+    INSTANTIATE(Tk, uintl)
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/wrap.hpp b/src/backend/oneapi/kernel/wrap.hpp
index ba503a1f56..5f2c92c641 100644
--- a/src/backend/oneapi/kernel/wrap.hpp
+++ b/src/backend/oneapi/kernel/wrap.hpp
@@ -15,6 +15,7 @@
 #include <debug_oneapi.hpp>
 #include <kernel/default_config.hpp>
 #include <math.hpp>
+#include <sycl/sycl.hpp>
 
 #include <sycl/sycl.hpp>
 
@@ -82,8 +83,8 @@ class wrapCreateKernel {
         // / stride Each previous index has the value appear "stride" locations
         // earlier We work our way back from the last index
 
-        const int x_end = fmin(pidx0 / sx_, nx_ - 1);
-        const int y_end = fmin(pidx1 / sy_, ny_ - 1);
+        const int x_end = sycl::min(pidx0 / sx_, nx_ - 1);
+        const int y_end = sycl::min(pidx1 / sy_, ny_ - 1);
 
         const int x_off = pidx0 - sx_ * x_end;
         const int y_off = pidx1 - sy_ * y_end;
diff --git a/src/backend/oneapi/kernel/wrap_dilated.hpp b/src/backend/oneapi/kernel/wrap_dilated.hpp
index c479316968..dae994e371 100644
--- a/src/backend/oneapi/kernel/wrap_dilated.hpp
+++ b/src/backend/oneapi/kernel/wrap_dilated.hpp
@@ -16,6 +16,7 @@
 #include <kernel/default_config.hpp>
 #include <math.hpp>
 
+#include <sycl/sycl.hpp>
 #include <string>
 #include <vector>
 
@@ -87,10 +88,10 @@ class wrapDilatedCreateKernel {
         // earlier We work our way back from the last index
 
         const int y_start = (pidx1 < eff_wy) ? 0 : (pidx1 - eff_wy) / sy_ + 1;
-        const int y_end   = fmin(pidx1 / sy_ + 1, ny_);
+        const int y_end   = sycl::min(pidx1 / sy_ + 1, ny_);
 
         const int x_start = (pidx0 < eff_wx) ? 0 : (pidx0 - eff_wx) / sx_ + 1;
-        const int x_end   = fmin(pidx0 / sx_ + 1, nx_);
+        const int x_end   = sycl::min(pidx0 / sx_ + 1, nx_);
 
         T val   = (T)0;
         int idx = 1;
diff --git a/src/backend/oneapi/sort.cpp b/src/backend/oneapi/sort.cpp
index 599d23c896..002385a320 100644
--- a/src/backend/oneapi/sort.cpp
+++ b/src/backend/oneapi/sort.cpp
@@ -7,10 +7,11 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#include <kernel/sort.hpp>
+
 #include <Array.hpp>
 #include <copy.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/sort.hpp>
 #include <math.hpp>
 #include <reorder.hpp>
 #include <sort.hpp>
@@ -18,19 +19,18 @@
 
 namespace arrayfire {
 namespace oneapi {
+
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending) {
-    ONEAPI_NOT_SUPPORTED("sort Not supported");
-
     try {
         Array<T> out = copyArray<T>(in);
-        // switch (dim) {
-        //     case 0: kernel::sort0<T>(out, isAscending); break;
-        //     case 1: kernel::sortBatched<T>(out, 1, isAscending); break;
-        //     case 2: kernel::sortBatched<T>(out, 2, isAscending); break;
-        //     case 3: kernel::sortBatched<T>(out, 3, isAscending); break;
-        //     default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
-        // }
+        switch (dim) {
+            case 0: kernel::sort0<T>(out, isAscending); break;
+            case 1: kernel::sortBatched<T>(out, 1, isAscending); break;
+            case 2: kernel::sortBatched<T>(out, 2, isAscending); break;
+            case 3: kernel::sortBatched<T>(out, 3, isAscending); break;
+            default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
+        }
 
         if (dim != 0) {
             af::dim4 preorderDims = out.dims();
diff --git a/src/backend/oneapi/sort_by_key.cpp b/src/backend/oneapi/sort_by_key.cpp
index 00a5bb55fa..9ec60130cd 100644
--- a/src/backend/oneapi/sort_by_key.cpp
+++ b/src/backend/oneapi/sort_by_key.cpp
@@ -10,7 +10,7 @@
 #include <Array.hpp>
 #include <copy.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/sort_by_key.hpp>
+#include <kernel/sort_by_key.hpp>
 #include <math.hpp>
 #include <reorder.hpp>
 #include <sort_by_key.hpp>
@@ -21,7 +21,35 @@ namespace oneapi {
 template<typename Tk, typename Tv>
 void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
                  const Array<Tv> &ival, const unsigned dim, bool isAscending) {
-    ONEAPI_NOT_SUPPORTED("");
+    okey = copyArray<Tk>(ikey);
+    oval = copyArray<Tv>(ival);
+
+    switch (dim) {
+        case 0: kernel::sort0ByKey<Tk, Tv>(okey, oval, isAscending); break;
+        case 1:
+        case 2:
+        case 3:
+            kernel::sortByKeyBatched<Tk, Tv>(okey, oval, dim, isAscending);
+            break;
+        default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
+    }
+
+    if (dim != 0) {
+        af::dim4 preorderDims = okey.dims();
+        af::dim4 reorderDims(0, 1, 2, 3);
+        reorderDims[dim] = 0;
+        preorderDims[0]  = okey.dims()[dim];
+        for (int i = 1; i <= (int)dim; i++) {
+            reorderDims[i - 1] = i;
+            preorderDims[i]    = okey.dims()[i - 1];
+        }
+
+        okey.setDataDims(preorderDims);
+        oval.setDataDims(preorderDims);
+
+        okey = reorder<Tk>(okey, reorderDims);
+        oval = reorder<Tv>(oval, reorderDims);
+    }
 }
 
 #define INSTANTIATE(Tk, Tv)                                        \
diff --git a/src/backend/oneapi/sort_index.cpp b/src/backend/oneapi/sort_index.cpp
index c0df0fb9de..17de33fbad 100644
--- a/src/backend/oneapi/sort_index.cpp
+++ b/src/backend/oneapi/sort_index.cpp
@@ -11,35 +11,33 @@
 #include <common/half.hpp>
 #include <copy.hpp>
 #include <err_oneapi.hpp>
-// #include <kernel/sort_by_key.hpp>
+#include <kernel/sort_by_key.hpp>
 #include <math.hpp>
 #include <range.hpp>
 #include <reorder.hpp>
 #include <sort_index.hpp>
 #include <stdexcept>
 
-using arrayfire::common::half;
-
 namespace arrayfire {
 namespace oneapi {
 template<typename T>
 void sort_index(Array<T> &okey, Array<uint> &oval, const Array<T> &in,
                 const uint dim, bool isAscending) {
-    ONEAPI_NOT_SUPPORTED("sort_index Not supported");
-
     try {
         // okey contains values, oval contains indices
         okey = copyArray<T>(in);
         oval = range<uint>(in.dims(), dim);
         oval.eval();
 
-        // switch (dim) {
-        //     case 0: kernel::sort0ByKey<T, uint>(okey, oval, isAscending);
-        //     break; case 1: case 2: case 3:
-        //         kernel::sortByKeyBatched<T, uint>(okey, oval, dim,
-        //         isAscending); break;
-        //     default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
-        // }
+        switch (dim) {
+            case 0: kernel::sort0ByKey<T, uint>(okey, oval, isAscending); break;
+            case 1:
+            case 2:
+            case 3:
+                kernel::sortByKeyBatched<T, uint>(okey, oval, dim, isAscending);
+                break;
+            default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
+        }
 
         if (dim != 0) {
             af::dim4 preorderDims = okey.dims();
@@ -75,7 +73,7 @@ INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
-INSTANTIATE(half)
+INSTANTIATE(arrayfire::common::half)
 
 }  // namespace oneapi
 }  // namespace arrayfire
diff --git a/src/backend/oneapi/topk.cpp b/src/backend/oneapi/topk.cpp
index 35c0b66975..17a14ce810 100644
--- a/src/backend/oneapi/topk.cpp
+++ b/src/backend/oneapi/topk.cpp
@@ -20,8 +20,6 @@
 #include <numeric>
 #include <vector>
 
-// using cl::Buffer;
-// using cl::Event;
 using arrayfire::common::half;
 
 using std::iota;
@@ -49,125 +47,12 @@ vector<af_index_t> indexForTopK(const int k) {
 template<typename T>
 void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
           const int k, const int dim, const af::topkFunction order) {
-    ONEAPI_NOT_SUPPORTED("topk Not supported");
-
-    // if (getDeviceType() == CL_DEVICE_TYPE_CPU) {
-    //     // This branch optimizes for CPU devices by first mapping the buffer
-    //     // and calling partial sort on the buffer
-
-    //     // TODO(umar): implement this in the kernel namespace
-
-    //     // The out_dims is of size k along the dimension of the topk
-    //     operation
-    //     // and the same as the input dimension otherwise.
-    //     dim4 out_dims(1);
-    //     int ndims = in.dims().ndims();
-    //     for (int i = 0; i < ndims; i++) {
-    //         if (i == dim) {
-    //             out_dims[i] = min(k, (int)in.dims()[i]);
-    //         } else {
-    //             out_dims[i] = in.dims()[i];
-    //         }
-    //     }
-
-    //     auto values          = createEmptyArray<T>(out_dims);
-    //     auto indices         = createEmptyArray<unsigned>(out_dims);
-    //     const Buffer* in_buf = in.get();
-    //     Buffer* ibuf         = indices.get();
-    //     Buffer* vbuf         = values.get();
-
-    //     cl::Event ev_in, ev_val, ev_ind;
-
-    //     T* ptr     = static_cast<T*>(getQueue().enqueueMapBuffer(
-    //         *in_buf, CL_FALSE, CL_MAP_READ, 0, in.elements() * sizeof(T),
-    //         nullptr, &ev_in));
-    //     uint* iptr = static_cast<uint*>(getQueue().enqueueMapBuffer(
-    //         *ibuf, CL_FALSE, CL_MAP_READ | CL_MAP_WRITE, 0, k * sizeof(uint),
-    //         nullptr, &ev_ind));
-    //     T* vptr    = static_cast<T*>(getQueue().enqueueMapBuffer(
-    //         *vbuf, CL_FALSE, CL_MAP_WRITE, 0, k * sizeof(T), nullptr,
-    //         &ev_val));
-
-    //     vector<uint> idx(in.elements());
-
-    //     // Create a linear index
-    //     iota(begin(idx), end(idx), 0);
-    //     cl::Event::waitForEvents({ev_in, ev_ind});
-
-    //     int iter = in.dims()[1] * in.dims()[2] * in.dims()[3];
-    //     for (int i = 0; i < iter; i++) {
-    //         auto idx_itr = begin(idx) + i * in.strides()[1];
-    //         auto kiptr   = iptr + k * i;
-
-    //         if (order & AF_TOPK_MIN) {
-    //             if (order & AF_TOPK_STABLE) {
-    //                 partial_sort_copy(
-    //                     idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
-    //                     [ptr](const uint lhs, const uint rhs) -> bool {
-    //                         return (compute_t<T>(ptr[lhs]) <
-    //                                 compute_t<T>(ptr[rhs]))
-    //                                    ? true
-    //                                : compute_t<T>(ptr[lhs]) ==
-    //                                        compute_t<T>(ptr[rhs])
-    //                                    ? (lhs < rhs)
-    //                                    : false;
-    //                     });
-    //             } else {
-    //                 // Sort the top k values in each column
-    //                 partial_sort_copy(
-    //                     idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
-    //                     [ptr](const uint lhs, const uint rhs) -> bool {
-    //                         return compute_t<T>(ptr[lhs]) <
-    //                                compute_t<T>(ptr[rhs]);
-    //                     });
-    //             }
-    //         } else {
-    //             if (order & AF_TOPK_STABLE) {
-    //                 partial_sort_copy(
-    //                     idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
-    //                     [ptr](const uint lhs, const uint rhs) -> bool {
-    //                         return (compute_t<T>(ptr[lhs]) >
-    //                                 compute_t<T>(ptr[rhs]))
-    //                                    ? true
-    //                                : compute_t<T>(ptr[lhs]) ==
-    //                                        compute_t<T>(ptr[rhs])
-    //                                    ? (lhs < rhs)
-    //                                    : false;
-    //                     });
-    //             } else {
-    //                 partial_sort_copy(
-    //                     idx_itr, idx_itr + in.strides()[1], kiptr, kiptr + k,
-    //                     [ptr](const uint lhs, const uint rhs) -> bool {
-    //                         return compute_t<T>(ptr[lhs]) >
-    //                                compute_t<T>(ptr[rhs]);
-    //                     });
-    //             }
-    //         }
-    //         ev_val.wait();
-
-    //         auto kvptr = vptr + k * i;
-    //         for (int j = 0; j < k; j++) {
-    //             // Update the value arrays with the original values
-    //             kvptr[j] = ptr[kiptr[j]];
-    //             // Convert linear indices back to column indices
-    //             kiptr[j] -= i * in.strides()[1];
-    //         }
-    //     }
-
-    //     getQueue().enqueueUnmapMemObject(*ibuf, iptr);
-    //     getQueue().enqueueUnmapMemObject(*vbuf, vptr);
-    //     getQueue().enqueueUnmapMemObject(*in_buf, ptr);
-
-    //     vals = values;
-    //     idxs = indices;
-    // } else {
-    //     auto values  = createEmptyArray<T>(in.dims());
-    //     auto indices = createEmptyArray<unsigned>(in.dims());
-    //     sort_index(values, indices, in, dim, order & AF_TOPK_MIN);
-    //     auto indVec = indexForTopK(k);
-    //     vals        = index<T>(values, indVec.data());
-    //     idxs        = index<unsigned>(indices, indVec.data());
-    // }
+    auto values  = createEmptyArray<T>(in.dims());
+    auto indices = createEmptyArray<unsigned>(in.dims());
+    sort_index(values, indices, in, dim, order & AF_TOPK_MIN);
+    auto indVec = indexForTopK(k);
+    vals        = index<T>(values, indVec.data());
+    idxs        = index<unsigned>(indices, indVec.data());
 }
 
 #define INSTANTIATE(T)                                                  \
@@ -181,5 +66,6 @@ INSTANTIATE(uint)
 INSTANTIATE(long long)
 INSTANTIATE(unsigned long long)
 INSTANTIATE(half)
+
 }  // namespace oneapi
 }  // namespace arrayfire

From 3d2ad9857083dee8af838dfcca41016215a1b6e5 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 10 Apr 2023 20:21:23 -0400
Subject: [PATCH 257/473] adds iir, pad_array_borders kernels

---
 src/backend/oneapi/CMakeLists.txt             |   2 +
 src/backend/oneapi/copy.hpp                   |   4 +-
 src/backend/oneapi/iir.cpp                    |  27 ++-
 src/backend/oneapi/kernel/iir.hpp             | 150 ++++++++++++
 .../oneapi/kernel/pad_array_borders.hpp       | 216 ++++++++++++++++++
 5 files changed, 394 insertions(+), 5 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/iir.hpp
 create mode 100644 src/backend/oneapi/kernel/pad_array_borders.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 7a58966711..831234a5a8 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -217,6 +217,7 @@ target_sources(afoneapi
     kernel/diagonal.hpp
     kernel/diff.hpp
     kernel/histogram.hpp
+    kernel/iir.hpp
     kernel/identity.hpp
     kernel/interp.hpp
     kernel/iota.hpp
@@ -224,6 +225,7 @@ target_sources(afoneapi
     kernel/lu_split.hpp
     kernel/memcopy.hpp
     kernel/mean.hpp
+    kernel/pad_array_borders.hpp
     kernel/random_engine.hpp
     kernel/random_engine_write.hpp
     kernel/random_engine_mersenne.hpp
diff --git a/src/backend/oneapi/copy.hpp b/src/backend/oneapi/copy.hpp
index 4b05151dbd..85b3b861ea 100644
--- a/src/backend/oneapi/copy.hpp
+++ b/src/backend/oneapi/copy.hpp
@@ -9,7 +9,7 @@
 #pragma once
 
 #include <Array.hpp>
-// #include <kernel/pad_array_borders.hpp>
+#include <kernel/pad_array_borders.hpp>
 
 namespace arrayfire {
 namespace oneapi {
@@ -55,7 +55,7 @@ Array<T> padArrayBorders(Array<T> const &in, dim4 const &lowerBoundPadding,
 
     auto ret = createEmptyArray<T>(oDims);
 
-    // kernel::padBorders<T>(ret, in, lowerBoundPadding, btype);
+    kernel::padBorders<T>(ret, in, lowerBoundPadding, btype);
 
     return ret;
 }
diff --git a/src/backend/oneapi/iir.cpp b/src/backend/oneapi/iir.cpp
index e38a70294f..f60db52e8e 100644
--- a/src/backend/oneapi/iir.cpp
+++ b/src/backend/oneapi/iir.cpp
@@ -12,7 +12,7 @@
 #include <convolve.hpp>
 #include <err_oneapi.hpp>
 #include <iir.hpp>
-// #include <kernel/iir.hpp>
+#include <kernel/iir.hpp>
 #include <math.hpp>
 #include <af/dim4.hpp>
 
@@ -22,8 +22,29 @@ namespace arrayfire {
 namespace oneapi {
 template<typename T>
 Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> y = createEmptyArray<T>(dim4(1));
+    AF_BATCH_KIND type = x.ndims() == 1 ? AF_BATCH_NONE : AF_BATCH_SAME;
+    if (x.ndims() != b.ndims()) {
+        type = (x.ndims() < b.ndims()) ? AF_BATCH_RHS : AF_BATCH_LHS;
+    }
+
+    // Extract the first N elements
+    Array<T> c = convolve<T, T>(x, b, type, 1, true);
+    dim4 cdims = c.dims();
+    cdims[0]   = x.dims()[0];
+    c.resetDims(cdims);
+
+    int num_a = a.dims()[0];
+
+    if (num_a == 1) { return c; }
+
+    dim4 ydims = c.dims();
+    Array<T> y = createEmptyArray<T>(ydims);
+
+    if (a.ndims() > 1) {
+        kernel::iir<T, true>(y, c, a);
+    } else {
+        kernel::iir<T, false>(y, c, a);
+    }
     return y;
 }
 
diff --git a/src/backend/oneapi/kernel/iir.hpp b/src/backend/oneapi/kernel/iir.hpp
new file mode 100644
index 0000000000..ab00655fec
--- /dev/null
+++ b/src/backend/oneapi/kernel/iir.hpp
@@ -0,0 +1,150 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+
+#include <sycl/sycl.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+constexpr int MAX_A_SIZE = 1024;
+
+template<typename T, bool batch_a>
+class iirKernel {
+   public:
+    iirKernel(write_accessor<T> y, KParam yInfo, read_accessor<T> c,
+              KParam cInfo, read_accessor<T> a, KParam aInfo,
+              sycl::local_accessor<T> s_z, sycl::local_accessor<T> s_a,
+              sycl::local_accessor<T> s_y, int groups_y)
+        : y_(y)
+        , yInfo_(yInfo)
+        , c_(c)
+        , cInfo_(cInfo)
+        , a_(a)
+        , aInfo_(aInfo)
+        , s_z_(s_z)
+        , s_a_(s_a)
+        , s_y_(s_y)
+        , groups_y_(groups_y) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        const int idz = g.get_group_id(0);
+        const int idw = g.get_group_id(1) / groups_y_;
+        const int idy = g.get_group_id(1) - idw * groups_y_;
+
+        const int tx    = it.get_local_id(0);
+        const int num_a = aInfo_.dims[0];
+
+        int y_off = idw * yInfo_.strides[3] + idz * yInfo_.strides[2] +
+                    idy * yInfo_.strides[1];
+        int c_off = idw * cInfo_.strides[3] + idz * cInfo_.strides[2] +
+                    idy * cInfo_.strides[1];
+        int a_off = 0;
+
+        if (batch_a)
+            a_off = idw * aInfo_.strides[3] + idz * aInfo_.strides[2] +
+                    idy * aInfo_.strides[1];
+
+        T *d_y       = y_.get_pointer() + y_off;
+        const T *d_c = c_.get_pointer() + c_off;
+        const T *d_a = a_.get_pointer() + a_off;
+        const int repeat =
+            (num_a + g.get_local_range(0) - 1) / g.get_local_range(0);
+
+        for (int ii = 0; ii < MAX_A_SIZE / g.get_local_range(0); ii++) {
+            int id   = ii * g.get_local_range(0) + tx;
+            s_z_[id] = scalar<T>(0);
+            s_a_[id] = (id < num_a) ? d_a[id] : scalar<T>(0);
+        }
+        group_barrier(g);
+
+        for (int i = 0; i < yInfo_.dims[0]; i++) {
+            if (tx == 0) {
+                s_y_[0] = (d_c[i] + s_z_[0]) / s_a_[0];
+                d_y[i]  = s_y_[0];
+            }
+            group_barrier(g);
+
+#pragma unroll
+            for (int ii = 0; ii < repeat; ii++) {
+                int id = ii * g.get_local_range(0) + tx + 1;
+
+                T z = s_z_[id] - s_a_[id] * s_y_[0];
+                group_barrier(g);
+
+                s_z_[id - 1] = z;
+                group_barrier(g);
+            }
+        }
+    }
+
+   protected:
+    write_accessor<T> y_;
+    KParam yInfo_;
+    read_accessor<T> c_;
+    KParam cInfo_;
+    read_accessor<T> a_;
+    KParam aInfo_;
+    sycl::local_accessor<T> s_z_;
+    sycl::local_accessor<T> s_a_;
+    sycl::local_accessor<T> s_y_;
+    int groups_y_;
+};
+
+template<typename T, bool batch_a>
+void iir(Param<T> y, Param<T> c, Param<T> a) {
+    const int groups_y = y.info.dims[1];
+    const int groups_x = y.info.dims[2];
+
+    int threads = 256;
+    while (threads > y.info.dims[0] && threads > 32) threads /= 2;
+    sycl::range<2> local = sycl::range{threads, 1};
+
+    sycl::range<2> global =
+        sycl::range<2>{groups_x * local[0], groups_y * y.info.dims[3]};
+
+    getQueue().submit([&](sycl::handler &h) {
+        write_accessor<T> yAcc{*y.data, h};
+        read_accessor<T> cAcc{*c.data, h};
+        read_accessor<T> aAcc{*a.data, h};
+
+        auto s_z = sycl::local_accessor<T>(MAX_A_SIZE, h);
+        auto s_a = sycl::local_accessor<T>(MAX_A_SIZE, h);
+        auto s_y = sycl::local_accessor<T>(1, h);
+
+        if (batch_a) {
+            h.parallel_for(sycl::nd_range{global, local},
+                           iirKernel<T, true>(yAcc, y.info, cAcc, c.info, aAcc,
+                                              a.info, s_z, s_a, s_y, groups_y));
+        } else {
+            h.parallel_for(
+                sycl::nd_range{global, local},
+                iirKernel<T, false>(yAcc, y.info, cAcc, c.info, aAcc, a.info,
+                                    s_z, s_a, s_y, groups_y));
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/pad_array_borders.hpp b/src/backend/oneapi/kernel/pad_array_borders.hpp
new file mode 100644
index 0000000000..620352f352
--- /dev/null
+++ b/src/backend/oneapi/kernel/pad_array_borders.hpp
@@ -0,0 +1,216 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <af/defines.h>
+
+#include <sycl/sycl.hpp>
+
+#include <array>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T, int BType>
+class padBordersKernel {
+   public:
+    padBordersKernel(write_accessor<T> out, KParam oInfo, read_accessor<T> in,
+                     KParam iInfo, const dim_t l0, const dim_t l1,
+                     const dim_t l2, const dim_t l3, const int groups_x,
+                     const int groups_y)
+        : out_(out)
+        , oInfo_(oInfo)
+        , in_(in)
+        , iInfo_(iInfo)
+        , l0_(l0)
+        , l1_(l1)
+        , l2_(l2)
+        , l3_(l3)
+        , groups_x_(groups_x)
+        , groups_y_(groups_y) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        const int lx  = it.get_local_id(0);
+        const int ly  = it.get_local_id(1);
+        const int k   = g.get_group_id(0) / groups_x_;
+        const int l   = g.get_group_id(1) / groups_y_;
+
+        const int blockIdx_x = g.get_group_id(0) - (groups_x_)*k;
+        const int blockIdx_y = g.get_group_id(1) - (groups_y_)*l;
+        const int i          = blockIdx_x * g.get_local_range(0) + lx;
+        const int j          = blockIdx_y * g.get_local_range(1) + ly;
+
+        const size_t d0 = iInfo_.dims[0];
+        const size_t d1 = iInfo_.dims[1];
+        const size_t d2 = iInfo_.dims[2];
+        const size_t d3 = iInfo_.dims[3];
+        const size_t s0 = iInfo_.strides[0];
+        const size_t s1 = iInfo_.strides[1];
+        const size_t s2 = iInfo_.strides[2];
+        const size_t s3 = iInfo_.strides[3];
+
+        const T* src = in_.get_pointer() + iInfo_.offset;
+        T* dst       = out_.get_pointer();
+
+        bool isNotPadding =
+            (l >= l3_ && l < (d3 + l3_)) && (k >= l2_ && k < (d2 + l2_)) &&
+            (j >= l1_ && j < (d1 + l1_)) && (i >= l0_ && i < (d0 + l0_));
+
+        T value = scalar<T>(0);
+        if (isNotPadding) {
+            unsigned iLOff = (l - l3_) * s3;
+            unsigned iKOff = (k - l2_) * s2;
+            unsigned iJOff = (j - l1_) * s1;
+            unsigned iIOff = (i - l0_) * s0;
+
+            value = src[iLOff + iKOff + iJOff + iIOff];
+        } else if (BType != AF_PAD_ZERO) {
+            unsigned iLOff =
+                padBordersKernel<T, BType>::idxByndEdge(l, l3_, d3) * s3;
+            unsigned iKOff =
+                padBordersKernel<T, BType>::idxByndEdge(k, l2_, d2) * s2;
+            unsigned iJOff =
+                padBordersKernel<T, BType>::idxByndEdge(j, l1_, d1) * s1;
+            unsigned iIOff =
+                padBordersKernel<T, BType>::idxByndEdge(i, l0_, d0) * s0;
+
+            value = src[iLOff + iKOff + iJOff + iIOff];
+        }
+
+        size_t xlim = oInfo_.dims[0];
+        size_t ylim = oInfo_.dims[1];
+        size_t zlim = oInfo_.dims[2];
+        size_t wlim = oInfo_.dims[3];
+
+        size_t woStrides = oInfo_.strides[3];
+        size_t zoStrides = oInfo_.strides[2];
+        size_t yoStrides = oInfo_.strides[1];
+        size_t xoStrides = oInfo_.strides[0];
+
+        if (i < xlim && j < ylim && k < zlim && l < wlim) {
+            unsigned off =
+                (l * woStrides + k * zoStrides + j * yoStrides + i * xoStrides);
+            dst[off] = value;
+        }
+    }
+
+    static int trimIndex(int idx, const int len) {
+        int ret_val = idx;
+        if (ret_val < 0) {
+            int offset = (abs(ret_val) - 1) % len;
+            ret_val    = offset;
+        } else if (ret_val >= len) {
+            int offset = abs(ret_val) % len;
+            ret_val    = len - offset - 1;
+        }
+        return ret_val;
+    }
+
+    static int idxByndEdge(const int i, const int lb, const int len) {
+        uint retVal;
+        switch (BType) {
+            case AF_PAD_SYM:
+                retVal = padBordersKernel<T, BType>::trimIndex(i - lb, len);
+                break;
+            case AF_PAD_CLAMP_TO_EDGE:
+                retVal = sycl::clamp(i - lb, 0, len - 1);
+                break;
+            case AF_PAD_PERIODIC: {
+                int rem   = (i - lb) % len;
+                bool cond = rem < 0;
+                retVal    = cond * (rem + len) + (1 - cond) * rem;
+            } break;
+            default: retVal = 0; break;  // AF_PAD_ZERO
+        }
+        return retVal;
+    }
+
+   protected:
+    write_accessor<T> out_;
+    KParam oInfo_;
+    read_accessor<T> in_;
+    KParam iInfo_;
+    const dim_t l0_;
+    const dim_t l1_;
+    const dim_t l2_;
+    const dim_t l3_;
+    const int groups_x_;
+    const int groups_y_;
+};
+
+static const int PADB_THREADS_X = 32;
+static const int PADB_THREADS_Y = 8;
+
+template<typename T>
+void padBorders(Param<T> out, Param<T> in, dim4 const lBoundPadding,
+                const af::borderType btype) {
+    sycl::range<2> local(PADB_THREADS_X, PADB_THREADS_Y);
+
+    int groups_x = divup(out.info.dims[0], PADB_THREADS_X);
+    int groups_y = divup(out.info.dims[1], PADB_THREADS_Y);
+
+    sycl::range<2> global(groups_x * out.info.dims[2] * local[0],
+                          groups_y * out.info.dims[3] * local[1]);
+
+    getQueue().submit([&](sycl::handler& h) {
+        read_accessor<T> iData{*in.data, h};
+        write_accessor<T> oData{*out.data, h};
+
+        switch (btype) {
+            case AF_PAD_ZERO:
+                h.parallel_for(
+                    sycl::nd_range{global, local},
+                    padBordersKernel<T, AF_PAD_ZERO>(
+                        oData, out.info, iData, in.info, lBoundPadding[0],
+                        lBoundPadding[1], lBoundPadding[2], lBoundPadding[3],
+                        groups_x, groups_y));
+                break;
+            case AF_PAD_SYM:
+                h.parallel_for(
+                    sycl::nd_range{global, local},
+                    padBordersKernel<T, AF_PAD_SYM>(
+                        oData, out.info, iData, in.info, lBoundPadding[0],
+                        lBoundPadding[1], lBoundPadding[2], lBoundPadding[3],
+                        groups_x, groups_y));
+                break;
+            case AF_PAD_CLAMP_TO_EDGE:
+                h.parallel_for(
+                    sycl::nd_range{global, local},
+                    padBordersKernel<T, AF_PAD_CLAMP_TO_EDGE>(
+                        oData, out.info, iData, in.info, lBoundPadding[0],
+                        lBoundPadding[1], lBoundPadding[2], lBoundPadding[3],
+                        groups_x, groups_y));
+                break;
+            case AF_PAD_PERIODIC:
+                h.parallel_for(
+                    sycl::nd_range{global, local},
+                    padBordersKernel<T, AF_PAD_PERIODIC>(
+                        oData, out.info, iData, in.info, lBoundPadding[0],
+                        lBoundPadding[1], lBoundPadding[2], lBoundPadding[3],
+                        groups_x, groups_y));
+                break;
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire

From b1d7f6405466773bcf07aedf64d2f393e718432e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 16:46:23 -0400
Subject: [PATCH 258/473] Fix invalid offsets in the copy kernel invocation in
 oneAPI

---
 src/backend/oneapi/copy.cpp | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index d9d2fba2c5..cd7d38396e 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -18,6 +18,13 @@
 using arrayfire::common::half;
 using arrayfire::common::is_complex;
 
+using sycl::access_mode;
+using sycl::accessor;
+using sycl::buffer;
+using sycl::id;
+using sycl::range;
+using sycl::target;
+
 namespace arrayfire {
 namespace oneapi {
 
@@ -106,11 +113,11 @@ struct copyWrapper<T, T> {
     void operator()(Array<T> &out, Array<T> const &in) {
         if (out.isLinear() && in.isLinear() &&
             out.elements() == in.elements()) {
-            dim_t in_offset  = in.getOffset() * sizeof(T);
-            dim_t out_offset = out.getOffset() * sizeof(T);
+            dim_t in_offset  = in.getOffset();
+            dim_t out_offset = out.getOffset();
 
-            const sycl::buffer<T> *in_buf = in.get();
-            sycl::buffer<T> *out_buf      = out.get();
+            sycl::buffer<T> *in_buf  = in.get();
+            sycl::buffer<T> *out_buf = out.get();
 
             getQueue()
                 .submit([=](sycl::handler &h) {
@@ -119,10 +126,11 @@ struct copyWrapper<T, T> {
                     sycl::id out_offset_id(out_offset);
 
                     auto offset_acc_in =
-                        const_cast<sycl::buffer<T> *>(in_buf)->get_access(
+                        in_buf->template get_access<access_mode::read>(
                             h, rr, in_offset_id);
                     auto offset_acc_out =
-                        out_buf->get_access(h, rr, out_offset_id);
+                        out_buf->template get_access<access_mode::write>(
+                            h, rr, out_offset_id);
 
                     h.copy(offset_acc_in, offset_acc_out);
                 })

From cd14280a3fe941dbc8a30e09aac02f082bdab37a Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 16:47:46 -0400
Subject: [PATCH 259/473] Address copy operation with unevaled arrays

---
 src/backend/oneapi/copy.cpp | 54 ++++++++++++++++++++++---------------
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index cd7d38396e..e61dbbc8db 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -69,28 +69,40 @@ Array<T> copyArray(const Array<T> &A) {
     if (A.elements() == 0) { return out; }
 
     dim_t offset = A.getOffset();
-    if (A.isLinear()) {
-        // FIXME: Add checks
-
-        const sycl::buffer<T> *A_buf = A.get();
-        sycl::buffer<T> *out_buf     = out.get();
-
-        getQueue()
-            .submit([=](sycl::handler &h) {
-                sycl::range rr(A.elements());
-                sycl::id offset_id(offset);
-                auto offset_acc_A =
-                    const_cast<sycl::buffer<T> *>(A_buf)->get_access(h, rr,
-                                                                     offset_id);
-                auto acc_out = out_buf->get_access(h);
-
-                h.copy(offset_acc_A, acc_out);
-            })
-            .wait();
+    if (A.isReady()) {
+        if (A.isLinear()) {
+            // FIXME: Add checks
+
+            sycl::buffer<T> *A_buf   = A.get();
+            sycl::buffer<T> *out_buf = out.get();
+
+            size_t aelem = A.elements();
+            getQueue()
+                .submit([=](sycl::handler &h) {
+                    range rr(aelem);
+                    id offset_id(offset);
+                    accessor offset_acc_A =
+                        A_buf->template get_access<access_mode::read>(
+                            h, rr, offset_id);
+                    accessor acc_out =
+                        out_buf->template get_access<access_mode::write>(h);
+
+                    h.copy(offset_acc_A, acc_out);
+                })
+                .wait();
+        } else {
+            kernel::memcopy<T>(out.get(), out.strides().get(), A.get(),
+                               A.dims().get(), A.strides().get(), offset,
+                               (uint)A.ndims());
+        }
     } else {
-        kernel::memcopy<T>(out.get(), out.strides().get(), A.get(),
-                           A.dims().get(), A.strides().get(), offset,
-                           (uint)A.ndims());
+        Param info = {out.get(),
+                      {{A.dims().dims[0], A.dims().dims[1], A.dims().dims[2],
+                        A.dims().dims[3]},
+                       {out.strides().dims[0], out.strides().dims[1],
+                        out.strides().dims[2], out.strides().dims[3]},
+                       0}};
+        evalNodes(info, A.getNode().get());
     }
     return out;
 }

From 1a907a6c83a560d890569178ca292341b92c6ba1 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 16:48:40 -0400
Subject: [PATCH 260/473] Workaround long long issue for the assign kernel in
 oneAPI

---
 src/backend/oneapi/kernel/assign.hpp | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 1ab8c42732..e37e95c7bd 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -76,8 +76,14 @@ class assignKernel {
         const int gy =
             g.get_local_range(1) * (g.get_group_id(1) - gw * nBBS1_) +
             it.get_local_id(1);
-        if (gx < iInfo_.dims[0] && gy < iInfo_.dims[1] && gz < iInfo_.dims[2] &&
-            gw < iInfo_.dims[3]) {
+
+        size_t idims0 = iInfo_.dims[0];
+        size_t idims1 = iInfo_.dims[1];
+        size_t idims2 = iInfo_.dims[2];
+        size_t idims3 = iInfo_.dims[3];
+
+        if (gx < idims0 && gy < idims1 && gz < idims2 &&
+            gw < idims3) {
             // calculate pointer offsets for input
             int i = p_.strds[0] *
                     trimIndex(s0 ? gx + p_.offs[0] : ptr0_[gx], oInfo_.dims[0]);

From 80d0eedcacd43047638a684f1f6f0e261c4d9713 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 16:49:51 -0400
Subject: [PATCH 261/473] Fix empty set issue error in the where function in
 oneAPI

---
 src/backend/oneapi/kernel/where.hpp | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index 64b25ec211..c5a0172134 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -148,19 +148,16 @@ static void where(Param<uint> &out, Param<T> in) {
 
     // Get output size and allocate output
     uint total;
-    sycl::buffer retBuffer(&total, {1},
-                           {sycl::property::buffer::use_host_ptr()});
 
     getQueue()
         .submit([&](sycl::handler &h) {
             auto acc_in  = rtmp.data->get_access(h, sycl::range{1},
                                                  sycl::id{rtmp_elements - 1});
-            auto acc_out = retBuffer.get_access();
-            h.copy(acc_in, acc_out);
+            h.copy(acc_in, &total);
         })
         .wait();
 
-    auto out_alloc = memAlloc<uint>(total);
+    auto out_alloc = memAlloc<uint>(std::max(1U,total));
     out.data       = out_alloc.get();
 
     out.info.dims[0]    = total;

From eecf0504b626dc4f08b13d72ae3fbd2451621ab9 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 16:51:01 -0400
Subject: [PATCH 262/473] Update Index.Docs_Util_C_API test to return -1
 instead of throw

This is a C API example which shouldn't have throw calls. I wrapped this
funciton in a lambda so it is now returning a negative number on errors. The
lambda is not exposed in the example and it is only necessary for the return
functionallity.
---
 test/index.cpp | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/test/index.cpp b/test/index.cpp
index a593348773..c8e1a7ffb9 100644
--- a/test/index.cpp
+++ b/test/index.cpp
@@ -586,12 +586,13 @@ TYPED_TEST(Indexing, 3D_to_1D) {
 }
 
 TEST(Index, Docs_Util_C_API) {
+    // clang-format off
+    ASSERT_EQ(0, ([]() -> int {
     //![ex_index_util_0]
     af_index_t *indexers = 0;
-    af_err err           = af_create_indexers(
-                  &indexers);  // Memory is allocated on heap by the callee
-    // by default all the indexers span all the elements along the given
-    // dimension
+    af_err err = af_create_indexers(&indexers); // Memory is allocated on heap by the callee
+                                                // by default all the indexers span all the elements along
+                                                // the given dimension
 
     // Create array
     af_array a;
@@ -613,12 +614,11 @@ TEST(Index, Docs_Util_C_API) {
 
     // index with indexers
     af_array out;
-    af_index_gen(&out, a, 2,
-                 indexers);  // number of indexers should be two since
-                             // we have set only second af_index_t
+    err = af_index_gen(&out, a, 2, indexers);  // number of indexers should be two since
+                                               // we have set only second af_index_t
     if (err != AF_SUCCESS) {
         printf("Failed in af_index_gen: %d\n", err);
-        throw;
+        return 1;
     }
     af_print_array(out);
     af_release_array(out);
@@ -630,7 +630,7 @@ TEST(Index, Docs_Util_C_API) {
     err = af_index_gen(&out, a, 2, indexers);
     if (err != AF_SUCCESS) {
         printf("Failed in af_index_gen: %d\n", err);
-        throw;
+        return 1;
     }
     af_print_array(out);
 
@@ -638,7 +638,10 @@ TEST(Index, Docs_Util_C_API) {
     af_release_array(a);
     af_release_array(idx);
     af_release_array(out);
+    return 0;
     //![ex_index_util_0]
+    }()));
+    // clang-format on
 }
 
 //////////////////////////////// CPP ////////////////////////////////

From bacc43e147ddc038749fb874d0a8a7b8ec6259db Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 17:27:37 -0400
Subject: [PATCH 263/473] Implement generalized indexing in the oneAPI backend

---
 src/backend/oneapi/index.cpp                  |  39 ++++-
 src/backend/oneapi/kernel/assign.hpp          |   3 +-
 .../oneapi/kernel/assign_kernel_param.hpp     |  25 +++
 src/backend/oneapi/kernel/index.hpp           | 156 ++++++++++++++++++
 4 files changed, 219 insertions(+), 4 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/assign_kernel_param.hpp
 create mode 100644 src/backend/oneapi/kernel/index.hpp

diff --git a/src/backend/oneapi/index.cpp b/src/backend/oneapi/index.cpp
index f0eb5e1cc4..bec65902d8 100644
--- a/src/backend/oneapi/index.cpp
+++ b/src/backend/oneapi/index.cpp
@@ -12,18 +12,53 @@
 #include <Array.hpp>
 #include <err_oneapi.hpp>
 #include <handle.hpp>
+#include <kernel/assign_kernel_param.hpp>
+#include <kernel/index.hpp>
 #include <memory.hpp>
 #include <af/dim4.hpp>
 
 using arrayfire::common::half;
+using arrayfire::oneapi::IndexKernelParam;
 
 namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
 Array<T> index(const Array<T>& in, const af_index_t idxrs[]) {
-    ONEAPI_NOT_SUPPORTED("Indexing not supported");
-    Array<T> out = createEmptyArray<T>(af::dim4(1));
+    IndexKernelParam p;
+    std::vector<af_seq> seqs(4, af_span);
+    // create seq vector to retrieve output
+    // dimensions, offsets & offsets
+    for (dim_t x = 0; x < 4; ++x) {
+        if (idxrs[x].isSeq) { seqs[x] = idxrs[x].idx.seq; }
+    }
+
+    // retrieve dimensions, strides and offsets
+    const dim4& iDims = in.dims();
+    dim4 dDims        = in.getDataDims();
+    dim4 oDims        = toDims(seqs, iDims);
+    dim4 iOffs        = toOffset(seqs, dDims);
+    dim4 iStrds       = in.strides();
+
+    for (dim_t i = 0; i < 4; ++i) {
+        p.isSeq[i] = idxrs[i].isSeq;
+        p.offs[i]  = iOffs[i];
+        p.strds[i] = iStrds[i];
+    }
+
+    std::vector<Array<uint>> idxArrs(4, createEmptyArray<uint>(dim4(1)));
+    // look through indexs to read af_array indexs
+    for (dim_t x = 0; x < 4; ++x) {
+        if (!p.isSeq[x]) {
+            idxArrs[x] = castArray<uint>(idxrs[x].idx.arr);
+            oDims[x]   = idxArrs[x].elements();
+        }
+    }
+
+    Array<T> out = createEmptyArray<T>(oDims);
+    if (oDims.elements() == 0) { return out; }
+    kernel::index<T>(out, in, p, idxArrs);
+
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index e37e95c7bd..27c4a58f1c 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -82,8 +82,7 @@ class assignKernel {
         size_t idims2 = iInfo_.dims[2];
         size_t idims3 = iInfo_.dims[3];
 
-        if (gx < idims0 && gy < idims1 && gz < idims2 &&
-            gw < idims3) {
+        if (gx < idims0 && gy < idims1 && gz < idims2 && gw < idims3) {
             // calculate pointer offsets for input
             int i = p_.strds[0] *
                     trimIndex(s0 ? gx + p_.offs[0] : ptr0_[gx], oInfo_.dims[0]);
diff --git a/src/backend/oneapi/kernel/assign_kernel_param.hpp b/src/backend/oneapi/kernel/assign_kernel_param.hpp
new file mode 100644
index 0000000000..e4c8a8c83a
--- /dev/null
+++ b/src/backend/oneapi/kernel/assign_kernel_param.hpp
@@ -0,0 +1,25 @@
+
+#include <sycl/sycl.hpp>
+
+#include <array>
+
+#pragma once
+
+namespace arrayfire {
+namespace oneapi {
+
+typedef struct {
+    int offs[4];
+    int strds[4];
+    bool isSeq[4];
+    std::array<sycl::accessor<unsigned int, 1, sycl::access::mode::read,
+                              sycl::access::target::device>,
+               4>
+        ptr;
+
+} AssignKernelParam;
+
+using IndexKernelParam = AssignKernelParam;
+
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/index.hpp b/src/backend/oneapi/kernel/index.hpp
new file mode 100644
index 0000000000..6e90d392ad
--- /dev/null
+++ b/src/backend/oneapi/kernel/index.hpp
@@ -0,0 +1,156 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <kernel/assign_kernel_param.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+class indexKernel {
+    sycl::accessor<T, 1, sycl::access::mode::write> out;
+    KParam outp;
+    sycl::accessor<T, 1, sycl::access::mode::read> in;
+    KParam inp;
+    IndexKernelParam p;
+    int nBBS0;
+    int nBBS1;
+
+   public:
+    indexKernel(sycl::accessor<T, 1, sycl::access::mode::write> out_,
+                KParam outp_,
+                sycl::accessor<T, 1, sycl::access::mode::read> in_, KParam inp_,
+                const IndexKernelParam p_, const int nBBS0_, const int nBBS1_)
+        : out(out_)
+        , outp(outp_)
+        , in(in_)
+        , inp(inp_)
+        , p(p_)
+        , nBBS0(nBBS0_)
+        , nBBS1(nBBS1_) {}
+
+    int trimIndex(int idx, const int len) const {
+        int ret_val = idx;
+        if (ret_val < 0) {
+            int offset = (abs(ret_val) - 1) % len;
+            ret_val    = offset;
+        } else if (ret_val >= len) {
+            int offset = abs(ret_val) % len;
+            ret_val    = len - offset - 1;
+        }
+        return ret_val;
+    }
+
+    void operator()(sycl::nd_item<3> it) const {
+        // retrieve index pointers
+        // these can be 0 where af_array index is not used
+        sycl::group g    = it.get_group();
+        const uint* ptr0 = p.ptr[0].get_pointer();
+        const uint* ptr1 = p.ptr[1].get_pointer();
+        const uint* ptr2 = p.ptr[2].get_pointer();
+        const uint* ptr3 = p.ptr[3].get_pointer();
+        // retrive booleans that tell us which index to use
+        const bool s0 = p.isSeq[0];
+        const bool s1 = p.isSeq[1];
+        const bool s2 = p.isSeq[2];
+        const bool s3 = p.isSeq[3];
+
+        const int gz = g.get_group_id(0) / nBBS0;
+        const int gx = g.get_local_range(0) * (g.get_group_id(0) - gz * nBBS0) +
+                       it.get_local_id(0);
+
+        const int gw =
+            (g.get_group_id(1) + g.get_group_id(2) * g.get_group_range(1)) /
+            nBBS1;
+        const int gy =
+            g.get_local_range(1) * ((g.get_group_id(1) +
+                                     g.get_group_id(2) * g.get_group_range(1)) -
+                                    gw * nBBS1) +
+            it.get_local_id(1);
+
+        size_t odims0 = outp.dims[0];
+        size_t odims1 = outp.dims[1];
+        size_t odims2 = outp.dims[2];
+        size_t odims3 = outp.dims[3];
+
+        if (gx < odims0 && gy < odims1 && gz < odims2 && gw < odims3) {
+            // calculate pointer offsets for input
+            int i = p.strds[0] *
+                    trimIndex(s0 ? gx + p.offs[0] : ptr0[gx], inp.dims[0]);
+            int j = p.strds[1] *
+                    trimIndex(s1 ? gy + p.offs[1] : ptr1[gy], inp.dims[1]);
+            int k = p.strds[2] *
+                    trimIndex(s2 ? gz + p.offs[2] : ptr2[gz], inp.dims[2]);
+            int l = p.strds[3] *
+                    trimIndex(s3 ? gw + p.offs[3] : ptr3[gw], inp.dims[3]);
+            // offset input and output pointers
+            const T* src = (const T*)in.get_pointer() + (i + j + k + l);
+            T* dst       = (T*)out.get_pointer() +
+                     (gx * outp.strides[0] + gy * outp.strides[1] +
+                      gz * outp.strides[2] + gw * outp.strides[3]);
+            // set the output
+            dst[0] = src[0];
+        }
+    }
+};
+
+template<typename T>
+void index(Param<T> out, Param<T> in, IndexKernelParam& p,
+           std::vector<Array<uint>>& idxArrs) {
+    sycl::range<3> threads(0, 0, 1);
+    switch (out.info.dims[1]) {
+        case 1: threads[1] = 1; break;
+        case 2: threads[1] = 2; break;
+        case 3:
+        case 4: threads[1] = 4; break;
+        default: threads[1] = 8; break;
+    }
+    threads[0] = static_cast<unsigned>(256.f / threads[1]);
+
+    int blks_x = divup(out.info.dims[0], threads[0]);
+    int blks_y = divup(out.info.dims[1], threads[1]);
+
+    sycl::range<3> blocks(blks_x * out.info.dims[2], blks_y * out.info.dims[3],
+                          1);
+
+    const size_t maxBlocksY =
+        getDevice().get_info<sycl::info::device::max_work_item_sizes<3>>()[2];
+    blocks[2] = divup(blocks[1], maxBlocksY);
+    blocks[1] = divup(blocks[1], blocks[2]) * threads[1];
+    blocks[1] = blocks[1] * threads[1];
+    blocks[0] *= threads[0];
+
+    sycl::nd_range<3> marange(blocks, threads);
+    getQueue().submit([=](sycl::handler& h) {
+        auto pp = p;
+        for (dim_t x = 0; x < 4; ++x) {
+            pp.ptr[x] =
+                idxArrs[x].get()->get_access<sycl::access::mode::read>(h);
+        }
+
+        h.parallel_for(
+            marange,
+            indexKernel<T>(
+                out.data->template get_access<sycl::access::mode::write>(h),
+                out.info,
+                in.data->template get_access<sycl::access::mode::read>(h),
+                in.info, pp, blks_x, blks_y));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire

From bb4fdb6d3bffaddea115a285c71abc3a2504fedb Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 17:28:27 -0400
Subject: [PATCH 264/473] Update assign to use the same AssignKernelParam
 object as index

---
 src/backend/oneapi/assign.cpp        |  2 +-
 src/backend/oneapi/kernel/assign.hpp | 58 +++++++++++++---------------
 2 files changed, 27 insertions(+), 33 deletions(-)

diff --git a/src/backend/oneapi/assign.cpp b/src/backend/oneapi/assign.cpp
index 0f2b96e5d5..def9378d2d 100644
--- a/src/backend/oneapi/assign.cpp
+++ b/src/backend/oneapi/assign.cpp
@@ -25,7 +25,7 @@ namespace oneapi {
 
 template<typename T>
 void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs) {
-    kernel::AssignKernelParam_t p;
+    AssignKernelParam p;
     std::vector<af_seq> seqs(4, af_span);
     // create seq vector to retrieve output
     // dimensions, offsets & offsets
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 27c4a58f1c..2bddb4cccf 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/assign_kernel_param.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -23,12 +24,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-typedef struct {
-    int offs[4];
-    int strds[4];
-    char isSeq[4];
-} AssignKernelParam_t;
-
 static int trimIndex(int idx, const int len) {
     int ret_val = idx;
     if (ret_val < 0) {
@@ -45,18 +40,13 @@ template<typename T>
 class assignKernel {
    public:
     assignKernel(sycl::accessor<T> out, KParam oInfo, sycl::accessor<T> in,
-                 KParam iInfo, AssignKernelParam_t p, sycl::accessor<uint> ptr0,
-                 sycl::accessor<uint> ptr1, sycl::accessor<uint> ptr2,
-                 sycl::accessor<uint> ptr3, const int nBBS0, const int nBBS1)
+                 KParam iInfo, AssignKernelParam p, const int nBBS0,
+                 const int nBBS1)
         : out_(out)
         , in_(in)
         , oInfo_(oInfo)
         , iInfo_(iInfo)
         , p_(p)
-        , ptr0_(ptr0)
-        , ptr1_(ptr1)
-        , ptr2_(ptr2)
-        , ptr3_(ptr3)
         , nBBS0_(nBBS0)
         , nBBS1_(nBBS1) {}
 
@@ -84,14 +74,18 @@ class assignKernel {
 
         if (gx < idims0 && gy < idims1 && gz < idims2 && gw < idims3) {
             // calculate pointer offsets for input
-            int i = p_.strds[0] *
-                    trimIndex(s0 ? gx + p_.offs[0] : ptr0_[gx], oInfo_.dims[0]);
-            int j = p_.strds[1] *
-                    trimIndex(s1 ? gy + p_.offs[1] : ptr1_[gy], oInfo_.dims[1]);
-            int k = p_.strds[2] *
-                    trimIndex(s2 ? gz + p_.offs[2] : ptr2_[gz], oInfo_.dims[2]);
-            int l = p_.strds[3] *
-                    trimIndex(s3 ? gw + p_.offs[3] : ptr3_[gw], oInfo_.dims[3]);
+            int i =
+                p_.strds[0] *
+                trimIndex(s0 ? gx + p_.offs[0] : p_.ptr[0][gx], oInfo_.dims[0]);
+            int j =
+                p_.strds[1] *
+                trimIndex(s1 ? gy + p_.offs[1] : p_.ptr[1][gy], oInfo_.dims[1]);
+            int k =
+                p_.strds[2] *
+                trimIndex(s2 ? gz + p_.offs[2] : p_.ptr[2][gz], oInfo_.dims[2]);
+            int l =
+                p_.strds[3] *
+                trimIndex(s3 ? gw + p_.offs[3] : p_.ptr[3][gw], oInfo_.dims[3]);
 
             T* iptr = in_.get_pointer();
             // offset input and output pointers
@@ -110,16 +104,16 @@ class assignKernel {
    protected:
     sycl::accessor<T> out_, in_;
     KParam oInfo_, iInfo_;
-    AssignKernelParam_t p_;
-    sycl::accessor<uint> ptr0_, ptr1_, ptr2_, ptr3_;
+    AssignKernelParam p_;
     const int nBBS0_, nBBS1_;
 };
 
 template<typename T>
-void assign(Param<T> out, const Param<T> in, const AssignKernelParam_t& p,
+void assign(Param<T> out, const Param<T> in, const AssignKernelParam& p,
             sycl::buffer<uint>* bPtr[4]) {
     constexpr int THREADS_X = 32;
     constexpr int THREADS_Y = 8;
+    using sycl::access_mode;
 
     sycl::range<2> local(THREADS_X, THREADS_Y);
 
@@ -130,18 +124,18 @@ void assign(Param<T> out, const Param<T> in, const AssignKernelParam_t& p,
                           blk_y * in.info.dims[3] * THREADS_Y);
 
     getQueue().submit([=](sycl::handler& h) {
+        auto pp      = p;
         auto out_acc = out.data->get_access(h);
         auto in_acc  = in.data->get_access(h);
 
-        auto bptr0 = bPtr[0]->get_access(h);
-        auto bptr1 = bPtr[1]->get_access(h);
-        auto bptr2 = bPtr[2]->get_access(h);
-        auto bptr3 = bPtr[3]->get_access(h);
+        pp.ptr[0] = bPtr[0]->template get_access<access_mode::read>(h);
+        pp.ptr[1] = bPtr[1]->template get_access<access_mode::read>(h);
+        pp.ptr[2] = bPtr[2]->template get_access<access_mode::read>(h);
+        pp.ptr[3] = bPtr[3]->template get_access<access_mode::read>(h);
 
-        h.parallel_for(
-            sycl::nd_range<2>(global, local),
-            assignKernel<T>(out_acc, out.info, in_acc, in.info, p, bptr0, bptr1,
-                            bptr2, bptr3, blk_x, blk_y));
+        h.parallel_for(sycl::nd_range<2>(global, local),
+                       assignKernel<T>(out_acc, out.info, in_acc, in.info, pp,
+                                       blk_x, blk_y));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }

From b3670f27ed0adff68a7459abeb283a95c4ba7dd1 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 17:28:51 -0400
Subject: [PATCH 265/473] Workaround long long compiler error for where

---
 src/backend/oneapi/kernel/where.hpp | 33 ++++++++++++++++-------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index c5a0172134..dd18189ae0 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -78,18 +78,21 @@ class whereKernel {
         iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] +
                 yid * iInfo_.strides[1];
 
-        bool cond = (yid < otInfo_.dims[1]) && (zid < otInfo_.dims[2]) &&
-                    (wid < otInfo_.dims[3]);
-        T zero = scalar<T>(0);
-
-        if (!cond) return;
-
-        uint accum = (bid == 0) ? 0 : rtptr[bid - 1];
-
-        for (uint k = 0, id = xid; k < lim_ && id < otInfo_.dims[0];
-             k++, id += g.get_local_range(0)) {
-            uint idx = otptr[id] + accum;
-            if (iptr[id] != zero) out_acc_[idx - 1] = (off + id);
+        size_t odims0 = otInfo_.dims[0];
+        size_t odims1 = otInfo_.dims[1];
+        size_t odims2 = otInfo_.dims[2];
+        size_t odims3 = otInfo_.dims[3];
+        bool cond     = (yid < odims1) && (zid < odims2) && (wid < odims3);
+        T zero        = scalar<T>(0);
+
+        if (cond) {
+            uint accum = (bid == 0) ? 0 : rtptr[bid - 1];
+
+            for (uint k = 0, id = xid; k < lim_ && id < odims0;
+                 k++, id += g.get_local_range(0)) {
+                uint idx = otptr[id] + accum;
+                if (iptr[id] != zero) out_acc_[idx - 1] = (off + id);
+            }
         }
     }
 
@@ -151,13 +154,13 @@ static void where(Param<uint> &out, Param<T> in) {
 
     getQueue()
         .submit([&](sycl::handler &h) {
-            auto acc_in  = rtmp.data->get_access(h, sycl::range{1},
-                                                 sycl::id{rtmp_elements - 1});
+            auto acc_in = rtmp.data->get_access(h, sycl::range{1},
+                                                sycl::id{rtmp_elements - 1});
             h.copy(acc_in, &total);
         })
         .wait();
 
-    auto out_alloc = memAlloc<uint>(std::max(1U,total));
+    auto out_alloc = memAlloc<uint>(std::max(1U, total));
     out.data       = out_alloc.get();
 
     out.info.dims[0]    = total;

From 04d9f8418576461c76a5d8c89610fa8810d0f1ba Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 18:06:05 -0400
Subject: [PATCH 266/473] Catch sycl exceptions in the processException
 function

---
 src/backend/common/err_common.cpp | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index c7dc95b8fd..9e2b2e8a2f 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -24,6 +24,8 @@
 #ifdef AF_OPENCL
 #include <errorcodes.hpp>
 #include <platform.hpp>
+#elif defined(AF_ONEAPI)
+#include <sycl/sycl.hpp>
 #endif
 
 using boost::stacktrace::stacktrace;
@@ -161,6 +163,14 @@ af_err processException() {
         if (is_stacktrace_enabled()) { ss << ex.getStacktrace(); }
 
         err = set_global_error_string(ss.str(), ex.getError());
+#ifdef AF_ONEAPI
+    } catch (const sycl::exception &ex) {
+        char oneapi_err_msg[1024];
+        snprintf(oneapi_err_msg, sizeof(oneapi_err_msg),
+                 "oneAPI Error (%d): %s", ex.code().value(), ex.what());
+
+        err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL);
+#endif
 #ifdef AF_OPENCL
     } catch (const cl::Error &ex) {
         char opencl_err_msg[1024];

From aafb995546e96ff559977a67e4f5116ebafbbcf8 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 11 Apr 2023 23:05:54 -0400
Subject: [PATCH 267/473] Fix iir errors and some other warnings in oneAPI

---
 src/backend/oneapi/jit.cpp                      | 2 --
 src/backend/oneapi/kernel/iir.hpp               | 7 ++++---
 src/backend/oneapi/kernel/pad_array_borders.hpp | 1 +
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 2190dd8070..4fc0e978ae 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -294,9 +294,7 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
     }
     if (numOutElems == 0) { return; }
 
-    const af::dtype outputType{output_nodes[0]->getType()};
     for (Node* node : output_nodes) {
-        assert(node->getType() == outputType);
         const int id{node->getNodesMap(nodes, full_nodes, full_ids)};
         output_ids.push_back(id);
     }
diff --git a/src/backend/oneapi/kernel/iir.hpp b/src/backend/oneapi/kernel/iir.hpp
index ab00655fec..88b515fe86 100644
--- a/src/backend/oneapi/kernel/iir.hpp
+++ b/src/backend/oneapi/kernel/iir.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <math.hpp>
 
 #include <sycl/sycl.hpp>
 
@@ -112,10 +113,10 @@ class iirKernel {
 
 template<typename T, bool batch_a>
 void iir(Param<T> y, Param<T> c, Param<T> a) {
-    const int groups_y = y.info.dims[1];
-    const int groups_x = y.info.dims[2];
+    const size_t groups_y = y.info.dims[1];
+    const size_t groups_x = y.info.dims[2];
 
-    int threads = 256;
+    size_t threads = 256;
     while (threads > y.info.dims[0] && threads > 32) threads /= 2;
     sycl::range<2> local = sycl::range{threads, 1};
 
diff --git a/src/backend/oneapi/kernel/pad_array_borders.hpp b/src/backend/oneapi/kernel/pad_array_borders.hpp
index 620352f352..129f9bf381 100644
--- a/src/backend/oneapi/kernel/pad_array_borders.hpp
+++ b/src/backend/oneapi/kernel/pad_array_borders.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <math.hpp>
 #include <af/defines.h>
 
 #include <sycl/sycl.hpp>

From 488f4e5abeea0ff007e1494e5146403bae13c06a Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Sun, 9 Apr 2023 22:02:49 +0200
Subject: [PATCH 268/473] Corrects exceptions thrown in opencl tests qr_dense,
 rank_dense & solve_dense

---
 src/backend/opencl/kernel/laset.hpp    |  2 +-
 src/backend/opencl/kernel/swapdblk.hpp |  9 ++++-----
 src/backend/opencl/magma/magma_data.h  | 14 +++++++-------
 src/backend/opencl/magma/ungqr.cpp     |  7 ++++++-
 4 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/backend/opencl/kernel/laset.hpp b/src/backend/opencl/kernel/laset.hpp
index 63e9a66526..5e4588c41f 100644
--- a/src/backend/opencl/kernel/laset.hpp
+++ b/src/backend/opencl/kernel/laset.hpp
@@ -69,7 +69,7 @@ void laset(int m, int n, T offdiag, T diag, cl_mem dA, size_t dA_offset,
     // retain the cl_mem object during cl::Buffer creation
     cl::Buffer dAObj(dA, true);
 
-    cl::CommandQueue q(queue);
+    cl::CommandQueue q(queue, true);
     lasetOp(cl::EnqueueArgs(q, global, local), m, n, offdiag, diag, dAObj,
             dA_offset, ldda);
 }
diff --git a/src/backend/opencl/kernel/swapdblk.hpp b/src/backend/opencl/kernel/swapdblk.hpp
index 0b8b43fb72..a6c96ea940 100644
--- a/src/backend/opencl/kernel/swapdblk.hpp
+++ b/src/backend/opencl/kernel/swapdblk.hpp
@@ -34,6 +34,9 @@ void swapdblk(int n, int nb, cl_mem dA, size_t dA_offset, int ldda, int inca,
     using std::string;
     using std::vector;
 
+    int nblocks = n / nb;
+    if (nblocks == 0) return;
+
     vector<TemplateArg> targs = {
         TemplateTypename<T>(),
     };
@@ -45,10 +48,6 @@ void swapdblk(int n, int nb, cl_mem dA, size_t dA_offset, int ldda, int inca,
     auto swapdblk =
         common::getKernel("swapdblk", {{swapdblk_cl_src}}, targs, compileOpts);
 
-    int nblocks = n / nb;
-
-    if (nblocks == 0) return;
-
     int info = 0;
     if (n < 0) {
         info = -1;
@@ -75,7 +74,7 @@ void swapdblk(int n, int nb, cl_mem dA, size_t dA_offset, int ldda, int inca,
     Buffer dAObj(dA, true);
     Buffer dBObj(dB, true);
 
-    CommandQueue q(queue);
+    CommandQueue q(queue, true);
     swapdblk(EnqueueArgs(q, global, local), nb, dAObj, dA_offset, ldda, inca,
              dBObj, dB_offset, lddb, incb);
     CL_DEBUG_FINISH(getQueue());
diff --git a/src/backend/opencl/magma/magma_data.h b/src/backend/opencl/magma/magma_data.h
index 69bd5e36a8..6ee5ac053e 100644
--- a/src/backend/opencl/magma/magma_data.h
+++ b/src/backend/opencl/magma/magma_data.h
@@ -79,7 +79,7 @@ static magma_int_t magma_malloc(magma_ptr* ptrPtr, int num) {
 
 // --------------------
 // Free GPU memory allocated by magma_malloc.
-static inline magma_int_t magma_free(cl_mem ptr) {
+static inline magma_int_t magma_free(magma_ptr ptr) {
     cl_int err = clReleaseMemObject(ptr);
     if (err != CL_SUCCESS) { return MAGMA_ERR_INVALID_PTR; }
     return MAGMA_SUCCESS;
@@ -321,9 +321,9 @@ static void magma_setmatrix_async(magma_int_t m, magma_int_t n, T const* hA_src,
     size_t host_orig[3]     = {0, 0, 0};
     size_t region[3]        = {m * sizeof(T), (size_t)n, 1};
     cl_int err              = clEnqueueWriteBufferRect(
-                     queue, dB_dst, CL_FALSE,  // non-blocking
-                     buffer_origin, host_orig, region, lddb * sizeof(T), 0, ldha * sizeof(T),
-                     0, hA_src, 0, NULL, event);
+        queue, dB_dst, CL_FALSE,  // non-blocking
+        buffer_origin, host_orig, region, lddb * sizeof(T), 0, ldha * sizeof(T),
+        0, hA_src, 0, NULL, event);
     clFlush(queue);
     check_error(err);
 }
@@ -357,9 +357,9 @@ static void magma_getmatrix_async(magma_int_t m, magma_int_t n, cl_mem dA_src,
     size_t host_orig[3]     = {0, 0, 0};
     size_t region[3]        = {m * sizeof(T), (size_t)n, 1};
     cl_int err              = clEnqueueReadBufferRect(
-                     queue, dA_src, CL_FALSE,  // non-blocking
-                     buffer_origin, host_orig, region, ldda * sizeof(T), 0, ldhb * sizeof(T),
-                     0, hB_dst, 0, NULL, event);
+        queue, dA_src, CL_FALSE,  // non-blocking
+        buffer_origin, host_orig, region, ldda * sizeof(T), 0, ldhb * sizeof(T),
+        0, hB_dst, 0, NULL, event);
     clFlush(queue);
     check_error(err);
 }
diff --git a/src/backend/opencl/magma/ungqr.cpp b/src/backend/opencl/magma/ungqr.cpp
index 8976758786..3f0ef001d2 100644
--- a/src/backend/opencl/magma/ungqr.cpp
+++ b/src/backend/opencl/magma/ungqr.cpp
@@ -129,7 +129,12 @@ magma_int_t magma_ungqr_gpu(magma_int_t m, magma_int_t n, magma_int_t k,
     // ((n+31)/32*32)*nb for dW larfb workspace.
     lddwork = std::min(m, n);
     cl_mem dW;
-    magma_malloc<Ty>(&dW, (((n + 31) / 32) * 32) * nb);
+    if (MAGMA_SUCCESS != magma_malloc<Ty>(&dW, (((n + 31) / 32) * 32) * nb)) {
+        magma_free_cpu(work);
+        magma_free(dV);
+        *info = MAGMA_ERR_DEVICE_ALLOC;
+        return *info;
+    }
 
     cpu_lapack_ungqr_work_func<Ty> cpu_lapack_ungqr;
 

From 0398c55d4ebc603ba28c2fefeedba947f99814bd Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 14 Apr 2023 10:07:25 -0400
Subject: [PATCH 269/473] Update clang-format version to 15 in GitHub actions

---
 .github/workflows/unix_cpu_build.yml      |  4 ++--
 src/api/unified/symbol_manager.hpp        |  2 +-
 src/backend/common/graphics_common.cpp    |  2 +-
 src/backend/cpu/convolve.cpp              |  4 ++--
 src/backend/cuda/convolveNN.cpp           |  4 ++--
 src/backend/cuda/kernel/random_engine.hpp |  4 ++--
 src/backend/oneapi/compile_module.cpp     | 17 -----------------
 src/backend/oneapi/exampleFunction.cpp    | 10 +++++-----
 src/backend/oneapi/jit.cpp                | 16 ++++++++++++++++
 src/backend/opencl/convolve.cpp           |  4 ++--
 src/backend/opencl/memory.cpp             |  4 ++--
 src/backend/opencl/svd.cpp                |  4 ++--
 src/backend/opencl/topk.cpp               |  6 +++---
 13 files changed, 40 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 01051f7e8f..3146358772 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -17,11 +17,11 @@ jobs:
               uses: actions/checkout@master
 
             - name: Check Sources
-              uses: DoozyX/clang-format-lint-action@v0.14
+              uses: DoozyX/clang-format-lint-action@v0.15
               with:
                 source: './src ./test ./examples'
                 extensions: 'h,cpp,hpp'
-                clangFormatVersion: 14
+                clangFormatVersion: 15
 
     documentation:
         name: Documentation
diff --git a/src/api/unified/symbol_manager.hpp b/src/api/unified/symbol_manager.hpp
index df5d77705c..7f96f586e2 100644
--- a/src/api/unified/symbol_manager.hpp
+++ b/src/api/unified/symbol_manager.hpp
@@ -156,7 +156,7 @@ bool checkArrays(af_backend activeBackend, T a, Args... arg) {
         if (index_ != arrayfire::unified::getActiveBackend()) {                  \
             index_ = arrayfire::unified::getActiveBackend();                     \
             func   = (af_func)arrayfire::common::getFunctionPointer(             \
-                  arrayfire::unified::getActiveHandle(), __func__);              \
+                arrayfire::unified::getActiveHandle(), __func__);              \
         }                                                                        \
         return func(__VA_ARGS__);                                                \
     } else {                                                                     \
diff --git a/src/backend/common/graphics_common.cpp b/src/backend/common/graphics_common.cpp
index 07084c43b2..217722eb36 100644
--- a/src/backend/common/graphics_common.cpp
+++ b/src/backend/common/graphics_common.cpp
@@ -260,7 +260,7 @@ fg_window ForgeManager::getMainWindow() {
             }
             fg_window w = nullptr;
             forgeError  = this->mPlugin->fg_create_window(
-                 &w, WIDTH, HEIGHT, "ArrayFire", NULL, true);
+                &w, WIDTH, HEIGHT, "ArrayFire", NULL, true);
             if (forgeError != FG_ERR_NONE) { return; }
             this->setWindowChartGrid(w, 1, 1);
             this->mPlugin->fg_make_window_current(w);
diff --git a/src/backend/cpu/convolve.cpp b/src/backend/cpu/convolve.cpp
index a57ace15f6..20138fd9e5 100644
--- a/src/backend/cpu/convolve.cpp
+++ b/src/backend/cpu/convolve.cpp
@@ -193,7 +193,7 @@ Array<T> conv2DataGradient(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     Array<T> res =
         matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE, AF_MAT_TRANS);
@@ -232,7 +232,7 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     Array<T> res =
         matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE);
diff --git a/src/backend/cuda/convolveNN.cpp b/src/backend/cuda/convolveNN.cpp
index 1110d81506..d4be5d9616 100644
--- a/src/backend/cuda/convolveNN.cpp
+++ b/src/backend/cuda/convolveNN.cpp
@@ -260,7 +260,7 @@ Array<T> data_gradient_base(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     T alpha        = scalar<T>(1.0);
     T beta         = scalar<T>(0.0);
@@ -390,7 +390,7 @@ Array<T> filter_gradient_base(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     T alpha        = scalar<T>(1.0);
     T beta         = scalar<T>(0.0);
diff --git a/src/backend/cuda/kernel/random_engine.hpp b/src/backend/cuda/kernel/random_engine.hpp
index 7fddcbfd20..07ba4163a2 100644
--- a/src/backend/cuda/kernel/random_engine.hpp
+++ b/src/backend/cuda/kernel/random_engine.hpp
@@ -60,9 +60,9 @@ static const int THREADS = 256;
 #define HALF_HALF_FACTOR __ushort_as_half(0x80)
 
 // Conversion to half adapted from Random123
-//#define SIGNED_HALF_FACTOR                                \
+// #define SIGNED_HALF_FACTOR                                \
     //((1.0f) / (std::numeric_limits<short>::max() + (1.0f)))
-//#define SIGNED_HALF_HALF_FACTOR ((0.5f) * SIGNED_HALF_FACTOR)
+// #define SIGNED_HALF_HALF_FACTOR ((0.5f) * SIGNED_HALF_FACTOR)
 //
 // NOTE: The following constants for half were calculated using the formulas
 // above. This is done so that we can avoid unnecessary computations because the
diff --git a/src/backend/oneapi/compile_module.cpp b/src/backend/oneapi/compile_module.cpp
index 2737909208..016b2d7dcf 100644
--- a/src/backend/oneapi/compile_module.cpp
+++ b/src/backend/oneapi/compile_module.cpp
@@ -71,23 +71,6 @@ string getProgramBuildLog(const kernel_bundle<bundle_state::executable> &prog) {
 namespace arrayfire {
 namespace oneapi {
 
-// const static string DEFAULT_MACROS_STR(
-// "\n\
-                                           //#ifdef USE_DOUBLE\n\
-                                           //#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n\
-                                           //#endif\n                     \
-                                           //#ifdef USE_HALF\n\
-                                           //#pragma OPENCL EXTENSION cl_khr_fp16 : enable\n\
-                                           //#else\n                     \
-                                           //#define half short\n          \
-                                           //#endif\n                      \
-                                           //#ifndef M_PI\n               \
-                                           //#define
-// M_PI 3.1415926535897932384626433832795028841971693993751058209749445923078164\n
-// \
-                                           //#endif\n                     \
-                                           //");
-
 /*
 get_kernel_bundle<>() needs sycl::context
 kernel_bundle<bundle_state::executable> buildProgram(const vector<string>
diff --git a/src/backend/oneapi/exampleFunction.cpp b/src/backend/oneapi/exampleFunction.cpp
index 9e6d81e9d5..6159d9d1d4 100644
--- a/src/backend/oneapi/exampleFunction.cpp
+++ b/src/backend/oneapi/exampleFunction.cpp
@@ -16,11 +16,11 @@
 #include <err_oneapi.hpp>  // error check functions and Macros
                            // specific to oneapi backend
 
-//#include <kernel/exampleFunction.hpp>  // this header under the folder
-// src/oneapi/kernel
-// defines the OneAPI kernel wrapper
-// function to which the main computation of your
-// algorithm should be relayed to
+// #include <kernel/exampleFunction.hpp>  // this header under the folder
+//  src/oneapi/kernel
+//  defines the OneAPI kernel wrapper
+//  function to which the main computation of your
+//  algorithm should be relayed to
 
 using af::dim4;
 
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 4fc0e978ae..0da9dfaf22 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -62,6 +62,22 @@ using sycl::backend;
 namespace arrayfire {
 
 namespace opencl {
+
+const static string DEFAULT_MACROS_STR(R"JIT(
+#ifdef USE_DOUBLE
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+#endif
+#ifdef USE_HALF
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#else
+#define half short
+#endif
+#ifndef M_PI
+#define
+ M_PI 3.1415926535897932384626433832795028841971693993751058209749445923078164
+#endif
+)JIT");
+
 string getKernelString(const string& funcName, const vector<Node*>& full_nodes,
                        const vector<Node_ids>& full_ids,
                        const vector<int>& output_ids, const bool is_linear,
diff --git a/src/backend/opencl/convolve.cpp b/src/backend/opencl/convolve.cpp
index edc28e4e35..f826102caf 100644
--- a/src/backend/opencl/convolve.cpp
+++ b/src/backend/opencl/convolve.cpp
@@ -185,7 +185,7 @@ Array<T> conv2DataGradient(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     Array<T> res =
         matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE, AF_MAT_TRANS);
@@ -224,7 +224,7 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
     Array<T> collapsed_gradient = incoming_gradient;
     collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
     collapsed_gradient          = modDims(
-                 collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
 
     Array<T> res =
         matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE);
diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index 68ae43c5e8..d2e0190431 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -204,8 +204,8 @@ size_t Allocator::getMaxMemorySize(int id) {
 void *Allocator::nativeAlloc(const size_t bytes) {
     cl_int err = CL_SUCCESS;
     auto ptr   = static_cast<void *>(clCreateBuffer(
-          getContext()(), CL_MEM_READ_WRITE,  // NOLINT(hicpp-signed-bitwise)
-          bytes, nullptr, &err));
+        getContext()(), CL_MEM_READ_WRITE,  // NOLINT(hicpp-signed-bitwise)
+        bytes, nullptr, &err));
 
     if (err != CL_SUCCESS) {
         auto str = fmt::format("Failed to allocate device memory of size {}",
diff --git a/src/backend/opencl/svd.cpp b/src/backend/opencl/svd.cpp
index 7bda5306ca..b8bea727d0 100644
--- a/src/backend/opencl/svd.cpp
+++ b/src/backend/opencl/svd.cpp
@@ -137,8 +137,8 @@ void svd(Array<T> &arrU, Array<Tr> &arrS, Array<T> &arrVT, Array<T> &arrA,
 
     if (want_vectors) {
         mappedU  = static_cast<T *>(getQueue().enqueueMapBuffer(
-             *arrU.get(), CL_FALSE, CL_MAP_WRITE, sizeof(T) * arrU.getOffset(),
-             sizeof(T) * arrU.elements()));
+            *arrU.get(), CL_FALSE, CL_MAP_WRITE, sizeof(T) * arrU.getOffset(),
+            sizeof(T) * arrU.elements()));
         mappedVT = static_cast<T *>(getQueue().enqueueMapBuffer(
             *arrVT.get(), CL_TRUE, CL_MAP_WRITE, sizeof(T) * arrVT.getOffset(),
             sizeof(T) * arrVT.elements()));
diff --git a/src/backend/opencl/topk.cpp b/src/backend/opencl/topk.cpp
index 9ff966ed65..18e03d2f0d 100644
--- a/src/backend/opencl/topk.cpp
+++ b/src/backend/opencl/topk.cpp
@@ -76,13 +76,13 @@ void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
         cl::Event ev_in, ev_val, ev_ind;
 
         T* ptr     = static_cast<T*>(getQueue().enqueueMapBuffer(
-                *in_buf, CL_FALSE, CL_MAP_READ, 0, in.elements() * sizeof(T),
-                nullptr, &ev_in));
+            *in_buf, CL_FALSE, CL_MAP_READ, 0, in.elements() * sizeof(T),
+            nullptr, &ev_in));
         uint* iptr = static_cast<uint*>(getQueue().enqueueMapBuffer(
             *ibuf, CL_FALSE, CL_MAP_READ | CL_MAP_WRITE, 0, k * sizeof(uint),
             nullptr, &ev_ind));
         T* vptr    = static_cast<T*>(getQueue().enqueueMapBuffer(
-               *vbuf, CL_FALSE, CL_MAP_WRITE, 0, k * sizeof(T), nullptr, &ev_val));
+            *vbuf, CL_FALSE, CL_MAP_WRITE, 0, k * sizeof(T), nullptr, &ev_val));
 
         vector<uint> idx(in.elements());
 

From 64586e04c8a5c7c3fe63b42c3a8514c810c18979 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 14 Apr 2023 10:48:16 -0400
Subject: [PATCH 270/473] Add macros to enable fp16 and fp64 in JIT kernels in
 oneAPI

---
 src/backend/oneapi/jit.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 0da9dfaf22..6c4d4c1828 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -231,7 +231,7 @@ __kernel void )JIT";
     }
 
     thread_local stringstream kerStream;
-    kerStream << kernelVoid << funcName << "(\n"
+    kerStream << DEFAULT_MACROS_STR << kernelVoid << funcName << "(\n"
               << inParamStream.str() << outParamStream.str() << dimParams << ")"
               << blockStart;
     if (is_linear) {

From febbe06f990abf58d7afe069cbb59cab6787da8f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 18 Apr 2023 13:15:02 -0400
Subject: [PATCH 271/473] Implement memory manager for the oneAPI backend

---
 src/backend/oneapi/Array.cpp  |  12 +--
 src/backend/oneapi/memory.cpp | 135 ++++++++++++----------------------
 src/backend/oneapi/memory.hpp |   8 +-
 3 files changed, 55 insertions(+), 100 deletions(-)

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 93c9e0df7e..4682df50f1 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -91,7 +91,7 @@ template<typename T>
 Array<T>::Array(const dim4 &dims)
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(memAlloc<T>(info.elements()).release(), bufferFree<T>)
+    , data(memAlloc<T>(info.elements()).release(), memFree<T>)
     , data_dims(dims)
     , node()
     , owner(true) {}
@@ -112,7 +112,7 @@ template<typename T>
 Array<T>::Array(const dim4 &dims, const T *const in_data)
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
-    , data(memAlloc<T>(info.elements()).release(), bufferFree<T>)
+    , data(memAlloc<T>(info.elements()).release(), memFree<T>)
     , data_dims(dims)
     , node()
     , owner(true) {
@@ -138,7 +138,7 @@ Array<T>::Array(const af::dim4 &dims, buffer<T> *const mem, size_t offset,
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
     , data(copy ? memAlloc<T>(info.elements()).release() : new buffer<T>(*mem),
-           bufferFree<T>)
+           memFree<T>)
     , data_dims(dims)
     , node()
     , owner(true) {
@@ -171,7 +171,7 @@ Array<T>::Array(Param<T> &tmp, bool owner_)
                 tmp.info.strides[3]),
            static_cast<af_dtype>(dtype_traits<T>::af_type))
     , data(
-          tmp.data, owner_ ? bufferFree<T> : [](buffer<T> * /*unused*/) {})
+          tmp.data, owner_ ? memFree<T> : [](sycl::buffer<T> * /*unused*/) {})
     , data_dims(dim4(tmp.info.dims[0], tmp.info.dims[1], tmp.info.dims[2],
                      tmp.info.dims[3]))
     , node()
@@ -205,7 +205,7 @@ void Array<T>::eval() {
 
     this->setId(getActiveDeviceId());
     data = std::shared_ptr<sycl::buffer<T>>(
-        memAlloc<T>(info.elements()).release(), bufferFree<T>);
+        memAlloc<T>(info.elements()).release(), memFree<T>);
 
     // Do not replace this with cast operator
     KParam info = {{dims()[0], dims()[1], dims()[2], dims()[3]},
@@ -256,7 +256,7 @@ void evalMultiple(vector<Array<T> *> arrays) {
 
         array->setId(getActiveDeviceId());
         array->data = std::shared_ptr<buffer<T>>(
-            memAlloc<T>(info.elements()).release(), bufferFree<T>);
+            memAlloc<T>(info.elements()).release(), memFree<T>);
 
         // Do not replace this with cast operator
         KParam kInfo = {
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index 971fa05b64..2b383b9520 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -61,27 +61,47 @@ template<typename T>
 // unique_ptr<int, function<void(int *)>> memAlloc(
 std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>>
 memAlloc(const size_t &elements) {
-    return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>(
-        new sycl::buffer<T>(sycl::range(elements)), bufferFree<T>);
+    if (elements) {
+        dim4 dims(elements * sizeof(T));
+
+        // The alloc function returns a pointer to a buffer<std::byte> object.
+        // We need to reinterpret that object into buffer<T> while keeping the
+        // same pointer value for memory accounting purposes. We acheive this
+        // assigning the renterpreted buffer back into the original pointer.
+        // This would delete the buffer<std::byte> object and replace it with
+        // the buffer<T> object. We do the reverse in the memFree function
+        auto *ptr = static_cast<sycl::buffer<std::byte> *>(
+            memoryManager().alloc(false, 1, dims.get(), 1));
+        sycl::buffer<T> *optr = static_cast<sycl::buffer<T> *>((void *)ptr);
+        size_t bytes          = ptr->byte_size();
+
+        // TODO(umar): This could be a DANGEROUS function becasue we are calling
+        // delete on the reniterpreted buffer<T> instead of the orignal
+        // buffer<byte> object
+        *optr = ptr->template reinterpret<T>(sycl::range(bytes / sizeof(T)));
+        return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>(
+            optr, memFree<T>);
+    } else {
+        return unique_ptr<sycl::buffer<T>, function<void(sycl::buffer<T> *)>>(
+            nullptr, memFree<T>);
+    }
 }
 
 void *memAllocUser(const size_t &bytes) {
-    ONEAPI_NOT_SUPPORTED("memAllocUser Not supported");
-    return nullptr;
-
-    // dim4 dims(bytes);
-    // void *ptr = memoryManager().alloc(true, 1, dims.get(), 1);
-    // auto buf  = static_cast<cl_mem>(ptr);
-    // return new cl::Buffer(buf, true);
+    dim4 dims(bytes);
+    void *ptr = memoryManager().alloc(true, 1, dims.get(), 1);
+    return ptr;
 }
 
-void memFree(void *ptr) {
-    ONEAPI_NOT_SUPPORTED("memFree Not supported");
-
-    // cl::Buffer *buf = reinterpret_cast<cl::Buffer *>(ptr);
-    // cl_mem mem      = static_cast<cl_mem>((*buf)());
-    // delete buf;
-    // return memoryManager().unlock(static_cast<void *>(mem), false);
+template<typename T>
+void memFree(sycl::buffer<T> *ptr) {
+    if (ptr) {
+        sycl::buffer<std::byte> *optr =
+            static_cast<sycl::buffer<std::byte> *>((void *)ptr);
+        size_t bytes = ptr->byte_size();
+        *optr        = ptr->template reinterpret<std::byte>(sycl::range(bytes));
+        memoryManager().unlock(optr, false);
+    }
 }
 
 void memFreeUser(void *ptr) {
@@ -90,49 +110,17 @@ void memFreeUser(void *ptr) {
     // cl::Buffer *buf = static_cast<cl::Buffer *>(ptr);
     // cl_mem mem      = (*buf)();
     // delete buf;
-    // memoryManager().unlock(mem, true);
-}
-
-template<typename T>
-sycl::buffer<T> *bufferAlloc(const size_t &bytes) {
-    ONEAPI_NOT_SUPPORTED("bufferAlloc Not supported");
-    return nullptr;
-
-    // dim4 dims(bytes);
-    // if (bytes) {
-    //     void *ptr       = memoryManager().alloc(false, 1, dims.get(), 1);
-    //     cl_mem mem      = static_cast<cl_mem>(ptr);
-    //     cl::Buffer *buf = new cl::Buffer(mem, true);
-    //     return buf;
-    // } else {
-    //     return nullptr;
-    // }
-}
-
-template<typename T>
-void bufferFree(sycl::buffer<T> *buf) {
-    if (buf) { delete buf; }
-    // if (buf) {
-    //     cl_mem mem = (*buf)();
-    //     delete buf;
-    //     memoryManager().unlock(static_cast<void *>(mem), false);
-    // }
+    memoryManager().unlock(ptr, true);
 }
 
 template<typename T>
 void memLock(const sycl::buffer<T> *ptr) {
-    ONEAPI_NOT_SUPPORTED("memLock Not supported");
-
-    // cl_mem mem = static_cast<cl_mem>((*ptr)());
-    // memoryManager().userLock(static_cast<void *>(mem));
+    memoryManager().userLock(static_cast<const void *>(ptr));
 }
 
 template<typename T>
 void memUnlock(const sycl::buffer<T> *ptr) {
-    ONEAPI_NOT_SUPPORTED("memUnlock Not supported");
-
-    // cl_mem mem = static_cast<cl_mem>((*ptr)());
-    // memoryManager().userUnlock(static_cast<void *>(mem));
+    memoryManager().userUnlock(static_cast<const void *>(ptr));
 }
 
 bool isLocked(const void *ptr) {
@@ -161,7 +149,6 @@ void pinnedFree(void *ptr) { pinnedMemoryManager().unlock(ptr, false); }
                              std::function<void(sycl::buffer<T> *)>> \
     memAlloc(const size_t &elements);                                \
     template T *pinnedAlloc(const size_t &elements);                 \
-    template void bufferFree(sycl::buffer<T> *buf);                  \
     template void memLock(const sycl::buffer<T> *buf);               \
     template void memUnlock(const sycl::buffer<T> *buf);
 
@@ -191,18 +178,7 @@ void *pinnedAlloc<void>(const size_t &elements) {
 
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
-void Allocator::shutdown() {
-    ONEAPI_NOT_SUPPORTED("Allocator::shutdown Not supported");
-
-    // for (int n = 0; n < opencl::getDeviceCount(); n++) {
-    //     try {
-    //         opencl::setDevice(n);
-    //         shutdownMemoryManager();
-    //     } catch (const AfError &err) {
-    //         continue;  // Do not throw any errors while shutting down
-    //     }
-    // }
-}
+void Allocator::shutdown() {}
 
 int Allocator::getActiveDeviceId() { return oneapi::getActiveDeviceId(); }
 
@@ -211,33 +187,16 @@ size_t Allocator::getMaxMemorySize(int id) {
 }
 
 void *Allocator::nativeAlloc(const size_t bytes) {
-    ONEAPI_NOT_SUPPORTED("Allocator::nativeAlloc Not supported");
-    return nullptr;
-
-    // cl_int err = CL_SUCCESS;
-    // auto ptr   = static_cast<void *>(clCreateBuffer(
-    //     getContext()(), CL_MEM_READ_WRITE,  // NOLINT(hicpp-signed-bitwise)
-    //     bytes, nullptr, &err));
-
-    // if (err != CL_SUCCESS) {
-    //     auto str = fmt::format("Failed to allocate device memory of size {}",
-    //                            bytesToString(bytes));
-    //     AF_ERROR(str, AF_ERR_NO_MEM);
-    // }
-
-    // AF_TRACE("nativeAlloc: {} {}", bytesToString(bytes), ptr);
-    // return ptr;
+    auto *ptr = new sycl::buffer<std::byte>(sycl::range(bytes));
+    AF_TRACE("nativeAlloc: {} {}", bytesToString(bytes),
+             static_cast<void *>(ptr));
+    return ptr;
 }
 
 void Allocator::nativeFree(void *ptr) {
-    ONEAPI_NOT_SUPPORTED("Allocator::nativeFree Not supported");
-
-    // cl_mem buffer = static_cast<cl_mem>(ptr);
-    // AF_TRACE("nativeFree:          {}", ptr);
-    // cl_int err = clReleaseMemObject(buffer);
-    // if (err != CL_SUCCESS) {
-    //     AF_ERROR("Failed to release device memory.", AF_ERR_RUNTIME);
-    // }
+    auto *buf = static_cast<sycl::buffer<std::byte> *>(ptr);
+    AF_TRACE("nativeFree:          {}", ptr);
+    delete buf;
 }
 
 AllocatorPinned::AllocatorPinned() { logger = common::loggerFactory("mem"); }
diff --git a/src/backend/oneapi/memory.hpp b/src/backend/oneapi/memory.hpp
index dea5e62f5a..ebe5f2403b 100644
--- a/src/backend/oneapi/memory.hpp
+++ b/src/backend/oneapi/memory.hpp
@@ -20,11 +20,6 @@
 
 namespace arrayfire {
 namespace oneapi {
-template<typename T>
-sycl::buffer<T> *bufferAlloc(const size_t &bytes);
-
-template<typename T>
-void bufferFree(sycl::buffer<T> *buf);
 
 template<typename T>
 using bufptr =
@@ -37,7 +32,8 @@ void *memAllocUser(const size_t &bytes);
 // Need these as 2 separate function and not a default argument
 // This is because it is used as the deleter in shared pointer
 // which cannot support default arguments
-void memFree(void *ptr);
+template<typename T>
+void memFree(sycl::buffer<T> *ptr);
 void memFreeUser(void *ptr);
 
 template<typename T>

From a3344ee5e706d8a542e72ffcae39aadb18f7e43d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 18 Apr 2023 13:38:50 -0400
Subject: [PATCH 272/473] Handle 0 element array in math and arith functions

---
 src/api/c/binary.cpp  | 22 +++++++++++++++++++++-
 src/api/c/complex.cpp |  9 ++++++++-
 src/api/c/unary.cpp   |  7 +++++++
 3 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/src/api/c/binary.cpp b/src/api/c/binary.cpp
index b9f9393421..566a4b22b5 100644
--- a/src/api/c/binary.cpp
+++ b/src/api/c/binary.cpp
@@ -128,6 +128,9 @@ static af_err af_arith(af_array *out, const af_array lhs, const af_array rhs,
 
         if (batchMode || linfo.dims() == rinfo.dims()) {
             dim4 odims = getOutDims(linfo.dims(), rinfo.dims(), batchMode);
+            if (odims.ndims() == 0) {
+                return af_create_handle(out, 0, nullptr, otype);
+            }
 
             switch (otype) {
                 case f32: res = arithOp<float, op>(lhs, rhs, odims); break;
@@ -146,6 +149,9 @@ static af_err af_arith(af_array *out, const af_array lhs, const af_array rhs,
                 default: TYPE_ERROR(0, otype);
             }
         } else {
+            if (linfo.ndims() == 0 && rinfo.ndims() == 0) {
+                return af_create_handle(out, 0, nullptr, otype);
+            }
             switch (otype) {
                 case f32: res = arithOpBroadcast<float, op>(lhs, rhs); break;
                 case f64: res = arithOpBroadcast<double, op>(lhs, rhs); break;
@@ -178,8 +184,11 @@ static af_err af_arith_real(af_array *out, const af_array lhs,
         const ArrayInfo &rinfo = getInfo(rhs);
 
         dim4 odims = getOutDims(linfo.dims(), rinfo.dims(), batchMode);
-
         const af_dtype otype = implicit(linfo.getType(), rinfo.getType());
+        if (odims.ndims() == 0) {
+            return af_create_handle(out, 0, nullptr, otype);
+        }
+
         af_array res;
         switch (otype) {
             case f32: res = arithOp<float, op>(lhs, rhs, odims); break;
@@ -462,6 +471,9 @@ af_err af_atan2(af_array *out, const af_array lhs, const af_array rhs,
         const ArrayInfo &rinfo = getInfo(rhs);
 
         dim4 odims = getOutDims(linfo.dims(), rinfo.dims(), batchMode);
+        if (odims.ndims() == 0) {
+            return af_create_handle(out, 0, nullptr, type);
+        }
 
         af_array res;
         switch (type) {
@@ -491,6 +503,10 @@ af_err af_hypot(af_array *out, const af_array lhs, const af_array rhs,
 
         dim4 odims = getOutDims(linfo.dims(), rinfo.dims(), batchMode);
 
+        if (odims.ndims() == 0) {
+            return af_create_handle(out, 0, nullptr, type);
+        }
+
         af_array res;
         switch (type) {
             case f32: res = arithOp<float, af_hypot_t>(lhs, rhs, odims); break;
@@ -523,6 +539,10 @@ static af_err af_logic(af_array *out, const af_array lhs, const af_array rhs,
 
         dim4 odims = getOutDims(linfo.dims(), rinfo.dims(), batchMode);
 
+        if (odims.ndims() == 0) {
+            return af_create_handle(out, 0, nullptr, type);
+        }
+
         af_array res;
         switch (type) {
             case f32: res = logicOp<float, op>(lhs, rhs, odims); break;
diff --git a/src/api/c/complex.cpp b/src/api/c/complex.cpp
index c7a4c4e2bc..afa24d8483 100644
--- a/src/api/c/complex.cpp
+++ b/src/api/c/complex.cpp
@@ -47,9 +47,11 @@ af_err af_cplx2(af_array *out, const af_array lhs, const af_array rhs,
         }
 
         if (type != f64) { type = f32; }
-
         dim4 odims =
             getOutDims(getInfo(lhs).dims(), getInfo(rhs).dims(), batchMode);
+        if (odims.ndims() == 0) {
+            return af_create_handle(out, 0, nullptr, type);
+        }
 
         af_array res;
         switch (type) {
@@ -72,6 +74,7 @@ af_err af_cplx(af_array *out, const af_array in) {
         if (type == c32 || type == c64) {
             AF_ERROR("Inputs to cplx2 can not be of complex type", AF_ERR_ARG);
         }
+        if (info.ndims() == 0) { return af_retain_array(out, in); }
 
         af_array tmp;
         AF_CHECK(af_constant(&tmp, 0, info.ndims(), info.dims().get(), type));
@@ -98,6 +101,7 @@ af_err af_real(af_array *out, const af_array in) {
         af_dtype type         = info.getType();
 
         if (type != c32 && type != c64) { return af_retain_array(out, in); }
+        if (info.ndims() == 0) { return af_retain_array(out, in); }
 
         af_array res;
         switch (type) {
@@ -125,6 +129,7 @@ af_err af_imag(af_array *out, const af_array in) {
         if (type != c32 && type != c64) {
             return af_constant(out, 0, info.ndims(), info.dims().get(), type);
         }
+        if (info.ndims() == 0) { return af_retain_array(out, in); }
 
         af_array res;
         switch (type) {
@@ -150,6 +155,7 @@ af_err af_conjg(af_array *out, const af_array in) {
         af_dtype type         = info.getType();
 
         if (type != c32 && type != c64) { return af_retain_array(out, in); }
+        if (info.ndims() == 0) { return af_retain_array(out, in); }
 
         af_array res;
         switch (type) {
@@ -178,6 +184,7 @@ af_err af_abs(af_array *out, const af_array in) {
         // Convert all inputs to floats / doubles
         af_dtype type = implicit(in_type, f32);
         if (in_type == f16) { type = f16; }
+        if (in_info.ndims() == 0) { return af_retain_array(out, in); }
 
         switch (type) {
             // clang-format off
diff --git a/src/api/c/unary.cpp b/src/api/c/unary.cpp
index af18031eab..6d8b584ace 100644
--- a/src/api/c/unary.cpp
+++ b/src/api/c/unary.cpp
@@ -79,6 +79,7 @@ static af_err af_unary(af_array *out, const af_array in) {
         // Convert all inputs to floats / doubles
         af_dtype type = implicit(in_type, f32);
         if (in_type == f16) { type = f16; }
+        if (in_info.ndims() == 0) { return af_retain_array(out, in); }
 
         switch (type) {
             case f16: res = unaryOp<half, op>(in); break;
@@ -104,6 +105,7 @@ static af_err af_unary_complex(af_array *out, const af_array in) {
         // Convert all inputs to floats / doubles
         af_dtype type = implicit(in_type, f32);
         if (in_type == f16) { type = f16; }
+        if (in_info.ndims() == 0) { return af_retain_array(out, in); }
 
         switch (type) {
             case f32: res = unaryOp<float, op>(in); break;
@@ -562,6 +564,7 @@ af_err af_not(af_array *out, const af_array in) {
     try {
         af_array tmp;
         const ArrayInfo &in_info = getInfo(in);
+        if (in_info.ndims() == 0) { return af_retain_array(out, in); }
 
         AF_CHECK(af_constant(&tmp, 0, in_info.ndims(), in_info.dims().get(),
                              in_info.getType()));
@@ -613,6 +616,7 @@ af_err af_bitnot(af_array *out, const af_array in) {
 af_err af_arg(af_array *out, const af_array in) {
     try {
         const ArrayInfo &in_info = getInfo(in);
+        if (in_info.ndims() == 0) { return af_retain_array(out, in); }
 
         if (!in_info.isComplex()) {
             return af_constant(out, 0, in_info.ndims(), in_info.dims().get(),
@@ -639,6 +643,7 @@ af_err af_pow2(af_array *out, const af_array in) {
     try {
         af_array two;
         const ArrayInfo &in_info = getInfo(in);
+        if (in_info.ndims() == 0) { return af_retain_array(out, in); }
 
         AF_CHECK(af_constant(&two, 2, in_info.ndims(), in_info.dims().get(),
                              in_info.getType()));
@@ -656,6 +661,7 @@ af_err af_factorial(af_array *out, const af_array in) {
     try {
         af_array one;
         const ArrayInfo &in_info = getInfo(in);
+        if (in_info.ndims() == 0) { return af_retain_array(out, in); }
 
         AF_CHECK(af_constant(&one, 1, in_info.ndims(), in_info.dims().get(),
                              in_info.getType()));
@@ -722,6 +728,7 @@ static af_err af_check(af_array *out, const af_array in) {
         // Convert all inputs to floats / doubles / complex
         af_dtype type = implicit(in_type, f32);
         if (in_type == f16) { type = f16; }
+        if (in_info.ndims() == 0) { return af_retain_array(out, in); }
 
         switch (type) {
             case f32: res = checkOp<float, op>(in); break;

From 48296c0a267609ce73146f093b213be2d3c9411f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 18 Apr 2023 13:39:33 -0400
Subject: [PATCH 273/473] Fix unroll warning in scan_first kernel

---
 src/backend/oneapi/kernel/scan_first.hpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index 8660494657..3a5b113914 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -105,7 +105,6 @@ class scanFirstKernel {
             group_barrier(g);
 
             int start = 0;
-#pragma unroll
             for (int off = 1; off < DIMX_; off *= 2) {
                 if (lidx >= off) val = binop(val, sptr[(start - off) + lidx]);
                 start              = DIMX_ - start;

From 913ff69b2c8d2cd932639d923fd18204a3e3ef7f Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 18 Apr 2023 20:49:21 -0400
Subject: [PATCH 274/473] enables join in oneapi backend

---
 src/backend/oneapi/join.cpp           | 233 ++++++++++++++++++++++++--
 src/backend/oneapi/kernel/memcopy.hpp |  31 ++--
 2 files changed, 240 insertions(+), 24 deletions(-)

diff --git a/src/backend/oneapi/join.cpp b/src/backend/oneapi/join.cpp
index 9e8aa2f743..2633c43a62 100644
--- a/src/backend/oneapi/join.cpp
+++ b/src/backend/oneapi/join.cpp
@@ -11,13 +11,18 @@
 #include <common/half.hpp>
 #include <err_oneapi.hpp>
 #include <join.hpp>
+#include <kernel/memcopy.hpp>
+#include <platform.hpp>
 
 #include <algorithm>
+#include <map>
 #include <stdexcept>
 #include <vector>
 
 using af::dim4;
 using arrayfire::common::half;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ptr;
 using std::transform;
 using std::vector;
 
@@ -33,21 +38,229 @@ dim4 calcOffset(const dim4 &dims, int dim) {
 }
 
 template<typename T>
-Array<T> join(const int dim, const Array<T> &first, const Array<T> &second) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> out = createEmptyArray<T>(af::dim4(1));
+Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
+    // All dimensions except join dimension must be equal
+    const dim4 &fdims{first.dims()};
+    const dim4 &sdims{second.dims()};
+
+    // Compute output dims
+    dim4 odims(fdims);
+    odims.dims[jdim] += sdims.dims[jdim];
+    Array<T> out = createEmptyArray<T>(odims);
+
+    // topspeed is achieved when byte size(in+out) ~= L2CacheSize
+    //
+    // 1 array: memcpy always copies 1 array.  topspeed
+    //      --> size(in) <= L2CacheSize/2
+    // 2 arrays: topspeeds
+    //      - size(in) < L2CacheSize/2/2
+    //          --> JIT can copy 2 arrays in // and is fastest
+    //              (condition: array sizes have to be identical)
+    //      - size(in) < L2CacheSize/2
+    //          --> memcpy will achieve highest speed, although the kernel
+    //              has to be called twice
+    //      - size(in) >= L2CacheSize/2
+    //          --> memcpy will achieve veryLargeArray speed.  The kernel
+    //              will be called twice
+    if (fdims.dims[jdim] == sdims.dims[jdim]) {
+        const size_t L2CacheSize{getL2CacheSize(oneapi::getDevice())};
+        if (!(first.isReady() || second.isReady()) ||
+            (fdims.elements() * sizeof(T) * 2 * 2 < L2CacheSize)) {
+            // Both arrays have same size & everything fits into the cache,
+            // so thread in 1 JIT kernel, iso individual copies which is
+            // always slower
+            const dim_t *outStrides{out.strides().dims};
+            vector<Param<T>> outputs{
+                {out.get(),
+                 {{fdims.dims[0], fdims.dims[1], fdims.dims[2], fdims.dims[3]},
+                  {outStrides[0], outStrides[1], outStrides[2], outStrides[3]},
+                  0}},
+                {out.get(),
+                 {{sdims.dims[0], sdims.dims[1], sdims.dims[2], sdims.dims[3]},
+                  {outStrides[0], outStrides[1], outStrides[2], outStrides[3]},
+                  fdims.dims[jdim] * outStrides[jdim]}}};
+            // Extend the life of the returned node, bij saving the
+            // corresponding shared_ptr
+            const Node_ptr fNode{first.getNode()};
+            const Node_ptr sNode{second.getNode()};
+            vector<Node *> nodes{fNode.get(), sNode.get()};
+            evalNodes(outputs, nodes);
+            return out;
+        }
+        // continue because individually processing is faster
+    }
+
+    // Handle each array individually
+    if (first.isReady()) {
+        if (1LL + jdim >= first.ndims() && first.isLinear()) {
+            // first & out are linear
+            getQueue()
+                .submit([=](sycl::handler &h) {
+                    sycl::range sz(first.elements());
+                    sycl::id src_offset(first.getOffset());
+                    sycl::accessor offset_acc_src =
+                        first.get()
+                            ->template get_access<sycl::access_mode::read>(
+                                h, sz, src_offset);
+                    sycl::id dst_offset(0);
+                    sycl::accessor offset_acc_dst =
+                        out.get()
+                            ->template get_access<sycl::access_mode::write>(
+                                h, sz, dst_offset);
+                    h.copy(offset_acc_src, offset_acc_dst);
+                })
+                .wait();
+        } else {
+            kernel::memcopy<T>(out.get(), out.strides().get(), first.get(),
+                               fdims.get(), first.strides().get(),
+                               first.getOffset(), first.ndims());
+        }
+    } else {
+        // Write the result directly in the out array
+        const dim_t *outStrides{out.strides().dims};
+        Param<T> output{
+            out.get(),
+            {{fdims.dims[0], fdims.dims[1], fdims.dims[2], fdims.dims[3]},
+             {outStrides[0], outStrides[1], outStrides[2], outStrides[3]},
+             0}};
+        evalNodes(output, first.getNode().get());
+    }
+
+    if (second.isReady()) {
+        if (1LL + jdim >= second.ndims() && second.isLinear()) {
+            // second & out are linear
+            getQueue()
+                .submit([=](sycl::handler &h) {
+                    sycl::range sz(second.elements());
+                    sycl::id src_offset(second.getOffset());
+                    sycl::accessor offset_acc_src =
+                        second.get()
+                            ->template get_access<sycl::access_mode::read>(
+                                h, sz, src_offset);
+                    sycl::id dst_offset(fdims.dims[jdim] *
+                                        out.strides().dims[jdim]);
+                    sycl::accessor offset_acc_dst =
+                        out.get()
+                            ->template get_access<sycl::access_mode::write>(
+                                h, sz, dst_offset);
+                    h.copy(offset_acc_src, offset_acc_dst);
+                })
+                .wait();
+        } else {
+            kernel::memcopy<T>(out.get(), out.strides().get(), second.get(),
+                               sdims.get(), second.strides().get(),
+                               second.getOffset(), second.ndims(),
+                               fdims.dims[jdim] * out.strides().dims[jdim]);
+        }
+    } else {
+        // Write the result directly in the out array
+        const dim_t *outStrides{out.strides().dims};
+        Param<T> output{
+            out.get(),
+            {{sdims.dims[0], sdims.dims[1], sdims.dims[2], sdims.dims[3]},
+             {outStrides[0], outStrides[1], outStrides[2], outStrides[3]},
+             fdims.dims[jdim] * outStrides[jdim]}};
+        evalNodes(output, second.getNode().get());
+    }
     return out;
 }
 
 template<typename T>
-void join_wrapper(const int dim, Array<T> &out,
-                  const vector<Array<T>> &inputs) {
-    ONEAPI_NOT_SUPPORTED("");
-}
+void join(Array<T> &out, const int jdim, const vector<Array<T>> &inputs) {
+    class eval {
+       public:
+        vector<Param<T>> outputs;
+        vector<Node_ptr> nodePtrs;
+        vector<Node *> nodes;
+        vector<const Array<T> *> ins;
+    };
+    std::map<dim_t, eval> evals;
+    const dim_t *ostrides{out.strides().dims};
+    const size_t L2CacheSize{getL2CacheSize(oneapi::getDevice())};
 
-template<typename T>
-void join(Array<T> &out, const int dim, const vector<Array<T>> &inputs) {
-    ONEAPI_NOT_SUPPORTED("");
+    // topspeed is achieved when byte size(in+out) ~= L2CacheSize
+    //
+    // 1 array: memcpy always copies 1 array.  topspeed
+    //      --> size(in) <= L2CacheSize/2
+    // 2 arrays: topspeeds
+    //      - size(in) < L2CacheSize/2/2
+    //          --> JIT can copy 2 arrays in // and is fastest
+    //              (condition: array sizes have to be identical)
+    //      - size(in) < L2CacheSize/2
+    //          --> memcpy will achieve highest speed, although the kernel
+    //              has to be called twice
+    //      - size(in) >= L2CacheSize/2
+    //          --> memcpy will achieve veryLargeArray speed.  The kernel
+    //              will be called twice
+
+    // Group all arrays according to size
+    dim_t outOffset{0};
+    for (const Array<T> &iArray : inputs) {
+        const dim_t *idims{iArray.dims().dims};
+        eval &e{evals[idims[jdim]]};
+        const Param output{
+            out.get(),
+            {{idims[0], idims[1], idims[2], idims[3]},
+             {ostrides[0], ostrides[1], ostrides[2], ostrides[3]},
+             outOffset}};
+        e.outputs.push_back(output);
+        // Extend life of the returned node by saving the corresponding
+        // shared_ptr
+        e.nodePtrs.emplace_back(iArray.getNode());
+        e.nodes.push_back(e.nodePtrs.back().get());
+        e.ins.push_back(&iArray);
+        outOffset += idims[jdim] * ostrides[jdim];
+    }
+
+    for (auto &eval : evals) {
+        auto &s{eval.second};
+        if (s.ins.size() == 1 ||
+            s.ins[0]->elements() * sizeof(T) * 2 * 2 > L2CacheSize) {
+            // Process (evaluate arrays) individually for
+            //  - single small array
+            //  - very large arrays
+            auto nodeIt{begin(s.nodes)};
+            auto outputIt{begin(s.outputs)};
+            for (const Array<T> *in : s.ins) {
+                if (in->isReady()) {
+                    if (1LL + jdim >= in->ndims() && in->isLinear()) {
+                        getQueue()
+                            .submit([=](sycl::handler &h) {
+                                sycl::range sz(in->elements());
+                                sycl::id src_offset(in->getOffset());
+                                sycl::accessor offset_acc_src =
+                                    in->get()
+                                        ->template get_access<
+                                            sycl::access_mode::read>(
+                                            h, sz, src_offset);
+                                sycl::id dst_offset(outputIt->info.offset);
+                                sycl::accessor offset_acc_dst =
+                                    outputIt->data->template get_access<
+                                        sycl::access_mode::write>(h, sz,
+                                                                  dst_offset);
+                                h.copy(offset_acc_src, offset_acc_dst);
+                            })
+                            .wait();
+                    } else {
+                        kernel::memcopy<T>(
+                            outputIt->data,
+                            af::dim4(4, outputIt->info.strides).get(),
+                            in->get(), in->dims().get(), in->strides().get(),
+                            in->getOffset(), in->ndims(),
+                            outputIt->info.offset);
+                    }
+                    // eliminate this array from the list, so that it will
+                    // not be processed in bulk via JIT
+                    outputIt = s.outputs.erase(outputIt);
+                    nodeIt   = s.nodes.erase(nodeIt);
+                } else {
+                    ++outputIt;
+                    ++nodeIt;
+                }
+            }
+        }
+        evalNodes(s.outputs, s.nodes);
+    }
 }
 
 #define INSTANTIATE(T)                                              \
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index c3b317ef17..482c7cd366 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -34,15 +34,16 @@ typedef struct {
 template<typename T>
 class memCopy {
    public:
-    memCopy(sycl::accessor<T> out, dims_t ostrides, sycl::accessor<T> in,
-            dims_t idims, dims_t istrides, int offset, int groups_0,
-            int groups_1)
+    memCopy(sycl::accessor<T> out, dims_t ostrides, int ooffset,
+            sycl::accessor<T> in, dims_t idims, dims_t istrides, int ioffset,
+            int groups_0, int groups_1)
         : out_(out)
-        , in_(in)
         , ostrides_(ostrides)
+        , ooffset_(ooffset)
+        , in_(in)
         , idims_(idims)
         , istrides_(istrides)
-        , offset_(offset)
+        , ioffset_(ioffset)
         , groups_0_(groups_0)
         , groups_1_(groups_1) {}
 
@@ -59,14 +60,13 @@ class memCopy {
         const int id1        = group_id_1 * gg.get_local_range(1) + lid1;
 
         T *iptr = in_.get_pointer();
-        iptr += offset_;
         // FIXME: Do more work per work group
 
         T *optr = out_.get_pointer();
         optr += id3 * ostrides_.dim[3] + id2 * ostrides_.dim[2] +
-                id1 * ostrides_.dim[1];
+                id1 * ostrides_.dim[1] + ooffset_;
         iptr += id3 * istrides_.dim[3] + id2 * istrides_.dim[2] +
-                id1 * istrides_.dim[1];
+                id1 * istrides_.dim[1] + ioffset_;
 
         int istride0 = istrides_.dim[0];
         size_t idd0  = idims_.dim[0];
@@ -81,9 +81,11 @@ class memCopy {
 
    protected:
     sycl::accessor<T> out_;
+    dims_t ostrides_;
+    int ooffset_;
     sycl::accessor<T> in_;
-    dims_t ostrides_, idims_, istrides_;
-    int offset_, groups_0_, groups_1_;
+    dims_t idims_, istrides_;
+    int ioffset_, groups_0_, groups_1_;
 };
 
 constexpr uint DIM0 = 32;
@@ -92,13 +94,14 @@ constexpr uint DIM1 = 8;
 template<typename T>
 void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
              const sycl::buffer<T> *in, const dim_t *idims,
-             const dim_t *istrides, int offset, uint ndims) {
+             const dim_t *istrides, dim_t ioffset, uint indims,
+             dim_t ooffset = 0) {
     dims_t _ostrides = {{ostrides[0], ostrides[1], ostrides[2], ostrides[3]}};
     dims_t _istrides = {{istrides[0], istrides[1], istrides[2], istrides[3]}};
     dims_t _idims    = {{idims[0], idims[1], idims[2], idims[3]}};
 
     size_t local_size[2] = {DIM0, DIM1};
-    if (ndims == 1) {
+    if (indims == 1) {
         local_size[0] *= local_size[1];
         local_size[1] = 1;
     }
@@ -116,8 +119,8 @@ void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
         auto in_acc  = const_cast<sycl::buffer<T> *>(in)->get_access(h);
 
         h.parallel_for(ndrange,
-                       memCopy<T>(out_acc, _ostrides, in_acc, _idims, _istrides,
-                                  offset, groups_0, groups_1));
+                       memCopy<T>(out_acc, _ostrides, ooffset, in_acc, _idims,
+                                  _istrides, ioffset, groups_0, groups_1));
     });
     ONEAPI_DEBUG_FINISH(getQueue());
 }

From 5042b885cc649c5e10f2d2222fc059465eef8564 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 19 Apr 2023 14:52:06 -0400
Subject: [PATCH 275/473] Update ArrayFireConfig file to enable oneAPI if
 available

---
 CMakeModules/ArrayFireConfig.cmake.in | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/CMakeModules/ArrayFireConfig.cmake.in b/CMakeModules/ArrayFireConfig.cmake.in
index 0d3cdda048..c258d19ed3 100644
--- a/CMakeModules/ArrayFireConfig.cmake.in
+++ b/CMakeModules/ArrayFireConfig.cmake.in
@@ -20,6 +20,8 @@
 #   Target for the ArrayFire CPU backend.
 # ``ArrayFire::afcuda``
 #   Target for the ArrayFire CUDA backend.
+# ``ArrayFire::afoneapi``
+#   Target for the ArrayFire oneAPI backend.
 # ``ArrayFire::afopencl``
 #   Target for the ArrayFire OpenCL backend.
 #
@@ -60,6 +62,11 @@
 # ``ArrayFire_CUDA_LIBRARIES``
 #   Location of ArrayFire's CUDA library, if found
 #
+# ``ArrayFire_oneAPI_FOUND``
+#   True of the ArrayFire oneAPI library has been found.
+# ``ArrayFire_oneAPI_LIBRARIES``
+#   Location of ArrayFire's oneAPI library, if found
+#
 # ``ArrayFire_OpenCL_FOUND``
 #   True of the ArrayFire OpenCL library has been found.
 # ``ArrayFire_OpenCL_LIBRARIES``
@@ -85,7 +92,7 @@
 
 set_and_check(ArrayFire_INCLUDE_DIRS @PACKAGE_INCLUDE_DIRS@)
 
-foreach(backend Unified CPU OpenCL CUDA)
+foreach(backend Unified CPU oneAPI OpenCL CUDA)
   if(backend STREQUAL "Unified")
     set(lowerbackend "")
   else()
@@ -140,4 +147,4 @@ foreach(_comp ${ArrayFire_FIND_COMPONENTS})
   endif()
 endforeach()
 
-check_required_components(CPU OpenCL CUDA Unified)
+check_required_components(CPU oneAPI OpenCL CUDA Unified)

From 58e4ff17e198a4fffca22a6a47a6314c39985145 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 19 Apr 2023 14:53:40 -0400
Subject: [PATCH 276/473] Enable the building of oneAPI examples

---
 examples/benchmarks/CMakeLists.txt       | 16 +++++++--
 examples/computer_vision/CMakeLists.txt  | 14 ++++++++
 examples/financial/CMakeLists.txt        | 11 ++++++
 examples/getting_started/CMakeLists.txt  | 14 ++++++++
 examples/graphics/CMakeLists.txt         | 30 ++++++++++++++++
 examples/helloworld/CMakeLists.txt       |  5 +++
 examples/image_processing/CMakeLists.txt | 44 ++++++++++++++++++++++++
 examples/lin_algebra/CMakeLists.txt      | 14 ++++++++
 examples/machine_learning/CMakeLists.txt | 32 +++++++++++++++++
 examples/pde/CMakeLists.txt              |  5 +++
 10 files changed, 183 insertions(+), 2 deletions(-)

diff --git a/examples/benchmarks/CMakeLists.txt b/examples/benchmarks/CMakeLists.txt
index c5b717f41a..d5ece4b562 100644
--- a/examples/benchmarks/CMakeLists.txt
+++ b/examples/benchmarks/CMakeLists.txt
@@ -26,7 +26,6 @@ if(ArrayFire_CPU_FOUND)
   target_link_libraries(pi_cpu ArrayFire::afcpu)
 endif()
 
-
 if(ArrayFire_CUDA_FOUND)
   add_executable(blas_cuda blas.cpp)
   target_link_libraries(blas_cuda ArrayFire::afcuda)
@@ -41,7 +40,6 @@ if(ArrayFire_CUDA_FOUND)
   target_link_libraries(pi_cuda ArrayFire::afcuda)
 endif()
 
-
 if(ArrayFire_OpenCL_FOUND)
   add_executable(blas_opencl blas.cpp)
   target_link_libraries(blas_opencl ArrayFire::afopencl)
@@ -55,3 +53,17 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(pi_opencl pi.cpp)
   target_link_libraries(pi_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(blas_oneapi blas.cpp)
+  target_link_libraries(blas_oneapi ArrayFire::afoneapi)
+
+  add_executable(cg_oneapi cg.cpp)
+  target_link_libraries(cg_oneapi ArrayFire::afoneapi)
+
+  add_executable(fft_oneapi fft.cpp)
+  target_link_libraries(fft_oneapi ArrayFire::afoneapi)
+
+  add_executable(pi_oneapi pi.cpp)
+  target_link_libraries(pi_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/computer_vision/CMakeLists.txt b/examples/computer_vision/CMakeLists.txt
index 521f7dc0a3..7314d29148 100644
--- a/examples/computer_vision/CMakeLists.txt
+++ b/examples/computer_vision/CMakeLists.txt
@@ -59,3 +59,17 @@ if (ArrayFire_OpenCL_FOUND)
   add_executable(susan_opencl susan.cpp)
   target_link_libraries(susan_opencl ArrayFire::afopencl)
 endif()
+
+if (ArrayFire_oneAPI_FOUND)
+  add_executable(fast_oneapi fast.cpp)
+  target_link_libraries(fast_oneapi ArrayFire::afoneapi)
+
+  add_executable(harris_oneapi harris.cpp)
+  target_link_libraries(harris_oneapi ArrayFire::afoneapi)
+
+  add_executable(matching_oneapi matching.cpp)
+  target_link_libraries(matching_oneapi ArrayFire::afoneapi)
+
+  add_executable(susan_oneapi susan.cpp)
+  target_link_libraries(susan_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/financial/CMakeLists.txt b/examples/financial/CMakeLists.txt
index 7c65c63595..9cc2435b25 100644
--- a/examples/financial/CMakeLists.txt
+++ b/examples/financial/CMakeLists.txt
@@ -47,3 +47,14 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(heston_model_opencl heston_model.cpp)
   target_link_libraries(heston_model_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(monte_carlo_options_oneapi monte_carlo_options.cpp)
+  target_link_libraries(monte_carlo_options_oneapi ArrayFire::afoneapi)
+
+  add_executable(black_scholes_options_oneapi black_scholes_options.cpp input.h)
+  target_link_libraries(black_scholes_options_oneapi ArrayFire::afoneapi)
+
+  add_executable(heston_model_oneapi heston_model.cpp)
+  target_link_libraries(heston_model_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/getting_started/CMakeLists.txt b/examples/getting_started/CMakeLists.txt
index 63bd043cd0..f0ee51249a 100644
--- a/examples/getting_started/CMakeLists.txt
+++ b/examples/getting_started/CMakeLists.txt
@@ -57,3 +57,17 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(vectorize_opencl vectorize.cpp)
   target_link_libraries(vectorize_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(convolve_oneapi convolve.cpp)
+  target_link_libraries(convolve_oneapi ArrayFire::afoneapi)
+
+  add_executable(integer_oneapi integer.cpp)
+  target_link_libraries(integer_oneapi ArrayFire::afoneapi)
+
+  add_executable(rainfall_oneapi rainfall.cpp)
+  target_link_libraries(rainfall_oneapi ArrayFire::afoneapi)
+
+  add_executable(vectorize_oneapi vectorize.cpp)
+  target_link_libraries(vectorize_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/graphics/CMakeLists.txt b/examples/graphics/CMakeLists.txt
index e7186cd1a7..d59a506278 100644
--- a/examples/graphics/CMakeLists.txt
+++ b/examples/graphics/CMakeLists.txt
@@ -111,3 +111,33 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(surface_opencl surface.cpp)
   target_link_libraries(surface_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(conway_oneapi conway.cpp)
+  target_link_libraries(conway_oneapi ArrayFire::afoneapi)
+
+  add_executable(conway_pretty_oneapi conway_pretty.cpp)
+  target_link_libraries(conway_pretty_oneapi ArrayFire::afoneapi)
+
+  add_executable(field_oneapi field.cpp)
+  target_link_libraries(field_oneapi ArrayFire::afoneapi)
+
+  add_executable(fractal_oneapi fractal.cpp)
+  target_link_libraries(fractal_oneapi ArrayFire::afoneapi)
+
+  add_executable(gravity_sim_oneapi gravity_sim.cpp gravity_sim_init.h)
+  target_link_libraries(gravity_sim_oneapi ArrayFire::afoneapi)
+
+  add_executable(histogram_oneapi histogram.cpp)
+  target_compile_definitions(histogram_oneapi PRIVATE "ASSETS_DIR=\"${ASSETS_DIR}\"")
+  target_link_libraries(histogram_oneapi ArrayFire::afoneapi)
+
+  add_executable(plot2d_oneapi plot2d.cpp)
+  target_link_libraries(plot2d_oneapi ArrayFire::afoneapi)
+
+  add_executable(plot3_oneapi plot3.cpp)
+  target_link_libraries(plot3_oneapi ArrayFire::afoneapi)
+
+  add_executable(surface_oneapi surface.cpp)
+  target_link_libraries(surface_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/helloworld/CMakeLists.txt b/examples/helloworld/CMakeLists.txt
index 64e9a6aa6a..3567873958 100644
--- a/examples/helloworld/CMakeLists.txt
+++ b/examples/helloworld/CMakeLists.txt
@@ -27,3 +27,8 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(helloworld_opencl helloworld.cpp)
   target_link_libraries(helloworld_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(helloworld_oneapi helloworld.cpp)
+  target_link_libraries(helloworld_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/image_processing/CMakeLists.txt b/examples/image_processing/CMakeLists.txt
index ffffe17fa7..12307b679f 100644
--- a/examples/image_processing/CMakeLists.txt
+++ b/examples/image_processing/CMakeLists.txt
@@ -156,3 +156,47 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(deconvolution_opencl deconvolution.cpp)
   target_link_libraries(deconvolution_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(adaptive_thresholding_oneapi adaptive_thresholding.cpp)
+  target_link_libraries(adaptive_thresholding_oneapi ArrayFire::afoneapi)
+
+  add_executable(binary_thresholding_oneapi binary_thresholding.cpp)
+  target_link_libraries(binary_thresholding_oneapi ArrayFire::afoneapi)
+
+  add_executable(brain_segmentation_oneapi brain_segmentation.cpp)
+  target_link_libraries(brain_segmentation_oneapi ArrayFire::afoneapi)
+
+  add_executable(confidence_connected_components_oneapi
+      confidence_connected_components.cpp)
+  target_link_libraries(confidence_connected_components_oneapi ArrayFire::afoneapi)
+
+  add_executable(edge_oneapi edge.cpp)
+  target_link_libraries(edge_oneapi ArrayFire::afoneapi)
+
+  add_executable(filters_oneapi filters.cpp)
+  target_link_libraries(filters_oneapi ArrayFire::afoneapi)
+
+  add_executable(image_demo_oneapi image_demo.cpp)
+  target_link_libraries(image_demo_oneapi ArrayFire::afoneapi)
+
+  add_executable(image_editing_oneapi image_editing.cpp)
+  target_link_libraries(image_editing_oneapi ArrayFire::afoneapi)
+
+  add_executable(morphing_oneapi morphing.cpp)
+  target_link_libraries(morphing_oneapi ArrayFire::afoneapi)
+
+  add_executable(optical_flow_oneapi optical_flow.cpp)
+  target_link_libraries(optical_flow_oneapi ArrayFire::afoneapi)
+
+  add_executable(pyramids_oneapi pyramids.cpp)
+  target_link_libraries(pyramids_oneapi ArrayFire::afoneapi)
+
+  # Gradient anisotropic diffusion example
+  add_executable(gradient_diffusion_oneapi gradient_diffusion.cpp)
+  target_link_libraries(gradient_diffusion_oneapi ArrayFire::afoneapi)
+
+  #Image Deconvolution Example
+  add_executable(deconvolution_oneapi deconvolution.cpp)
+  target_link_libraries(deconvolution_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/lin_algebra/CMakeLists.txt b/examples/lin_algebra/CMakeLists.txt
index 59aa2cbcd9..baba1a4181 100644
--- a/examples/lin_algebra/CMakeLists.txt
+++ b/examples/lin_algebra/CMakeLists.txt
@@ -57,3 +57,17 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(svd_opencl svd.cpp)
   target_link_libraries(svd_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(cholesky_oneapi cholesky.cpp)
+  target_link_libraries(cholesky_oneapi ArrayFire::afoneapi)
+
+  add_executable(lu_oneapi lu.cpp)
+  target_link_libraries(lu_oneapi ArrayFire::afoneapi)
+
+  add_executable(qr_oneapi qr.cpp)
+  target_link_libraries(qr_oneapi ArrayFire::afoneapi)
+
+  add_executable(svd_oneapi svd.cpp)
+  target_link_libraries(svd_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/machine_learning/CMakeLists.txt b/examples/machine_learning/CMakeLists.txt
index 136e9338a0..9c2c3ade6c 100644
--- a/examples/machine_learning/CMakeLists.txt
+++ b/examples/machine_learning/CMakeLists.txt
@@ -119,3 +119,35 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(softmax_regression_opencl softmax_regression.cpp)
   target_link_libraries(softmax_regression_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(bagging_oneapi bagging.cpp)
+  target_link_libraries(bagging_oneapi ArrayFire::afoneapi)
+
+  add_executable(deep_belief_net_oneapi deep_belief_net.cpp)
+  target_link_libraries(deep_belief_net_oneapi ArrayFire::afoneapi)
+
+  add_executable(geneticalgorithm_oneapi geneticalgorithm.cpp)
+  target_link_libraries(geneticalgorithm_oneapi ArrayFire::afoneapi)
+
+  add_executable(kmeans_oneapi kmeans.cpp)
+  target_link_libraries(kmeans_oneapi ArrayFire::afoneapi)
+
+  add_executable(logistic_regression_oneapi logistic_regression.cpp)
+  target_link_libraries(logistic_regression_oneapi ArrayFire::afoneapi)
+
+  add_executable(naive_bayes_oneapi naive_bayes.cpp)
+  target_link_libraries(naive_bayes_oneapi ArrayFire::afoneapi)
+
+  add_executable(neural_network_oneapi neural_network.cpp)
+  target_link_libraries(neural_network_oneapi ArrayFire::afoneapi)
+
+  add_executable(perceptron_oneapi perceptron.cpp)
+  target_link_libraries(perceptron_oneapi ArrayFire::afoneapi)
+
+  add_executable(rbm_oneapi rbm.cpp)
+  target_link_libraries(rbm_oneapi ArrayFire::afoneapi)
+
+  add_executable(softmax_regression_oneapi softmax_regression.cpp)
+  target_link_libraries(softmax_regression_oneapi ArrayFire::afoneapi)
+endif()
diff --git a/examples/pde/CMakeLists.txt b/examples/pde/CMakeLists.txt
index 345afeabfb..0b74e6165f 100644
--- a/examples/pde/CMakeLists.txt
+++ b/examples/pde/CMakeLists.txt
@@ -27,3 +27,8 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(swe_opencl swe.cpp)
   target_link_libraries(swe_opencl ArrayFire::afopencl)
 endif()
+
+if(ArrayFire_oneAPI_FOUND)
+  add_executable(swe_oneapi swe.cpp)
+  target_link_libraries(swe_oneapi ArrayFire::afoneapi)
+endif()

From e0f042342baa4abb84268951a9a526691253d266 Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Mon, 24 Apr 2023 15:42:36 -0400
Subject: [PATCH 277/473] convolve2 and convolve_separable oneapi ports (#3409)

* convolve2 (not separable) stubs filled in. half is not supported

---------

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
---
 src/backend/oneapi/CMakeLists.txt             |   1 +
 src/backend/oneapi/convolve.cpp               | 115 +++++++++-
 src/backend/oneapi/convolve_separable.cpp     |  35 ++-
 src/backend/oneapi/kernel/convolve2.hpp       |  10 +
 src/backend/oneapi/kernel/convolve3.hpp       |  10 +
 .../oneapi/kernel/convolve_separable.cpp      | 217 ++++++++++++++++++
 .../oneapi/kernel/convolve_separable.hpp      |  29 +++
 7 files changed, 403 insertions(+), 14 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/convolve_separable.cpp
 create mode 100644 src/backend/oneapi/kernel/convolve_separable.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 831234a5a8..b1ab64d87e 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -214,6 +214,7 @@ target_sources(afoneapi
     kernel/approx2.hpp
     kernel/assign.hpp
     kernel/bilateral.hpp
+    kernel/convolve_separable.cpp
     kernel/diagonal.hpp
     kernel/diff.hpp
     kernel/histogram.hpp
diff --git a/src/backend/oneapi/convolve.cpp b/src/backend/oneapi/convolve.cpp
index ac940f501d..69c120569b 100644
--- a/src/backend/oneapi/convolve.cpp
+++ b/src/backend/oneapi/convolve.cpp
@@ -110,8 +110,38 @@ template<typename T>
 Array<T> convolve2_unwrap(const Array<T> &signal, const Array<T> &filter,
                           const dim4 &stride, const dim4 &padding,
                           const dim4 &dilation) {
-    Array<T> out =
-        convolve2_unwrap<T>(signal, filter, stride, padding, dilation);
+    dim4 sDims = signal.dims();
+    dim4 fDims = filter.dims();
+
+    dim_t outputWidth =
+        1 + (sDims[0] + 2 * padding[0] - (((fDims[0] - 1) * dilation[0]) + 1)) /
+                stride[0];
+    dim_t outputHeight =
+        1 + (sDims[1] + 2 * padding[1] - (((fDims[1] - 1) * dilation[1]) + 1)) /
+                stride[1];
+
+    const bool retCols = false;
+    Array<T> unwrapped =
+        unwrap(signal, fDims[0], fDims[1], stride[0], stride[1], padding[0],
+               padding[1], dilation[0], dilation[1], retCols);
+
+    unwrapped  = reorder(unwrapped, dim4(1, 2, 0, 3));
+    dim4 uDims = unwrapped.dims();
+
+    unwrapped =
+        modDims(unwrapped, dim4(uDims[0] * uDims[1], uDims[2] * uDims[3]));
+
+    Array<T> collapsedFilter = filter;
+
+    collapsedFilter = flip(collapsedFilter, {1, 1, 0, 0});
+    collapsedFilter = modDims(collapsedFilter,
+                              dim4(fDims[0] * fDims[1] * fDims[2], fDims[3]));
+
+    Array<T> res =
+        matmul(unwrapped, collapsedFilter, AF_MAT_TRANS, AF_MAT_NONE);
+    res = modDims(res, dim4(outputWidth, outputHeight, signal.dims()[3],
+                            collapsedFilter.dims()[1]));
+    Array<T> out = reorder(res, dim4(0, 1, 3, 2));
 
     return out;
 }
@@ -119,9 +149,15 @@ Array<T> convolve2_unwrap(const Array<T> &signal, const Array<T> &filter,
 template<typename T>
 Array<T> convolve2(Array<T> const &signal, Array<T> const &filter,
                    const dim4 stride, const dim4 padding, const dim4 dilation) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> out = createEmptyArray<T>(dim4(1));
-    return out;
+    if constexpr (!std::is_same<T, half>::value) {
+        Array<T> out =
+            convolve2_unwrap<T>(signal, filter, stride, padding, dilation);
+        return out;
+    } else {
+        ONEAPI_NOT_SUPPORTED("");
+        Array<T> out = createEmptyArray<T>(dim4(1));
+        return out;
+    }
 }
 
 #define INSTANTIATE(T)                                                        \
@@ -141,9 +177,39 @@ Array<T> conv2DataGradient(const Array<T> &incoming_gradient,
                            const Array<T> & /*convolved_output*/,
                            af::dim4 stride, af::dim4 padding,
                            af::dim4 dilation) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> out = createEmptyArray<T>(dim4(1));
-    return out;
+    if constexpr (!std::is_same<T, half>::value) {
+        const dim4 &cDims = incoming_gradient.dims();
+        const dim4 &sDims = original_signal.dims();
+        const dim4 &fDims = original_filter.dims();
+
+        Array<T> collapsed_filter = original_filter;
+
+        collapsed_filter = flip(collapsed_filter, {1, 1, 0, 0});
+        collapsed_filter = modDims(
+            collapsed_filter, dim4(fDims[0] * fDims[1] * fDims[2], fDims[3]));
+
+        Array<T> collapsed_gradient = incoming_gradient;
+        collapsed_gradient = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
+        collapsed_gradient = modDims(
+            collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+
+        Array<T> res = matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE,
+                              AF_MAT_TRANS);
+        res          = modDims(res, dim4(res.dims()[0] / sDims[3], sDims[3],
+                                         fDims[0] * fDims[1], sDims[2]));
+        res          = reorder(res, dim4(0, 2, 3, 1));
+
+        const bool retCols = false;
+        res = wrap_dilated(res, sDims[0], sDims[1], fDims[0], fDims[1],
+                           stride[0], stride[1], padding[0], padding[1],
+                           dilation[0], dilation[1], retCols);
+
+        return res;
+    } else {
+        ONEAPI_NOT_SUPPORTED("");
+        Array<T> out = createEmptyArray<T>(dim4(1));
+        return out;
+    }
 }
 
 template<typename T>
@@ -153,9 +219,36 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
                              const Array<T> & /*convolved_output*/,
                              af::dim4 stride, af::dim4 padding,
                              af::dim4 dilation) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> out = createEmptyArray<T>(dim4(1));
-    return out;
+    if constexpr (!std::is_same<T, half>::value) {
+        const dim4 &cDims = incoming_gradient.dims();
+        const dim4 &fDims = original_filter.dims();
+
+        const bool retCols = false;
+        Array<T> unwrapped =
+            unwrap(original_signal, fDims[0], fDims[1], stride[0], stride[1],
+                   padding[0], padding[1], dilation[0], dilation[1], retCols);
+
+        unwrapped  = reorder(unwrapped, dim4(1, 2, 0, 3));
+        dim4 uDims = unwrapped.dims();
+        unwrapped =
+            modDims(unwrapped, dim4(uDims[0] * uDims[1], uDims[2] * uDims[3]));
+
+        Array<T> collapsed_gradient = incoming_gradient;
+        collapsed_gradient = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
+        collapsed_gradient = modDims(
+            collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+
+        Array<T> res =
+            matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE);
+        res = modDims(res, dim4(fDims[0], fDims[1], fDims[2], fDims[3]));
+
+        auto out = flip(res, {1, 1, 0, 0});
+        return out;
+    } else {
+        ONEAPI_NOT_SUPPORTED("");
+        Array<T> out = createEmptyArray<T>(dim4(1));
+        return out;
+    }
 }
 
 #define INSTANTIATE(T)                                                      \
diff --git a/src/backend/oneapi/convolve_separable.cpp b/src/backend/oneapi/convolve_separable.cpp
index 969aff66e2..fdf9fc952f 100644
--- a/src/backend/oneapi/convolve_separable.cpp
+++ b/src/backend/oneapi/convolve_separable.cpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2022, ArrayFire
+ * Copyright (c) 2023, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -11,6 +11,7 @@
 
 #include <Array.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/convolve_separable.hpp>
 #include <af/dim4.hpp>
 
 using af::dim4;
@@ -21,8 +22,36 @@ namespace oneapi {
 template<typename T, typename accT>
 Array<T> convolve2(Array<T> const& signal, Array<accT> const& c_filter,
                    Array<accT> const& r_filter, const bool expand) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> out = createEmptyArray<T>(dim4(1));
+    const auto cflen = c_filter.elements();
+    const auto rflen = r_filter.elements();
+
+    if ((cflen > kernel::MAX_SCONV_FILTER_LEN) ||
+        (rflen > kernel::MAX_SCONV_FILTER_LEN)) {
+        // TODO call upon fft
+        char errMessage[256];
+        snprintf(errMessage, sizeof(errMessage),
+                 "\noneAPI Separable convolution doesn't support %llu(coloumn) "
+                 "%llu(row) filters\n",
+                 cflen, rflen);
+        ONEAPI_NOT_SUPPORTED(errMessage);
+    }
+
+    const dim4& sDims = signal.dims();
+    dim4 tDims        = sDims;
+    dim4 oDims        = sDims;
+
+    if (expand) {
+        tDims[0] += cflen - 1;
+        oDims[0] += cflen - 1;
+        oDims[1] += rflen - 1;
+    }
+
+    Array<T> temp = createEmptyArray<T>(tDims);
+    Array<T> out  = createEmptyArray<T>(oDims);
+
+    kernel::convSep<T, accT>(temp, signal, c_filter, 0, expand);
+    kernel::convSep<T, accT>(out, temp, r_filter, 1, expand);
+
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/convolve2.hpp b/src/backend/oneapi/kernel/convolve2.hpp
index fc5db9c06a..b216e50917 100644
--- a/src/backend/oneapi/kernel/convolve2.hpp
+++ b/src/backend/oneapi/kernel/convolve2.hpp
@@ -1,3 +1,13 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
 template<typename T, typename aT>
 class conv2HelperCreateKernel {
    public:
diff --git a/src/backend/oneapi/kernel/convolve3.hpp b/src/backend/oneapi/kernel/convolve3.hpp
index 30861a2a63..3ac4a50aa2 100644
--- a/src/backend/oneapi/kernel/convolve3.hpp
+++ b/src/backend/oneapi/kernel/convolve3.hpp
@@ -1,3 +1,13 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
 int index(int i, int j, int k, int jstride, int kstride) {
     return i + j * jstride + k * kstride;
 }
diff --git a/src/backend/oneapi/kernel/convolve_separable.cpp b/src/backend/oneapi/kernel/convolve_separable.cpp
new file mode 100644
index 0000000000..712570a558
--- /dev/null
+++ b/src/backend/oneapi/kernel/convolve_separable.cpp
@@ -0,0 +1,217 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
+
+template<typename T, typename accType>
+class convolveSeparableCreateKernel {
+   public:
+    convolveSeparableCreateKernel(write_accessor<T> out, KParam oInfo,
+                                  read_accessor<T> signal, KParam sInfo,
+                                  read_accessor<accType> impulse, int nBBS0,
+                                  int nBBS1, const int FLEN, const int CONV_DIM,
+                                  const bool EXPAND,
+                                  sycl::local_accessor<T> localMem)
+        : out_(out)
+        , oInfo_(oInfo)
+        , signal_(signal)
+        , sInfo_(sInfo)
+        , impulse_(impulse)
+        , nBBS0_(nBBS0)
+        , nBBS1_(nBBS1)
+        , FLEN_(FLEN)
+        , CONV_DIM_(CONV_DIM)
+        , EXPAND_(EXPAND)
+        , localMem_(localMem) {}
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        const int radius  = FLEN_ - 1;
+        const int padding = 2 * radius;
+        const int s0      = sInfo_.strides[0];
+        const int s1      = sInfo_.strides[1];
+        const int d0      = sInfo_.dims[0];
+        const int d1      = sInfo_.dims[1];
+        const int shrdLen =
+            g.get_local_range(0) + (CONV_DIM_ == 0 ? padding : 0);
+
+        unsigned b2 = g.get_group_id(0) / nBBS0_;
+        unsigned b3 = g.get_group_id(1) / nBBS1_;
+        T *dst      = out_.get_pointer() +
+                 (b2 * oInfo_.strides[2] + b3 * oInfo_.strides[3]);
+        const T *src = signal_.get_pointer() +
+                       (b2 * sInfo_.strides[2] + b3 * sInfo_.strides[3]) +
+                       sInfo_.offset;
+
+        int lx = it.get_local_id(0);
+        int ly = it.get_local_id(1);
+        int ox = g.get_local_range(0) * (g.get_group_id(0) - b2 * nBBS0_) + lx;
+        int oy = g.get_local_range(1) * (g.get_group_id(1) - b3 * nBBS1_) + ly;
+        int gx = ox;
+        int gy = oy;
+
+        // below if-else statement is based on MACRO value passed while kernel
+        // compilation
+        if (CONV_DIM_ == 0) {
+            gx += (EXPAND_ ? 0 : FLEN_ >> 1);
+            int endX = ((FLEN_ - 1) << 1) + g.get_local_range(0);
+#pragma unroll
+            for (int lx = it.get_local_id(0), glb_x = gx; lx < endX;
+                 lx += g.get_local_range(0), glb_x += g.get_local_range(0)) {
+                int i     = glb_x - radius;
+                int j     = gy;
+                bool is_i = i >= 0 && i < d0;
+                bool is_j = j >= 0 && j < d1;
+                localMem_[ly * shrdLen + lx] =
+                    (is_i && is_j ? src[i * s0 + j * s1] : (T)(0));
+            }
+
+        } else if (CONV_DIM_ == 1) {
+            gy += (EXPAND_ ? 0 : FLEN_ >> 1);
+            int endY = ((FLEN_ - 1) << 1) + g.get_local_range(1);
+#pragma unroll
+            for (int ly = it.get_local_id(1), glb_y = gy; ly < endY;
+                 ly += g.get_local_range(1), glb_y += g.get_local_range(1)) {
+                int i     = gx;
+                int j     = glb_y - radius;
+                bool is_i = i >= 0 && i < d0;
+                bool is_j = j >= 0 && j < d1;
+                localMem_[ly * shrdLen + lx] =
+                    (is_i && is_j ? src[i * s0 + j * s1] : (T)(0));
+            }
+        }
+        it.barrier();
+
+        if (ox < oInfo_.dims[0] && oy < oInfo_.dims[1]) {
+            // below conditional statement is based on MACRO value passed while
+            // kernel compilation
+            int i         = (CONV_DIM_ == 0 ? lx : ly) + radius;
+            accType accum = (accType)(0);
+#pragma unroll
+            for (int f = 0; f < FLEN_; ++f) {
+                accType f_val = impulse_[f];
+                // below conditional statement is based on MACRO value passed
+                // while kernel compilation
+                int s_idx = (CONV_DIM_ == 0 ? (ly * shrdLen + (i - f))
+                                            : ((i - f) * shrdLen + lx));
+                T s_val   = localMem_[s_idx];
+
+                // binOp omitted from OpenCL implementation (see
+                // convolve_separable.cl)
+                accum = accum + (accType)s_val * (accType)f_val;
+            }
+            dst[oy * oInfo_.strides[1] + ox] = (T)accum;
+        }
+    }
+
+   private:
+    write_accessor<T> out_;
+    KParam oInfo_;
+    read_accessor<T> signal_;
+    KParam sInfo_;
+    read_accessor<accType> impulse_;
+    int nBBS0_;
+    int nBBS1_;
+    const int FLEN_;
+    const int CONV_DIM_;
+    const bool EXPAND_;
+    sycl::local_accessor<T> localMem_;
+};
+
+template<typename T>
+void memcpyBuffer(sycl::buffer<T, 1> &dest, sycl::buffer<T, 1> &src,
+                  const size_t n, const size_t srcOffset) {
+    getQueue().submit([&](auto &h) {
+        sycl::accessor srcAcc{src, h, sycl::range{n}, sycl::id{srcOffset},
+                              sycl::read_only};
+        sycl::accessor destAcc{
+            dest,         h, sycl::range{n}, sycl::id{0}, sycl::write_only,
+            sycl::no_init};
+        h.copy(srcAcc, destAcc);
+    });
+}
+
+template<typename T, typename accType>
+void convSep(Param<T> out, const Param<T> signal, const Param<accType> filter,
+             const int conv_dim, const bool expand) {
+    if (!(conv_dim == 0 || conv_dim == 1)) {
+        AF_ERROR(
+            "Separable convolution accepts only 0 or 1 as convolution "
+            "dimension",
+            AF_ERR_NOT_SUPPORTED);
+    }
+    constexpr int THREADS_X = 16;
+    constexpr int THREADS_Y = 16;
+    constexpr bool IsComplex =
+        std::is_same<T, cfloat>::value || std::is_same<T, cdouble>::value;
+
+    const int fLen       = filter.info.dims[0] * filter.info.dims[1];
+    const size_t C0_SIZE = (THREADS_X + 2 * (fLen - 1)) * THREADS_Y;
+    const size_t C1_SIZE = (THREADS_Y + 2 * (fLen - 1)) * THREADS_X;
+    size_t locSize       = (conv_dim == 0 ? C0_SIZE : C1_SIZE);
+
+    auto local = sycl::range(THREADS_X, THREADS_Y);
+
+    int blk_x = divup(out.info.dims[0], THREADS_X);
+    int blk_y = divup(out.info.dims[1], THREADS_Y);
+
+    auto global = sycl::range(blk_x * signal.info.dims[2] * THREADS_X,
+                              blk_y * signal.info.dims[3] * THREADS_Y);
+
+    sycl::buffer<accType> mBuff = {sycl::range(fLen * sizeof(accType))};
+    memcpyBuffer(mBuff, *filter.data, fLen, 0);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_signal{*signal.data, h, sycl::read_only};
+        sycl::accessor d_out{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor d_mBuff{mBuff, h, sycl::read_only};
+        sycl::local_accessor<T> localMem(locSize, h);
+        h.parallel_for(sycl::nd_range{global, local},
+                       convolveSeparableCreateKernel<T, accType>(
+                           d_out, out.info, d_signal, signal.info, d_mBuff,
+                           blk_x, blk_y, fLen, conv_dim, expand, localMem));
+    });
+}
+
+#define INSTANTIATE(T, accT)                                          \
+    template void convSep<T, accT>(Param<T>, const Param<T>,          \
+                                   const Param<accT> filt, const int, \
+                                   const bool);
+
+INSTANTIATE(cdouble, cdouble)
+INSTANTIATE(cfloat, cfloat)
+INSTANTIATE(double, double)
+INSTANTIATE(float, float)
+INSTANTIATE(uint, float)
+INSTANTIATE(int, float)
+INSTANTIATE(uchar, float)
+INSTANTIATE(char, float)
+INSTANTIATE(ushort, float)
+INSTANTIATE(short, float)
+INSTANTIATE(uintl, float)
+INSTANTIATE(intl, float)
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/convolve_separable.hpp b/src/backend/oneapi/kernel/convolve_separable.hpp
new file mode 100644
index 0000000000..0339c9c614
--- /dev/null
+++ b/src/backend/oneapi/kernel/convolve_separable.hpp
@@ -0,0 +1,29 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+// below shared MAX_*_LEN's are calculated based on
+// a maximum shared memory configuration of 48KB per block
+// considering complex types as well
+constexpr int MAX_SCONV_FILTER_LEN = 31;
+
+template<typename T, typename accT>
+void convSep(Param<T> out, const Param<T> sig, const Param<accT> filt,
+             const int cDim, const bool expand);
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire

From c6f5947213c0aec5ecb592271a40c6037ed31aa4 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 26 Apr 2023 02:07:52 -0400
Subject: [PATCH 278/473] corrects missing oneapi accessor semantics

---
 src/backend/oneapi/CMakeLists.txt             |   1 +
 src/backend/oneapi/join.cpp                   |  90 +++---
 src/backend/oneapi/kernel/accessors.hpp       |  17 ++
 src/backend/oneapi/kernel/approx1.hpp         |  12 +-
 src/backend/oneapi/kernel/approx2.hpp         |  12 +-
 src/backend/oneapi/kernel/assign.hpp          |  12 +-
 .../oneapi/kernel/assign_kernel_param.hpp     |  12 +-
 src/backend/oneapi/kernel/bilateral.hpp       |  33 +--
 src/backend/oneapi/kernel/convolve.hpp        |   7 +-
 src/backend/oneapi/kernel/diagonal.hpp        |  30 +-
 src/backend/oneapi/kernel/diff.hpp            |  16 +-
 src/backend/oneapi/kernel/gradient.hpp        |  15 +-
 src/backend/oneapi/kernel/histogram.hpp       |  27 +-
 src/backend/oneapi/kernel/identity.hpp        |   4 +-
 src/backend/oneapi/kernel/iir.hpp             |   8 +-
 src/backend/oneapi/kernel/index.hpp           |  12 +-
 src/backend/oneapi/kernel/interp.hpp          |   9 +-
 src/backend/oneapi/kernel/iota.hpp            |  34 +--
 src/backend/oneapi/kernel/ireduce.hpp         | 275 +++++++++---------
 src/backend/oneapi/kernel/lookup.hpp          |   6 +-
 src/backend/oneapi/kernel/lu_split.hpp        |   6 +-
 src/backend/oneapi/kernel/mean.hpp            |  44 ++-
 src/backend/oneapi/kernel/meanshift.hpp       |   6 +-
 src/backend/oneapi/kernel/memcopy.hpp         |  27 +-
 .../oneapi/kernel/pad_array_borders.hpp       |   6 +-
 src/backend/oneapi/kernel/random_engine.hpp   |  25 +-
 .../oneapi/kernel/random_engine_mersenne.hpp  |  49 ++--
 .../oneapi/kernel/random_engine_philox.hpp    |   9 +-
 .../oneapi/kernel/random_engine_threefry.hpp  |   9 +-
 src/backend/oneapi/kernel/range.hpp           |   7 +-
 src/backend/oneapi/kernel/reduce.hpp          |   1 +
 src/backend/oneapi/kernel/reduce_all.hpp      |  27 +-
 src/backend/oneapi/kernel/reduce_dim.hpp      |  21 +-
 src/backend/oneapi/kernel/reduce_first.hpp    |  22 +-
 src/backend/oneapi/kernel/reorder.hpp         |   6 +-
 src/backend/oneapi/kernel/resize.hpp          |   6 +-
 src/backend/oneapi/kernel/rotate.hpp          |   6 +-
 src/backend/oneapi/kernel/scan_dim.hpp        |  26 +-
 src/backend/oneapi/kernel/scan_first.hpp      |  23 +-
 src/backend/oneapi/kernel/select.hpp          |   6 +-
 src/backend/oneapi/kernel/tile.hpp            |   6 +-
 src/backend/oneapi/kernel/transform.hpp       |   6 +-
 src/backend/oneapi/kernel/transpose.hpp       |  12 +-
 .../oneapi/kernel/transpose_inplace.hpp       |  21 +-
 src/backend/oneapi/kernel/triangle.hpp        |  16 +-
 src/backend/oneapi/kernel/where.hpp           |   9 +-
 src/backend/oneapi/kernel/wrap.hpp            |  10 +-
 src/backend/oneapi/kernel/wrap_dilated.hpp    |   9 +-
 48 files changed, 439 insertions(+), 614 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/accessors.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index b1ab64d87e..8ea40564e9 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -210,6 +210,7 @@ add_library(afoneapi
 target_sources(afoneapi
   PRIVATE
     kernel/KParam.hpp
+    kernel/accessors.hpp
     kernel/approx1.hpp
     kernel/approx2.hpp
     kernel/assign.hpp
diff --git a/src/backend/oneapi/join.cpp b/src/backend/oneapi/join.cpp
index 2633c43a62..ecbcae0ba4 100644
--- a/src/backend/oneapi/join.cpp
+++ b/src/backend/oneapi/join.cpp
@@ -94,22 +94,18 @@ Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     if (first.isReady()) {
         if (1LL + jdim >= first.ndims() && first.isLinear()) {
             // first & out are linear
-            getQueue()
-                .submit([=](sycl::handler &h) {
-                    sycl::range sz(first.elements());
-                    sycl::id src_offset(first.getOffset());
-                    sycl::accessor offset_acc_src =
-                        first.get()
-                            ->template get_access<sycl::access_mode::read>(
-                                h, sz, src_offset);
-                    sycl::id dst_offset(0);
-                    sycl::accessor offset_acc_dst =
-                        out.get()
-                            ->template get_access<sycl::access_mode::write>(
-                                h, sz, dst_offset);
-                    h.copy(offset_acc_src, offset_acc_dst);
-                })
-                .wait();
+            getQueue().submit([=](sycl::handler &h) {
+                sycl::range sz(first.elements());
+                sycl::id src_offset(first.getOffset());
+                sycl::accessor offset_acc_src =
+                    first.get()->template get_access<sycl::access_mode::read>(
+                        h, sz, src_offset);
+                sycl::id dst_offset(0);
+                sycl::accessor offset_acc_dst =
+                    out.get()->template get_access<sycl::access_mode::write>(
+                        h, sz, dst_offset);
+                h.copy(offset_acc_src, offset_acc_dst);
+            });
         } else {
             kernel::memcopy<T>(out.get(), out.strides().get(), first.get(),
                                fdims.get(), first.strides().get(),
@@ -129,23 +125,19 @@ Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     if (second.isReady()) {
         if (1LL + jdim >= second.ndims() && second.isLinear()) {
             // second & out are linear
-            getQueue()
-                .submit([=](sycl::handler &h) {
-                    sycl::range sz(second.elements());
-                    sycl::id src_offset(second.getOffset());
-                    sycl::accessor offset_acc_src =
-                        second.get()
-                            ->template get_access<sycl::access_mode::read>(
-                                h, sz, src_offset);
-                    sycl::id dst_offset(fdims.dims[jdim] *
-                                        out.strides().dims[jdim]);
-                    sycl::accessor offset_acc_dst =
-                        out.get()
-                            ->template get_access<sycl::access_mode::write>(
-                                h, sz, dst_offset);
-                    h.copy(offset_acc_src, offset_acc_dst);
-                })
-                .wait();
+            getQueue().submit([=](sycl::handler &h) {
+                sycl::range sz(second.elements());
+                sycl::id src_offset(second.getOffset());
+                sycl::accessor offset_acc_src =
+                    second.get()->template get_access<sycl::access_mode::read>(
+                        h, sz, src_offset);
+                sycl::id dst_offset(fdims.dims[jdim] *
+                                    out.strides().dims[jdim]);
+                sycl::accessor offset_acc_dst =
+                    out.get()->template get_access<sycl::access_mode::write>(
+                        h, sz, dst_offset);
+                h.copy(offset_acc_src, offset_acc_dst);
+            });
         } else {
             kernel::memcopy<T>(out.get(), out.strides().get(), second.get(),
                                sdims.get(), second.strides().get(),
@@ -224,23 +216,21 @@ void join(Array<T> &out, const int jdim, const vector<Array<T>> &inputs) {
             for (const Array<T> *in : s.ins) {
                 if (in->isReady()) {
                     if (1LL + jdim >= in->ndims() && in->isLinear()) {
-                        getQueue()
-                            .submit([=](sycl::handler &h) {
-                                sycl::range sz(in->elements());
-                                sycl::id src_offset(in->getOffset());
-                                sycl::accessor offset_acc_src =
-                                    in->get()
-                                        ->template get_access<
-                                            sycl::access_mode::read>(
-                                            h, sz, src_offset);
-                                sycl::id dst_offset(outputIt->info.offset);
-                                sycl::accessor offset_acc_dst =
-                                    outputIt->data->template get_access<
-                                        sycl::access_mode::write>(h, sz,
-                                                                  dst_offset);
-                                h.copy(offset_acc_src, offset_acc_dst);
-                            })
-                            .wait();
+                        getQueue().submit([=](sycl::handler &h) {
+                            sycl::range sz(in->elements());
+                            sycl::id src_offset(in->getOffset());
+                            sycl::accessor offset_acc_src =
+                                in->get()
+                                    ->template get_access<
+                                        sycl::access_mode::read>(h, sz,
+                                                                 src_offset);
+                            sycl::id dst_offset(outputIt->info.offset);
+                            sycl::accessor offset_acc_dst =
+                                outputIt->data->template get_access<
+                                    sycl::access_mode::write>(h, sz,
+                                                              dst_offset);
+                            h.copy(offset_acc_src, offset_acc_dst);
+                        });
                     } else {
                         kernel::memcopy<T>(
                             outputIt->data,
diff --git a/src/backend/oneapi/kernel/accessors.hpp b/src/backend/oneapi/kernel/accessors.hpp
new file mode 100644
index 0000000000..902f48b0e0
--- /dev/null
+++ b/src/backend/oneapi/kernel/accessors.hpp
@@ -0,0 +1,17 @@
+/*******************************************************
+ * Copyright (c) 2022 ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+#include <sycl/sycl.hpp>
+
+template<typename T>
+using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
+
+template<typename T>
+using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
diff --git a/src/backend/oneapi/kernel/approx1.hpp b/src/backend/oneapi/kernel/approx1.hpp
index 3f0e2cfbe5..ed2290ffc9 100644
--- a/src/backend/oneapi/kernel/approx1.hpp
+++ b/src/backend/oneapi/kernel/approx1.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/interp.hpp>
 #include <traits.hpp>
 #include <af/constants.h>
@@ -30,17 +31,6 @@ constexpr int TILE_DIM  = 32;
 constexpr int THREADS_X = TILE_DIM;
 constexpr int THREADS_Y = 256 / TILE_DIM;
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename Ty, typename Tp, int order>
 class approx1Kernel {
    public:
diff --git a/src/backend/oneapi/kernel/approx2.hpp b/src/backend/oneapi/kernel/approx2.hpp
index 8713d87d20..c173b527b1 100644
--- a/src/backend/oneapi/kernel/approx2.hpp
+++ b/src/backend/oneapi/kernel/approx2.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/interp.hpp>
 #include <traits.hpp>
 #include <af/constants.h>
@@ -30,17 +31,6 @@ constexpr int TILE_DIM  = 32;
 constexpr int THREADS_X = TILE_DIM;
 constexpr int THREADS_Y = 256 / TILE_DIM;
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename Ty, typename Tp, int order>
 class approx2Kernel {
    public:
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 2bddb4cccf..6d553f18ad 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/assign_kernel_param.hpp>
 #include <traits.hpp>
 
@@ -39,7 +40,7 @@ static int trimIndex(int idx, const int len) {
 template<typename T>
 class assignKernel {
    public:
-    assignKernel(sycl::accessor<T> out, KParam oInfo, sycl::accessor<T> in,
+    assignKernel(write_accessor<T> out, KParam oInfo, read_accessor<T> in,
                  KParam iInfo, AssignKernelParam p, const int nBBS0,
                  const int nBBS1)
         : out_(out)
@@ -102,7 +103,8 @@ class assignKernel {
     }
 
    protected:
-    sycl::accessor<T> out_, in_;
+    write_accessor<T> out_;
+    read_accessor<T> in_;
     KParam oInfo_, iInfo_;
     AssignKernelParam p_;
     const int nBBS0_, nBBS1_;
@@ -124,9 +126,9 @@ void assign(Param<T> out, const Param<T> in, const AssignKernelParam& p,
                           blk_y * in.info.dims[3] * THREADS_Y);
 
     getQueue().submit([=](sycl::handler& h) {
-        auto pp      = p;
-        auto out_acc = out.data->get_access(h);
-        auto in_acc  = in.data->get_access(h);
+        auto pp = p;
+        write_accessor<T> out_acc{*out.data, h};
+        read_accessor<T> in_acc{*in.data, h};
 
         pp.ptr[0] = bPtr[0]->template get_access<access_mode::read>(h);
         pp.ptr[1] = bPtr[1]->template get_access<access_mode::read>(h);
diff --git a/src/backend/oneapi/kernel/assign_kernel_param.hpp b/src/backend/oneapi/kernel/assign_kernel_param.hpp
index e4c8a8c83a..e2539ed2b3 100644
--- a/src/backend/oneapi/kernel/assign_kernel_param.hpp
+++ b/src/backend/oneapi/kernel/assign_kernel_param.hpp
@@ -1,10 +1,18 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
 
 #include <sycl/sycl.hpp>
 
 #include <array>
 
-#pragma once
-
 namespace arrayfire {
 namespace oneapi {
 
diff --git a/src/backend/oneapi/kernel/bilateral.hpp b/src/backend/oneapi/kernel/bilateral.hpp
index 8c340ccb81..210c92e911 100644
--- a/src/backend/oneapi/kernel/bilateral.hpp
+++ b/src/backend/oneapi/kernel/bilateral.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -24,11 +25,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 template<typename outType, bool USE_NATIVE_EXP>
 auto exp_native_nonnative(float in) {
     if constexpr (USE_NATIVE_EXP)
@@ -40,10 +36,10 @@ auto exp_native_nonnative(float in) {
 template<typename outType, typename inType, bool USE_NATIVE_EXP>
 class bilateralKernel {
    public:
-    bilateralKernel(sycl::accessor<outType> d_dst, KParam oInfo,
-                    sycl::accessor<inType> d_src, KParam iInfo,
-                    local_accessor<outType, 1> localMem,
-                    local_accessor<outType, 1> gauss2d, float sigma_space,
+    bilateralKernel(write_accessor<outType> d_dst, KParam oInfo,
+                    read_accessor<inType> d_src, KParam iInfo,
+                    sycl::local_accessor<outType, 1> localMem,
+                    sycl::local_accessor<outType, 1> gauss2d, float sigma_space,
                     float sigma_color, int gaussOff, int nBBS0, int nBBS1)
         : d_dst_(d_dst)
         , oInfo_(oInfo)
@@ -148,7 +144,7 @@ class bilateralKernel {
         return (v < lo) ? lo : (hi < v) ? hi : v;
     }
 
-    void load2LocalMem(local_accessor<outType, 1> shrd, const inType* in,
+    void load2LocalMem(sycl::local_accessor<outType, 1> shrd, const inType* in,
                        int lx, int ly, int shrdStride, int dim0, int dim1,
                        int gx, int gy, int inStride1, int inStride0) const {
         int gx_ = sycl::clamp(gx, 0, dim0 - 1);
@@ -158,12 +154,12 @@ class bilateralKernel {
     }
 
    private:
-    sycl::accessor<outType> d_dst_;
+    write_accessor<outType> d_dst_;
     KParam oInfo_;
-    sycl::accessor<inType> d_src_;
+    read_accessor<inType> d_src_;
     KParam iInfo_;
-    local_accessor<outType, 1> localMem_;
-    local_accessor<outType, 1> gauss2d_;
+    sycl::local_accessor<outType, 1> localMem_;
+    sycl::local_accessor<outType, 1> gauss2d_;
     float sigma_space_;
     float sigma_color_;
     int gaussOff_;
@@ -203,18 +199,17 @@ void bilateral(Param<outType> out, const Param<inType> in, const float s_sigma,
     }
 
     getQueue().submit([&](sycl::handler& h) {
-        auto inAcc  = in.data->get_access(h);
-        auto outAcc = out.data->get_access(h);
+        read_accessor<inType> inAcc{*in.data, h};
+        write_accessor<outType> outAcc{*out.data, h};
 
-        auto localMem = local_accessor<outType, 1>(num_shrd_elems, h);
-        auto gauss2d  = local_accessor<outType, 1>(num_shrd_elems, h);
+        auto localMem = sycl::local_accessor<outType, 1>(num_shrd_elems, h);
+        auto gauss2d  = sycl::local_accessor<outType, 1>(num_shrd_elems, h);
 
         h.parallel_for(sycl::nd_range{global, local},
                        bilateralKernel<outType, inType, UseNativeExp>(
                            outAcc, out.info, inAcc, in.info, localMem, gauss2d,
                            s_sigma, c_sigma, num_shrd_elems, blk_x, blk_y));
     });
-
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 
diff --git a/src/backend/oneapi/kernel/convolve.hpp b/src/backend/oneapi/kernel/convolve.hpp
index 276c84c3af..ebec7dbe88 100644
--- a/src/backend/oneapi/kernel/convolve.hpp
+++ b/src/backend/oneapi/kernel/convolve.hpp
@@ -9,10 +9,10 @@
 
 #pragma once
 #include <Param.hpp>
-#include <accessor.hpp>
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <af/defines.h>
 
 #include <sycl/sycl.hpp>
@@ -109,11 +109,6 @@ void memcpyBuffer(sycl::buffer<T, 1> &dest, sycl::buffer<T, 1> &src,
     });
 }
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 #include "convolve1.hpp"
 #include "convolve2.hpp"
 #include "convolve3.hpp"
diff --git a/src/backend/oneapi/kernel/diagonal.hpp b/src/backend/oneapi/kernel/diagonal.hpp
index 8da78dba70..91db3fbda1 100644
--- a/src/backend/oneapi/kernel/diagonal.hpp
+++ b/src/backend/oneapi/kernel/diagonal.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -24,16 +25,11 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 template<typename T>
 class diagCreateKernel {
    public:
-    diagCreateKernel(sycl::accessor<T> oData, KParam oInfo,
-                     sycl::accessor<T> iData, KParam iInfo, int num,
+    diagCreateKernel(write_accessor<T> oData, KParam oInfo,
+                     read_accessor<T> iData, KParam iInfo, int num,
                      int groups_x)
         : oData_(oData)
         , oInfo_(oInfo)
@@ -65,9 +61,9 @@ class diagCreateKernel {
     }
 
    private:
-    sycl::accessor<T> oData_;
+    write_accessor<T> oData_;
     KParam oInfo_;
-    sycl::accessor<T> iData_;
+    read_accessor<T> iData_;
     KParam iInfo_;
     int num_;
     int groups_x_;
@@ -82,8 +78,8 @@ static void diagCreate(Param<T> out, Param<T> in, int num) {
                               groups_y * local[1]};
 
     getQueue().submit([&](sycl::handler &h) {
-        auto oData = out.data->get_access(h);
-        auto iData = in.data->get_access(h);
+        write_accessor<T> oData{*out.data, h};
+        read_accessor<T> iData{*in.data, h};
 
         h.parallel_for(sycl::nd_range{global, local},
                        diagCreateKernel<T>(oData, out.info, iData, in.info, num,
@@ -95,8 +91,8 @@ static void diagCreate(Param<T> out, Param<T> in, int num) {
 template<typename T>
 class diagExtractKernel {
    public:
-    diagExtractKernel(sycl::accessor<T> oData, KParam oInfo,
-                      sycl::accessor<T> iData, KParam iInfo, int num,
+    diagExtractKernel(write_accessor<T> oData, KParam oInfo,
+                      read_accessor<T> iData, KParam iInfo, int num,
                       int groups_z)
         : oData_(oData)
         , oInfo_(oInfo)
@@ -133,9 +129,9 @@ class diagExtractKernel {
     }
 
    private:
-    sycl::accessor<T> oData_;
+    write_accessor<T> oData_;
     KParam oInfo_;
-    sycl::accessor<T> iData_;
+    read_accessor<T> iData_;
     KParam iInfo_;
     int num_;
     int groups_z_;
@@ -150,8 +146,8 @@ static void diagExtract(Param<T> out, Param<T> in, int num) {
                               groups_z * local[1] * out.info.dims[3]};
 
     getQueue().submit([&](sycl::handler &h) {
-        auto oData = out.data->get_access(h);
-        auto iData = in.data->get_access(h);
+        write_accessor<T> oData{*out.data, h};
+        read_accessor<T> iData{*in.data, h};
 
         h.parallel_for(sycl::nd_range{global, local},
                        diagExtractKernel<T>(oData, out.info, iData, in.info,
diff --git a/src/backend/oneapi/kernel/diff.hpp b/src/backend/oneapi/kernel/diff.hpp
index 478da588c0..5276786646 100644
--- a/src/backend/oneapi/kernel/diff.hpp
+++ b/src/backend/oneapi/kernel/diff.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -24,15 +25,10 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 template<typename T>
 class diffKernel {
    public:
-    diffKernel(sycl::accessor<T> outAcc, const sycl::accessor<T> inAcc,
+    diffKernel(write_accessor<T> outAcc, const read_accessor<T> inAcc,
                const KParam op, const KParam ip, const int oElem,
                const int blocksPerMatX, const int blocksPerMatY,
                const bool isDiff2, const unsigned DIM)
@@ -82,8 +78,8 @@ class diffKernel {
     }
 
    private:
-    sycl::accessor<T> outAcc_;
-    const sycl::accessor<T> inAcc_;
+    write_accessor<T> outAcc_;
+    const read_accessor<T> inAcc_;
     const KParam op_;
     const KParam ip_;
     const int oElem_;
@@ -111,8 +107,8 @@ void diff(Param<T> out, const Param<T> in, const unsigned indims,
                       out.info.dims[3];
 
     getQueue().submit([&](sycl::handler &h) {
-        auto inAcc  = in.data->get_access(h);
-        auto outAcc = out.data->get_access(h);
+        read_accessor<T> inAcc   = {*in.data, h};
+        write_accessor<T> outAcc = {*out.data, h};
 
         h.parallel_for(
             sycl::nd_range{global, local},
diff --git a/src/backend/oneapi/kernel/gradient.hpp b/src/backend/oneapi/kernel/gradient.hpp
index 7f29b4cec3..f8ae841444 100644
--- a/src/backend/oneapi/kernel/gradient.hpp
+++ b/src/backend/oneapi/kernel/gradient.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/default_config.hpp>
 
 #include <sycl/sycl.hpp>
@@ -21,14 +22,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
-                                      sycl::access::target::local>;
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 #define sidx(y, x) scratch_[((y + 1) * (TX + 2)) + (x + 1)]
 
 template<typename T, int TX, int TY>
@@ -38,7 +31,7 @@ class gradientCreateKernel {
                          write_accessor<T> d_grad1, const KParam grad1,
                          read_accessor<T> d_in, const KParam in,
                          const int blocksPerMatX, const int blocksPerMatY,
-                         local_accessor<T> scratch)
+                         sycl::local_accessor<T> scratch)
         : d_grad0_(d_grad0)
         , grad0_(grad0)
         , d_grad1_(d_grad1)
@@ -132,7 +125,7 @@ class gradientCreateKernel {
     const KParam in_;
     const int blocksPerMatX_;
     const int blocksPerMatY_;
-    local_accessor<T> scratch_;
+    sycl::local_accessor<T> scratch_;
 };
 
 template<typename T>
@@ -151,7 +144,7 @@ void gradient(Param<T> grad0, Param<T> grad1, const Param<T> in) {
         write_accessor<T> grad0Acc{*grad0.data, h};
         write_accessor<T> grad1Acc{*grad1.data, h};
         read_accessor<T> inAcc{*in.data, h};
-        auto scratch = local_accessor<T>((TY + 2) * (TX + 2), h);
+        auto scratch = sycl::local_accessor<T>((TY + 2) * (TX + 2), h);
         h.parallel_for(sycl::nd_range{global, local},
                        gradientCreateKernel<T, TX, TY>(
                            grad0Acc, grad0.info, grad1Acc, grad1.info, inAcc,
diff --git a/src/backend/oneapi/kernel/histogram.hpp b/src/backend/oneapi/kernel/histogram.hpp
index 35f21fc9b6..bd574c9e2d 100644
--- a/src/backend/oneapi/kernel/histogram.hpp
+++ b/src/backend/oneapi/kernel/histogram.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -42,18 +43,14 @@ using global_atomic_ref =
     sycl::atomic_ref<T, sycl::memory_order::relaxed, sycl::memory_scope::system,
                      sycl::access::address_space::global_space>;
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 template<typename T>
 class histogramKernel {
    public:
-    histogramKernel(sycl::accessor<uint> d_dst, KParam oInfo,
-                    const sycl::accessor<T> d_src, KParam iInfo,
-                    local_accessor<uint, 1> localMemAcc, int len, int nbins,
-                    float minval, float maxval, int nBBS, const bool isLinear)
+    histogramKernel(write_accessor<uint> d_dst, KParam oInfo,
+                    const read_accessor<T> d_src, KParam iInfo,
+                    sycl::local_accessor<uint, 1> localMemAcc, int len,
+                    int nbins, float minval, float maxval, int nBBS,
+                    const bool isLinear)
         : d_dst_(d_dst)
         , oInfo_(oInfo)
         , d_src_(d_src)
@@ -118,11 +115,11 @@ class histogramKernel {
     }
 
    private:
-    sycl::accessor<uint> d_dst_;
+    write_accessor<uint> d_dst_;
     KParam oInfo_;
-    sycl::accessor<T> d_src_;
+    read_accessor<T> d_src_;
     KParam iInfo_;
-    local_accessor<uint, 1> localMemAcc_;
+    sycl::local_accessor<uint, 1> localMemAcc_;
     int len_;
     int nbins_;
     float minval_;
@@ -144,10 +141,10 @@ void histogram(Param<uint> out, const Param<T> in, int nbins, float minval,
     auto global          = sycl::range{global0, global1};
 
     getQueue().submit([&](sycl::handler &h) {
-        auto inAcc  = in.data->get_access(h);
-        auto outAcc = out.data->get_access(h);
+        read_accessor<T> inAcc{*in.data, h};
+        write_accessor<uint> outAcc{*out.data, h};
 
-        auto localMem = local_accessor<uint, 1>(locSize, h);
+        auto localMem = sycl::local_accessor<uint, 1>(locSize, h);
 
         h.parallel_for(
             sycl::nd_range{global, local},
diff --git a/src/backend/oneapi/kernel/identity.hpp b/src/backend/oneapi/kernel/identity.hpp
index 20553a2149..0f6911606a 100644
--- a/src/backend/oneapi/kernel/identity.hpp
+++ b/src/backend/oneapi/kernel/identity.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <math.hpp>
 #include <types.hpp>
 
@@ -19,9 +20,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 class identityKernel {
    public:
diff --git a/src/backend/oneapi/kernel/iir.hpp b/src/backend/oneapi/kernel/iir.hpp
index 88b515fe86..38769ad46a 100644
--- a/src/backend/oneapi/kernel/iir.hpp
+++ b/src/backend/oneapi/kernel/iir.hpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2023, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <math.hpp>
 
 #include <sycl/sycl.hpp>
@@ -20,11 +21,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 constexpr int MAX_A_SIZE = 1024;
 
 template<typename T, bool batch_a>
diff --git a/src/backend/oneapi/kernel/index.hpp b/src/backend/oneapi/kernel/index.hpp
index 6e90d392ad..ef2b837b75 100644
--- a/src/backend/oneapi/kernel/index.hpp
+++ b/src/backend/oneapi/kernel/index.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/assign_kernel_param.hpp>
 
 namespace arrayfire {
@@ -20,19 +21,18 @@ namespace kernel {
 
 template<typename T>
 class indexKernel {
-    sycl::accessor<T, 1, sycl::access::mode::write> out;
+    write_accessor<T> out;
     KParam outp;
-    sycl::accessor<T, 1, sycl::access::mode::read> in;
+    read_accessor<T> in;
     KParam inp;
     IndexKernelParam p;
     int nBBS0;
     int nBBS1;
 
    public:
-    indexKernel(sycl::accessor<T, 1, sycl::access::mode::write> out_,
-                KParam outp_,
-                sycl::accessor<T, 1, sycl::access::mode::read> in_, KParam inp_,
-                const IndexKernelParam p_, const int nBBS0_, const int nBBS1_)
+    indexKernel(write_accessor<T> out_, KParam outp_, read_accessor<T> in_,
+                KParam inp_, const IndexKernelParam p_, const int nBBS0_,
+                const int nBBS1_)
         : out(out_)
         , outp(outp_)
         , in(in_)
diff --git a/src/backend/oneapi/kernel/interp.hpp b/src/backend/oneapi/kernel/interp.hpp
index 516acea466..bfc894dfdf 100644
--- a/src/backend/oneapi/kernel/interp.hpp
+++ b/src/backend/oneapi/kernel/interp.hpp
@@ -7,7 +7,10 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#pragma once
+
 #include <Param.hpp>
+#include <kernel/accessors.hpp>
 #include <math.hpp>
 #include <types.hpp>
 #include <af/constants.h>
@@ -19,12 +22,6 @@
 namespace arrayfire {
 namespace oneapi {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 struct itype_t {
     typedef float wtype;
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index 87dbfc923c..97018b6a1d 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 #include <types.hpp>
 #include <af/dim4.hpp>
@@ -29,7 +30,7 @@ namespace kernel {
 template<typename T>
 class iotaKernel {
    public:
-    iotaKernel(sycl::accessor<T> out, KParam oinfo, const int s0, const int s1,
+    iotaKernel(write_accessor<T> out, KParam oinfo, const int s0, const int s1,
                const int s2, const int s3, const int blocksPerMatX,
                const int blocksPerMatY)
         : out_(out)
@@ -79,7 +80,7 @@ class iotaKernel {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     KParam oinfo_;
     int s0_, s1_, s2_, s3_;
     int blocksPerMatX_, blocksPerMatY_;
@@ -100,24 +101,17 @@ void iota(Param<T> out, const af::dim4& sdims) {
                           local[1] * blocksPerMatY * out.info.dims[3]);
     sycl::nd_range<2> ndrange(global, local);
 
-    try {
-        getQueue()
-            .submit([=](sycl::handler& h) {
-                auto out_acc = out.data->get_access(h);
-
-                h.parallel_for(
-                    ndrange,
-                    iotaKernel<T>(out_acc, out.info, static_cast<int>(sdims[0]),
-                                  static_cast<int>(sdims[1]),
-                                  static_cast<int>(sdims[2]),
-                                  static_cast<int>(sdims[3]), blocksPerMatX,
-                                  blocksPerMatY));
-            })
-            .wait();
-        ONEAPI_DEBUG_FINISH(getQueue());
-    } catch (sycl::exception& e) {
-        std::cout << e.what() << std::endl;
-    } catch (std::exception& e) { std::cout << e.what() << std::endl; }
+    getQueue().submit([=](sycl::handler& h) {
+        write_accessor<T> out_acc{*out.data, h};
+
+        h.parallel_for(ndrange, iotaKernel<T>(out_acc, out.info,
+                                              static_cast<int>(sdims[0]),
+                                              static_cast<int>(sdims[1]),
+                                              static_cast<int>(sdims[2]),
+                                              static_cast<int>(sdims[3]),
+                                              blocksPerMatX, blocksPerMatY));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 }  // namespace kernel
diff --git a/src/backend/oneapi/kernel/ireduce.hpp b/src/backend/oneapi/kernel/ireduce.hpp
index e047826b08..0c6ae70383 100644
--- a/src/backend/oneapi/kernel/ireduce.hpp
+++ b/src/backend/oneapi/kernel/ireduce.hpp
@@ -8,12 +8,14 @@
  ********************************************************/
 
 #pragma once
+
 #include <Param.hpp>
 #include <backend.hpp>
 #include <common/Binary.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/reduce_config.hpp>
 #include <math.hpp>
 #include <memory.hpp>
@@ -32,17 +34,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T, af_op_t op, uint dim, bool is_first, uint DIMY>
 class ireduceDimKernelSMEM {
    public:
@@ -52,8 +43,8 @@ class ireduceDimKernelSMEM {
                          read_accessor<uint> iloc, KParam ilocInfo,
                          uint groups_x, uint groups_y, uint groups_dim,
                          read_accessor<uint> rlen, KParam rlenInfo,
-                         local_accessor<compute_t<T>, 1> s_val,
-                         local_accessor<uint, 1> s_idx)
+                         sycl::local_accessor<compute_t<T>, 1> s_val,
+                         sycl::local_accessor<uint, 1> s_idx)
         : out_(out)
         , oInfo_(oInfo)
         , oloc_(oloc)
@@ -215,8 +206,8 @@ class ireduceDimKernelSMEM {
     uint groups_x_, groups_y_, groups_dim_;
     read_accessor<uint> rlen_;
     KParam rlenInfo_;
-    local_accessor<compute_t<T>, 1> s_val_;
-    local_accessor<uint, 1> s_idx_;
+    sycl::local_accessor<compute_t<T>, 1> s_val_;
+    sycl::local_accessor<uint, 1> s_idx_;
 };
 
 template<typename T, af_op_t op, int dim, bool is_first>
@@ -228,73 +219,70 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                           groups_dim[1] * groups_dim[3] * local[1]);
 
     sycl::buffer<uint, 1> empty{sycl::range<1>(1)};
-    try {
-        getQueue().submit([&](sycl::handler &h) {
-            write_accessor<T> out_acc{*out.data, h};
-            write_accessor<uint> oloc_acc{*oloc.data, h};
-            read_accessor<T> in_acc{*in.data, h};
-
-            read_accessor<uint> iloc_acc{empty, h};
-            if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] *
-                    iloc.info.dims[3] >
-                0) {
-                iloc_acc = read_accessor<uint>{*iloc.data, h};
-            }
+    getQueue().submit([&](sycl::handler &h) {
+        write_accessor<T> out_acc{*out.data, h};
+        write_accessor<uint> oloc_acc{*oloc.data, h};
+        read_accessor<T> in_acc{*in.data, h};
+
+        read_accessor<uint> iloc_acc{empty, h};
+        if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] *
+                iloc.info.dims[3] >
+            0) {
+            iloc_acc = read_accessor<uint>{*iloc.data, h};
+        }
 
-            read_accessor<uint> rlen_acc{empty, h};
-            if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] *
-                    rlen.info.dims[3] >
-                0) {
-                rlen_acc = read_accessor<uint>{*rlen.data, h};
-            }
+        read_accessor<uint> rlen_acc{empty, h};
+        if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] *
+                rlen.info.dims[3] >
+            0) {
+            rlen_acc = read_accessor<uint>{*rlen.data, h};
+        }
 
-            auto shrdVal =
-                local_accessor<compute_t<T>, 1>(creduce::THREADS_PER_BLOCK, h);
-            auto shrdLoc =
-                local_accessor<uint, 1>(creduce::THREADS_PER_BLOCK, h);
-
-            switch (threads_y) {
-                case 8:
-                    h.parallel_for(
-                        sycl::nd_range<2>(global, local),
-                        ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
-                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
-                            in.info, iloc_acc, iloc.info, groups_dim[0],
-                            groups_dim[1], groups_dim[dim], rlen_acc, rlen.info,
-                            shrdVal, shrdLoc));
-                    break;
-                case 4:
-                    h.parallel_for(
-                        sycl::nd_range<2>(global, local),
-                        ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
-                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
-                            in.info, iloc_acc, iloc.info, groups_dim[0],
-                            groups_dim[1], groups_dim[dim], rlen_acc, rlen.info,
-                            shrdVal, shrdLoc));
-                    break;
-                case 2:
-                    h.parallel_for(
-                        sycl::nd_range<2>(global, local),
-                        ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
-                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
-                            in.info, iloc_acc, iloc.info, groups_dim[0],
-                            groups_dim[1], groups_dim[dim], rlen_acc, rlen.info,
-                            shrdVal, shrdLoc));
-                    break;
-                case 1:
-                    h.parallel_for(
-                        sycl::nd_range<2>(global, local),
-                        ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
-                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
-                            in.info, iloc_acc, iloc.info, groups_dim[0],
-                            groups_dim[1], groups_dim[dim], rlen_acc, rlen.info,
-                            shrdVal, shrdLoc));
-                    break;
-            }
-        });
-        getQueue().wait_and_throw();
-        ONEAPI_DEBUG_FINISH(getQueue());
-    } catch (sycl::exception &e) { std::cout << e.what() << std::endl; }
+        auto shrdVal = sycl::local_accessor<compute_t<T>, 1>(
+            creduce::THREADS_PER_BLOCK, h);
+        auto shrdLoc =
+            sycl::local_accessor<uint, 1>(creduce::THREADS_PER_BLOCK, h);
+
+        switch (threads_y) {
+            case 8:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                        out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
+                        iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
+                        groups_dim[dim], rlen_acc, rlen.info, shrdVal,
+                        shrdLoc));
+                break;
+            case 4:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                        out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
+                        iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
+                        groups_dim[dim], rlen_acc, rlen.info, shrdVal,
+                        shrdLoc));
+                break;
+            case 2:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                        out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
+                        iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
+                        groups_dim[dim], rlen_acc, rlen.info, shrdVal,
+                        shrdLoc));
+                break;
+            case 1:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                        out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
+                        iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
+                        groups_dim[dim], rlen_acc, rlen.info, shrdVal,
+                        shrdLoc));
+                break;
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 template<typename T, af_op_t op, int dim>
@@ -348,8 +336,8 @@ class ireduceFirstKernelSMEM {
                            read_accessor<uint> iloc, KParam ilocInfo,
                            uint groups_x, uint groups_y, uint repeat,
                            read_accessor<uint> rlen, KParam rlenInfo,
-                           local_accessor<compute_t<T>, 1> s_val,
-                           local_accessor<uint, 1> s_idx)
+                           sycl::local_accessor<compute_t<T>, 1> s_val,
+                           sycl::local_accessor<uint, 1> s_idx)
         : out_(out)
         , oInfo_(oInfo)
         , oloc_(oloc)
@@ -515,8 +503,8 @@ class ireduceFirstKernelSMEM {
     uint groups_x_, groups_y_, repeat_;
     read_accessor<uint> rlen_;
     KParam rlenInfo_;
-    local_accessor<compute_t<T>, 1> s_val_;
-    local_accessor<uint, 1> s_idx_;
+    sycl::local_accessor<compute_t<T>, 1> s_val_;
+    sycl::local_accessor<uint, 1> s_idx_;
 };
 
 template<typename T, af_op_t op, bool is_first>
@@ -531,69 +519,66 @@ void ireduce_first_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
     uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
 
     sycl::buffer<uint, 1> empty{sycl::range<1>(1)};
-    try {
-        getQueue().submit([&](sycl::handler &h) {
-            write_accessor<T> out_acc{*out.data, h};
-            write_accessor<uint> oloc_acc{*oloc.data, h};
-            read_accessor<T> in_acc{*in.data, h};
-
-            read_accessor<uint> iloc_acc{empty, h};
-            if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] *
-                    iloc.info.dims[3] >
-                0) {
-                iloc_acc = read_accessor<uint>{*iloc.data, h};
-            }
+    getQueue().submit([&](sycl::handler &h) {
+        write_accessor<T> out_acc{*out.data, h};
+        write_accessor<uint> oloc_acc{*oloc.data, h};
+        read_accessor<T> in_acc{*in.data, h};
+
+        read_accessor<uint> iloc_acc{empty, h};
+        if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] *
+                iloc.info.dims[3] >
+            0) {
+            iloc_acc = read_accessor<uint>{*iloc.data, h};
+        }
 
-            read_accessor<uint> rlen_acc{empty, h};
-            if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] *
-                    rlen.info.dims[3] >
-                0) {
-                rlen_acc = read_accessor<uint>{*rlen.data, h};
-            }
+        read_accessor<uint> rlen_acc{empty, h};
+        if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] *
+                rlen.info.dims[3] >
+            0) {
+            rlen_acc = read_accessor<uint>{*rlen.data, h};
+        }
 
-            auto shrdVal =
-                local_accessor<compute_t<T>, 1>(creduce::THREADS_PER_BLOCK, h);
-            auto shrdLoc =
-                local_accessor<uint, 1>(creduce::THREADS_PER_BLOCK, h);
-
-            switch (threads_x) {
-                case 32:
-                    h.parallel_for(
-                        sycl::nd_range<2>(global, local),
-                        ireduceFirstKernelSMEM<T, op, is_first, 32>(
-                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
-                            in.info, iloc_acc, iloc.info, groups_x, groups_y,
-                            repeat, rlen_acc, rlen.info, shrdVal, shrdLoc));
-                    break;
-                case 64:
-                    h.parallel_for(
-                        sycl::nd_range<2>(global, local),
-                        ireduceFirstKernelSMEM<T, op, is_first, 64>(
-                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
-                            in.info, iloc_acc, iloc.info, groups_x, groups_y,
-                            repeat, rlen_acc, rlen.info, shrdVal, shrdLoc));
-                    break;
-                case 128:
-                    h.parallel_for(
-                        sycl::nd_range<2>(global, local),
-                        ireduceFirstKernelSMEM<T, op, is_first, 128>(
-                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
-                            in.info, iloc_acc, iloc.info, groups_x, groups_y,
-                            repeat, rlen_acc, rlen.info, shrdVal, shrdLoc));
-                    break;
-                case 256:
-                    h.parallel_for(
-                        sycl::nd_range<2>(global, local),
-                        ireduceFirstKernelSMEM<T, op, is_first, 256>(
-                            out_acc, out.info, oloc_acc, oloc.info, in_acc,
-                            in.info, iloc_acc, iloc.info, groups_x, groups_y,
-                            repeat, rlen_acc, rlen.info, shrdVal, shrdLoc));
-                    break;
-            }
-        });
-        getQueue().wait_and_throw();
-        ONEAPI_DEBUG_FINISH(getQueue());
-    } catch (sycl::exception &e) { std::cout << e.what() << std::endl; }
+        auto shrdVal = sycl::local_accessor<compute_t<T>, 1>(
+            creduce::THREADS_PER_BLOCK, h);
+        auto shrdLoc =
+            sycl::local_accessor<uint, 1>(creduce::THREADS_PER_BLOCK, h);
+
+        switch (threads_x) {
+            case 32:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    ireduceFirstKernelSMEM<T, op, is_first, 32>(
+                        out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
+                        iloc_acc, iloc.info, groups_x, groups_y, repeat,
+                        rlen_acc, rlen.info, shrdVal, shrdLoc));
+                break;
+            case 64:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    ireduceFirstKernelSMEM<T, op, is_first, 64>(
+                        out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
+                        iloc_acc, iloc.info, groups_x, groups_y, repeat,
+                        rlen_acc, rlen.info, shrdVal, shrdLoc));
+                break;
+            case 128:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    ireduceFirstKernelSMEM<T, op, is_first, 128>(
+                        out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
+                        iloc_acc, iloc.info, groups_x, groups_y, repeat,
+                        rlen_acc, rlen.info, shrdVal, shrdLoc));
+                break;
+            case 256:
+                h.parallel_for(
+                    sycl::nd_range<2>(global, local),
+                    ireduceFirstKernelSMEM<T, op, is_first, 256>(
+                        out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
+                        iloc_acc, iloc.info, groups_x, groups_y, repeat,
+                        rlen_acc, rlen.info, shrdVal, shrdLoc));
+                break;
+        }
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 template<typename T, af_op_t op>
diff --git a/src/backend/oneapi/kernel/lookup.hpp b/src/backend/oneapi/kernel/lookup.hpp
index a5d29fea09..f3e2fcdcde 100644
--- a/src/backend/oneapi/kernel/lookup.hpp
+++ b/src/backend/oneapi/kernel/lookup.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 
 #include <sycl/sycl.hpp>
 
@@ -23,11 +24,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 int trimIndex(int idx, const int len) {
     int ret_val = idx;
     if (ret_val < 0) {
diff --git a/src/backend/oneapi/kernel/lu_split.hpp b/src/backend/oneapi/kernel/lu_split.hpp
index f42cf8644c..fb69001ebc 100644
--- a/src/backend/oneapi/kernel/lu_split.hpp
+++ b/src/backend/oneapi/kernel/lu_split.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 
 #include <math.hpp>
 #include <array>
@@ -20,11 +21,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T, bool same_dims>
 class luSplitKernel {
    public:
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 1d58458e46..7d622e611c 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -15,6 +15,7 @@
 #include <common/half.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/default_config.hpp>
 #include <kernel/reduce_config.hpp>
 #include <math.hpp>
@@ -41,17 +42,6 @@ __device__ auto operator/(__half lhs, float rhs) -> __half {
 
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename To, typename Tw>
 void stable_mean(To *lhs, Tw *l_wt, To rhs, Tw r_wt) {
     if (((*l_wt) != (Tw)0) || (r_wt != (Tw)0)) {
@@ -71,9 +61,10 @@ class meanDimKernelSMEM {
                       write_accessor<Tw> owt, KParam owInfo,
                       read_accessor<Ti> in, KParam iInfo, read_accessor<Tw> iwt,
                       KParam iwInfo, uint groups_x, uint groups_y,
-                      uint offset_dim, local_accessor<compute_t<To>, 1> s_val,
-                      local_accessor<compute_t<Tw>, 1> s_idx, bool input_weight,
-                      bool output_weight)
+                      uint offset_dim,
+                      sycl::local_accessor<compute_t<To>, 1> s_val,
+                      sycl::local_accessor<compute_t<Tw>, 1> s_idx,
+                      bool input_weight, bool output_weight)
         : out_(out)
         , owt_(owt)
         , in_(in)
@@ -213,8 +204,8 @@ class meanDimKernelSMEM {
     read_accessor<Tw> iwt_;
     KParam oInfo_, owInfo_, iInfo_, iwInfo_;
     const uint groups_x_, groups_y_, offset_dim_;
-    local_accessor<compute_t<To>, 1> s_val_;
-    local_accessor<compute_t<Tw>, 1> s_idx_;
+    sycl::local_accessor<compute_t<To>, 1> s_val_;
+    sycl::local_accessor<compute_t<Tw>, 1> s_idx_;
     bool input_weight_, output_weight_;
 };
 
@@ -231,8 +222,10 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        auto s_val = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
-        auto s_idx = local_accessor<compute_t<Tw>, 1>(THREADS_PER_BLOCK, h);
+        auto s_val =
+            sycl::local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
+        auto s_idx =
+            sycl::local_accessor<compute_t<Tw>, 1>(THREADS_PER_BLOCK, h);
 
         bool input_weight = ((iwt.info.dims[0] * iwt.info.dims[1] *
                               iwt.info.dims[2] * iwt.info.dims[3]) != 0);
@@ -327,8 +320,8 @@ class meanFirstKernelSMEM {
                         read_accessor<Tw> iwt, KParam iwInfo, const uint DIMX,
                         const uint groups_x, const uint groups_y,
                         const uint repeat,
-                        local_accessor<compute_t<To>, 1> s_val,
-                        local_accessor<compute_t<Tw>, 1> s_idx,
+                        sycl::local_accessor<compute_t<To>, 1> s_val,
+                        sycl::local_accessor<compute_t<Tw>, 1> s_idx,
                         bool input_weight, bool output_weight)
         : out_(out)
         , owt_(owt)
@@ -485,8 +478,8 @@ class meanFirstKernelSMEM {
     read_accessor<Tw> iwt_;
     KParam oInfo_, owInfo_, iInfo_, iwInfo_;
     const uint DIMX_, groups_x_, groups_y_, repeat_;
-    local_accessor<compute_t<To>, 1> s_val_;
-    local_accessor<compute_t<Tw>, 1> s_idx_;
+    sycl::local_accessor<compute_t<To>, 1> s_val_;
+    sycl::local_accessor<compute_t<Tw>, 1> s_idx_;
     bool input_weight_, output_weight_;
 };
 
@@ -505,8 +498,10 @@ void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        auto s_val = local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
-        auto s_idx = local_accessor<compute_t<Tw>, 1>(THREADS_PER_BLOCK, h);
+        auto s_val =
+            sycl::local_accessor<compute_t<To>, 1>(THREADS_PER_BLOCK, h);
+        auto s_idx =
+            sycl::local_accessor<compute_t<Tw>, 1>(THREADS_PER_BLOCK, h);
 
         bool input_weight = ((iwt.info.dims[0] * iwt.info.dims[1] *
                               iwt.info.dims[2] * iwt.info.dims[3]) != 0);
@@ -626,6 +621,7 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         sycl::buffer hwBuffer(h_wptr.data(), {tmp_elements},
                               {sycl::property::buffer::use_host_ptr()});
 
+        // TODO: fix when addressing other mean errors
         auto e1 = getQueue().submit([&](sycl::handler &h) {
             auto acc_in =
                 tmpOut.get()->get_access(h, sycl::range{tmp_elements});
diff --git a/src/backend/oneapi/kernel/meanshift.hpp b/src/backend/oneapi/kernel/meanshift.hpp
index 2211d81b73..ef28998d4d 100644
--- a/src/backend/oneapi/kernel/meanshift.hpp
+++ b/src/backend/oneapi/kernel/meanshift.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 
 #include <sycl/sycl.hpp>
 
@@ -24,11 +25,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 inline int convert_int_rtz(float number) { return ((int)(number)); }
 
 template<typename T, typename AccType, const int MAX_CHANNELS>
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 482c7cd366..33a53fc160 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -14,6 +14,7 @@
 #include <common/half.hpp>
 #include <common/traits.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <sycl/sycl.hpp>
 #include <traits.hpp>
 
@@ -34,8 +35,8 @@ typedef struct {
 template<typename T>
 class memCopy {
    public:
-    memCopy(sycl::accessor<T> out, dims_t ostrides, int ooffset,
-            sycl::accessor<T> in, dims_t idims, dims_t istrides, int ioffset,
+    memCopy(write_accessor<T> out, dims_t ostrides, int ooffset,
+            read_accessor<T> in, dims_t idims, dims_t istrides, int ioffset,
             int groups_0, int groups_1)
         : out_(out)
         , ostrides_(ostrides)
@@ -80,10 +81,10 @@ class memCopy {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     dims_t ostrides_;
     int ooffset_;
-    sycl::accessor<T> in_;
+    read_accessor<T> in_;
     dims_t idims_, istrides_;
     int ioffset_, groups_0_, groups_1_;
 };
@@ -115,8 +116,8 @@ void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
     sycl::nd_range<2> ndrange(global, local);
 
     getQueue().submit([=](sycl::handler &h) {
-        auto out_acc = out->get_access(h);
-        auto in_acc  = const_cast<sycl::buffer<T> *>(in)->get_access(h);
+        write_accessor<T> out_acc{*out, h};
+        read_accessor<T> in_acc{*const_cast<sycl::buffer<T> *>(in), h};
 
         h.parallel_for(ndrange,
                        memCopy<T>(out_acc, _ostrides, ooffset, in_acc, _idims,
@@ -198,8 +199,8 @@ OTHER_SPECIALIZATIONS(arrayfire::common::half)
 template<typename inType, typename outType, bool SAMEDIMS>
 class reshapeCopy {
    public:
-    reshapeCopy(sycl::accessor<outType> dst, KParam oInfo,
-                sycl::accessor<inType> src, KParam iInfo, outType default_value,
+    reshapeCopy(write_accessor<outType> dst, KParam oInfo,
+                read_accessor<inType> src, KParam iInfo, outType default_value,
                 float factor, dims_t trgt, int blk_x, int blk_y)
         : dst_(dst)
         , src_(src)
@@ -261,8 +262,8 @@ class reshapeCopy {
     }
 
    protected:
-    sycl::accessor<outType> dst_;
-    sycl::accessor<inType> src_;
+    write_accessor<outType> dst_;
+    read_accessor<inType> src_;
     KParam oInfo_, iInfo_;
     outType default_value_;
     float factor_;
@@ -303,9 +304,9 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
     }
 
     getQueue().submit([=](sycl::handler &h) {
-        auto dst_acc = dst.data->get_access(h);
-        auto src_acc =
-            const_cast<sycl::buffer<inType> *>(src.data)->get_access(h);
+        write_accessor<outType> dst_acc{*dst.data, h};
+        read_accessor<inType> src_acc{
+            *const_cast<sycl::buffer<inType> *>(src.data), h};
 
         if (same_dims) {
             h.parallel_for(ndrange, reshapeCopy<inType, outType, true>(
diff --git a/src/backend/oneapi/kernel/pad_array_borders.hpp b/src/backend/oneapi/kernel/pad_array_borders.hpp
index 129f9bf381..c5401a65c2 100644
--- a/src/backend/oneapi/kernel/pad_array_borders.hpp
+++ b/src/backend/oneapi/kernel/pad_array_borders.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <math.hpp>
 #include <af/defines.h>
 
@@ -23,11 +24,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T, int BType>
 class padBordersKernel {
    public:
diff --git a/src/backend/oneapi/kernel/random_engine.hpp b/src/backend/oneapi/kernel/random_engine.hpp
index 329387eef5..b416827a7d 100644
--- a/src/backend/oneapi/kernel/random_engine.hpp
+++ b/src/backend/oneapi/kernel/random_engine.hpp
@@ -12,6 +12,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/random_engine_mersenne.hpp>
 #include <kernel/random_engine_philox.hpp>
 #include <kernel/random_engine_threefry.hpp>
@@ -56,7 +57,7 @@ void uniformDistributionCBRNG(Param<T> out, const size_t elements,
     switch (type) {
         case AF_RANDOM_ENGINE_PHILOX_4X32_10:
             getQueue().submit([=](sycl::handler &h) {
-                auto out_acc = out.data->get_access(h);
+                write_accessor<T> out_acc{*out.data, h};
 
                 h.parallel_for(ndrange,
                                uniformPhilox<T>(out_acc, hi, lo, hic, loc,
@@ -66,7 +67,7 @@ void uniformDistributionCBRNG(Param<T> out, const size_t elements,
             break;
         case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
             getQueue().submit([=](sycl::handler &h) {
-                auto out_acc = out.data->get_access(h);
+                write_accessor<T> out_acc{*out.data, h};
 
                 h.parallel_for(ndrange,
                                uniformThreefry<T>(out_acc, hi, lo, hic, loc,
@@ -96,7 +97,7 @@ void normalDistributionCBRNG(Param<T> out, const size_t elements,
     switch (type) {
         case AF_RANDOM_ENGINE_PHILOX_4X32_10:
             getQueue().submit([=](sycl::handler &h) {
-                auto out_acc = out.data->get_access(h);
+                write_accessor<T> out_acc{*out.data, h};
 
                 h.parallel_for(ndrange,
                                normalPhilox<T>(out_acc, hi, lo, hic, loc,
@@ -105,7 +106,7 @@ void normalDistributionCBRNG(Param<T> out, const size_t elements,
             break;
         case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
             getQueue().submit([=](sycl::handler &h) {
-                auto out_acc = out.data->get_access(h);
+                write_accessor<T> out_acc{*out.data, h};
 
                 h.parallel_for(ndrange,
                                normalThreefry<T>(out_acc, hi, lo, hic, loc,
@@ -134,7 +135,7 @@ void uniformDistributionMT(Param<T> out, const size_t elements,
     sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads),
                               sycl::range<1>(threads));
     getQueue().submit([=](sycl::handler &h) {
-        auto out_acc       = out.data->get_access(h);
+        write_accessor<T> out_acc{*out.data, h};
         auto state_acc     = state.data->get_access(h);
         auto pos_acc       = pos.data->get_access(h);
         auto sh1_acc       = sh1.data->get_access(h);
@@ -142,9 +143,9 @@ void uniformDistributionMT(Param<T> out, const size_t elements,
         auto recursion_acc = sh2.data->get_access(h);
         auto temper_acc    = sh2.data->get_access(h);
 
-        auto lstate_acc     = local_accessor<uint, 1>(STATE_SIZE, h);
-        auto lrecursion_acc = local_accessor<uint, 1>(TABLE_SIZE, h);
-        auto ltemper_acc    = local_accessor<uint, 1>(TABLE_SIZE, h);
+        auto lstate_acc     = sycl::local_accessor<uint, 1>(STATE_SIZE, h);
+        auto lrecursion_acc = sycl::local_accessor<uint, 1>(TABLE_SIZE, h);
+        auto ltemper_acc    = sycl::local_accessor<uint, 1>(TABLE_SIZE, h);
 
         h.parallel_for(
             ndrange, uniformMersenne<T>(
@@ -170,7 +171,7 @@ void normalDistributionMT(Param<T> out, const size_t elements,
     sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads),
                               sycl::range<1>(threads));
     getQueue().submit([=](sycl::handler &h) {
-        auto out_acc       = out.data->get_access(h);
+        write_accessor<T> out_acc{*out.data, h};
         auto state_acc     = state.data->get_access(h);
         auto pos_acc       = pos.data->get_access(h);
         auto sh1_acc       = sh1.data->get_access(h);
@@ -178,9 +179,9 @@ void normalDistributionMT(Param<T> out, const size_t elements,
         auto recursion_acc = sh2.data->get_access(h);
         auto temper_acc    = sh2.data->get_access(h);
 
-        auto lstate_acc     = local_accessor<uint, 1>(STATE_SIZE, h);
-        auto lrecursion_acc = local_accessor<uint, 1>(TABLE_SIZE, h);
-        auto ltemper_acc    = local_accessor<uint, 1>(TABLE_SIZE, h);
+        auto lstate_acc     = sycl::local_accessor<uint, 1>(STATE_SIZE, h);
+        auto lrecursion_acc = sycl::local_accessor<uint, 1>(TABLE_SIZE, h);
+        auto ltemper_acc    = sycl::local_accessor<uint, 1>(TABLE_SIZE, h);
 
         h.parallel_for(
             ndrange, normalMersenne<T>(out_acc, state_acc, pos_acc, sh1_acc,
diff --git a/src/backend/oneapi/kernel/random_engine_mersenne.hpp b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
index f78bc8d732..f36b2b60d0 100644
--- a/src/backend/oneapi/kernel/random_engine_mersenne.hpp
+++ b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
@@ -42,6 +42,7 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *******************************************************/
 #pragma once
+#include <kernel/accessors.hpp>
 #include <kernel/random_engine_write.hpp>
 
 #include <sycl/sycl.hpp>
@@ -55,11 +56,6 @@ constexpr int BLOCKS     = 32;
 constexpr int STATE_SIZE = (256 * 3);
 constexpr int TABLE_SIZE = 16;
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 // Utils
 static inline void read_table(uint *const sharedTable, const uint *const table,
                               size_t groupId, size_t localId) {
@@ -108,8 +104,8 @@ static inline uint temper(const uint *const temper_table, const uint v,
 // Initialization
 class initMersenneKernel {
    public:
-    initMersenneKernel(sycl::accessor<uint> state, sycl::accessor<uint> tbl,
-                       local_accessor<uint, 1> lstate, uintl seed)
+    initMersenneKernel(write_accessor<uint> state, read_accessor<uint> tbl,
+                       sycl::local_accessor<uint, 1> lstate, uintl seed)
         : state_(state), tbl_(tbl), lstate_(lstate), seed_(seed) {}
 
     void operator()(sycl::nd_item<1> it) const {
@@ -141,17 +137,18 @@ class initMersenneKernel {
     }
 
    protected:
-    sycl::accessor<uint> state_, tbl_;
-    local_accessor<uint, 1> lstate_;
+    write_accessor<uint> state_;
+    read_accessor<uint> tbl_;
+    sycl::local_accessor<uint, 1> lstate_;
     uintl seed_;
 };
 
 void initMersenneState(Param<uint> state, const Param<uint> tbl, uintl seed) {
     sycl::nd_range<1> ndrange({BLOCKS * N}, {N});
     getQueue().submit([=](sycl::handler &h) {
-        auto state_acc  = state.data->get_access(h);
-        auto tbl_acc    = tbl.data->get_access(h);
-        auto lstate_acc = local_accessor<uint, 1>(N, h);
+        write_accessor<uint> state_acc{*state.data, h};
+        read_accessor<uint> tbl_acc{*tbl.data, h};
+        auto lstate_acc = sycl::local_accessor<uint, 1>(N, h);
 
         h.parallel_for(
             ndrange, initMersenneKernel(state_acc, tbl_acc, lstate_acc, seed));
@@ -164,16 +161,16 @@ void initMersenneState(Param<uint> state, const Param<uint> tbl, uintl seed) {
 template<typename T>
 class uniformMersenne {
    public:
-    uniformMersenne(sycl::accessor<T> out, sycl::accessor<uint> gState,
+    uniformMersenne(write_accessor<T> out, sycl::accessor<uint> gState,
                     sycl::accessor<uint> pos_tbl, sycl::accessor<uint> sh1_tbl,
                     sycl::accessor<uint> sh2_tbl, uint mask,
                     sycl::accessor<uint> g_recursion_table,
                     sycl::accessor<uint> g_temper_table,
                     // local memory caches of global state
-                    local_accessor<uint, 1> state,
-                    local_accessor<uint, 1> recursion_table,
-                    local_accessor<uint, 1> temper_table, uint elementsPerBlock,
-                    size_t elements)
+                    sycl::local_accessor<uint, 1> state,
+                    sycl::local_accessor<uint, 1> recursion_table,
+                    sycl::local_accessor<uint, 1> temper_table,
+                    uint elementsPerBlock, size_t elements)
         : out_(out)
         , gState_(gState)
         , pos_tbl_(pos_tbl)
@@ -248,12 +245,12 @@ class uniformMersenne {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     sycl::accessor<uint> gState_;
     sycl::accessor<uint> pos_tbl_, sh1_tbl_, sh2_tbl_;
     uint mask_;
     sycl::accessor<uint> g_recursion_table_, g_temper_table_;
-    local_accessor<uint, 1> state_, recursion_table_, temper_table_;
+    sycl::local_accessor<uint, 1> state_, recursion_table_, temper_table_;
     uint elementsPerBlock_;
     size_t elements_;
 };
@@ -261,16 +258,16 @@ class uniformMersenne {
 template<typename T>
 class normalMersenne {
    public:
-    normalMersenne(sycl::accessor<T> out, sycl::accessor<uint> gState,
+    normalMersenne(write_accessor<T> out, sycl::accessor<uint> gState,
                    sycl::accessor<uint> pos_tbl, sycl::accessor<uint> sh1_tbl,
                    sycl::accessor<uint> sh2_tbl, uint mask,
                    sycl::accessor<uint> g_recursion_table,
                    sycl::accessor<uint> g_temper_table,
                    // local memory caches of global state
-                   local_accessor<uint, 1> state,
-                   local_accessor<uint, 1> recursion_table,
-                   local_accessor<uint, 1> temper_table, uint elementsPerBlock,
-                   size_t elements)
+                   sycl::local_accessor<uint, 1> state,
+                   sycl::local_accessor<uint, 1> recursion_table,
+                   sycl::local_accessor<uint, 1> temper_table,
+                   uint elementsPerBlock, size_t elements)
         : out_(out)
         , gState_(gState)
         , pos_tbl_(pos_tbl)
@@ -346,12 +343,12 @@ class normalMersenne {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     sycl::accessor<uint> gState_;
     sycl::accessor<uint> pos_tbl_, sh1_tbl_, sh2_tbl_;
     uint mask_;
     sycl::accessor<uint> g_recursion_table_, g_temper_table_;
-    local_accessor<uint, 1> state_, recursion_table_, temper_table_;
+    sycl::local_accessor<uint, 1> state_, recursion_table_, temper_table_;
     uint elementsPerBlock_;
     size_t elements_;
 };
diff --git a/src/backend/oneapi/kernel/random_engine_philox.hpp b/src/backend/oneapi/kernel/random_engine_philox.hpp
index 3bfe44251d..afa29394e2 100644
--- a/src/backend/oneapi/kernel/random_engine_philox.hpp
+++ b/src/backend/oneapi/kernel/random_engine_philox.hpp
@@ -45,6 +45,7 @@
  *********************************************************/
 
 #pragma once
+#include <kernel/accessors.hpp>
 #include <kernel/random_engine_write.hpp>
 
 namespace arrayfire {
@@ -106,7 +107,7 @@ static inline void philox(uint key[2], uint ctr[4]) {
 template<typename T>
 class uniformPhilox {
    public:
-    uniformPhilox(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
+    uniformPhilox(write_accessor<T> out, uint hi, uint lo, uint hic, uint loc,
                   uint elementsPerBlock, uint elements)
         : out_(out)
         , hi_(hi)
@@ -138,7 +139,7 @@ class uniformPhilox {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
 };
@@ -146,7 +147,7 @@ class uniformPhilox {
 template<typename T>
 class normalPhilox {
    public:
-    normalPhilox(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
+    normalPhilox(write_accessor<T> out, uint hi, uint lo, uint hic, uint loc,
                  uint elementsPerBlock, uint elements)
         : out_(out)
         , hi_(hi)
@@ -180,7 +181,7 @@ class normalPhilox {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
 };
diff --git a/src/backend/oneapi/kernel/random_engine_threefry.hpp b/src/backend/oneapi/kernel/random_engine_threefry.hpp
index 919f04d010..1969bf3b69 100644
--- a/src/backend/oneapi/kernel/random_engine_threefry.hpp
+++ b/src/backend/oneapi/kernel/random_engine_threefry.hpp
@@ -45,6 +45,7 @@
  *********************************************************/
 
 #pragma once
+#include <kernel/accessors.hpp>
 #include <kernel/random_engine_write.hpp>
 
 namespace arrayfire {
@@ -161,7 +162,7 @@ void threefry(uint k[2], uint c[2], uint X[2]) {
 template<typename T>
 class uniformThreefry {
    public:
-    uniformThreefry(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
+    uniformThreefry(write_accessor<T> out, uint hi, uint lo, uint hic, uint loc,
                     uint elementsPerBlock, uint elements)
         : out_(out)
         , hi_(hi)
@@ -198,7 +199,7 @@ class uniformThreefry {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
 };
@@ -206,7 +207,7 @@ class uniformThreefry {
 template<typename T>
 class normalThreefry {
    public:
-    normalThreefry(sycl::accessor<T> out, uint hi, uint lo, uint hic, uint loc,
+    normalThreefry(write_accessor<T> out, uint hi, uint lo, uint hic, uint loc,
                    uint elementsPerBlock, uint elements)
         : out_(out)
         , hi_(hi)
@@ -243,7 +244,7 @@ class normalThreefry {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     uint hi_, lo_, hic_, loc_;
     uint elementsPerBlock_, elements_;
 };
diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
index 1c8512be0b..f052abb48c 100644
--- a/src/backend/oneapi/kernel/range.hpp
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -15,6 +15,7 @@
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 #include <af/dim4.hpp>
 
@@ -30,7 +31,7 @@ namespace kernel {
 template<typename T>
 class rangeOp {
    public:
-    rangeOp(sycl::accessor<T> out, KParam oinfo, const int dim,
+    rangeOp(write_accessor<T> out, KParam oinfo, const int dim,
             const int blocksPerMatX, const int blocksPerMatY)
         : out_(out)
         , oinfo_(oinfo)
@@ -82,7 +83,7 @@ class rangeOp {
     }
 
    protected:
-    sycl::accessor<T> out_;
+    write_accessor<T> out_;
     KParam oinfo_;
     int dim_;
     int blocksPerMatX_, blocksPerMatY_;
@@ -104,7 +105,7 @@ void range(Param<T> out, const int dim) {
     sycl::nd_range<2> ndrange(global, local);
 
     getQueue().submit([=](sycl::handler& h) {
-        auto out_acc = out.data->get_access(h);
+        write_accessor<T> out_acc{*out.data, h};
 
         h.parallel_for(ndrange, rangeOp<T>(out_acc, out.info, dim,
                                            blocksPerMatX, blocksPerMatY));
diff --git a/src/backend/oneapi/kernel/reduce.hpp b/src/backend/oneapi/kernel/reduce.hpp
index 6807a68396..7089cb9b4e 100644
--- a/src/backend/oneapi/kernel/reduce.hpp
+++ b/src/backend/oneapi/kernel/reduce.hpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #pragma once
+
 #include <Param.hpp>
 #include <backend.hpp>
 #include <common/Binary.hpp>
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index 0878f33329..4bc3d5254d 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #pragma once
+
 #include <Param.hpp>
 #include <backend.hpp>
 #include <common/Binary.hpp>
@@ -15,6 +16,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/reduce_config.hpp>
 #include <math.hpp>
 #include <memory.hpp>
@@ -31,22 +33,11 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 template<typename T>
 using global_atomic_ref =
     sycl::atomic_ref<T, sycl::memory_order::relaxed, sycl::memory_scope::system,
                      sycl::access::address_space::global_space>;
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename Ti, typename To, af_op_t op>
 class reduceAllKernelSMEM {
    public:
@@ -56,8 +47,8 @@ class reduceAllKernelSMEM {
                         read_accessor<Ti> in, KParam iInfo, uint DIMX,
                         uint groups_x, uint groups_y, uint repeat,
                         bool change_nan, To nanval,
-                        local_accessor<compute_t<To>, 1> s_ptr,
-                        local_accessor<bool, 1> amLast)
+                        sycl::local_accessor<compute_t<To>, 1> s_ptr,
+                        sycl::local_accessor<bool, 1> amLast)
         : out_(out)
         , retCount_(retCount)
         , tmp_(tmp)
@@ -237,8 +228,8 @@ class reduceAllKernelSMEM {
     uint groups_x_, groups_y_;
     bool change_nan_;
     To nanval_;
-    local_accessor<compute_t<To>, 1> s_ptr_;
-    local_accessor<bool, 1> amLast_;
+    sycl::local_accessor<compute_t<To>, 1> s_ptr_;
+    sycl::local_accessor<bool, 1> amLast_;
 };
 
 template<typename Ti, typename To, af_op_t op>
@@ -267,9 +258,9 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
         auto tmp_acc      = tmp.get()->get_access(h);
         read_accessor<Ti> in_acc{*in.data, h};
 
-        auto shrdMem =
-            local_accessor<compute_t<To>, 1>(creduce::THREADS_PER_BLOCK, h);
-        auto amLast = local_accessor<bool, 1>(1, h);
+        auto shrdMem = sycl::local_accessor<compute_t<To>, 1>(
+            creduce::THREADS_PER_BLOCK, h);
+        auto amLast = sycl::local_accessor<bool, 1>(1, h);
         h.parallel_for(
             sycl::nd_range<2>(global, local),
             reduceAllKernelSMEM<Ti, To, op>(
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index 22b9c0f8dc..926a7205e9 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #pragma once
+
 #include <Param.hpp>
 #include <backend.hpp>
 #include <common/Binary.hpp>
@@ -15,6 +16,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/reduce_config.hpp>
 #include <math.hpp>
 #include <memory.hpp>
@@ -29,23 +31,12 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename Ti, typename To, af_op_t op, uint dim, uint DIMY>
 class reduceDimKernelSMEM {
    public:
     reduceDimKernelSMEM(Param<To> out, Param<Ti> in, uint groups_x,
                         uint groups_y, uint offset_dim, bool change_nan,
-                        To nanval, local_accessor<compute_t<To>, 1> s_val,
+                        To nanval, sycl::local_accessor<compute_t<To>, 1> s_val,
                         sycl::handler &h)
         : out_(out.template get_accessor<sycl::access::mode::write>(h))
         , in_(in.template get_accessor<sycl::access::mode::read>(h))
@@ -141,7 +132,7 @@ class reduceDimKernelSMEM {
     uint groups_x_, groups_y_, offset_dim_;
     bool change_nan_;
     To nanval_;
-    local_accessor<compute_t<To>, 1> s_val_;
+    sycl::local_accessor<compute_t<To>, 1> s_val_;
 };
 
 template<typename Ti, typename To, af_op_t op, uint dim>
@@ -154,8 +145,8 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
                           blocks_dim[1] * blocks_dim[3] * local[1]);
 
     getQueue().submit([=](sycl::handler &h) {
-        auto shrdMem =
-            local_accessor<compute_t<To>, 1>(creduce::THREADS_X * threads_y, h);
+        auto shrdMem = sycl::local_accessor<compute_t<To>, 1>(
+            creduce::THREADS_X * threads_y, h);
 
         switch (threads_y) {
             case 8:
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index 42ffb9199d..27143aa24b 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #pragma once
+
 #include <Param.hpp>
 #include <backend.hpp>
 #include <common/Binary.hpp>
@@ -15,6 +16,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/reduce_config.hpp>
 #include <math.hpp>
 #include <memory.hpp>
@@ -31,24 +33,14 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename Ti, typename To, af_op_t op, uint DIMX>
 class reduceFirstKernelSMEM {
    public:
     reduceFirstKernelSMEM(write_accessor<To> out, KParam oInfo,
                           read_accessor<Ti> in, KParam iInfo, uint groups_x,
                           uint groups_y, uint repeat, bool change_nan,
-                          To nanval, local_accessor<compute_t<To>, 1> s_val)
+                          To nanval,
+                          sycl::local_accessor<compute_t<To>, 1> s_val)
         : out_(out)
         , oInfo_(oInfo)
         , iInfo_(iInfo)
@@ -145,7 +137,7 @@ class reduceFirstKernelSMEM {
     uint groups_x_, groups_y_, repeat_;
     bool change_nan_;
     To nanval_;
-    local_accessor<compute_t<To>, 1> s_val_;
+    sycl::local_accessor<compute_t<To>, 1> s_val_;
 };
 
 template<typename Ti, typename To, af_op_t op>
@@ -163,8 +155,8 @@ void reduce_first_launcher_default(Param<To> out, Param<Ti> in,
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        auto shrdMem =
-            local_accessor<compute_t<To>, 1>(creduce::THREADS_PER_BLOCK, h);
+        auto shrdMem = sycl::local_accessor<compute_t<To>, 1>(
+            creduce::THREADS_PER_BLOCK, h);
 
         switch (threads_x) {
             case 32:
diff --git a/src/backend/oneapi/kernel/reorder.hpp b/src/backend/oneapi/kernel/reorder.hpp
index 1064047f77..adf1c8f57b 100644
--- a/src/backend/oneapi/kernel/reorder.hpp
+++ b/src/backend/oneapi/kernel/reorder.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -23,11 +24,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 class reorderCreateKernel {
    public:
diff --git a/src/backend/oneapi/kernel/resize.hpp b/src/backend/oneapi/kernel/resize.hpp
index b14ceafe14..50cc041ab5 100644
--- a/src/backend/oneapi/kernel/resize.hpp
+++ b/src/backend/oneapi/kernel/resize.hpp
@@ -13,6 +13,7 @@
 #include <common/complex.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -33,11 +34,6 @@ std::complex<double> mul(AT a, std::complex<double> b) {
     return std::complex<double>(a * b.real(), a * b.imag());
 }
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 using wtype_t = typename std::conditional<std::is_same<T, double>::value,
                                           double, float>::type;
diff --git a/src/backend/oneapi/kernel/rotate.hpp b/src/backend/oneapi/kernel/rotate.hpp
index 84641a3f76..a6d255d369 100644
--- a/src/backend/oneapi/kernel/rotate.hpp
+++ b/src/backend/oneapi/kernel/rotate.hpp
@@ -12,6 +12,7 @@
 #include <common/complex.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/interp.hpp>
 #include <math.hpp>
 #include <traits.hpp>
@@ -22,11 +23,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 typedef struct {
     float tmat[6];
 } tmat_t;
diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp
index a9ce3d7838..eea34ffff7 100644
--- a/src/backend/oneapi/kernel/scan_dim.hpp
+++ b/src/backend/oneapi/kernel/scan_dim.hpp
@@ -14,6 +14,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/default_config.hpp>
 #include <memory.hpp>
 
@@ -23,17 +24,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename Ti, typename To, af_op_t op, int dim>
 class scanDimKernel {
    public:
@@ -42,8 +32,8 @@ class scanDimKernel {
                   read_accessor<Ti> in_acc, KParam iInfo, const uint groups_x,
                   const uint groups_y, const uint blocks_dim, const uint lim,
                   const bool isFinalPass, const uint DIMY,
-                  const bool inclusive_scan, local_accessor<To, 1> s_val,
-                  local_accessor<To, 1> s_tmp)
+                  const bool inclusive_scan, sycl::local_accessor<To, 1> s_val,
+                  sycl::local_accessor<To, 1> s_tmp)
         : out_acc_(out_acc)
         , tmp_acc_(tmp_acc)
         , in_acc_(in_acc)
@@ -161,8 +151,8 @@ class scanDimKernel {
     KParam oInfo_, tInfo_, iInfo_;
     const uint groups_x_, groups_y_, blocks_dim_, lim_, DIMY_;
     const bool isFinalPass_, inclusive_scan_;
-    local_accessor<To, 1> s_val_;
-    local_accessor<To, 1> s_tmp_;
+    sycl::local_accessor<To, 1> s_val_;
+    sycl::local_accessor<To, 1> s_tmp_;
 };
 
 template<typename To, af_op_t op, int dim>
@@ -262,9 +252,9 @@ static void scan_dim_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
         write_accessor<To> tmp_acc{*tmp.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
-        auto s_val =
-            local_accessor<compute_t<To>, 1>(THREADS_X * threads_y * 2, h);
-        auto s_tmp = local_accessor<compute_t<To>, 1>(THREADS_X, h);
+        auto s_val = sycl::local_accessor<compute_t<To>, 1>(
+            THREADS_X * threads_y * 2, h);
+        auto s_tmp = sycl::local_accessor<compute_t<To>, 1>(THREADS_X, h);
 
         h.parallel_for(
             sycl::nd_range<2>(global, local),
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index 3a5b113914..649e031b03 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -14,6 +14,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/default_config.hpp>
 #include <memory.hpp>
 
@@ -23,17 +24,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename Ti, typename To, af_op_t op>
 class scanFirstKernel {
    public:
@@ -42,7 +32,8 @@ class scanFirstKernel {
                     read_accessor<Ti> in_acc, KParam iInfo, const uint groups_x,
                     const uint groups_y, const uint lim, const bool isFinalPass,
                     const uint DIMX, const bool inclusive_scan,
-                    local_accessor<To, 1> s_val, local_accessor<To, 1> s_tmp)
+                    sycl::local_accessor<To, 1> s_val,
+                    sycl::local_accessor<To, 1> s_tmp)
         : out_acc_(out_acc)
         , tmp_acc_(tmp_acc)
         , in_acc_(in_acc)
@@ -138,8 +129,8 @@ class scanFirstKernel {
     KParam oInfo_, tInfo_, iInfo_;
     const uint groups_x_, groups_y_, lim_, DIMX_;
     const bool isFinalPass_, inclusive_scan_;
-    local_accessor<To, 1> s_val_;
-    local_accessor<To, 1> s_tmp_;
+    sycl::local_accessor<To, 1> s_val_;
+    sycl::local_accessor<To, 1> s_tmp_;
 };
 
 template<typename To, af_op_t op>
@@ -220,8 +211,8 @@ static void scan_first_launcher(Param<To> out, Param<To> tmp, Param<Ti> in,
 
         const int DIMY            = THREADS_PER_BLOCK / threads_x;
         const int SHARED_MEM_SIZE = (2 * threads_x + 1) * (DIMY);
-        auto s_val = local_accessor<compute_t<To>, 1>(SHARED_MEM_SIZE, h);
-        auto s_tmp = local_accessor<compute_t<To>, 1>(DIMY, h);
+        auto s_val = sycl::local_accessor<compute_t<To>, 1>(SHARED_MEM_SIZE, h);
+        auto s_tmp = sycl::local_accessor<compute_t<To>, 1>(DIMY, h);
 
         // TODO threads_x as template arg for #pragma unroll?
         h.parallel_for(sycl::nd_range<2>(global, local),
diff --git a/src/backend/oneapi/kernel/select.hpp b/src/backend/oneapi/kernel/select.hpp
index abba384f80..b5a6ae5954 100644
--- a/src/backend/oneapi/kernel/select.hpp
+++ b/src/backend/oneapi/kernel/select.hpp
@@ -12,6 +12,7 @@
 #include <Param.hpp>
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
+#include <kernel/accessors.hpp>
 #include <math.hpp>
 
 #include <sycl/sycl.hpp>
@@ -23,11 +24,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 constexpr uint DIMX  = 32;
 constexpr uint DIMY  = 8;
 constexpr int REPEAT = 64;
diff --git a/src/backend/oneapi/kernel/tile.hpp b/src/backend/oneapi/kernel/tile.hpp
index 2c44594a34..39cea65af3 100644
--- a/src/backend/oneapi/kernel/tile.hpp
+++ b/src/backend/oneapi/kernel/tile.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 
 #include <sycl/sycl.hpp>
 
@@ -23,11 +24,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 class tileCreateKernel {
    public:
diff --git a/src/backend/oneapi/kernel/transform.hpp b/src/backend/oneapi/kernel/transform.hpp
index 6760e1a489..07f70a3a62 100644
--- a/src/backend/oneapi/kernel/transform.hpp
+++ b/src/backend/oneapi/kernel/transform.hpp
@@ -13,6 +13,7 @@
 #include <common/complex.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/interp.hpp>
 #include <math.hpp>
 #include <traits.hpp>
@@ -26,11 +27,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 using wtype_t = typename std::conditional<std::is_same<T, double>::value,
                                           double, float>::type;
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index eeb9387145..bf7c7a874b 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -44,11 +45,6 @@ cdouble getConjugate(const cdouble &in) {
     return std::conj(in);
 }
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 template<typename T>
 class transposeKernel {
    public:
@@ -57,7 +53,7 @@ class transposeKernel {
                     const sycl::accessor<T, 1, sycl::access::mode::read> iData,
                     const KParam in, const int blocksPerMatX,
                     const int blocksPerMatY, const bool conjugate,
-                    const bool IS32MULTIPLE, local_accessor<T, 1> shrdMem)
+                    const bool IS32MULTIPLE, sycl::local_accessor<T, 1> shrdMem)
         : oData_(oData)
         , out_(out)
         , iData_(iData)
@@ -135,7 +131,7 @@ class transposeKernel {
     int blocksPerMatY_;
     bool conjugate_;
     bool IS32MULTIPLE_;
-    local_accessor<T, 1> shrdMem_;
+    sycl::local_accessor<T, 1> shrdMem_;
 };
 
 template<typename T>
@@ -153,7 +149,7 @@ void transpose(Param<T> out, const Param<T> in, const bool conjugate,
         auto r = in.data->template get_access<sycl::access::mode::read>(h);
         auto q = out.data->template get_access<sycl::access::mode::write>(h);
 
-        auto shrdMem = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
+        auto shrdMem = sycl::local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
 
         h.parallel_for(sycl::nd_range{global, local},
                        transposeKernel<T>(q, out.info, r, in.info, blk_x, blk_y,
diff --git a/src/backend/oneapi/kernel/transpose_inplace.hpp b/src/backend/oneapi/kernel/transpose_inplace.hpp
index 23f04c6559..721a3befb9 100644
--- a/src/backend/oneapi/kernel/transpose_inplace.hpp
+++ b/src/backend/oneapi/kernel/transpose_inplace.hpp
@@ -47,19 +47,14 @@ constexpr dim_t TILE_DIM  = 16;
 constexpr dim_t THREADS_X = TILE_DIM;
 constexpr dim_t THREADS_Y = 256 / TILE_DIM;
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 template<typename T>
 class transposeInPlaceKernel {
    public:
     transposeInPlaceKernel(const sycl::accessor<T> iData, const KParam in,
                            const int blocksPerMatX, const int blocksPerMatY,
                            const bool conjugate, const bool IS32MULTIPLE,
-                           local_accessor<T, 1> shrdMem_s,
-                           local_accessor<T, 1> shrdMem_d)
+                           sycl::local_accessor<T, 1> shrdMem_s,
+                           sycl::local_accessor<T, 1> shrdMem_d)
         : iData_(iData)
         , in_(in)
         , blocksPerMatX_(blocksPerMatX)
@@ -163,8 +158,8 @@ class transposeInPlaceKernel {
     int blocksPerMatY_;
     bool conjugate_;
     bool IS32MULTIPLE_;
-    local_accessor<T, 1> shrdMem_s_;
-    local_accessor<T, 1> shrdMem_d_;
+    sycl::local_accessor<T, 1> shrdMem_s_;
+    sycl::local_accessor<T, 1> shrdMem_d_;
 };
 
 template<typename T>
@@ -179,9 +174,11 @@ void transpose_inplace(Param<T> in, const bool conjugate,
                               blk_y * local[1] * in.info.dims[3]};
 
     getQueue().submit([&](sycl::handler &h) {
-        auto r         = in.data->get_access(h);
-        auto shrdMem_s = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
-        auto shrdMem_d = local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
+        auto r = in.data->get_access(h);
+        auto shrdMem_s =
+            sycl::local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
+        auto shrdMem_d =
+            sycl::local_accessor<T, 1>(TILE_DIM * (TILE_DIM + 1), h);
 
         h.parallel_for(
             sycl::nd_range{global, local},
diff --git a/src/backend/oneapi/kernel/triangle.hpp b/src/backend/oneapi/kernel/triangle.hpp
index f4705035b3..4634f69570 100644
--- a/src/backend/oneapi/kernel/triangle.hpp
+++ b/src/backend/oneapi/kernel/triangle.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <traits.hpp>
 
 #include <sycl/sycl.hpp>
@@ -24,15 +25,10 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T, int dimensions>
-using local_accessor =
-    sycl::accessor<T, dimensions, sycl::access::mode::read_write,
-                   sycl::access::target::local>;
-
 template<typename T>
 class triangleKernel {
    public:
-    triangleKernel(sycl::accessor<T> rAcc, KParam rinfo, sycl::accessor<T> iAcc,
+    triangleKernel(write_accessor<T> rAcc, KParam rinfo, read_accessor<T> iAcc,
                    KParam iinfo, const int groups_x, const int groups_y,
                    const bool is_upper, const bool is_unit_diag)
         : rAcc_(rAcc)
@@ -82,9 +78,9 @@ class triangleKernel {
     }
 
    private:
-    sycl::accessor<T> rAcc_;
+    write_accessor<T> rAcc_;
     KParam rinfo_;
-    sycl::accessor<T> iAcc_;
+    read_accessor<T> iAcc_;
     KParam iinfo_;
     const int groups_x_;
     const int groups_y_;
@@ -109,8 +105,8 @@ void triangle(Param<T> out, const Param<T> in, bool is_upper,
                               groups_y * out.info.dims[3] * local[1]};
 
     getQueue().submit([&](sycl::handler &h) {
-        auto iAcc = in.data->get_access(h);
-        auto rAcc = out.data->get_access(h);
+        read_accessor<T> iAcc{*in.data, h};
+        write_accessor<T> rAcc{*out.data, h};
 
         h.parallel_for(
             sycl::nd_range{global, local},
diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index dd18189ae0..b65e0d9333 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -7,11 +7,14 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#pragma once
+
 #include <Param.hpp>
 #include <backend.hpp>
 #include <common/dispatch.hpp>
 #include <debug_oneapi.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/default_config.hpp>
 #include <kernel/scan_first.hpp>
 #include <memory.hpp>
@@ -26,12 +29,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 class whereKernel {
    public:
diff --git a/src/backend/oneapi/kernel/wrap.hpp b/src/backend/oneapi/kernel/wrap.hpp
index 5f2c92c641..ef8d2eba21 100644
--- a/src/backend/oneapi/kernel/wrap.hpp
+++ b/src/backend/oneapi/kernel/wrap.hpp
@@ -13,9 +13,9 @@
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/default_config.hpp>
 #include <math.hpp>
-#include <sycl/sycl.hpp>
 
 #include <sycl/sycl.hpp>
 
@@ -26,14 +26,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
-                                      sycl::access::target::local>;
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 class wrapCreateKernel {
    public:
diff --git a/src/backend/oneapi/kernel/wrap_dilated.hpp b/src/backend/oneapi/kernel/wrap_dilated.hpp
index dae994e371..63bdf342a8 100644
--- a/src/backend/oneapi/kernel/wrap_dilated.hpp
+++ b/src/backend/oneapi/kernel/wrap_dilated.hpp
@@ -13,6 +13,7 @@
 #include <common/dispatch.hpp>
 #include <common/kernel_cache.hpp>
 #include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/default_config.hpp>
 #include <math.hpp>
 
@@ -24,14 +25,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-template<typename T>
-using local_accessor = sycl::accessor<T, 1, sycl::access::mode::read_write,
-                                      sycl::access::target::local>;
-template<typename T>
-using read_accessor = sycl::accessor<T, 1, sycl::access::mode::read>;
-template<typename T>
-using write_accessor = sycl::accessor<T, 1, sycl::access::mode::write>;
-
 template<typename T>
 class wrapDilatedCreateKernel {
    public:

From 7a393d7fa4b06b3088abf20e16d91d7e6f087732 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 25 Apr 2023 14:32:10 -0400
Subject: [PATCH 279/473] Add simple caching for oneAPI backend

---
 src/backend/oneapi/jit.cpp | 123 ++++++++++++++++++++++---------------
 1 file changed, 73 insertions(+), 50 deletions(-)

diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 6c4d4c1828..17b1a63c3f 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -37,6 +37,7 @@
 #include <sstream>
 #include <stdexcept>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 using arrayfire::common::getFuncName;
@@ -51,10 +52,14 @@ using arrayfire::oneapi::getActiveDeviceBaseBuildFlags;
 using arrayfire::oneapi::jit::BufferNode;
 
 using std::array;
+using std::begin;
+using std::end;
+using std::find;
 using std::find_if;
 using std::string;
 using std::stringstream;
 using std::to_string;
+using std::unordered_map;
 using std::vector;
 
 using sycl::backend;
@@ -78,10 +83,12 @@ const static string DEFAULT_MACROS_STR(R"JIT(
 #endif
 )JIT");
 
-string getKernelString(const string& funcName, const vector<Node*>& full_nodes,
-                       const vector<Node_ids>& full_ids,
-                       const vector<int>& output_ids, const bool is_linear,
-                       const bool loop0, const bool loop1, const bool loop3) {
+string getKernelString(const string& funcName,
+                       const nonstd::span<Node* const> full_nodes,
+                       nonstd::span<const Node_ids> full_ids,
+                       const nonstd::span<int const> output_ids,
+                       const bool is_linear, const bool loop0, const bool loop1,
+                       const bool loop3) {
     // Common OpenCL code
     // This part of the code does not change with the kernel.
 
@@ -163,7 +170,6 @@ __kernel void )JIT";
     int id1 = get_global_id(1);
     const int id0End = oInfo.dims[0];
     const int id1End = oInfo.dims[1];
-    //printf("id0: %d  id1: %d id0End: %d, id1End: %d\n")
     if ((id0 < id0End) & (id1 < id1End)) {
         const int id2 = get_global_id(2);
 #define id3 0
@@ -280,6 +286,48 @@ __kernel void )JIT";
 //     return common::getKernel("", "", true).get();
 // }
 
+template<typename T>
+cl_kernel getKernel(std::string funcName, cl_context ctx, cl_device_id dev,
+                    cl_command_queue q,
+                    const nonstd::span<Node* const> full_nodes,
+                    nonstd::span<Node_ids const> full_ids,
+                    nonstd::span<int const> output_ids,
+                    nonstd::span<oneapi::AParam<T> const> ap, bool is_linear) {
+    static unordered_map<std::string, cl_kernel> kernel_map;
+
+    vector<cl_kernel> kernels(10);
+    if (kernel_map.find(funcName) == end(kernel_map)) {
+        string jitstr = arrayfire::opencl::getKernelString(
+            funcName, full_nodes, full_ids, output_ids, is_linear, false, false,
+            ap[0].dims[2] > 1);
+
+        cl_int err;
+        vector<const char*> jitsources = {
+            {arrayfire::oneapi::opencl::KParam_hpp,
+             arrayfire::oneapi::opencl::jit_cl, jitstr.c_str()}};
+        vector<size_t> jitsizes = {arrayfire::oneapi::opencl::KParam_hpp_len,
+                                   arrayfire::oneapi::opencl::jit_cl_len,
+                                   jitstr.size()};
+
+        cl_program prog = clCreateProgramWithSource(
+            ctx, jitsources.size(), jitsources.data(), jitsizes.data(), &err);
+
+        std::string options = getActiveDeviceBaseBuildFlags();
+
+        CL_CHECK_BUILD(
+            clBuildProgram(prog, 1, &dev, options.c_str(), nullptr, nullptr));
+
+        cl_uint ret_kernels = 0;
+        CL_CHECK(
+            clCreateKernelsInProgram(prog, 1, kernels.data(), &ret_kernels));
+        kernel_map[funcName] = kernels[0];
+        CL_CHECK(clReleaseProgram(prog));
+    } else {
+        kernels[0] = kernel_map[funcName];
+    }
+    return kernels[0];
+}
+
 }  // namespace opencl
 
 namespace oneapi {
@@ -432,10 +480,6 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                          funcName](sycl::interop_handle hh) {
                 switch (hh.get_backend()) {
                     case backend::opencl: {
-                        string jitstr = arrayfire::opencl::getKernelString(
-                            funcName, full_nodes, full_ids, output_ids,
-                            is_linear, false, false, ap[0].dims[2] > 1);
-
                         cl_command_queue q =
                             hh.get_native_queue<backend::opencl>();
                         cl_context ctx =
@@ -443,35 +487,15 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                         cl_device_id dev =
                             hh.get_native_device<backend::opencl>();
 
-                        cl_int err;
-                        vector<const char*> jitsources = {
-                            {arrayfire::oneapi::opencl::KParam_hpp,
-                             arrayfire::oneapi::opencl::jit_cl,
-                             jitstr.c_str()}};
-                        vector<size_t> jitsizes = {
-                            arrayfire::oneapi::opencl::KParam_hpp_len,
-                            arrayfire::oneapi::opencl::jit_cl_len,
-                            jitstr.size()};
-
-                        cl_program prog = clCreateProgramWithSource(
-                            ctx, jitsources.size(), jitsources.data(),
-                            jitsizes.data(), &err);
-
-                        std::string options = getActiveDeviceBaseBuildFlags();
-
-                        CL_CHECK_BUILD(clBuildProgram(
-                            prog, 1, &dev, options.c_str(), nullptr, nullptr));
-
-                        vector<cl_kernel> kernels(10);
-                        cl_uint ret_kernels = 0;
-                        CL_CHECK(clCreateKernelsInProgram(
-                            prog, 1, kernels.data(), &ret_kernels));
+                        cl_kernel kernel = arrayfire::opencl::getKernel<T>(
+                            funcName, ctx, dev, q, full_nodes, full_ids,
+                            output_ids, ap, is_linear);
                         int nargs{0};
                         for (Node* node : full_nodes) {
                             if (node->isBuffer()) {
                                 nargs = node->setArgs(
                                     nargs, is_linear,
-                                    [&kernels, &hh, &is_linear](
+                                    [&kernel, &hh, &is_linear](
                                         int id, const void* ptr,
                                         size_t arg_size) {
                                         AParam<T>* info =
@@ -482,27 +506,27 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                                                 info->data);
                                         if (is_linear) {
                                             CL_CHECK(clSetKernelArg(
-                                                kernels[0], id++,
-                                                sizeof(cl_mem), &mem[0]));
+                                                kernel, id++, sizeof(cl_mem),
+                                                &mem[0]));
                                             CL_CHECK(clSetKernelArg(
-                                                kernels[0], id++, sizeof(dim_t),
+                                                kernel, id++, sizeof(dim_t),
                                                 &info->offset));
                                         } else {
                                             CL_CHECK(clSetKernelArg(
-                                                kernels[0], id++,
-                                                sizeof(cl_mem), &mem[0]));
+                                                kernel, id++, sizeof(cl_mem),
+                                                &mem[0]));
                                             KParam ooo = *info;
                                             CL_CHECK(clSetKernelArg(
-                                                kernels[0], id++,
-                                                sizeof(KParam), &ooo));
+                                                kernel, id++, sizeof(KParam),
+                                                &ooo));
                                         }
                                     });
                             } else {
                                 nargs = node->setArgs(
                                     nargs, is_linear,
-                                    [&kernels](int id, const void* ptr,
-                                               size_t arg_size) {
-                                        CL_CHECK(clSetKernelArg(kernels[0], id,
+                                    [&kernel](int id, const void* ptr,
+                                              size_t arg_size) {
+                                        CL_CHECK(clSetKernelArg(kernel, id,
                                                                 arg_size, ptr));
                                     });
                             }
@@ -514,15 +538,15 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                             mem =
                                 hh.get_native_mem<backend::opencl>(output.data);
                             cl_mem mmm = mem[0];
-                            CL_CHECK(clSetKernelArg(kernels[0], nargs++,
+                            CL_CHECK(clSetKernelArg(kernel, nargs++,
                                                     sizeof(cl_mem), &mmm));
                             int off = output.offset;
-                            CL_CHECK(clSetKernelArg(kernels[0], nargs++,
+                            CL_CHECK(clSetKernelArg(kernel, nargs++,
                                                     sizeof(int), &off));
                         }
                         const KParam ooo = ap[0];
-                        CL_CHECK(clSetKernelArg(kernels[0], nargs++,
-                                                sizeof(KParam), &ooo));
+                        CL_CHECK(clSetKernelArg(kernel, nargs++, sizeof(KParam),
+                                                &ooo));
                         array<size_t, 3> offset{0, 0, 0};
                         array<size_t, 3> global;
                         int ndims = 0;
@@ -537,11 +561,10 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                         }
                         // SHOW(global);
                         CL_CHECK(clEnqueueNDRangeKernel(
-                            q, kernels[0], ndims, offset.data(), global.data(),
+                            q, kernel, ndims, offset.data(), global.data(),
                             nullptr, 0, nullptr, nullptr));
 
-                        CL_CHECK(clReleaseKernel(kernels[0]));
-                        CL_CHECK(clReleaseProgram(prog));
+                        // CL_CHECK(clReleaseKernel(kernel));
                         CL_CHECK(clReleaseDevice(dev));
                         CL_CHECK(clReleaseContext(ctx));
                         CL_CHECK(clReleaseCommandQueue(q));

From feb60fa536724104c62ef734b5d8a12807aba99a Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Sat, 29 Apr 2023 07:34:07 -0400
Subject: [PATCH 280/473] fix fft.cpp example wrong comment

---
 examples/benchmarks/fft.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/benchmarks/fft.cpp b/examples/benchmarks/fft.cpp
index 490a1fa18e..b28873f16a 100644
--- a/examples/benchmarks/fft.cpp
+++ b/examples/benchmarks/fft.cpp
@@ -17,7 +17,7 @@ using namespace af;
 // create a small wrapper to benchmark
 static array A;  // populated before each timing
 static void fn() {
-    array B = fft2(A);  // matrix multiply
+    array B = fft2(A);  // 2d fft
     B.eval();           // ensure evaluated
 }
 

From 4ec5f289d4e6628cfe2d33b57ae33e79949c8ed8 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 3 May 2023 17:22:06 -0400
Subject: [PATCH 281/473] Fix synchronize issues related to OpenCL oneAPI JIT

---
 src/backend/oneapi/Array.cpp                 |   2 +-
 src/backend/oneapi/Array.hpp                 |  15 +-
 src/backend/oneapi/Param.hpp                 |  17 +-
 src/backend/oneapi/copy.cpp                  |  70 ++----
 src/backend/oneapi/jit.cpp                   | 249 +++++++++----------
 src/backend/oneapi/jit/BufferNode.hpp        |   4 +-
 src/backend/oneapi/jit/kernel_generators.hpp |   3 +-
 7 files changed, 170 insertions(+), 190 deletions(-)

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 4682df50f1..3a9fbff3be 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -279,7 +279,7 @@ template<typename T>
 Node_ptr Array<T>::getNode() {
     if (node) { return node; }
 
-    AParam<T> info = *this;
+    AParam<T, sycl::access_mode::read> info = *this;
     unsigned bytes = this->dims().elements() * sizeof(T);
     auto nn        = bufferNodePtr<T>();
     nn->setData(info, data, bytes, isLinear());
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index d3f81bff2c..a6ca6c402c 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -41,7 +41,7 @@ namespace oneapi {
 
 template<typename T>
 struct Param;
-template<typename T>
+template<typename T, sycl::access_mode AM>
 struct AParam;
 
 template<typename T>
@@ -254,7 +254,7 @@ class Array {
     }
 
     sycl::buffer<T> *get() const {
-        if (!isReady()) eval();
+        if (!isReady()) { eval(); }
         return data.get();
     }
 
@@ -277,8 +277,15 @@ class Array {
         return out;
     }
 
-    operator AParam<T>() {
-        AParam<T> out(*getData(), dims().get(), strides().get(), getOffset());
+    operator AParam<T, sycl::access_mode::write>() {
+        AParam<T, sycl::access_mode::write> out(*getData(), dims().get(),
+                                                strides().get(), getOffset());
+        return out;
+    }
+
+    operator AParam<T, sycl::access_mode::read>() const {
+        AParam<T, sycl::access_mode::read> out(*getData(), dims().get(),
+                                               strides().get(), getOffset());
         return out;
     }
 
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index 447e8fb117..7df0a73f85 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -8,13 +8,12 @@
  ********************************************************/
 
 #pragma once
+#include <sycl/sycl.hpp>
 
 #include <kernel/KParam.hpp>
 #include <types.hpp>
 #include <af/dim4.hpp>
 
-#include <sycl/sycl.hpp>
-
 #include <optional>
 
 namespace arrayfire {
@@ -43,9 +42,9 @@ struct Param {
     ~Param() = default;
 };
 
-template<typename T>
+template<typename T, sycl::access_mode AM>
 struct AParam {
-    sycl::accessor<T, 1, sycl::access_mode::read_write, sycl::target::device,
+    sycl::accessor<T, 1, AM, sycl::target::device,
                    sycl::access::placeholder::true_t>
         data;
     af::dim4 dims;
@@ -60,17 +59,11 @@ struct AParam {
 
     AParam(sycl::buffer<T, 1>& data_, const dim_t dims_[4],
            const dim_t strides_[4], dim_t offset_)
-        : data(data_.get_access())
-        , dims(4, dims_)
-        , strides(4, strides_)
-        , offset(offset_) {}
+        : data(data_), dims(4, dims_), strides(4, strides_), offset(offset_) {}
     // AF_DEPRECATED("Use Array<T>")
     AParam(sycl::handler& h, sycl::buffer<T, 1>& data_, const dim_t dims_[4],
            const dim_t strides_[4], dim_t offset_)
-        : data(data_.get_access())
-        , dims(4, dims_)
-        , strides(4, strides_)
-        , offset(offset_) {
+        : data(data_), dims(4, dims_), strides(4, strides_), offset(offset_) {
         require(h);
     }
 
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index e61dbbc8db..a70cc3a6f4 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -29,38 +29,22 @@ namespace arrayfire {
 namespace oneapi {
 
 template<typename T>
-void copyData(T *data, const Array<T> &A) {
-    if (A.elements() == 0) { return; }
-
-    // FIXME: Merge this with copyArray
-    A.eval();
-
-    dim_t offset = 0;
-    const sycl::buffer<T> *buf;
-    Array<T> out = A;
-
-    if (A.isLinear() ||  // No offsets, No strides
-        A.ndims() == 1   // Simple offset, no strides.
-    ) {
-        buf    = A.get();
-        offset = A.getOffset();
-    } else {
-        // FIXME: Think about implementing eval
-        out    = copyArray(A);
-        buf    = out.get();
-        offset = 0;
+void copyData(T *data, const Array<T> &src) {
+    if (src.elements() > 0) {
+        Array<T> lin = src.isReady() && src.isLinear() ? src : copyArray(src);
+        size_t elements = lin.elements();
+        Param<T> p      = lin;
+        getQueue()
+            .submit([&](sycl::handler &h) {
+                sycl::range rr(elements);
+                sycl::id offset_id(p.info.offset);
+                auto offset_acc =
+                    p.data->template get_access<sycl::access_mode::read_write>(
+                        h, rr, offset_id);
+                h.copy(offset_acc, data);
+            })
+            .wait();
     }
-
-    // FIXME: Add checks
-    getQueue()
-        .submit([=](sycl::handler &h) {
-            sycl::range rr(A.elements());
-            sycl::id offset_id(offset);
-            auto offset_acc = const_cast<sycl::buffer<T> *>(buf)->get_access(
-                h, rr, offset_id);
-            h.copy(offset_acc, data);
-        })
-        .wait();
 }
 
 template<typename T>
@@ -77,19 +61,17 @@ Array<T> copyArray(const Array<T> &A) {
             sycl::buffer<T> *out_buf = out.get();
 
             size_t aelem = A.elements();
-            getQueue()
-                .submit([=](sycl::handler &h) {
-                    range rr(aelem);
-                    id offset_id(offset);
-                    accessor offset_acc_A =
-                        A_buf->template get_access<access_mode::read>(
-                            h, rr, offset_id);
-                    accessor acc_out =
-                        out_buf->template get_access<access_mode::write>(h);
-
-                    h.copy(offset_acc_A, acc_out);
-                })
-                .wait();
+            getQueue().submit([=](sycl::handler &h) {
+                range rr(aelem);
+                id offset_id(offset);
+                accessor offset_acc_A =
+                    A_buf->template get_access<access_mode::read>(h, rr,
+                                                                  offset_id);
+                accessor acc_out =
+                    out_buf->template get_access<access_mode::write>(h);
+
+                h.copy(offset_acc_A, acc_out);
+            });
         } else {
             kernel::memcopy<T>(out.get(), out.strides().get(), A.get(),
                                A.dims().get(), A.strides().get(), offset,
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 17b1a63c3f..794bb7796f 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -287,12 +287,12 @@ __kernel void )JIT";
 // }
 
 template<typename T>
-cl_kernel getKernel(std::string funcName, cl_context ctx, cl_device_id dev,
-                    cl_command_queue q,
-                    const nonstd::span<Node* const> full_nodes,
-                    nonstd::span<Node_ids const> full_ids,
-                    nonstd::span<int const> output_ids,
-                    nonstd::span<oneapi::AParam<T> const> ap, bool is_linear) {
+cl_kernel getKernel(
+    std::string funcName, cl_context ctx, cl_device_id dev, cl_command_queue q,
+    const nonstd::span<Node* const> full_nodes,
+    nonstd::span<Node_ids const> full_ids, nonstd::span<int const> output_ids,
+    nonstd::span<oneapi::AParam<T, sycl::access_mode::write> const> ap,
+    bool is_linear) {
     static unordered_map<std::string, cl_kernel> kernel_map;
 
     vector<cl_kernel> kernels(10);
@@ -363,6 +363,10 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
         output_ids.push_back(id);
     }
 
+    node_clones.clear();
+    node_clones.reserve(full_nodes.size());
+    for (Node* node : full_nodes) { node_clones.emplace_back(node->clone()); }
+
     bool moddimsFound{false};
     for (const Node* node : full_nodes) {
         is_linear &= node->isLinear(outDims);
@@ -394,12 +398,6 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
     //  Avoid all cloning/copying when no moddims node is present (high
     //  chance)
     if (moddimsFound || emptyColumnsFound) {
-        node_clones.clear();
-        node_clones.reserve(full_nodes.size());
-        for (Node* node : full_nodes) {
-            node_clones.emplace_back(node->clone());
-        }
-
         for (const Node_ids& ids : full_ids) {
             auto& children{node_clones[ids.id]->m_children};
             for (int i{0}; i < Node::kMaxChildren && children[i] != nullptr;
@@ -452,129 +450,128 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                      });
             ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
         }
-
-        full_nodes.clear();
-        for (Node_ptr& node : node_clones) { full_nodes.push_back(node.get()); }
     }
 
+    full_nodes.clear();
+    for (Node_ptr& node : node_clones) { full_nodes.push_back(node.get()); }
+
     const string funcName{getFuncName(output_nodes, full_nodes, full_ids,
                                       is_linear, false, false, false,
                                       outputs[0].info.dims[2] > 1)};
 
-    getQueue()
-        .submit([=](sycl::handler& h) {
-            for (Node* node : full_nodes) {
-                if (node->isBuffer()) {
-                    BufferNode<T>* n = static_cast<BufferNode<T>*>(node);
-                    n->m_param.require(h);
-                }
+    getQueue().submit([&](sycl::handler& h) {
+        for (Node* node : full_nodes) {
+            if (node->isBuffer()) {
+                BufferNode<T>* n = static_cast<BufferNode<T>*>(node);
+                n->m_param.require(h);
             }
-            vector<AParam<T>> ap;
-            transform(begin(outputs), end(outputs), back_inserter(ap),
-                      [&](const Param<T>& p) {
-                          return AParam<T>(h, *p.data, p.info.dims,
-                                           p.info.strides, p.info.offset);
-                      });
-
-            h.host_task([ap, full_nodes, output_ids, full_ids, is_linear,
-                         funcName](sycl::interop_handle hh) {
-                switch (hh.get_backend()) {
-                    case backend::opencl: {
-                        cl_command_queue q =
-                            hh.get_native_queue<backend::opencl>();
-                        cl_context ctx =
-                            hh.get_native_context<backend::opencl>();
-                        cl_device_id dev =
-                            hh.get_native_device<backend::opencl>();
-
-                        cl_kernel kernel = arrayfire::opencl::getKernel<T>(
-                            funcName, ctx, dev, q, full_nodes, full_ids,
-                            output_ids, ap, is_linear);
-                        int nargs{0};
-                        for (Node* node : full_nodes) {
-                            if (node->isBuffer()) {
-                                nargs = node->setArgs(
-                                    nargs, is_linear,
-                                    [&kernel, &hh, &is_linear](
-                                        int id, const void* ptr,
-                                        size_t arg_size) {
-                                        AParam<T>* info =
-                                            static_cast<AParam<T>*>(
-                                                const_cast<void*>(ptr));
-                                        vector<cl_mem> mem =
-                                            hh.get_native_mem<backend::opencl>(
-                                                info->data);
-                                        if (is_linear) {
-                                            CL_CHECK(clSetKernelArg(
-                                                kernel, id++, sizeof(cl_mem),
-                                                &mem[0]));
-                                            CL_CHECK(clSetKernelArg(
-                                                kernel, id++, sizeof(dim_t),
-                                                &info->offset));
-                                        } else {
-                                            CL_CHECK(clSetKernelArg(
-                                                kernel, id++, sizeof(cl_mem),
-                                                &mem[0]));
-                                            KParam ooo = *info;
-                                            CL_CHECK(clSetKernelArg(
-                                                kernel, id++, sizeof(KParam),
-                                                &ooo));
-                                        }
-                                    });
-                            } else {
-                                nargs = node->setArgs(
-                                    nargs, is_linear,
-                                    [&kernel](int id, const void* ptr,
-                                              size_t arg_size) {
-                                        CL_CHECK(clSetKernelArg(kernel, id,
-                                                                arg_size, ptr));
-                                    });
-                            }
-                        }
-
-                        // Set output parameters
-                        vector<cl_mem> mem;
-                        for (const auto& output : ap) {
-                            mem =
-                                hh.get_native_mem<backend::opencl>(output.data);
-                            cl_mem mmm = mem[0];
-                            CL_CHECK(clSetKernelArg(kernel, nargs++,
-                                                    sizeof(cl_mem), &mmm));
-                            int off = output.offset;
-                            CL_CHECK(clSetKernelArg(kernel, nargs++,
-                                                    sizeof(int), &off));
-                        }
-                        const KParam ooo = ap[0];
-                        CL_CHECK(clSetKernelArg(kernel, nargs++, sizeof(KParam),
-                                                &ooo));
-                        array<size_t, 3> offset{0, 0, 0};
-                        array<size_t, 3> global;
-                        int ndims = 0;
-                        if (is_linear) {
-                            global = {(size_t)ap[0].dims.elements(), 0, 0};
-                            ndims  = 1;
+        }
+        vector<AParam<T, sycl::access_mode::write>> ap;
+        transform(begin(outputs), end(outputs), back_inserter(ap),
+                  [&](const Param<T>& p) {
+                      return AParam<T, sycl::access_mode::write>(
+                          h, *p.data, p.info.dims, p.info.strides,
+                          p.info.offset);
+                  });
+
+        h.host_task([ap, full_nodes, output_ids, full_ids, is_linear, funcName,
+                     node_clones, nodes, outputs](sycl::interop_handle hh) {
+            switch (hh.get_backend()) {
+                case backend::opencl: {
+                    auto ncc = node_clones;
+
+                    cl_command_queue q = hh.get_native_queue<backend::opencl>();
+                    cl_context ctx   = hh.get_native_context<backend::opencl>();
+                    cl_device_id dev = hh.get_native_device<backend::opencl>();
+
+                    cl_kernel kernel = arrayfire::opencl::getKernel<T>(
+                        funcName, ctx, dev, q, full_nodes, full_ids, output_ids,
+                        ap, is_linear);
+                    int nargs{0};
+                    for (Node* node : full_nodes) {
+                        if (node->isBuffer()) {
+                            nargs = node->setArgs(
+                                nargs, is_linear,
+                                [&kernel, &hh, &is_linear](
+                                    int id, const void* ptr, size_t arg_size) {
+                                    AParam<T, sycl::access_mode::read>* info =
+                                        static_cast<AParam<
+                                            T, sycl::access_mode::read>*>(
+                                            const_cast<void*>(ptr));
+                                    vector<cl_mem> mem =
+                                        hh.get_native_mem<backend::opencl>(
+                                            info->data);
+                                    if (is_linear) {
+                                        CL_CHECK(clSetKernelArg(kernel, id++,
+                                                                sizeof(cl_mem),
+                                                                &mem[0]));
+                                        CL_CHECK(clSetKernelArg(kernel, id++,
+                                                                sizeof(dim_t),
+                                                                &info->offset));
+                                    } else {
+                                        CL_CHECK(clSetKernelArg(kernel, id++,
+                                                                sizeof(cl_mem),
+                                                                &mem[0]));
+                                        KParam ooo = *info;
+                                        CL_CHECK(clSetKernelArg(kernel, id++,
+                                                                sizeof(KParam),
+                                                                &ooo));
+                                    }
+                                });
                         } else {
-                            global = {(size_t)ap[0].dims[0],
-                                      (size_t)ap[0].dims[1],
-                                      (size_t)ap[0].dims[2]};
-                            ndims  = 3;
+                            nargs = node->setArgs(
+                                nargs, is_linear,
+                                [&kernel](int id, const void* ptr,
+                                          size_t arg_size) {
+                                    CL_CHECK(clSetKernelArg(kernel, id,
+                                                            arg_size, ptr));
+                                });
                         }
-                        // SHOW(global);
-                        CL_CHECK(clEnqueueNDRangeKernel(
-                            q, kernel, ndims, offset.data(), global.data(),
-                            nullptr, 0, nullptr, nullptr));
-
-                        // CL_CHECK(clReleaseKernel(kernel));
-                        CL_CHECK(clReleaseDevice(dev));
-                        CL_CHECK(clReleaseContext(ctx));
-                        CL_CHECK(clReleaseCommandQueue(q));
-
-                    } break;
-                    default: ONEAPI_NOT_SUPPORTED("Backend not supported");
-                }
-            });
-        })
-        .wait();
+                    }
+
+                    // Set output parameters
+                    vector<cl_mem> mem;
+                    for (const auto& output : ap) {
+                        mem = hh.get_native_mem<backend::opencl>(output.data);
+                        cl_mem mmm = mem[0];
+                        CL_CHECK(clSetKernelArg(kernel, nargs++, sizeof(cl_mem),
+                                                &mmm));
+                        int off = output.offset;
+                        CL_CHECK(
+                            clSetKernelArg(kernel, nargs++, sizeof(int), &off));
+                    }
+                    const KParam ooo = ap[0];
+                    CL_CHECK(
+                        clSetKernelArg(kernel, nargs++, sizeof(KParam), &ooo));
+                    array<size_t, 3> offset{0, 0, 0};
+                    array<size_t, 3> global;
+                    int ndims = 0;
+                    if (is_linear) {
+                        global = {(size_t)ap[0].dims.elements(), 0, 0};
+                        ndims  = 1;
+                    } else {
+                        global = {(size_t)ap[0].dims[0], (size_t)ap[0].dims[1],
+                                  (size_t)ap[0].dims[2]};
+                        ndims  = 3;
+                    }
+                    // SHOW(global);
+                    cl_event kernel_event;
+                    CL_CHECK(clEnqueueNDRangeKernel(
+                        q, kernel, ndims, offset.data(), global.data(), nullptr,
+                        0, nullptr, &kernel_event));
+                    CL_CHECK(clEnqueueBarrierWithWaitList(q, 1, &kernel_event,
+                                                          nullptr));
+                    CL_CHECK(clReleaseEvent(kernel_event));
+
+                    CL_CHECK(clReleaseDevice(dev));
+                    CL_CHECK(clReleaseContext(ctx));
+                    CL_CHECK(clReleaseCommandQueue(q));
+
+                } break;
+                default: ONEAPI_NOT_SUPPORTED("Backend not supported");
+            }
+        });
+    });
 }
 
 template<typename T>
diff --git a/src/backend/oneapi/jit/BufferNode.hpp b/src/backend/oneapi/jit/BufferNode.hpp
index 8c8d61abf2..94655f23e7 100644
--- a/src/backend/oneapi/jit/BufferNode.hpp
+++ b/src/backend/oneapi/jit/BufferNode.hpp
@@ -18,8 +18,8 @@ namespace arrayfire {
 namespace oneapi {
 namespace jit {
 template<typename T>
-using BufferNode =
-    common::BufferNodeBase<std::shared_ptr<sycl::buffer<T>>, AParam<T>>;
+using BufferNode = common::BufferNodeBase<std::shared_ptr<sycl::buffer<T>>,
+                                          AParam<T, sycl::access_mode::read>>;
 }  // namespace jit
 }  // namespace oneapi
 
diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
index a69553acd3..bc12929fe6 100644
--- a/src/backend/oneapi/jit/kernel_generators.hpp
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -41,7 +41,8 @@ template<typename T>
 inline int setKernelArguments(
     int start_id, bool is_linear,
     std::function<void(int id, const void* ptr, size_t arg_size)>& setArg,
-    const std::shared_ptr<sycl::buffer<T>>& ptr, const AParam<T>& info) {
+    const std::shared_ptr<sycl::buffer<T>>& ptr,
+    const AParam<T, sycl::access_mode::read>& info) {
     setArg(start_id + 0, static_cast<const void*>(&info), sizeof(Param<T>));
     return start_id + 2;
 }

From 5a3ac34ef790f7b16a0004c0d9b6399e63f9648d Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 11 May 2023 22:39:39 +0200
Subject: [PATCH 282/473] Fixed compile error on MSVC 16.11.26

---
 src/backend/opencl/Array.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 311ec715b9..d479ac5752 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -342,7 +342,8 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
         // Setting the maximum to 5120 bytes to keep the compile times
         // resonable. This still results in large kernels but its not excessive.
         size_t max_param_size =
-            min(5120UL, device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>());
+            min(static_cast<cl::size_type>(5120),
+                device.getInfo<CL_DEVICE_MAX_PARAMETER_SIZE>());
         max_param_size -= base_param_size;
 
         struct tree_info {

From 8e3d1fa6cebd5df0bd62e15b80be5f430d9d8f88 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 11 May 2023 23:41:39 +0200
Subject: [PATCH 283/473] Fixed assignment with index after device change

---
 src/backend/opencl/assign.cpp | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/backend/opencl/assign.cpp b/src/backend/opencl/assign.cpp
index 9e0f8074a3..57ceeaab2d 100644
--- a/src/backend/opencl/assign.cpp
+++ b/src/backend/opencl/assign.cpp
@@ -23,6 +23,11 @@ using arrayfire::common::half;
 namespace arrayfire {
 namespace opencl {
 
+static std::mutex mtx;
+static std::map<std::pair<const cl::Context*, const int /* deviceId */>,
+                cl::Buffer*>
+    cachedEmptyBuffers;
+
 template<typename T>
 void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs) {
     kernel::AssignKernelParam_t p;
@@ -49,6 +54,27 @@ void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs) {
     cl::Buffer* bPtrs[4];
 
     std::vector<Array<uint>> idxArrs(4, createEmptyArray<uint>(dim4()));
+
+    // Prepare commonBuffer for empty indexes
+    // Buffer is dependent on the context.
+    // To avoid copying between devices, we add also deviceId as a dependency
+    cl::Buffer* emptyBuffer;
+    {
+        std::lock_guard<std::mutex> lck(mtx);
+        const auto dependent = std::make_pair<const cl::Context*, const int>(
+            &getContext(), getActiveDeviceId());
+        auto it = cachedEmptyBuffers.find(dependent);
+        if (it == cachedEmptyBuffers.end()) {
+            emptyBuffer = new cl::Buffer(
+                getContext(),
+                CL_MEM_READ_ONLY,  // NOLINT(hicpp-signed-bitwise)
+                sizeof(uint));
+            cachedEmptyBuffers[dependent] = emptyBuffer;
+        } else {
+            emptyBuffer = it->second;
+        }
+    }
+
     // look through indexs to read af_array indexs
     for (dim_t x = 0; x < 4; ++x) {
         // set index pointers were applicable
@@ -59,10 +85,7 @@ void assign(Array<T>& out, const af_index_t idxrs[], const Array<T>& rhs) {
             // alloc an 1-element buffer to avoid OpenCL from failing using
             // direct buffer allocation as opposed to mem manager to avoid
             // reference count desprepancies between different backends
-            static auto* empty = new cl::Buffer(
-                getContext(), CL_MEM_READ_ONLY,  // NOLINT(hicpp-signed-bitwise)
-                sizeof(uint));
-            bPtrs[x] = empty;
+            bPtrs[x] = emptyBuffer;
         }
     }
 

From 11af3076d2439080e62ee9c21c772eaa1829afdf Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Fri, 12 May 2023 00:09:22 +0200
Subject: [PATCH 284/473] Random after device change

---
 src/api/c/random.cpp | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/api/c/random.cpp b/src/api/c/random.cpp
index f1a85b2891..915e733974 100644
--- a/src/api/c/random.cpp
+++ b/src/api/c/random.cpp
@@ -19,7 +19,9 @@
 #include <af/array.h>
 #include <af/data.h>
 #include <af/defines.h>
+#include <af/device.h>
 #include <af/dim4.hpp>
+#include <map>
 #include <memory>
 
 using af::dim4;
@@ -128,8 +130,20 @@ af_err af_get_default_random_engine(af_random_engine *r) {
     try {
         AF_CHECK(af_init());
 
-        thread_local auto *re = new RandomEngine;
-        *r                    = static_cast<af_random_engine>(re);
+        // RandomEngine contains device buffers which are dependent on
+        // context|stream/device. Since nor context or stream are available at
+        // this level, we will only use the deviceId.
+        thread_local std::map<int /*deviceId*/, RandomEngine *>
+            cachedDefaultRandomEngines;
+        const int dependent = af::getDevice();
+        auto it             = cachedDefaultRandomEngines.find(dependent);
+        if (it == cachedDefaultRandomEngines.end()) {
+            RandomEngine *defaultRandomEngine     = new RandomEngine;
+            cachedDefaultRandomEngines[dependent] = defaultRandomEngine;
+            *r = static_cast<af_random_engine>(defaultRandomEngine);
+        } else {
+            *r = static_cast<af_random_engine>(it->second);
+        }
         return AF_SUCCESS;
     }
     CATCHALL;

From 8889ee0c954e581ee7fdd20b60d76c42c6e07390 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 11 May 2023 23:39:22 +0200
Subject: [PATCH 285/473] Fixed initialization error on gebrd

---
 src/backend/opencl/magma/gebrd.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/backend/opencl/magma/gebrd.cpp b/src/backend/opencl/magma/gebrd.cpp
index 4e88a498ae..c63be4a5bb 100644
--- a/src/backend/opencl/magma/gebrd.cpp
+++ b/src/backend/opencl/magma/gebrd.cpp
@@ -239,11 +239,17 @@ magma_int_t magma_gebrd_hybrid(magma_int_t m, magma_int_t n, Ty *a,
         return *info;
     }
 
-    if (MAGMA_SUCCESS != magma_malloc<Ty>(&dwork, (m + n) * nb)) {
+    const size_t size = (m + n) * nb;
+    if (MAGMA_SUCCESS != magma_malloc<Ty>(&dwork, size)) {
         *info = MAGMA_ERR_DEVICE_ALLOC;
         return *info;
     }
     size_t dwork_offset = 0;
+    // initialize dwork to 0.0
+    const float dfill = 0.0;
+    cl_int err = clEnqueueFillBuffer(queue, dwork, &dfill, sizeof(dfill), 0,
+                                     size * sizeof(Ty), 0, nullptr, nullptr);
+    check_error(err);
 
     cl_event event = 0;
 

From 852776d17769bcd55cce452a6637d666699e9a0f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 9 May 2023 18:54:48 -0400
Subject: [PATCH 286/473] Update accessor and fix segfault in mean

---
 src/backend/oneapi/kernel/mean.hpp | 109 +++++++++++++----------------
 1 file changed, 49 insertions(+), 60 deletions(-)

diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 7d622e611c..e7281f2e45 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -217,7 +217,8 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
     sycl::range<2> global(blocks_dim[0] * blocks_dim[2] * local[0],
                           blocks_dim[1] * blocks_dim[3] * local[1]);
 
-    sycl::buffer<Tw, 1> empty(sycl::range<1>{1});
+    auto empty  = memAlloc<Tw>(1);
+    auto oempty = memAlloc<Tw>(1);
     getQueue().submit([&](sycl::handler &h) {
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
@@ -233,8 +234,8 @@ void mean_dim_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
         bool output_weight = ((owt.info.dims[0] * owt.info.dims[1] *
                                owt.info.dims[2] * owt.info.dims[3]) != 0);
 
-        write_accessor<Tw> owt_acc{(output_weight) ? *owt.data : empty, h};
-        read_accessor<Tw> iwt_acc{(input_weight) ? *iwt.data : empty, h};
+        write_accessor<Tw> owt_acc{(output_weight) ? *owt.data : *oempty, h};
+        read_accessor<Tw> iwt_acc{(input_weight) ? *iwt.data : *empty, h};
 
         switch (threads_y) {
             case 8:
@@ -484,17 +485,18 @@ class meanFirstKernelSMEM {
 };
 
 template<typename Ti, typename Tw, typename To>
-void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
-                         Param<Tw> iwt, const uint groups_x,
-                         const uint groups_y, const uint threads_x) {
+sycl::event mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
+                                Param<Tw> iwt, const uint groups_x,
+                                const uint groups_y, const uint threads_x) {
     sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
     sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
                           groups_y * in.info.dims[3] * local[1]);
 
     uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
 
-    sycl::buffer<Tw, 1> empty(sycl::range<1>{1});
-    getQueue().submit([&](sycl::handler &h) {
+    auto empty  = memAlloc<Tw>(1);
+    auto oempty = memAlloc<Tw>(1);
+    return getQueue().submit([&](sycl::handler &h) {
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
@@ -509,8 +511,8 @@ void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
         bool output_weight = ((owt.info.dims[0] * owt.info.dims[1] *
                                owt.info.dims[2] * owt.info.dims[3]) != 0);
 
-        write_accessor<Tw> owt_acc{(output_weight) ? *owt.data : empty, h};
-        read_accessor<Tw> iwt_acc{(input_weight) ? *iwt.data : empty, h};
+        write_accessor<Tw> owt_acc{(output_weight) ? *owt.data : *oempty, h};
+        read_accessor<Tw> iwt_acc{(input_weight) ? *iwt.data : *empty, h};
 
         h.parallel_for(
             sycl::nd_range<2>(global, local),
@@ -519,7 +521,6 @@ void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                 iwt.info, threads_x, groups_x, groups_y, repeat, s_val, s_idx,
                 input_weight, output_weight));
     });
-    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 template<typename Ti, typename Tw, typename To>
@@ -612,26 +613,24 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         uintl tmp_elements = tmpOut.elements();
 
         mean_first_launcher<T, Tw, T>(tmpOut, tmpWt, in, iwt, blocks_x,
-                                      blocks_y, threads_x);
+                                      blocks_y, threads_x)
+            .wait();
 
         std::vector<T> h_ptr(tmp_elements);
         std::vector<Tw> h_wptr(tmp_elements);
-        sycl::buffer hBuffer(h_ptr.data(), {tmp_elements},
-                             {sycl::property::buffer::use_host_ptr()});
-        sycl::buffer hwBuffer(h_wptr.data(), {tmp_elements},
-                              {sycl::property::buffer::use_host_ptr()});
 
         // TODO: fix when addressing other mean errors
         auto e1 = getQueue().submit([&](sycl::handler &h) {
             auto acc_in =
-                tmpOut.get()->get_access(h, sycl::range{tmp_elements});
-            auto acc_out = hBuffer.get_access();
-            h.copy(acc_in, acc_out);
+                tmpOut.get()->template get_access<sycl::access_mode::read>(
+                    h, sycl::range{tmp_elements});
+            h.copy(acc_in, h_ptr.data());
         });
         auto e2 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in = tmpWt.get()->get_access(h, sycl::range{tmp_elements});
-            auto acc_out = hwBuffer.get_access();
-            h.copy(acc_in, acc_out);
+            auto acc_in =
+                tmpWt.get()->template get_access<sycl::access_mode::read>(
+                    h, sycl::range{tmp_elements});
+            h.copy(acc_in, h_wptr.data());
         });
         e1.wait();
         e2.wait();
@@ -649,20 +648,16 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         std::vector<T> h_ptr(in_elements);
         std::vector<Tw> h_wptr(in_elements);
 
-        sycl::buffer hBuffer(h_ptr.data(), {in_elements},
-                             {sycl::property::buffer::use_host_ptr()});
-        sycl::buffer hwBuffer(h_wptr.data(), {in_elements},
-                              {sycl::property::buffer::use_host_ptr()});
-
         auto e1 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in  = in.data->get_access(h, sycl::range{in_elements});
-            auto acc_out = hBuffer.get_access();
-            h.copy(acc_in, acc_out);
+            auto acc_in = in.data->template get_access<sycl::access_mode::read>(
+                h, sycl::range{in_elements});
+            h.copy(acc_in, h_ptr.data());
         });
         auto e2 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in  = iwt.data->get_access(h, sycl::range{in_elements});
-            auto acc_out = hwBuffer.get_access();
-            h.copy(acc_in, acc_out);
+            auto acc_in =
+                iwt.data->template get_access<sycl::access_mode::read>(
+                    h, sycl::range{in_elements});
+            h.copy(acc_in, h_wptr.data());
         });
         e1.wait();
         e2.wait();
@@ -720,21 +715,17 @@ To mean_all(Param<Ti> in) {
         std::vector<To> h_ptr(tmp_elements);
         std::vector<Tw> h_cptr(tmp_elements);
 
-        sycl::buffer hBuffer(h_ptr.data(), {tmp_elements},
-                             {sycl::property::buffer::use_host_ptr()});
-        sycl::buffer hcBuffer(h_cptr.data(), {tmp_elements},
-                              {sycl::property::buffer::use_host_ptr()});
-
         auto e1 = getQueue().submit([&](sycl::handler &h) {
             auto acc_in =
-                tmpOut.get()->get_access(h, sycl::range{tmp_elements});
-            auto acc_out = hBuffer.get_access();
-            h.copy(acc_in, acc_out);
+                tmpOut.get()->template get_access<sycl::access_mode::read>(
+                    h, sycl::range{tmp_elements});
+            h.copy(acc_in, h_ptr.data());
         });
         auto e2 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in = tmpCt.get()->get_access(h, sycl::range{tmp_elements});
-            auto acc_out = hcBuffer.get_access();
-            h.copy(acc_in, acc_out);
+            auto acc_in =
+                tmpCt.get()->template get_access<sycl::access_mode::read>(
+                    h, sycl::range{tmp_elements});
+            h.copy(acc_in, h_cptr.data());
         });
         e1.wait();
         e2.wait();
@@ -749,27 +740,25 @@ To mean_all(Param<Ti> in) {
 
         return static_cast<To>(val);
     } else {
-        std::vector<Ti> h_ptr(in_elements);
-        sycl::buffer outBuffer(h_ptr.data(), {in_elements},
-                               {sycl::property::buffer::use_host_ptr()});
-
+        compute_t<To> val;
         getQueue()
             .submit([&](sycl::handler &h) {
-                auto acc_in  = in.data->get_access(h);
-                auto acc_out = outBuffer.get_access();
-                h.copy(acc_in, acc_out);
+                auto acc_in =
+                    in.data->template get_access<sycl::access_mode::read,
+                                                 sycl::target::host_buffer>(h);
+                h.host_task([&]() {
+                    common::Transform<Ti, compute_t<To>, af_add_t> transform;
+                    compute_t<Tw> count = static_cast<compute_t<Tw>>(1);
+
+                    val                  = transform(acc_in[0]);
+                    compute_t<Tw> weight = count;
+                    for (int i = 1; i < in_elements; i++) {
+                        stable_mean(&val, &weight, transform(acc_in[i]), count);
+                    }
+                });
             })
             .wait();
 
-        common::Transform<Ti, compute_t<To>, af_add_t> transform;
-        compute_t<Tw> count = static_cast<compute_t<Tw>>(1);
-
-        compute_t<To> val    = transform(h_ptr[0]);
-        compute_t<Tw> weight = count;
-        for (int i = 1; i < in_elements; i++) {
-            stable_mean(&val, &weight, transform(h_ptr[i]), count);
-        }
-
         return static_cast<To>(val);
     }
 }

From 67c013742c144cc553cb8eb835775011c416b59b Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 9 May 2023 18:57:27 -0400
Subject: [PATCH 287/473] Fix segfault in isHalfSupported function

---
 src/backend/oneapi/platform.cpp | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index edd62e0d6a..d9b6f1d832 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -384,12 +384,8 @@ bool isDoubleSupported(unsigned device) {
 bool isHalfSupported(unsigned device) {
     DeviceManager& devMngr = DeviceManager::getInstance();
 
-    sycl::device dev;
-    {
-        common::lock_guard_t lock(devMngr.deviceMutex);
-        dev = *devMngr.mDevices[device];
-    }
-    return dev.has(sycl::aspect::fp16);
+    common::lock_guard_t lock(devMngr.deviceMutex);
+    return devMngr.mDevices[device]->has(sycl::aspect::fp16);
 }
 
 void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute) {

From 5191729bfa9598978b9c19976b8f7b46697725c5 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 9 May 2023 23:36:47 -0400
Subject: [PATCH 288/473] Update submit function lambdas to change to reference
 captures

---
 src/backend/oneapi/copy.cpp                          |  4 ++--
 src/backend/oneapi/join.cpp                          |  6 +++---
 src/backend/oneapi/kernel/assign.hpp                 |  2 +-
 src/backend/oneapi/kernel/index.hpp                  |  2 +-
 src/backend/oneapi/kernel/iota.hpp                   |  2 +-
 src/backend/oneapi/kernel/memcopy.hpp                |  4 ++--
 src/backend/oneapi/kernel/random_engine.hpp          | 12 ++++++------
 src/backend/oneapi/kernel/random_engine_mersenne.hpp |  2 +-
 src/backend/oneapi/kernel/range.hpp                  |  2 +-
 src/backend/oneapi/kernel/reduce_dim.hpp             |  2 +-
 src/backend/oneapi/kernel/reduce_first.hpp           |  2 +-
 11 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index a70cc3a6f4..f99f79854e 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -61,7 +61,7 @@ Array<T> copyArray(const Array<T> &A) {
             sycl::buffer<T> *out_buf = out.get();
 
             size_t aelem = A.elements();
-            getQueue().submit([=](sycl::handler &h) {
+            getQueue().submit([&](sycl::handler &h) {
                 range rr(aelem);
                 id offset_id(offset);
                 accessor offset_acc_A =
@@ -114,7 +114,7 @@ struct copyWrapper<T, T> {
             sycl::buffer<T> *out_buf = out.get();
 
             getQueue()
-                .submit([=](sycl::handler &h) {
+                .submit([&](sycl::handler &h) {
                     sycl::range rr(in.elements());
                     sycl::id in_offset_id(in_offset);
                     sycl::id out_offset_id(out_offset);
diff --git a/src/backend/oneapi/join.cpp b/src/backend/oneapi/join.cpp
index ecbcae0ba4..37c7c14fc9 100644
--- a/src/backend/oneapi/join.cpp
+++ b/src/backend/oneapi/join.cpp
@@ -94,7 +94,7 @@ Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     if (first.isReady()) {
         if (1LL + jdim >= first.ndims() && first.isLinear()) {
             // first & out are linear
-            getQueue().submit([=](sycl::handler &h) {
+            getQueue().submit([&](sycl::handler &h) {
                 sycl::range sz(first.elements());
                 sycl::id src_offset(first.getOffset());
                 sycl::accessor offset_acc_src =
@@ -125,7 +125,7 @@ Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     if (second.isReady()) {
         if (1LL + jdim >= second.ndims() && second.isLinear()) {
             // second & out are linear
-            getQueue().submit([=](sycl::handler &h) {
+            getQueue().submit([&](sycl::handler &h) {
                 sycl::range sz(second.elements());
                 sycl::id src_offset(second.getOffset());
                 sycl::accessor offset_acc_src =
@@ -216,7 +216,7 @@ void join(Array<T> &out, const int jdim, const vector<Array<T>> &inputs) {
             for (const Array<T> *in : s.ins) {
                 if (in->isReady()) {
                     if (1LL + jdim >= in->ndims() && in->isLinear()) {
-                        getQueue().submit([=](sycl::handler &h) {
+                        getQueue().submit([&](sycl::handler &h) {
                             sycl::range sz(in->elements());
                             sycl::id src_offset(in->getOffset());
                             sycl::accessor offset_acc_src =
diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 6d553f18ad..5e3ef6c666 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -125,7 +125,7 @@ void assign(Param<T> out, const Param<T> in, const AssignKernelParam& p,
     sycl::range<2> global(blk_x * in.info.dims[2] * THREADS_X,
                           blk_y * in.info.dims[3] * THREADS_Y);
 
-    getQueue().submit([=](sycl::handler& h) {
+    getQueue().submit([&](sycl::handler& h) {
         auto pp = p;
         write_accessor<T> out_acc{*out.data, h};
         read_accessor<T> in_acc{*in.data, h};
diff --git a/src/backend/oneapi/kernel/index.hpp b/src/backend/oneapi/kernel/index.hpp
index ef2b837b75..857b299aef 100644
--- a/src/backend/oneapi/kernel/index.hpp
+++ b/src/backend/oneapi/kernel/index.hpp
@@ -133,7 +133,7 @@ void index(Param<T> out, Param<T> in, IndexKernelParam& p,
     blocks[0] *= threads[0];
 
     sycl::nd_range<3> marange(blocks, threads);
-    getQueue().submit([=](sycl::handler& h) {
+    getQueue().submit([&](sycl::handler& h) {
         auto pp = p;
         for (dim_t x = 0; x < 4; ++x) {
             pp.ptr[x] =
diff --git a/src/backend/oneapi/kernel/iota.hpp b/src/backend/oneapi/kernel/iota.hpp
index 97018b6a1d..f334695ef5 100644
--- a/src/backend/oneapi/kernel/iota.hpp
+++ b/src/backend/oneapi/kernel/iota.hpp
@@ -101,7 +101,7 @@ void iota(Param<T> out, const af::dim4& sdims) {
                           local[1] * blocksPerMatY * out.info.dims[3]);
     sycl::nd_range<2> ndrange(global, local);
 
-    getQueue().submit([=](sycl::handler& h) {
+    getQueue().submit([&](sycl::handler& h) {
         write_accessor<T> out_acc{*out.data, h};
 
         h.parallel_for(ndrange, iotaKernel<T>(out_acc, out.info,
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index 33a53fc160..c6b8dbb04c 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -115,7 +115,7 @@ void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
                           groups_1 * idims[3] * local_size[1]);
     sycl::nd_range<2> ndrange(global, local);
 
-    getQueue().submit([=](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         write_accessor<T> out_acc{*out, h};
         read_accessor<T> in_acc{*const_cast<sycl::buffer<T> *>(in), h};
 
@@ -303,7 +303,7 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
         trgt_dims    = {{trgt_i, trgt_j, trgt_k, trgt_l}};
     }
 
-    getQueue().submit([=](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         write_accessor<outType> dst_acc{*dst.data, h};
         read_accessor<inType> src_acc{
             *const_cast<sycl::buffer<inType> *>(src.data), h};
diff --git a/src/backend/oneapi/kernel/random_engine.hpp b/src/backend/oneapi/kernel/random_engine.hpp
index b416827a7d..7e97a6fc59 100644
--- a/src/backend/oneapi/kernel/random_engine.hpp
+++ b/src/backend/oneapi/kernel/random_engine.hpp
@@ -56,7 +56,7 @@ void uniformDistributionCBRNG(Param<T> out, const size_t elements,
                               sycl::range<1>(threads));
     switch (type) {
         case AF_RANDOM_ENGINE_PHILOX_4X32_10:
-            getQueue().submit([=](sycl::handler &h) {
+            getQueue().submit([&](sycl::handler &h) {
                 write_accessor<T> out_acc{*out.data, h};
 
                 h.parallel_for(ndrange,
@@ -66,7 +66,7 @@ void uniformDistributionCBRNG(Param<T> out, const size_t elements,
             ONEAPI_DEBUG_FINISH(getQueue());
             break;
         case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
-            getQueue().submit([=](sycl::handler &h) {
+            getQueue().submit([&](sycl::handler &h) {
                 write_accessor<T> out_acc{*out.data, h};
 
                 h.parallel_for(ndrange,
@@ -96,7 +96,7 @@ void normalDistributionCBRNG(Param<T> out, const size_t elements,
                               sycl::range<1>(threads));
     switch (type) {
         case AF_RANDOM_ENGINE_PHILOX_4X32_10:
-            getQueue().submit([=](sycl::handler &h) {
+            getQueue().submit([&](sycl::handler &h) {
                 write_accessor<T> out_acc{*out.data, h};
 
                 h.parallel_for(ndrange,
@@ -105,7 +105,7 @@ void normalDistributionCBRNG(Param<T> out, const size_t elements,
             });
             break;
         case AF_RANDOM_ENGINE_THREEFRY_2X32_16:
-            getQueue().submit([=](sycl::handler &h) {
+            getQueue().submit([&](sycl::handler &h) {
                 write_accessor<T> out_acc{*out.data, h};
 
                 h.parallel_for(ndrange,
@@ -134,7 +134,7 @@ void uniformDistributionMT(Param<T> out, const size_t elements,
 
     sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads),
                               sycl::range<1>(threads));
-    getQueue().submit([=](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         write_accessor<T> out_acc{*out.data, h};
         auto state_acc     = state.data->get_access(h);
         auto pos_acc       = pos.data->get_access(h);
@@ -170,7 +170,7 @@ void normalDistributionMT(Param<T> out, const size_t elements,
 
     sycl::nd_range<1> ndrange(sycl::range<1>(blocks * threads),
                               sycl::range<1>(threads));
-    getQueue().submit([=](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         write_accessor<T> out_acc{*out.data, h};
         auto state_acc     = state.data->get_access(h);
         auto pos_acc       = pos.data->get_access(h);
diff --git a/src/backend/oneapi/kernel/random_engine_mersenne.hpp b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
index f36b2b60d0..acb56f3c9f 100644
--- a/src/backend/oneapi/kernel/random_engine_mersenne.hpp
+++ b/src/backend/oneapi/kernel/random_engine_mersenne.hpp
@@ -145,7 +145,7 @@ class initMersenneKernel {
 
 void initMersenneState(Param<uint> state, const Param<uint> tbl, uintl seed) {
     sycl::nd_range<1> ndrange({BLOCKS * N}, {N});
-    getQueue().submit([=](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         write_accessor<uint> state_acc{*state.data, h};
         read_accessor<uint> tbl_acc{*tbl.data, h};
         auto lstate_acc = sycl::local_accessor<uint, 1>(N, h);
diff --git a/src/backend/oneapi/kernel/range.hpp b/src/backend/oneapi/kernel/range.hpp
index f052abb48c..b8678179c2 100644
--- a/src/backend/oneapi/kernel/range.hpp
+++ b/src/backend/oneapi/kernel/range.hpp
@@ -104,7 +104,7 @@ void range(Param<T> out, const int dim) {
                           local[1] * blocksPerMatY * out.info.dims[3]);
     sycl::nd_range<2> ndrange(global, local);
 
-    getQueue().submit([=](sycl::handler& h) {
+    getQueue().submit([&](sycl::handler& h) {
         write_accessor<T> out_acc{*out.data, h};
 
         h.parallel_for(ndrange, rangeOp<T>(out_acc, out.info, dim,
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index 926a7205e9..6b51801fa7 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -144,7 +144,7 @@ void reduce_dim_launcher_default(Param<To> out, Param<Ti> in,
     sycl::range<2> global(blocks_dim[0] * blocks_dim[2] * local[0],
                           blocks_dim[1] * blocks_dim[3] * local[1]);
 
-    getQueue().submit([=](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         auto shrdMem = sycl::local_accessor<compute_t<To>, 1>(
             creduce::THREADS_X * threads_y, h);
 
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index 27143aa24b..f105d63671 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -151,7 +151,7 @@ void reduce_first_launcher_default(Param<To> out, Param<Ti> in,
 
     uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
 
-    getQueue().submit([=](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 

From efed24b9ce9773b9c1aa05751a058939a8755640 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 9 May 2023 23:38:57 -0400
Subject: [PATCH 289/473] Change basic_c's add_test function because its not a
 gtest test

The basic_c test binary is not a Google Test binary so the gtest
module in CMake fails when it tries to find tests.
---
 test/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 5b7c869eba..0cb3cbfe51 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -480,7 +480,7 @@ foreach(backend ${enabled_backends})
       PRIVATE
       ArrayFire::af${backend})
   endif()
-  af_add_test(${target} ${backend} ON)
+  add_test(NAME ${target} COMMAND ${target})
 endforeach()
 
 if(AF_TEST_WITH_MTX_FILES)

From 15893ab197199009d18b6d8921b35e0d887de5e5 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 10 May 2023 20:04:51 -0400
Subject: [PATCH 290/473] Perform host operation in host_tasks for mean

---
 src/backend/oneapi/kernel/mean.hpp | 150 ++++++++++++++---------------
 1 file changed, 70 insertions(+), 80 deletions(-)

diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index e7281f2e45..ef98cb0954 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -613,62 +613,55 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         uintl tmp_elements = tmpOut.elements();
 
         mean_first_launcher<T, Tw, T>(tmpOut, tmpWt, in, iwt, blocks_x,
-                                      blocks_y, threads_x)
-            .wait();
-
-        std::vector<T> h_ptr(tmp_elements);
-        std::vector<Tw> h_wptr(tmp_elements);
-
-        // TODO: fix when addressing other mean errors
-        auto e1 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in =
-                tmpOut.get()->template get_access<sycl::access_mode::read>(
-                    h, sycl::range{tmp_elements});
-            h.copy(acc_in, h_ptr.data());
-        });
-        auto e2 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in =
-                tmpWt.get()->template get_access<sycl::access_mode::read>(
-                    h, sycl::range{tmp_elements});
-            h.copy(acc_in, h_wptr.data());
-        });
-        e1.wait();
-        e2.wait();
-
-        compute_t<T> val     = static_cast<compute_t<T>>(h_ptr[0]);
-        compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_wptr[0]);
-
-        for (int i = 1; i < tmp_elements; i++) {
-            stable_mean(&val, &weight, compute_t<T>(h_ptr[i]),
-                        compute_t<Tw>(h_wptr[i]));
-        }
+                                      blocks_y, threads_x);
 
+        compute_t<T> val;
+        getQueue()
+            .submit([&](sycl::handler &h) {
+                auto acc_in =
+                    tmpOut.get()
+                        ->template get_access<sycl::access_mode::read,
+                                              sycl::target::host_buffer>(h);
+                auto acc_wt =
+                    tmpWt.get()
+                        ->template get_access<sycl::access_mode::read,
+                                              sycl::target::host_buffer>(h);
+
+                h.host_task([acc_in, acc_wt, tmp_elements, &val] {
+                    val = static_cast<compute_t<T>>(acc_in[0]);
+                    compute_t<Tw> weight =
+                        static_cast<compute_t<Tw>>(acc_wt[0]);
+
+                    for (int i = 1; i < tmp_elements; i++) {
+                        stable_mean(&val, &weight, compute_t<T>(acc_in[i]),
+                                    compute_t<Tw>(acc_wt[i]));
+                    }
+                });
+            })
+            .wait();
         return static_cast<T>(val);
     } else {
-        std::vector<T> h_ptr(in_elements);
-        std::vector<Tw> h_wptr(in_elements);
-
-        auto e1 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in = in.data->template get_access<sycl::access_mode::read>(
-                h, sycl::range{in_elements});
-            h.copy(acc_in, h_ptr.data());
-        });
-        auto e2 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in =
-                iwt.data->template get_access<sycl::access_mode::read>(
-                    h, sycl::range{in_elements});
-            h.copy(acc_in, h_wptr.data());
-        });
-        e1.wait();
-        e2.wait();
-
-        compute_t<T> val     = static_cast<compute_t<T>>(h_ptr[0]);
-        compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_wptr[0]);
-        for (int i = 1; i < in_elements; i++) {
-            stable_mean(&val, &weight, compute_t<T>(h_ptr[i]),
-                        compute_t<Tw>(h_wptr[i]));
-        }
-
+        compute_t<T> val;
+        getQueue()
+            .submit([&](sycl::handler &h) {
+                auto acc_in =
+                    in.data->template get_access<sycl::access_mode::read,
+                                                 sycl::target::host_buffer>(
+                        h, sycl::range{in_elements});
+                auto acc_wt =
+                    iwt.data->template get_access<sycl::access_mode::read>(
+                        h, sycl::range{in_elements});
+
+                h.host_task([acc_in, acc_wt, in_elements, &val]() {
+                    val                  = acc_in[0];
+                    compute_t<Tw> weight = acc_wt[0];
+                    for (int i = 1; i < in_elements; i++) {
+                        stable_mean(&val, &weight, compute_t<T>(acc_in[i]),
+                                    compute_t<Tw>(acc_wt[i]));
+                    }
+                });
+            })
+            .wait();
         return static_cast<T>(val);
     }
 }
@@ -712,32 +705,30 @@ To mean_all(Param<Ti> in) {
                                         blocks_y, threads_x);
 
         uintl tmp_elements = tmpOut.elements();
-        std::vector<To> h_ptr(tmp_elements);
-        std::vector<Tw> h_cptr(tmp_elements);
-
-        auto e1 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in =
-                tmpOut.get()->template get_access<sycl::access_mode::read>(
-                    h, sycl::range{tmp_elements});
-            h.copy(acc_in, h_ptr.data());
-        });
-        auto e2 = getQueue().submit([&](sycl::handler &h) {
-            auto acc_in =
-                tmpCt.get()->template get_access<sycl::access_mode::read>(
-                    h, sycl::range{tmp_elements});
-            h.copy(acc_in, h_cptr.data());
-        });
-        e1.wait();
-        e2.wait();
-
-        compute_t<To> val    = static_cast<compute_t<To>>(h_ptr[0]);
-        compute_t<Tw> weight = static_cast<compute_t<Tw>>(h_cptr[0]);
-
-        for (int i = 1; i < tmp_elements; i++) {
-            stable_mean(&val, &weight, compute_t<To>(h_ptr[i]),
-                        compute_t<Tw>(h_cptr[i]));
-        }
 
+        compute_t<To> val;
+        getQueue()
+            .submit([&](sycl::handler &h) {
+                auto out =
+                    tmpOut.get()
+                        ->template get_access<sycl::access_mode::read,
+                                              sycl::target::host_buffer>(h);
+                auto ct =
+                    tmpCt.get()
+                        ->template get_access<sycl::access_mode::read,
+                                              sycl::target::host_buffer>(h);
+
+                h.host_task([out, ct, tmp_elements, &val] {
+                    val                  = static_cast<compute_t<To>>(out[0]);
+                    compute_t<Tw> weight = static_cast<compute_t<Tw>>(ct[0]);
+
+                    for (int i = 1; i < tmp_elements; i++) {
+                        stable_mean(&val, &weight, compute_t<To>(out[i]),
+                                    compute_t<Tw>(ct[i]));
+                    }
+                });
+            })
+            .wait();
         return static_cast<To>(val);
     } else {
         compute_t<To> val;
@@ -746,7 +737,7 @@ To mean_all(Param<Ti> in) {
                 auto acc_in =
                     in.data->template get_access<sycl::access_mode::read,
                                                  sycl::target::host_buffer>(h);
-                h.host_task([&]() {
+                h.host_task([acc_in, in_elements, &val]() {
                     common::Transform<Ti, compute_t<To>, af_add_t> transform;
                     compute_t<Tw> count = static_cast<compute_t<Tw>>(1);
 
@@ -758,7 +749,6 @@ To mean_all(Param<Ti> in) {
                 });
             })
             .wait();
-
         return static_cast<To>(val);
     }
 }

From e9432c2788ab6a4bbe4af036224d4572c15c1e77 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 10 May 2023 20:05:16 -0400
Subject: [PATCH 291/473] Add formatters for dim4 and complex

---
 src/backend/common/ArrayFireTypesIO.hpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/backend/common/ArrayFireTypesIO.hpp b/src/backend/common/ArrayFireTypesIO.hpp
index 8d36aa54c1..e7a2e085ee 100644
--- a/src/backend/common/ArrayFireTypesIO.hpp
+++ b/src/backend/common/ArrayFireTypesIO.hpp
@@ -10,7 +10,9 @@
 #pragma once
 #include <common/Version.hpp>
 #include <spdlog/fmt/ostr.h>
+#include <af/dim4.hpp>
 #include <af/seq.h>
+#include <complex>
 
 template<>
 struct fmt::formatter<af_seq> {
@@ -33,6 +35,15 @@ struct fmt::formatter<af_seq> {
     }
 };
 
+#if FMT_VERSION >= 90000
+template<>
+struct fmt::formatter<af::dim4> : ostream_formatter {};
+template<>
+struct fmt::formatter<std::complex<float>> : ostream_formatter {};
+template<>
+struct fmt::formatter<std::complex<double>> : ostream_formatter {};
+#endif
+
 template<>
 struct fmt::formatter<arrayfire::common::Version> {
     // show major version

From f3887ea091bf896bbccf83e350782b3baadfdfff Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 16 May 2023 12:26:44 -0400
Subject: [PATCH 292/473] Fix some maxDims tests due to launch dimensions
 exceeding int range

---
 src/backend/oneapi/CMakeLists.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 8ea40564e9..f541bcb13b 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -294,6 +294,7 @@ target_include_directories(afoneapi
 target_compile_options(afoneapi
   PRIVATE
     -fsycl
+    -fno-sycl-id-queries-fit-in-int
     -sycl-std=2020
 )
 
@@ -317,9 +318,9 @@ target_link_libraries(afoneapi
     afcommon_interface
     OpenCL::OpenCL
     OpenCL::cl2hpp
-    -fsycl
     -fsycl-device-code-split=per_kernel
     -fsycl-link-huge-device-code
+    -fno-sycl-id-queries-fit-in-int
     MKL::MKL_DPCPP
   )
 

From 3fa27a58c63a91853c830d510fd5900cf7767965 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 16 May 2023 12:27:56 -0400
Subject: [PATCH 293/473] Improve oneAPI debug link times using -fno-sycl-rdc

---
 src/backend/oneapi/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index f541bcb13b..6de42d891c 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -296,6 +296,7 @@ target_compile_options(afoneapi
     -fsycl
     -fno-sycl-id-queries-fit-in-int
     -sycl-std=2020
+    -fno-sycl-rdc
 )
 
 target_compile_definitions(afoneapi
@@ -321,6 +322,7 @@ target_link_libraries(afoneapi
     -fsycl-device-code-split=per_kernel
     -fsycl-link-huge-device-code
     -fno-sycl-id-queries-fit-in-int
+    -fno-sycl-rdc
     MKL::MKL_DPCPP
   )
 

From 25d2b692f4f7374a7d93157da2611c2a91b2819a Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 16 May 2023 17:00:57 -0400
Subject: [PATCH 294/473] Add -fsycl-max-parallel-link-jobs flag to improve
 link times

---
 src/backend/oneapi/CMakeLists.txt | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 6de42d891c..46e20c88d4 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -309,6 +309,9 @@ target_compile_definitions(afoneapi
     CL_HPP_ENABLE_EXCEPTIONS
   )
 
+cmake_host_system_information(RESULT NumberOfThreads
+  QUERY NUMBER_OF_LOGICAL_CORES)
+
 target_link_libraries(afoneapi
   PRIVATE
     -fsycl
@@ -319,10 +322,11 @@ target_link_libraries(afoneapi
     afcommon_interface
     OpenCL::OpenCL
     OpenCL::cl2hpp
-    -fsycl-device-code-split=per_kernel
-    -fsycl-link-huge-device-code
     -fno-sycl-id-queries-fit-in-int
     -fno-sycl-rdc
+    -fsycl-device-code-split=per_kernel
+    -fsycl-link-huge-device-code
+    -fsycl-max-parallel-link-jobs=${NumberOfThreads}
     MKL::MKL_DPCPP
   )
 

From 46d50bc6f70e75b957e174c72955247d792fb391 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 11 May 2023 23:55:33 +0200
Subject: [PATCH 295/473] Integrated magma memory allocations into arrayfire
 memory mgt

---
 src/backend/opencl/magma/magma_data.h | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/backend/opencl/magma/magma_data.h b/src/backend/opencl/magma/magma_data.h
index 6ee5ac053e..04a1e5261c 100644
--- a/src/backend/opencl/magma/magma_data.h
+++ b/src/backend/opencl/magma/magma_data.h
@@ -55,6 +55,7 @@
 #ifndef MAGMA_DATA_H
 #define MAGMA_DATA_H
 
+#include <memory.hpp>
 #include <platform.hpp>
 #include "magma_types.h"
 
@@ -70,18 +71,18 @@ static magma_int_t magma_malloc(magma_ptr* ptrPtr, int num) {
     // malloc and free sometimes don't work for size=0, so allocate some minimal
     // size
     if (size == 0) size = sizeof(T);
-    cl_int err;
-    *ptrPtr = clCreateBuffer(arrayfire::opencl::getContext()(),
-                             CL_MEM_READ_WRITE, size, NULL, &err);
-    if (err != CL_SUCCESS) { return MAGMA_ERR_DEVICE_ALLOC; }
+    cl::Buffer* buf = arrayfire::opencl::bufferAlloc(size);
+    *ptrPtr         = static_cast<magma_ptr>(buf->get());
+    delete (buf);
+
+    if (ptrPtr == nullptr) { return MAGMA_ERR_DEVICE_ALLOC; };
     return MAGMA_SUCCESS;
 }
 
 // --------------------
 // Free GPU memory allocated by magma_malloc.
 static inline magma_int_t magma_free(magma_ptr ptr) {
-    cl_int err = clReleaseMemObject(ptr);
-    if (err != CL_SUCCESS) { return MAGMA_ERR_INVALID_PTR; }
+    arrayfire::opencl::memFree(ptr);
     return MAGMA_SUCCESS;
 }
 

From a5ad10b1219b34204d22c39da9ec9a1711081648 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 20 May 2023 15:37:45 -0400
Subject: [PATCH 296/473] Refactor isnan and is_nan functions to use standard
 isnan functions

---
 src/backend/common/complex.hpp             |  2 +-
 src/backend/common/half.hpp                |  1 +
 src/backend/cpu/kernel/ireduce.hpp         |  8 ++---
 src/backend/cpu/math.hpp                   | 30 ++++++++++++++++++
 src/backend/cuda/compile_module.cpp        |  2 +-
 src/backend/cuda/complex.hpp               |  2 ++
 src/backend/cuda/math.hpp                  | 36 ++++++++++++++++++++++
 src/backend/cuda/minmax_op.hpp             | 17 ++--------
 src/backend/oneapi/math.cpp                | 33 ++------------------
 src/backend/oneapi/math.hpp                | 32 +++++++++++++++++--
 src/backend/oneapi/minmax_op.hpp           | 16 +---------
 src/backend/opencl/kernel/sparse_arith.hpp |  1 +
 12 files changed, 111 insertions(+), 69 deletions(-)

diff --git a/src/backend/common/complex.hpp b/src/backend/common/complex.hpp
index b7663580dc..e6c5bb79ce 100644
--- a/src/backend/common/complex.hpp
+++ b/src/backend/common/complex.hpp
@@ -6,8 +6,8 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
+#pragma once
 
-#include <Array.hpp>
 #include <backend.hpp>
 #include <types.hpp>
 
diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index 67bd47829f..ac03ea6d89 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -88,6 +88,7 @@ using uint16_t = unsigned short;
 #else
 #include <af/compilers.h>
 #include <cmath>
+#include <cstdint>
 #include <cstring>
 #include <ostream>
 #include <string>
diff --git a/src/backend/cpu/kernel/ireduce.hpp b/src/backend/cpu/kernel/ireduce.hpp
index 9c371498c7..9d2598af4b 100644
--- a/src/backend/cpu/kernel/ireduce.hpp
+++ b/src/backend/cpu/kernel/ireduce.hpp
@@ -10,7 +10,9 @@
 #pragma once
 #include <Param.hpp>
 #include <common/Binary.hpp>
+#include <common/half.hpp>
 #include <algorithm>
+#include <cmath>
 
 namespace arrayfire {
 namespace cpu {
@@ -23,16 +25,13 @@ double cabs(const T in) {
 static double cabs(const char in) { return (double)(in > 0); }
 static double cabs(const cfloat &in) { return (double)abs(in); }
 static double cabs(const cdouble &in) { return (double)abs(in); }
-template<typename T>
-static bool is_nan(T in) {
-    return in != in;
-}
 
 template<af_op_t op, typename T>
 struct MinMaxOp {
     T m_val;
     uint m_idx;
     MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) {
+        using arrayfire::cpu::is_nan;
         if (is_nan(val)) { m_val = common::Binary<T, op>::init(); }
     }
 
@@ -50,6 +49,7 @@ struct MinMaxOp<af_max_t, T> {
     T m_val;
     uint m_idx;
     MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) {
+        using arrayfire::cpu::is_nan;
         if (is_nan(val)) { m_val = common::Binary<T, af_max_t>::init(); }
     }
 
diff --git a/src/backend/cpu/math.hpp b/src/backend/cpu/math.hpp
index d2735acd2a..16a4e2abbf 100644
--- a/src/backend/cpu/math.hpp
+++ b/src/backend/cpu/math.hpp
@@ -42,6 +42,36 @@ static inline T max(T lhs, T rhs) {
 cfloat max(cfloat lhs, cfloat rhs);
 cdouble max(cdouble lhs, cdouble rhs);
 
+template<typename T>
+static inline auto is_nan(const T &val) -> bool {
+    return false;
+}
+
+template<>
+inline auto is_nan<float>(const float &val) -> bool {
+    return std::isnan(val);
+}
+
+template<>
+inline auto is_nan<double>(const double &val) -> bool {
+    return std::isnan(val);
+}
+
+template<>
+inline auto is_nan<common::half>(const common::half &val) -> bool {
+    return isnan(val);
+}
+
+template<>
+inline auto is_nan<cfloat>(const cfloat &in) -> bool {
+    return std::isnan(real(in)) || std::isnan(imag(in));
+}
+
+template<>
+inline auto is_nan<cdouble>(const cdouble &in) -> bool {
+    return std::isnan(real(in)) || std::isnan(imag(in));
+}
+
 template<typename T>
 static inline T division(T lhs, double rhs) {
     return lhs / rhs;
diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp
index 36014049a8..06dfd0f377 100644
--- a/src/backend/cuda/compile_module.cpp
+++ b/src/backend/cuda/compile_module.cpp
@@ -176,7 +176,7 @@ Module compileModule(const string &moduleKey, span<const string> sources,
             "stdbool.h",       // DUMMY ENTRY TO SATISFY af/defines.h inclusion
             "stdlib.h",        // DUMMY ENTRY TO SATISFY af/defines.h inclusion
             "vector_types.h",  // DUMMY ENTRY TO SATISFY cuComplex_h inclusion
-            "utility",         // DUMMY ENTRY TO SATISFY cuda_fp16.hpp inclusion
+            "utility",         // DUMMY ENTRY TO SATISFY utility inclusion
             "backend.hpp",
             "cuComplex.h",
             "jit.cuh",
diff --git a/src/backend/cuda/complex.hpp b/src/backend/cuda/complex.hpp
index d9d143ddbf..81f39dd785 100644
--- a/src/backend/cuda/complex.hpp
+++ b/src/backend/cuda/complex.hpp
@@ -7,6 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#pragma once
+
 #include <Array.hpp>
 #include <binary.hpp>
 #include <common/jit/BinaryNode.hpp>
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index 3562565a86..f7b11347cc 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -260,6 +260,42 @@ __SDH__ double real(cdouble c) { return cuCreal(c); }
 __SDH__ float imag(cfloat c) { return cuCimagf(c); }
 __SDH__ double imag(cdouble c) { return cuCimag(c); }
 
+template<typename T>
+static inline __DH__ auto is_nan(const T &val) -> bool {
+    return false;
+}
+
+template<>
+inline __DH__ auto is_nan<float>(const float &val) -> bool {
+    return ::isnan(val);
+}
+
+template<>
+inline __DH__ auto is_nan<double>(const double &val) -> bool {
+    return ::isnan(val);
+}
+
+#ifdef __CUDA_ARCH__
+template<>
+inline __device__ auto is_nan<__half>(const __half &val) -> bool {
+#if __CUDA_ARCH__ >= 530
+    return __hisnan(val);
+#else
+    return ::isnan(__half2float(val));
+#endif
+}
+#endif
+
+template<>
+inline auto is_nan<cfloat>(const cfloat &in) -> bool {
+    return ::isnan(real(in)) || ::isnan(imag(in));
+}
+
+template<>
+inline auto is_nan<cdouble>(const cdouble &in) -> bool {
+    return ::isnan(real(in)) || ::isnan(imag(in));
+}
+
 template<typename T>
 T __SDH__ conj(T x) {
     return x;
diff --git a/src/backend/cuda/minmax_op.hpp b/src/backend/cuda/minmax_op.hpp
index 4fcc995c0b..a2b7149a07 100644
--- a/src/backend/cuda/minmax_op.hpp
+++ b/src/backend/cuda/minmax_op.hpp
@@ -34,26 +34,12 @@ double cabs<cdouble>(const cdouble &in) {
     return (double)abs(in);
 }
 
-template<typename T>
-static bool is_nan(const T &in) {
-    return in != in;
-}
-
-template<>
-bool is_nan<cfloat>(const cfloat &in) {
-    return in.x != in.x || in.y != in.y;
-}
-
-template<>
-bool is_nan<cdouble>(const cdouble &in) {
-    return in.x != in.x || in.y != in.y;
-}
-
 template<af_op_t op, typename T>
 struct MinMaxOp {
     T m_val;
     uint m_idx;
     MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) {
+        using arrayfire::cuda::is_nan;
         if (is_nan(val)) { m_val = common::Binary<compute_t<T>, op>::init(); }
     }
 
@@ -71,6 +57,7 @@ struct MinMaxOp<af_max_t, T> {
     T m_val;
     uint m_idx;
     MinMaxOp(T val, uint idx) : m_val(val), m_idx(idx) {
+        using arrayfire::cuda::is_nan;
         if (is_nan(val)) { m_val = common::Binary<T, af_max_t>::init(); }
     }
 
diff --git a/src/backend/oneapi/math.cpp b/src/backend/oneapi/math.cpp
index a673f9293b..18bafd324b 100644
--- a/src/backend/oneapi/math.cpp
+++ b/src/backend/oneapi/math.cpp
@@ -12,43 +12,14 @@
 
 namespace arrayfire {
 namespace oneapi {
-cfloat operator+(cfloat lhs, cfloat rhs) {
-    // cfloat res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
-    cfloat res;
-    return res;
-}
-
-cdouble operator+(cdouble lhs, cdouble rhs) {
-    // cdouble res = {{lhs.s[0] + rhs.s[0], lhs.s[1] + rhs.s[1]}};
-    cdouble res;
-    return res;
-}
-
-cfloat operator*(cfloat lhs, cfloat rhs) {
-    cfloat out;
-    // out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1];
-    // out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0];
-    return out;
-}
-
-cdouble operator*(cdouble lhs, cdouble rhs) {
-    cdouble out;
-    // out.s[0] = lhs.s[0] * rhs.s[0] - lhs.s[1] * rhs.s[1];
-    // out.s[1] = lhs.s[0] * rhs.s[1] + lhs.s[1] * rhs.s[0];
-    return out;
-}
 
 cfloat division(cfloat lhs, double rhs) {
-    cfloat retVal;
-    // retVal.s[0] = real(lhs) / rhs;
-    // retVal.s[1] = imag(lhs) / rhs;
+    cfloat retVal(real(lhs) / rhs, imag(lhs) / rhs);
     return retVal;
 }
 
 cdouble division(cdouble lhs, double rhs) {
-    cdouble retVal;
-    // retVal.s[0] = real(lhs) / rhs;
-    // retVal.s[1] = imag(lhs) / rhs;
+    cdouble retVal(real(lhs) / rhs, imag(lhs) / rhs);
     return retVal;
 }
 }  // namespace oneapi
diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp
index 063d82f370..83973994c9 100644
--- a/src/backend/oneapi/math.hpp
+++ b/src/backend/oneapi/math.hpp
@@ -71,6 +71,36 @@ inline cdouble min<cdouble>(cdouble lhs, cdouble rhs) {
     return abs(lhs) < abs(rhs) ? lhs : rhs;
 }
 
+template<typename T>
+static inline auto is_nan(const T &val) -> bool {
+    return false;
+}
+
+template<>
+inline auto is_nan<sycl::half>(const sycl::half &val) -> bool {
+    return sycl::isnan(val);
+}
+
+template<>
+inline auto is_nan<float>(const float &val) -> bool {
+    return std::isnan(val);
+}
+
+template<>
+inline auto is_nan<double>(const double &val) -> bool {
+    return std::isnan(val);
+}
+
+template<>
+inline auto is_nan<cfloat>(const cfloat &in) -> bool {
+    return std::isnan(real(in)) || std::isnan(imag(in));
+}
+
+template<>
+inline auto is_nan<cdouble>(const cdouble &in) -> bool {
+    return std::isnan(real(in)) || std::isnan(imag(in));
+}
+
 template<typename T>
 static T scalar(double val) {
     return (T)(val);
@@ -79,8 +109,6 @@ static T scalar(double val) {
 template<>
 inline cfloat scalar<cfloat>(double val) {
     cfloat cval(static_cast<float>(val));
-    // cval.real() = (float)val;
-    // cval.imag() = 0;
     return cval;
 }
 
diff --git a/src/backend/oneapi/minmax_op.hpp b/src/backend/oneapi/minmax_op.hpp
index f006ff419c..40159d3ec9 100644
--- a/src/backend/oneapi/minmax_op.hpp
+++ b/src/backend/oneapi/minmax_op.hpp
@@ -10,6 +10,7 @@
 #pragma once
 
 #include <common/Binary.hpp>
+#include <math.hpp>
 
 namespace arrayfire {
 namespace oneapi {
@@ -34,21 +35,6 @@ double cabs<cdouble>(const cdouble &in) {
     return (double)abs(in);
 }
 
-template<typename T>
-static bool is_nan(const T &in) {
-    return in != in;
-}
-
-template<>
-bool is_nan<cfloat>(const cfloat &in) {
-    return in.real() != in.real() || in.imag() != in.imag();
-}
-
-template<>
-bool is_nan<cdouble>(const cdouble &in) {
-    return in.real() != in.real() || in.imag() != in.imag();
-}
-
 template<af_op_t op, typename T>
 struct MinMaxOp {
     T m_val;
diff --git a/src/backend/opencl/kernel/sparse_arith.hpp b/src/backend/opencl/kernel/sparse_arith.hpp
index 313fa902d2..17cd67ca8a 100644
--- a/src/backend/opencl/kernel/sparse_arith.hpp
+++ b/src/backend/opencl/kernel/sparse_arith.hpp
@@ -20,6 +20,7 @@
 #include <kernel_headers/sparse_arith_csr.hpp>
 #include <kernel_headers/ssarith_calc_out_nnz.hpp>
 #include <math.hpp>
+#include <memory.hpp>
 #include <traits.hpp>
 
 #include <string>

From 279087c5796698a57181f05aebbf1823756d3c2e Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 20 May 2023 17:12:12 -0400
Subject: [PATCH 297/473] Fix ragged reductions by passing a bool that checks
 if accessor valid

---
 src/backend/oneapi/kernel/ireduce.hpp | 102 +++++++++++++++-----------
 1 file changed, 59 insertions(+), 43 deletions(-)

diff --git a/src/backend/oneapi/kernel/ireduce.hpp b/src/backend/oneapi/kernel/ireduce.hpp
index 0c6ae70383..2366264ea4 100644
--- a/src/backend/oneapi/kernel/ireduce.hpp
+++ b/src/backend/oneapi/kernel/ireduce.hpp
@@ -42,7 +42,8 @@ class ireduceDimKernelSMEM {
                          read_accessor<T> in, KParam iInfo,
                          read_accessor<uint> iloc, KParam ilocInfo,
                          uint groups_x, uint groups_y, uint groups_dim,
-                         read_accessor<uint> rlen, KParam rlenInfo,
+                         bool rlenValid, read_accessor<uint> rlen,
+                         KParam rlenInfo,
                          sycl::local_accessor<compute_t<T>, 1> s_val,
                          sycl::local_accessor<uint, 1> s_idx)
         : out_(out)
@@ -56,6 +57,7 @@ class ireduceDimKernelSMEM {
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , groups_dim_(groups_dim)
+        , rlenValid_(rlenValid)
         , rlen_(rlen)
         , rlenInfo_(rlenInfo)
         , s_val_(s_val)
@@ -90,8 +92,7 @@ class ireduceDimKernelSMEM {
         const bool rlen_valid =
             (ids[0] < rlenInfo_.dims[0]) && (ids[1] < rlenInfo_.dims[1]) &&
             (ids[2] < rlenInfo_.dims[2]) && (ids[3] < rlenInfo_.dims[3]);
-        const bool rlen_nonnull = (rlenInfo_.dims[0] * rlenInfo_.dims[1] *
-                                   rlenInfo_.dims[2] * rlenInfo_.dims[3]) > 0;
+        const bool rlen_nonnull = rlenValid_;
         uint *const rlenptr =
             (rlen_nonnull && rlen_valid)
                 ? rlen_.get_pointer() + ids[3] * rlenInfo_.strides[3] +
@@ -204,6 +205,7 @@ class ireduceDimKernelSMEM {
     read_accessor<uint> iloc_;
     KParam ilocInfo_;
     uint groups_x_, groups_y_, groups_dim_;
+    bool rlenValid_;
     read_accessor<uint> rlen_;
     KParam rlenInfo_;
     sycl::local_accessor<compute_t<T>, 1> s_val_;
@@ -218,25 +220,25 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
     sycl::range<2> global(groups_dim[0] * groups_dim[2] * local[0],
                           groups_dim[1] * groups_dim[3] * local[1]);
 
-    sycl::buffer<uint, 1> empty{sycl::range<1>(1)};
+    auto iempty = memAlloc<uint>(1);
+    auto rempty = memAlloc<uint>(1);
     getQueue().submit([&](sycl::handler &h) {
         write_accessor<T> out_acc{*out.data, h};
         write_accessor<uint> oloc_acc{*oloc.data, h};
         read_accessor<T> in_acc{*in.data, h};
 
-        read_accessor<uint> iloc_acc{empty, h};
+        read_accessor<uint> iloc_acc{*iempty, h};
         if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] *
                 iloc.info.dims[3] >
             0) {
             iloc_acc = read_accessor<uint>{*iloc.data, h};
         }
 
-        read_accessor<uint> rlen_acc{empty, h};
-        if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] *
-                rlen.info.dims[3] >
-            0) {
-            rlen_acc = read_accessor<uint>{*rlen.data, h};
-        }
+        read_accessor<uint> rlen_acc{*rempty, h};
+        bool rlenValid = (rlen.info.dims[0] * rlen.info.dims[1] *
+                              rlen.info.dims[2] * rlen.info.dims[3] >
+                          0);
+        if (rlenValid) { rlen_acc = read_accessor<uint>{*rlen.data, h}; }
 
         auto shrdVal = sycl::local_accessor<compute_t<T>, 1>(
             creduce::THREADS_PER_BLOCK, h);
@@ -250,8 +252,8 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                     ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
-                        groups_dim[dim], rlen_acc, rlen.info, shrdVal,
-                        shrdLoc));
+                        groups_dim[dim], rlenValid, rlen_acc, rlen.info,
+                        shrdVal, shrdLoc));
                 break;
             case 4:
                 h.parallel_for(
@@ -259,8 +261,8 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                     ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
-                        groups_dim[dim], rlen_acc, rlen.info, shrdVal,
-                        shrdLoc));
+                        groups_dim[dim], rlenValid, rlen_acc, rlen.info,
+                        shrdVal, shrdLoc));
                 break;
             case 2:
                 h.parallel_for(
@@ -268,8 +270,8 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                     ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
-                        groups_dim[dim], rlen_acc, rlen.info, shrdVal,
-                        shrdLoc));
+                        groups_dim[dim], rlenValid, rlen_acc, rlen.info,
+                        shrdVal, shrdLoc));
                 break;
             case 1:
                 h.parallel_for(
@@ -277,8 +279,8 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                     ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
-                        groups_dim[dim], rlen_acc, rlen.info, shrdVal,
-                        shrdLoc));
+                        groups_dim[dim], rlenValid, rlen_acc, rlen.info,
+                        shrdVal, shrdLoc));
                 break;
         }
     });
@@ -335,7 +337,8 @@ class ireduceFirstKernelSMEM {
                            read_accessor<T> in, KParam iInfo,
                            read_accessor<uint> iloc, KParam ilocInfo,
                            uint groups_x, uint groups_y, uint repeat,
-                           read_accessor<uint> rlen, KParam rlenInfo,
+                           bool rlenValid, read_accessor<uint> rlen,
+                           KParam rlenInfo,
                            sycl::local_accessor<compute_t<T>, 1> s_val,
                            sycl::local_accessor<uint, 1> s_idx)
         : out_(out)
@@ -349,6 +352,7 @@ class ireduceFirstKernelSMEM {
         , groups_x_(groups_x)
         , groups_y_(groups_y)
         , repeat_(repeat)
+        , rlenValid_(rlenValid)
         , rlen_(rlen)
         , rlenInfo_(rlenInfo)
         , s_val_(s_val)
@@ -372,23 +376,24 @@ class ireduceFirstKernelSMEM {
                         iInfo_.offset;
 
         T *optr = out_.get_pointer() + wid * oInfo_.strides[3] +
-                  zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
+                  zid * oInfo_.strides[2] + yid * oInfo_.strides[1] +
+                  oInfo_.offset;
 
-        const bool rlenvalid = (rlenInfo_.dims[0] * rlenInfo_.dims[1] *
-                                rlenInfo_.dims[2] * rlenInfo_.dims[3]) > 0;
-        uint *const rlenptr =
-            (rlenvalid)
-                ? rlen_.get_pointer() + wid * rlenInfo_.strides[3] +
-                      zid * rlenInfo_.strides[2] + yid * rlenInfo_.strides[1]
-                : nullptr;
+        const uint *rlenptr =
+            (rlenValid_) ? rlen_.get_pointer() + wid * rlenInfo_.strides[3] +
+                               zid * rlenInfo_.strides[2] +
+                               yid * rlenInfo_.strides[1] + rlenInfo_.offset
+                         : nullptr;
 
         uint *ilptr;
         if (!is_first) {
             ilptr = iloc_.get_pointer() + wid * iInfo_.strides[3] +
-                    zid * iInfo_.strides[2] + yid * iInfo_.strides[1];
+                    zid * iInfo_.strides[2] + yid * iInfo_.strides[1] +
+                    iInfo_.offset;
         }
         uint *olptr = oloc_.get_pointer() + wid * oInfo_.strides[3] +
-                      zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
+                      zid * oInfo_.strides[2] + yid * oInfo_.strides[1] +
+                      oInfo_.offset;
 
         size_t ylim   = iInfo_.dims[1];
         size_t zlim   = iInfo_.dims[2];
@@ -404,7 +409,7 @@ class ireduceFirstKernelSMEM {
         compute_t<T> out_val = common::Binary<compute_t<T>, op>::init();
         uint idx             = xid;
 
-        if (xid < lim) {
+        if (xid < lim && is_valid) {
             out_val = static_cast<compute_t<T>>(iptr[xid]);
             if (!is_first) idx = ilptr[xid];
         }
@@ -501,6 +506,7 @@ class ireduceFirstKernelSMEM {
     read_accessor<uint> iloc_;
     KParam ilocInfo_;
     uint groups_x_, groups_y_, repeat_;
+    bool rlenValid_;
     read_accessor<uint> rlen_;
     KParam rlenInfo_;
     sycl::local_accessor<compute_t<T>, 1> s_val_;
@@ -518,25 +524,25 @@ void ireduce_first_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
 
     uint repeat = divup(in.info.dims[0], (groups_x * threads_x));
 
-    sycl::buffer<uint, 1> empty{sycl::range<1>(1)};
+    auto iempty = memAlloc<uint>(1);
+    auto rempty = memAlloc<uint>(1);
     getQueue().submit([&](sycl::handler &h) {
         write_accessor<T> out_acc{*out.data, h};
         write_accessor<uint> oloc_acc{*oloc.data, h};
         read_accessor<T> in_acc{*in.data, h};
 
-        read_accessor<uint> iloc_acc{empty, h};
+        read_accessor<uint> iloc_acc{*iempty, h};
         if (iloc.info.dims[0] * iloc.info.dims[1] * iloc.info.dims[2] *
                 iloc.info.dims[3] >
             0) {
             iloc_acc = read_accessor<uint>{*iloc.data, h};
         }
 
-        read_accessor<uint> rlen_acc{empty, h};
-        if (rlen.info.dims[0] * rlen.info.dims[1] * rlen.info.dims[2] *
-                rlen.info.dims[3] >
-            0) {
-            rlen_acc = read_accessor<uint>{*rlen.data, h};
-        }
+        read_accessor<uint> rlen_acc{*rempty, h};
+        bool rlenValid = (rlen.info.dims[0] * rlen.info.dims[1] *
+                              rlen.info.dims[2] * rlen.info.dims[3] >
+                          0);
+        if (rlenValid) { rlen_acc = read_accessor<uint>{*rlen.data, h}; }
 
         auto shrdVal = sycl::local_accessor<compute_t<T>, 1>(
             creduce::THREADS_PER_BLOCK, h);
@@ -550,7 +556,7 @@ void ireduce_first_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                     ireduceFirstKernelSMEM<T, op, is_first, 32>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_x, groups_y, repeat,
-                        rlen_acc, rlen.info, shrdVal, shrdLoc));
+                        rlenValid, rlen_acc, rlen.info, shrdVal, shrdLoc));
                 break;
             case 64:
                 h.parallel_for(
@@ -558,7 +564,7 @@ void ireduce_first_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                     ireduceFirstKernelSMEM<T, op, is_first, 64>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_x, groups_y, repeat,
-                        rlen_acc, rlen.info, shrdVal, shrdLoc));
+                        rlenValid, rlen_acc, rlen.info, shrdVal, shrdLoc));
                 break;
             case 128:
                 h.parallel_for(
@@ -566,7 +572,7 @@ void ireduce_first_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                     ireduceFirstKernelSMEM<T, op, is_first, 128>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_x, groups_y, repeat,
-                        rlen_acc, rlen.info, shrdVal, shrdLoc));
+                        rlenValid, rlen_acc, rlen.info, shrdVal, shrdLoc));
                 break;
             case 256:
                 h.parallel_for(
@@ -574,7 +580,7 @@ void ireduce_first_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
                     ireduceFirstKernelSMEM<T, op, is_first, 256>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_x, groups_y, repeat,
-                        rlen_acc, rlen.info, shrdVal, shrdLoc));
+                        rlenValid, rlen_acc, rlen.info, shrdVal, shrdLoc));
                 break;
         }
     });
@@ -669,6 +675,16 @@ T ireduce_all(uint *idx, Param<T> in) {
 
     sycl::host_accessor h_ptr_raw{*tmp.get()};
     sycl::host_accessor h_lptr_raw{*tlptr.get()};
+    if (!is_linear) {
+        // Converting n-d index into a linear index
+        // in is of size   [   dims0, dims1, dims2, dims3]
+        // tidx is of size [blocks_x, dims1, dims2, dims3]
+        // i / blocks_x gives you the batch number "N"
+        // "N * dims0 + i" gives the linear index
+        for (int i = 0; i < tmp_elements; i++) {
+            h_lptr_raw[i] += (i / groups_x) * in.info.dims[0];
+        }
+    }
 
     MinMaxOp<op, T> Op(h_ptr_raw[0], h_lptr_raw[0]);
 

From 2f6bd933789f30fbce1e3ea9b0fdd144831d647d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 20 May 2023 17:13:49 -0400
Subject: [PATCH 298/473] Update half checks and add ASSERT_SUCCESS to
 reduction tests.

---
 test/reduce.cpp | 36 ++++++++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/test/reduce.cpp b/test/reduce.cpp
index fc16e60716..f01dafec45 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -408,7 +408,9 @@ class ReduceByKeyP : public ::testing::TestWithParam<reduce_by_key_params *> {
 
     void SetUp() {
         reduce_by_key_params *params = GetParam();
-        if (noHalfTests(params->vType_)) { return; }
+        if (noHalfTests(params->vType_)) {
+            GTEST_SKIP() << "Half not supported on this device";
+        }
 
         keys = ptrToArray(params->iSize, params->iKeys_, params->kType_);
         vals = ptrToArray(params->iSize, params->iVals_, params->vType_);
@@ -551,7 +553,12 @@ INSTANTIATE_TEST_SUITE_P(UniqueKeyTests, ReduceByKeyP,
                          testNameGenerator<ReduceByKeyP>);
 
 TEST_P(ReduceByKeyP, SumDim0) {
-    if (noHalfTests(GetParam()->vType_)) { return; }
+    if (noHalfTests(GetParam()->vType_)) {
+        GTEST_SKIP() << "Half not supported on this device";
+    }
+    if (noHalfTests(GetParam()->kType_)) {
+        GTEST_SKIP() << "Half not supported on this device";
+    }
     array keyRes, valsReduced;
     sumByKey(keyRes, valsReduced, keys, vals, 0, 0);
 
@@ -560,7 +567,12 @@ TEST_P(ReduceByKeyP, SumDim0) {
 }
 
 TEST_P(ReduceByKeyP, SumDim2) {
-    if (noHalfTests(GetParam()->vType_)) { return; }
+    if (noHalfTests(GetParam()->vType_)) {
+        GTEST_SKIP() << "Half not supported on this device";
+    }
+    if (noHalfTests(GetParam()->kType_)) {
+        GTEST_SKIP() << "Half not supported on this device";
+    }
     const int ntile = 2;
     vals            = tile(vals, 1, ntile, 1, 1);
     vals            = reorder(vals, 1, 2, 0, 3);
@@ -1946,7 +1958,9 @@ class RaggedReduceMaxRangeP : public ::testing::TestWithParam<ragged_params *> {
 
     void SetUp() {
         ragged_params *params = GetParam();
-        if (noHalfTests(params->vType_)) { return; }
+        if (noHalfTests(params->vType_)) {
+            GTEST_SKIP() << "Half not supported on this device";
+        }
 
         const size_t rdim_size = params->reduceDimLen_;
         const int dim          = params->reduceDim_;
@@ -2043,8 +2057,9 @@ INSTANTIATE_TEST_SUITE_P(RaggedReduceTests, RaggedReduceMaxRangeP,
                          testNameGeneratorRagged<RaggedReduceMaxRangeP>);
 
 TEST_P(RaggedReduceMaxRangeP, rangeMaxTest) {
-    if (noHalfTests(GetParam()->vType_)) { return; }
-
+    if (noHalfTests(GetParam()->vType_)) {
+        GTEST_SKIP() << "Half not supported on this device";
+    }
     array ragged_max, idx;
     const int dim = GetParam()->reduceDim_;
     max(ragged_max, idx, vals, ragged_lens, dim);
@@ -2308,20 +2323,21 @@ TEST(Reduce, nanval_issue_3255) {
     dim_t dims[1] = {8};
 
     int ikeys_src[8] = {0, 0, 1, 1, 1, 2, 2, 0};
-    af_create_array(&ikeys, ikeys_src, 1, dims, u32);
+    ASSERT_SUCCESS(af_create_array(&ikeys, ikeys_src, 1, dims, u32));
 
     int i;
     for (i = 0; i < 8; i++) {
         double ivals_src[8] = {1, 2, 3, 4, 5, 6, 7, 8};
         ivals_src[i]        = NAN;
-        af_create_array(&ivals, ivals_src, 1, dims, f64);
+        ASSERT_SUCCESS(af_create_array(&ivals, ivals_src, 1, dims, f64));
 
-        af_product_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0);
+        ASSERT_SUCCESS(
+            af_product_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0));
         af::array ovals_cpp(ovals);
         ASSERT_FALSE(af::anyTrue<bool>(af::isNaN(ovals_cpp)));
         ASSERT_SUCCESS(af_release_array(okeys));
 
-        af_sum_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0);
+        ASSERT_SUCCESS(af_sum_by_key_nan(&okeys, &ovals, ikeys, ivals, 0, 1.0));
         ovals_cpp = af::array(ovals);
 
         ASSERT_FALSE(af::anyTrue<bool>(af::isNaN(ovals_cpp)));

From f14d57c3ba141a3a64e92efa33010725919a5530 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 20 May 2023 21:07:22 -0400
Subject: [PATCH 299/473] Remove operator+ for common::half in oneAPI backend.
 update wrap

---
 src/backend/oneapi/kernel/wrap_dilated.hpp | 37 ++++++++++++----------
 src/backend/oneapi/math.hpp                |  5 ---
 2 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/src/backend/oneapi/kernel/wrap_dilated.hpp b/src/backend/oneapi/kernel/wrap_dilated.hpp
index 63bdf342a8..f8f9614d01 100644
--- a/src/backend/oneapi/kernel/wrap_dilated.hpp
+++ b/src/backend/oneapi/kernel/wrap_dilated.hpp
@@ -28,12 +28,13 @@ namespace kernel {
 template<typename T>
 class wrapDilatedCreateKernel {
    public:
-    wrapDilatedCreateKernel(write_accessor<T> optrAcc, KParam out,
-                            read_accessor<T> iptrAcc, KParam in, const int wx,
-                            const int wy, const int sx, const int sy,
-                            const int px, const int py, const int dx,
-                            const int dy, const int nx, const int ny,
-                            int groups_x, int groups_y, const bool is_column)
+    wrapDilatedCreateKernel(write_accessor<data_t<T>> optrAcc, KParam out,
+                            read_accessor<data_t<T>> iptrAcc, KParam in,
+                            const int wx, const int wy, const int sx,
+                            const int sy, const int px, const int py,
+                            const int dx, const int dy, const int nx,
+                            const int ny, int groups_x, int groups_y,
+                            const bool is_column)
         : optrAcc_(optrAcc)
         , out_(out)
         , iptrAcc_(iptrAcc)
@@ -63,10 +64,10 @@ class wrapDilatedCreateKernel {
         int oidx0 = it.get_local_id(0) + g.get_local_range(0) * groupId_x;
         int oidx1 = it.get_local_id(1) + g.get_local_range(1) * groupId_y;
 
-        T *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] +
-                  idx3 * out_.strides[3];
-        T *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] +
-                  idx3 * in_.strides[3] + in_.offset;
+        data_t<T> *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] +
+                          idx3 * out_.strides[3];
+        data_t<T> *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] +
+                          idx3 * in_.strides[3] + in_.offset;
 
         if (oidx0 >= out_.dims[0] || oidx1 >= out_.dims[1]) return;
 
@@ -86,7 +87,7 @@ class wrapDilatedCreateKernel {
         const int x_start = (pidx0 < eff_wx) ? 0 : (pidx0 - eff_wx) / sx_ + 1;
         const int x_end   = sycl::min(pidx0 / sx_ + 1, nx_);
 
-        T val   = (T)0;
+        compute_t<T> val(0);
         int idx = 1;
 
         for (int y = y_start; y < y_end; y++) {
@@ -111,8 +112,8 @@ class wrapDilatedCreateKernel {
                     idx = dim_end + win_end * in_.strides[1];
                 }
 
-                T ival;
-                ival = (yvalid && xvalid) ? iptr[idx] : (T)0;
+                compute_t<T> ival;
+                ival = (yvalid && xvalid) ? iptr[idx] : compute_t<T>(0);
                 val  = val + ival;
             }
         }
@@ -121,9 +122,9 @@ class wrapDilatedCreateKernel {
     }
 
    private:
-    write_accessor<T> optrAcc_;
+    write_accessor<data_t<T>> optrAcc_;
     KParam out_;
-    read_accessor<T> iptrAcc_;
+    read_accessor<data_t<T>> iptrAcc_;
     KParam in_;
     const int wx_;
     const int wy_;
@@ -158,8 +159,10 @@ void wrap_dilated(Param<T> out, const Param<T> in, const dim_t wx,
 
     auto Q = getQueue();
     Q.submit([&](sycl::handler &h) {
-        sycl::accessor outAcc{*out.data, h, sycl::write_only, sycl::no_init};
-        sycl::accessor inAcc{*in.data, h, sycl::read_only};
+        write_accessor<data_t<T>> outAcc =
+            out.template get_accessor<sycl::access_mode::write>(h);
+        read_accessor<data_t<T>> inAcc =
+            in.template get_accessor<sycl::access_mode::read>(h);
         h.parallel_for(sycl::nd_range{global, local},
                        wrapDilatedCreateKernel<T>(
                            outAcc, out.info, inAcc, in.info, wx, wy, sx, sy, px,
diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp
index 83973994c9..359b4ae9a3 100644
--- a/src/backend/oneapi/math.hpp
+++ b/src/backend/oneapi/math.hpp
@@ -170,11 +170,6 @@ static inline T imag(T in) {
     return std::imag(in);
 }
 
-inline arrayfire::common::half operator+(arrayfire::common::half lhs,
-                                         arrayfire::common::half rhs) noexcept {
-    return arrayfire::common::half(static_cast<float>(lhs) +
-                                   static_cast<float>(rhs));
-}
 }  // namespace oneapi
 }  // namespace arrayfire
 

From 435a55c7fb11872126b4ada425254b6d60963b87 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 19 May 2023 18:57:46 -0400
Subject: [PATCH 300/473] fix scale for non-double supported kernels in oneapi
 backend

---
 src/backend/oneapi/kernel/memcopy.hpp | 57 +++++++++++++++------------
 1 file changed, 31 insertions(+), 26 deletions(-)

diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index c6b8dbb04c..dea4fd000c 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -28,6 +28,27 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
+template<typename T>
+using factortypes = typename std::conditional<std::is_same_v<T, double> ||
+                                                  std::is_same_v<T, cdouble>,
+                                              double, float>::type;
+
+template<typename T, typename FACTORTYPE = factortypes<T>>
+inline T scale(T value, FACTORTYPE factor) {
+    return (T)(FACTORTYPE(value) * factor);
+}
+
+template<>
+inline cfloat scale<cfloat, float>(cfloat value, float factor) {
+    return cfloat{static_cast<float>(value.real() * factor),
+                  static_cast<float>(value.imag() * factor)};
+}
+
+template<>
+inline cdouble scale<cdouble, double>(cdouble value, double factor) {
+    return cdouble{value.real() * factor, value.imag() * factor};
+}
+
 typedef struct {
     dim_t dim[4];
 } dims_t;
@@ -126,22 +147,6 @@ void memcopy(sycl::buffer<T> *out, const dim_t *ostrides,
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 
-template<typename T>
-inline T scale(T value, double factor) {
-    return (T)(double(value) * factor);
-}
-
-template<>
-inline cfloat scale<cfloat>(cfloat value, double factor) {
-    return cfloat{static_cast<float>(value.real() * factor),
-                  static_cast<float>(value.imag() * factor)};
-}
-
-template<>
-inline cdouble scale<cdouble>(cdouble value, double factor) {
-    return cdouble{value.real() * factor, value.imag() * factor};
-}
-
 template<typename inType, typename outType>
 inline outType convertType(inType value) {
     return static_cast<outType>(value);
@@ -201,7 +206,7 @@ class reshapeCopy {
    public:
     reshapeCopy(write_accessor<outType> dst, KParam oInfo,
                 read_accessor<inType> src, KParam iInfo, outType default_value,
-                float factor, dims_t trgt, int blk_x, int blk_y)
+                factortypes<inType> factor, dims_t trgt, int blk_x, int blk_y)
         : dst_(dst)
         , src_(src)
         , oInfo_(oInfo)
@@ -266,7 +271,7 @@ class reshapeCopy {
     read_accessor<inType> src_;
     KParam oInfo_, iInfo_;
     outType default_value_;
-    float factor_;
+    factortypes<inType> factor_;
     dims_t trgt_;
     int blk_x_, blk_y_;
 };
@@ -309,15 +314,15 @@ void copy(Param<outType> dst, const Param<inType> src, const int ndims,
             *const_cast<sycl::buffer<inType> *>(src.data), h};
 
         if (same_dims) {
-            h.parallel_for(ndrange, reshapeCopy<inType, outType, true>(
-                                        dst_acc, dst.info, src_acc, src.info,
-                                        default_value, (float)factor, trgt_dims,
-                                        blk_x, blk_y));
+            h.parallel_for(ndrange,
+                           reshapeCopy<inType, outType, true>(
+                               dst_acc, dst.info, src_acc, src.info,
+                               default_value, factor, trgt_dims, blk_x, blk_y));
         } else {
-            h.parallel_for(ndrange, reshapeCopy<inType, outType, false>(
-                                        dst_acc, dst.info, src_acc, src.info,
-                                        default_value, (float)factor, trgt_dims,
-                                        blk_x, blk_y));
+            h.parallel_for(ndrange,
+                           reshapeCopy<inType, outType, false>(
+                               dst_acc, dst.info, src_acc, src.info,
+                               default_value, factor, trgt_dims, blk_x, blk_y));
         }
     });
     ONEAPI_DEBUG_FINISH(getQueue());

From 5a42d39468b49c336523862fd13dc65a55deb3c3 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 23 May 2023 10:18:43 -0400
Subject: [PATCH 301/473] Fix JIT failures due to reliance on stride[0]s

---
 src/backend/oneapi/jit/kernel_generators.hpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
index bc12929fe6..5a3321d0a0 100644
--- a/src/backend/oneapi/jit/kernel_generators.hpp
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -61,8 +61,9 @@ inline void generateBufferOffsets(std::stringstream& kerStream, int id,
                   << info_str << ".strides[3] * id3 + (id2 < " << info_str
                   << ".dims[2]) * " << info_str << ".strides[2] * id2 + (id1 < "
                   << info_str << ".dims[1]) * " << info_str
-                  << ".strides[1] * id1 + (id0 < " << info_str
-                  << ".dims[0]) * id0 + " << info_str << ".offset;\n";
+                  << ".strides[1] * id1 + (id0 < " << info_str << ".dims[0]) * "
+                  << info_str << ".strides[0]  * id0 + " << info_str
+                  << ".offset;\n";
     }
 }
 

From 40011556e423571b3b8bb40f3d5c317363a5cf2d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 23 May 2023 10:19:19 -0400
Subject: [PATCH 302/473] Fix wrap and unwrap failures due to invalid work
 group size

---
 src/backend/oneapi/kernel/unwrap.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/oneapi/kernel/unwrap.hpp b/src/backend/oneapi/kernel/unwrap.hpp
index 0c88bd4348..43301fd744 100644
--- a/src/backend/oneapi/kernel/unwrap.hpp
+++ b/src/backend/oneapi/kernel/unwrap.hpp
@@ -149,7 +149,7 @@ void unwrap(Param<T> out, const Param<T> in, const dim_t wx, const dim_t wy,
         reps = divup((wx * wy), TX);
     } else {
         TX   = THREADS_X;
-        TY   = THREADS_X;
+        TY   = THREADS_Y;
         BX   = divup(out.info.dims[0], TX);
         reps = divup((wx * wy), TY);
     }

From 2a2ca609708f6c44b72c508266890e57fe79ed7f Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 23 May 2023 14:07:56 -0400
Subject: [PATCH 303/473] fix multiblock offset in scanFirstBcastKernel

---
 src/backend/oneapi/kernel/scan_first.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index 649e031b03..dd483f069b 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -181,7 +181,7 @@ class scanFirstBcastKernel {
         // Shift broadcast one step to the right for exclusive scan (#2366)
         int offset = !inclusive_scan_;
         for (int k = 0, id = xid + offset; k < lim_ && id < oInfo_.dims[0];
-             k++, id += g.get_group_range(0)) {
+             k++, id += g.get_local_range(0)) {
             optr[id] = binop(accum, optr[id]);
         }
     }

From b7ce6153dd43c6c3e67b1c33d8748a2b4b0de8c7 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Wed, 24 May 2023 23:20:16 +0200
Subject: [PATCH 304/473] Fix cannyEdgeDetector for CUDA when compiled with
 AF_WITH_FAST_MATH option

---
 src/api/c/canny.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp
index ae1fa8add9..ef3ad029cd 100644
--- a/src/api/c/canny.cpp
+++ b/src/api/c/canny.cpp
@@ -93,7 +93,6 @@ Array<float> otsuThreshold(const Array<float>& in, const unsigned NUM_BINS,
     seqBegin[0] = af_make_seq(0, static_cast<double>(hDims[0] - 1), 1);
     seqRest[0]  = af_make_seq(0, static_cast<double>(hDims[0] - 1), 1);
 
-    Array<float> TWOS   = createValueArray<float>(oDims, 2.0f);
     Array<float> UnitP  = createValueArray<float>(oDims, 1.0f);
     Array<float> histf  = cast<float, uint>(hist);
     Array<float> totals = createValueArray<float>(hDims, inDims[0] * inDims[1]);
@@ -126,7 +125,7 @@ Array<float> otsuThreshold(const Array<float>& in, const unsigned NUM_BINS,
         auto muL   = arithOp<float, af_div_t>(_muL, qL, oDims);
         auto muH   = arithOp<float, af_div_t>(_muH, qH, oDims);
         auto diff  = arithOp<float, af_sub_t>(muL, muH, oDims);
-        auto sqrd  = arithOp<float, af_pow_t>(diff, TWOS, oDims);
+        auto sqrd  = arithOp<float, af_mul_t>(diff, diff, oDims);
         auto op2   = createSubArray(qLqH, sliceIndex, false);
         auto sigma = arithOp<float, af_mul_t>(sqrd, op2, oDims);
 

From 24f426273c3e4b87ef5c3e05beefa945e78bc3dc Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 24 May 2023 15:58:05 -0400
Subject: [PATCH 305/473] Fix const correctness in oneAPI kernels

---
 src/backend/oneapi/kernel/assign.hpp       |  2 +-
 src/backend/oneapi/kernel/ireduce.hpp      | 14 +++++++-------
 src/backend/oneapi/kernel/lu_split.hpp     | 12 ++++++------
 src/backend/oneapi/kernel/memcopy.hpp      |  2 +-
 src/backend/oneapi/kernel/reduce_dim.hpp   |  8 ++++----
 src/backend/oneapi/kernel/reduce_first.hpp |  2 +-
 src/backend/oneapi/kernel/select.hpp       | 15 ++++++++-------
 src/backend/oneapi/kernel/transpose.hpp    |  3 ++-
 src/backend/oneapi/kernel/wrap.hpp         |  4 ++--
 src/backend/oneapi/kernel/wrap_dilated.hpp |  4 ++--
 10 files changed, 34 insertions(+), 32 deletions(-)

diff --git a/src/backend/oneapi/kernel/assign.hpp b/src/backend/oneapi/kernel/assign.hpp
index 5e3ef6c666..1b69827d18 100644
--- a/src/backend/oneapi/kernel/assign.hpp
+++ b/src/backend/oneapi/kernel/assign.hpp
@@ -88,7 +88,7 @@ class assignKernel {
                 p_.strds[3] *
                 trimIndex(s3 ? gw + p_.offs[3] : p_.ptr[3][gw], oInfo_.dims[3]);
 
-            T* iptr = in_.get_pointer();
+            const T* iptr = in_.get_pointer();
             // offset input and output pointers
             const T* src =
                 iptr + (gx * iInfo_.strides[0] + gy * iInfo_.strides[1] +
diff --git a/src/backend/oneapi/kernel/ireduce.hpp b/src/backend/oneapi/kernel/ireduce.hpp
index 2366264ea4..5f8f96bfc8 100644
--- a/src/backend/oneapi/kernel/ireduce.hpp
+++ b/src/backend/oneapi/kernel/ireduce.hpp
@@ -93,7 +93,7 @@ class ireduceDimKernelSMEM {
             (ids[0] < rlenInfo_.dims[0]) && (ids[1] < rlenInfo_.dims[1]) &&
             (ids[2] < rlenInfo_.dims[2]) && (ids[3] < rlenInfo_.dims[3]);
         const bool rlen_nonnull = rlenValid_;
-        uint *const rlenptr =
+        const uint *rlenptr =
             (rlen_nonnull && rlen_valid)
                 ? rlen_.get_pointer() + ids[3] * rlenInfo_.strides[3] +
                       ids[2] * rlenInfo_.strides[2] +
@@ -105,10 +105,10 @@ class ireduceDimKernelSMEM {
         // add thread offset for reduced dim for inputs
         ids[dim] = ids[dim] * g.get_local_range(1) + lidy;
 
-        T *iptr = in_.get_pointer() + ids[3] * iInfo_.strides[3] +
-                  ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] +
-                  ids[0] + iInfo_.offset;
-        uint *ilptr;
+        const T *iptr = in_.get_pointer() + ids[3] * iInfo_.strides[3] +
+                        ids[2] * iInfo_.strides[2] +
+                        ids[1] * iInfo_.strides[1] + ids[0] + iInfo_.offset;
+        const uint *ilptr;
         if (!is_first) {
             ilptr = iloc_.get_pointer() + ids[3] * iInfo_.strides[3] +
                     ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] +
@@ -371,7 +371,7 @@ class ireduceFirstKernelSMEM {
         const uint xid = groupId_x * g.get_local_range(0) * repeat_ + lidx;
         const uint yid = groupId_y * g.get_local_range(1) + lidy;
 
-        T *const iptr = in_.get_pointer() + wid * iInfo_.strides[3] +
+        const T *iptr = in_.get_pointer() + wid * iInfo_.strides[3] +
                         zid * iInfo_.strides[2] + yid * iInfo_.strides[1] +
                         iInfo_.offset;
 
@@ -385,7 +385,7 @@ class ireduceFirstKernelSMEM {
                                yid * rlenInfo_.strides[1] + rlenInfo_.offset
                          : nullptr;
 
-        uint *ilptr;
+        const uint *ilptr;
         if (!is_first) {
             ilptr = iloc_.get_pointer() + wid * iInfo_.strides[3] +
                     zid * iInfo_.strides[2] + yid * iInfo_.strides[1] +
diff --git a/src/backend/oneapi/kernel/lu_split.hpp b/src/backend/oneapi/kernel/lu_split.hpp
index fb69001ebc..6d52fb3835 100644
--- a/src/backend/oneapi/kernel/lu_split.hpp
+++ b/src/backend/oneapi/kernel/lu_split.hpp
@@ -51,9 +51,9 @@ class luSplitKernel {
         const int incy = groupsPerMatY_ * g.get_local_range(1);
         const int incx = groupsPerMatX_ * g.get_local_range(0);
 
-        T *d_l = lower_.get_pointer();
-        T *d_u = upper_.get_pointer();
-        T *d_i = in_.get_pointer();
+        T *d_l       = lower_.get_pointer();
+        T *d_u       = upper_.get_pointer();
+        const T *d_i = in_.get_pointer();
 
         if (oz < iInfo_.dims[2] && ow < iInfo_.dims[3]) {
             d_i = d_i + oz * iInfo_.strides[2] + ow * iInfo_.strides[3];
@@ -61,9 +61,9 @@ class luSplitKernel {
             d_u = d_u + oz * uInfo_.strides[2] + ow * uInfo_.strides[3];
 
             for (int oy = yy; oy < iInfo_.dims[1]; oy += incy) {
-                T *Yd_i = d_i + oy * iInfo_.strides[1];
-                T *Yd_l = d_l + oy * lInfo_.strides[1];
-                T *Yd_u = d_u + oy * uInfo_.strides[1];
+                const T *Yd_i = d_i + oy * iInfo_.strides[1];
+                T *Yd_l       = d_l + oy * lInfo_.strides[1];
+                T *Yd_u       = d_u + oy * uInfo_.strides[1];
                 for (int ox = xx; ox < iInfo_.dims[0]; ox += incx) {
                     if (ox > oy) {
                         if (same_dims || oy < lInfo_.dims[1])
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index dea4fd000c..b400d04673 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -81,7 +81,7 @@ class memCopy {
         const int id0        = group_id_0 * gg.get_local_range(0) + lid0;
         const int id1        = group_id_1 * gg.get_local_range(1) + lid1;
 
-        T *iptr = in_.get_pointer();
+        const T *iptr = in_.get_pointer();
         // FIXME: Do more work per work group
 
         T *optr = out_.get_pointer();
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index 6b51801fa7..b1d3d81648 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -65,14 +65,14 @@ class reduceDimKernelSMEM {
         uint ids[4] = {xid, yid, zid, wid};
         using sycl::global_ptr;
 
-        global_ptr<data_t<To>> optr =
-            out_.get_pointer() + ids[3] * oInfo_.strides[3] +
-            ids[2] * oInfo_.strides[2] + ids[1] * oInfo_.strides[1] + ids[0];
+        data_t<To> *optr = out_.get_pointer() + ids[3] * oInfo_.strides[3] +
+                           ids[2] * oInfo_.strides[2] +
+                           ids[1] * oInfo_.strides[1] + ids[0];
 
         const uint groupIdx_dim = ids[dim];
         ids[dim]                = ids[dim] * g.get_local_range(1) + lidy;
 
-        global_ptr<data_t<Ti>> iptr =
+        const data_t<Ti> *iptr =
             in_.get_pointer() + ids[3] * iInfo_.strides[3] +
             ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + ids[0];
 
diff --git a/src/backend/oneapi/kernel/reduce_first.hpp b/src/backend/oneapi/kernel/reduce_first.hpp
index f105d63671..152120648b 100644
--- a/src/backend/oneapi/kernel/reduce_first.hpp
+++ b/src/backend/oneapi/kernel/reduce_first.hpp
@@ -68,7 +68,7 @@ class reduceFirstKernelSMEM {
         common::Binary<compute_t<To>, op> reduce;
         common::Transform<Ti, compute_t<To>, op> transform;
 
-        Ti *const iptr = in_.get_pointer() + wid * iInfo_.strides[3] +
+        const Ti *iptr = in_.get_pointer() + wid * iInfo_.strides[3] +
                          zid * iInfo_.strides[2] + yid * iInfo_.strides[1] +
                          iInfo_.offset;
 
diff --git a/src/backend/oneapi/kernel/select.hpp b/src/backend/oneapi/kernel/select.hpp
index b5a6ae5954..06db45ad79 100644
--- a/src/backend/oneapi/kernel/select.hpp
+++ b/src/backend/oneapi/kernel/select.hpp
@@ -59,9 +59,9 @@ class selectKernelCreateKernel {
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g = it.get_group();
 
-        char *cptr = cptr__.get_pointer() + cinfo_.offset;
-        T *aptr    = aptr__.get_pointer() + ainfo_.offset;
-        T *bptr    = bptr__.get_pointer() + binfo_.offset;
+        const char *cptr = cptr__.get_pointer() + cinfo_.offset;
+        const T *aptr    = aptr__.get_pointer() + ainfo_.offset;
+        const T *bptr    = bptr__.get_pointer() + binfo_.offset;
 
         const int idz = g.get_group_id(0) / groups_0_;
         const int idw = g.get_group_id(1) / groups_1_;
@@ -169,8 +169,8 @@ class selectScalarCreateKernel {
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g = it.get_group();
 
-        char *cptr = cptr__.get_pointer() + cinfo_.offset;
-        T *aptr    = aptr__.get_pointer() + ainfo_.offset;
+        const char *cptr = cptr__.get_pointer() + cinfo_.offset;
+        const T *aptr    = aptr__.get_pointer() + ainfo_.offset;
 
         const int idz = g.get_group_id(0) / groups_0_;
         const int idw = g.get_group_id(1) / groups_1_;
@@ -185,7 +185,8 @@ class selectScalarCreateKernel {
                         idy * oinfo_.strides[1];
 
         int ids[] = {idx0, idy, idz, idw};
-        optr_.get_pointer() += off;
+        T *optr   = optr_.get_pointer();
+        optr += off;
         aptr += getOffset(ainfo_.dims, ainfo_.strides, oinfo_.dims, ids);
         cptr += getOffset(cinfo_.dims, cinfo_.strides, oinfo_.dims, ids);
 
@@ -196,7 +197,7 @@ class selectScalarCreateKernel {
 
         for (int idx = idx0; idx < oinfo_.dims[0];
              idx += g.get_local_range(0) * groups_0_) {
-            optr_.get_pointer()[idx] = (cptr[idx] ^ flip_) ? aptr[idx] : b_;
+            optr[idx] = (cptr[idx] ^ flip_) ? aptr[idx] : b_;
         }
     }
 
diff --git a/src/backend/oneapi/kernel/transpose.hpp b/src/backend/oneapi/kernel/transpose.hpp
index bf7c7a874b..2752111534 100644
--- a/src/backend/oneapi/kernel/transpose.hpp
+++ b/src/backend/oneapi/kernel/transpose.hpp
@@ -95,7 +95,8 @@ class transposeKernel {
 
         // offset in_ and out_ based on batch id
         // also add the subBuffer offsets
-        T *iDataPtr = iData_.get_pointer(), *oDataPtr = oData_.get_pointer();
+        const T *iDataPtr = iData_.get_pointer();
+        T *oDataPtr       = oData_.get_pointer();
         iDataPtr += batchId_x * in_.strides[2] + batchId_y * in_.strides[3] +
                     in_.offset;
         oDataPtr += batchId_x * out_.strides[2] + batchId_y * out_.strides[3] +
diff --git a/src/backend/oneapi/kernel/wrap.hpp b/src/backend/oneapi/kernel/wrap.hpp
index ef8d2eba21..b5e5226035 100644
--- a/src/backend/oneapi/kernel/wrap.hpp
+++ b/src/backend/oneapi/kernel/wrap.hpp
@@ -63,8 +63,8 @@ class wrapCreateKernel {
 
         T *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] +
                   idx3 * out_.strides[3] + out_.offset;
-        T *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] +
-                  idx3 * in_.strides[3] + in_.offset;
+        const T *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] +
+                        idx3 * in_.strides[3] + in_.offset;
 
         if (oidx0 >= out_.dims[0] || oidx1 >= out_.dims[1]) return;
 
diff --git a/src/backend/oneapi/kernel/wrap_dilated.hpp b/src/backend/oneapi/kernel/wrap_dilated.hpp
index f8f9614d01..41112fbce4 100644
--- a/src/backend/oneapi/kernel/wrap_dilated.hpp
+++ b/src/backend/oneapi/kernel/wrap_dilated.hpp
@@ -66,8 +66,8 @@ class wrapDilatedCreateKernel {
 
         data_t<T> *optr = optrAcc_.get_pointer() + idx2 * out_.strides[2] +
                           idx3 * out_.strides[3];
-        data_t<T> *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] +
-                          idx3 * in_.strides[3] + in_.offset;
+        const data_t<T> *iptr = iptrAcc_.get_pointer() + idx2 * in_.strides[2] +
+                                idx3 * in_.strides[3] + in_.offset;
 
         if (oidx0 >= out_.dims[0] || oidx1 >= out_.dims[1]) return;
 

From 448a103d3d55ca4177162fffb27f75f0a3990bd1 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 24 May 2023 15:58:56 -0400
Subject: [PATCH 306/473] Add type checks in pinverse tests

---
 test/pinverse.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/pinverse.cpp b/test/pinverse.cpp
index 7258558bc2..13b2151836 100644
--- a/test/pinverse.cpp
+++ b/test/pinverse.cpp
@@ -124,6 +124,7 @@ TYPED_TEST_SUITE(Pinverse, TestTypes);
 // Test Moore-Penrose conditions in the following first 4 tests
 // See https://en.wikipedia.org/wiki/Moore%E2%80%93Penrose_inverse#Definition
 TYPED_TEST(Pinverse, AApinvA_A) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
     array in = readTestInput<TypeParam>(
         string(TEST_DIR "/pinverse/pinverse10x8.test"));
     array inpinv = pinverse(in);
@@ -132,6 +133,7 @@ TYPED_TEST(Pinverse, AApinvA_A) {
 }
 
 TYPED_TEST(Pinverse, ApinvAApinv_Apinv) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
     array in = readTestInput<TypeParam>(
         string(TEST_DIR "/pinverse/pinverse10x8.test"));
     array inpinv = pinverse(in);
@@ -140,6 +142,7 @@ TYPED_TEST(Pinverse, ApinvAApinv_Apinv) {
 }
 
 TYPED_TEST(Pinverse, AApinv_IsHermitian) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
     array in = readTestInput<TypeParam>(
         string(TEST_DIR "/pinverse/pinverse10x8.test"));
     array inpinv = pinverse(in);
@@ -149,6 +152,7 @@ TYPED_TEST(Pinverse, AApinv_IsHermitian) {
 }
 
 TYPED_TEST(Pinverse, ApinvA_IsHermitian) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
     array in = readTestInput<TypeParam>(
         string(TEST_DIR "/pinverse/pinverse10x8.test"));
     array inpinv = pinverse(in);
@@ -158,6 +162,7 @@ TYPED_TEST(Pinverse, ApinvA_IsHermitian) {
 }
 
 TYPED_TEST(Pinverse, Large) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
     array in = readTestInput<TypeParam>(
         string(TEST_DIR "/pinverse/pinv_640x480_inputs.test"));
     array inpinv = pinverse(in);
@@ -166,6 +171,7 @@ TYPED_TEST(Pinverse, Large) {
 }
 
 TYPED_TEST(Pinverse, LargeTall) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
     array in = readTestInput<TypeParam>(
                    string(TEST_DIR "/pinverse/pinv_640x480_inputs.test"))
                    .T();
@@ -227,6 +233,7 @@ TEST(Pinverse, SmallSigValExistsFloat) {
 }
 
 TEST(Pinverse, SmallSigValExistsDouble) {
+    SUPPORTED_TYPE_CHECK(double);
     array in =
         readTestInput<double>(string(TEST_DIR "/pinverse/pinverse10x8.test"));
     const dim_t dim0 = in.dims(0);

From cc51889ca73d6350ff15fb530f9866d94545d20e Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Thu, 25 May 2023 20:53:33 +0200
Subject: [PATCH 307/473] speedup complexNorm when compiled with
 AF_WITH_FAST_MATH flag

---
 src/api/c/deconvolution.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/api/c/deconvolution.cpp b/src/api/c/deconvolution.cpp
index d5327d1efe..f579eeadf8 100644
--- a/src/api/c/deconvolution.cpp
+++ b/src/api/c/deconvolution.cpp
@@ -68,9 +68,8 @@ const dim_t GREATEST_PRIME_FACTOR = 7;
 
 template<typename T, typename CT>
 Array<T> complexNorm(const Array<CT>& input) {
-    auto mag  = detail::abs<T, CT>(input);
-    auto TWOS = createValueArray(input.dims(), scalar<T>(2));
-    return arithOp<T, af_pow_t>(mag, TWOS, input.dims());
+    auto mag = detail::abs<T, CT>(input);
+    return arithOp<T, af_mul_t>(mag, mag, input.dims());
 }
 
 std::vector<af_seq> calcPadInfo(dim4& inLPad, dim4& psfLPad, dim4& inUPad,

From b31d8c68537e0335386ae3838737296577fb9729 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 24 May 2023 23:36:18 -0400
Subject: [PATCH 308/473] fix offset stride in meanDimKernel

---
 src/backend/oneapi/kernel/mean.hpp | 54 +++++++++++++-----------------
 1 file changed, 23 insertions(+), 31 deletions(-)

diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index ef98cb0954..7c0f6f3243 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -29,17 +29,6 @@
 namespace arrayfire {
 namespace oneapi {
 
-/*
-TODO: port half
-__device__ auto operator*(float lhs, __half rhs) -> __half {
-    return __float2half(lhs * __half2float(rhs));
-}
-
-__device__ auto operator/(__half lhs, float rhs) -> __half {
-    return __float2half(__half2float(lhs) / rhs);
-}
-*/
-
 namespace kernel {
 
 template<typename To, typename Tw>
@@ -101,7 +90,7 @@ class meanDimKernelSMEM {
         To *optr       = out_.get_pointer();
 
         uint ooffset = ids[3] * oInfo_.strides[3] + ids[2] * oInfo_.strides[2] +
-                       ids[1] * oInfo_.strides[1] + ids[0];
+                       ids[1] * oInfo_.strides[1] + ids[0] + oInfo_.offset;
         // There is only one element per block for out
         // There are blockDim.y elements per block for in
         // Hence increment ids[dim] just after offseting out and before
@@ -112,11 +101,11 @@ class meanDimKernelSMEM {
         ids[dim]                = ids[dim] * g.get_local_range(1) + lidy;
 
         uint ioffset = ids[3] * iInfo_.strides[3] + ids[2] * iInfo_.strides[2] +
-                       ids[1] * iInfo_.strides[1] + ids[0];
+                       ids[1] * iInfo_.strides[1] + ids[0] + iInfo_.offset;
         iptr += ioffset;
 
-        const Tw *iwptr;
-        Tw *owptr;
+        const Tw *iwptr = nullptr;
+        Tw *owptr       = nullptr;
 
         if (output_weight_) owptr = owt_.get_pointer() + ooffset;
         if (input_weight_) iwptr = iwt_.get_pointer() + ioffset;
@@ -135,7 +124,7 @@ class meanDimKernelSMEM {
 
         if (is_valid && id_dim_in < iInfo_.dims[dim]) {
             val = transform(*iptr);
-            if (iwptr != NULL) {
+            if (iwptr) {
                 weight = *iwptr;
             } else {
                 weight = (Tw)1;
@@ -143,14 +132,14 @@ class meanDimKernelSMEM {
         }
 
         const uint id_dim_in_start =
-            id_dim_in + offset_dim_ * g.get_local_range(0);
+            id_dim_in + offset_dim_ * g.get_local_range(1);
 
         for (int id = id_dim_in_start; is_valid && (id < iInfo_.dims[dim]);
-             id += offset_dim_ * g.get_local_range(0)) {
-            iptr = iptr + offset_dim_ * g.get_local_range(0) * istride_dim;
+             id += offset_dim_ * g.get_local_range(1)) {
+            iptr = iptr + offset_dim_ * g.get_local_range(1) * istride_dim;
             if (input_weight_) {
                 iwptr =
-                    iwptr + offset_dim_ * g.get_local_range(0) * istride_dim;
+                    iwptr + offset_dim_ * g.get_local_range(1) * istride_dim;
                 stable_mean(&val, &weight, transform(*iptr),
                             compute_t<Tw>(*iwptr));
             } else {
@@ -358,19 +347,21 @@ class meanFirstKernelSMEM {
         To *optr       = out_.get_pointer();
 
         iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] +
-                yid * iInfo_.strides[1];
+                yid * iInfo_.strides[1] + iInfo_.offset;
         optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] +
-                yid * oInfo_.strides[1];
+                yid * oInfo_.strides[1] + oInfo_.offset;
 
-        const Tw *iwptr;
-        Tw *owptr;
+        const Tw *iwptr = nullptr;
+        Tw *owptr       = nullptr;
         if (input_weight_)
             iwptr = iwt_.get_pointer() + wid * iwInfo_.strides[3] +
-                    zid * iwInfo_.strides[2] + yid * iwInfo_.strides[1];
+                    zid * iwInfo_.strides[2] + yid * iwInfo_.strides[1] +
+                    iwInfo_.offset;
 
         if (output_weight_)
-            owptr = owt_.get_pointer() + wid * oInfo_.strides[3] +
-                    zid * oInfo_.strides[2] + yid * oInfo_.strides[1];
+            owptr = owt_.get_pointer() + wid * owInfo_.strides[3] +
+                    zid * owInfo_.strides[2] + yid * owInfo_.strides[1] +
+                    owInfo_.offset;
 
         bool cond = (yid < iInfo_.dims[1] && zid < iInfo_.dims[2] &&
                      wid < iInfo_.dims[3]);
@@ -485,9 +476,9 @@ class meanFirstKernelSMEM {
 };
 
 template<typename Ti, typename Tw, typename To>
-sycl::event mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
-                                Param<Tw> iwt, const uint groups_x,
-                                const uint groups_y, const uint threads_x) {
+void mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
+                         Param<Tw> iwt, const uint groups_x,
+                         const uint groups_y, const uint threads_x) {
     sycl::range<2> local(threads_x, THREADS_PER_BLOCK / threads_x);
     sycl::range<2> global(groups_x * in.info.dims[2] * local[0],
                           groups_y * in.info.dims[3] * local[1]);
@@ -496,7 +487,7 @@ sycl::event mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
 
     auto empty  = memAlloc<Tw>(1);
     auto oempty = memAlloc<Tw>(1);
-    return getQueue().submit([&](sycl::handler &h) {
+    getQueue().submit([&](sycl::handler &h) {
         write_accessor<To> out_acc{*out.data, h};
         read_accessor<Ti> in_acc{*in.data, h};
 
@@ -521,6 +512,7 @@ sycl::event mean_first_launcher(Param<To> out, Param<Tw> owt, Param<Ti> in,
                 iwt.info, threads_x, groups_x, groups_y, repeat, s_val, s_idx,
                 input_weight, output_weight));
     });
+    ONEAPI_DEBUG_FINISH(getQueue());
 }
 
 template<typename Ti, typename Tw, typename To>

From d734fd1f7c420c9f2281daf69b9fab5d03e6ee52 Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Wed, 31 May 2023 15:32:15 -0400
Subject: [PATCH 309/473] fftconvolve oneapi port (includes fft fix) (#3426)

* fftconvolve oneapi port
* fix fftconvolve reorder

---------

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
---
 src/backend/oneapi/CMakeLists.txt             |   5 +
 src/backend/oneapi/fft.cpp                    |  12 +-
 src/backend/oneapi/fftconvolve.cpp            |  77 ++++++-
 .../oneapi/kernel/fftconvolve_common.hpp      |  74 +++++++
 .../oneapi/kernel/fftconvolve_multiply.hpp    | 155 ++++++++++++++
 .../oneapi/kernel/fftconvolve_pack.hpp        | 146 +++++++++++++
 src/backend/oneapi/kernel/fftconvolve_pad.hpp | 129 ++++++++++++
 .../oneapi/kernel/fftconvolve_reorder.hpp     | 193 ++++++++++++++++++
 8 files changed, 784 insertions(+), 7 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/fftconvolve_common.hpp
 create mode 100644 src/backend/oneapi/kernel/fftconvolve_multiply.hpp
 create mode 100644 src/backend/oneapi/kernel/fftconvolve_pack.hpp
 create mode 100644 src/backend/oneapi/kernel/fftconvolve_pad.hpp
 create mode 100644 src/backend/oneapi/kernel/fftconvolve_reorder.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 46e20c88d4..b13de94f95 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -218,6 +218,11 @@ target_sources(afoneapi
     kernel/convolve_separable.cpp
     kernel/diagonal.hpp
     kernel/diff.hpp
+    kernel/fftconvolve_common.hpp
+    kernel/fftconvolve_multiply.hpp
+    kernel/fftconvolve_pack.hpp
+    kernel/fftconvolve_pad.hpp
+    kernel/fftconvolve_reorder.hpp
     kernel/histogram.hpp
     kernel/iir.hpp
     kernel/identity.hpp
diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
index eff8770bfc..b32c801423 100644
--- a/src/backend/oneapi/fft.cpp
+++ b/src/backend/oneapi/fft.cpp
@@ -50,9 +50,9 @@ void fft_inplace(Array<T> &in, const int rank, const bool direction) {
 
     auto desc = [rank, &idims]() {
         if (rank == 1) return desc_ty(idims[0]);
-        if (rank == 2) return desc_ty({idims[0], idims[1]});
-        if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]});
-        return desc_ty({idims[0], idims[1], idims[2], idims[3]});
+        if (rank == 2) return desc_ty({idims[1], idims[0]});
+        if (rank == 3) return desc_ty({idims[2], idims[1], idims[0]});
+        return desc_ty({idims[3], idims[2], idims[1], idims[0]});
     }();
 
     desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT, DFTI_INPLACE);
@@ -139,9 +139,9 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank) {
 
     auto desc = [rank, &odims]() {
         if (rank == 1) return desc_ty(odims[0]);
-        if (rank == 2) return desc_ty({odims[0], odims[1]});
-        if (rank == 3) return desc_ty({odims[0], odims[1], odims[2]});
-        return desc_ty({odims[0], odims[1], odims[2], odims[3]});
+        if (rank == 2) return desc_ty({odims[1], odims[0]});
+        if (rank == 3) return desc_ty({odims[2], odims[1], odims[0]});
+        return desc_ty({odims[3], odims[2], odims[1], odims[0]});
     }();
 
     desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
diff --git a/src/backend/oneapi/fftconvolve.cpp b/src/backend/oneapi/fftconvolve.cpp
index c4aea5689c..de96d94c99 100644
--- a/src/backend/oneapi/fftconvolve.cpp
+++ b/src/backend/oneapi/fftconvolve.cpp
@@ -15,6 +15,12 @@
 #include <fft.hpp>
 #include <af/dim4.hpp>
 
+#include <kernel/fftconvolve_common.hpp>
+#include <kernel/fftconvolve_multiply.hpp>
+#include <kernel/fftconvolve_pack.hpp>
+#include <kernel/fftconvolve_pad.hpp>
+#include <kernel/fftconvolve_reorder.hpp>
+
 #include <cmath>
 #include <type_traits>
 #include <vector>
@@ -59,9 +65,78 @@ dim4 calcPackedSize(Array<T> const& i1, Array<T> const& i2, const dim_t rank) {
 template<typename T>
 Array<T> fftconvolve(Array<T> const& signal, Array<T> const& filter,
                      const bool expand, AF_BATCH_KIND kind, const int rank) {
-    ONEAPI_NOT_SUPPORTED("");
+    using convT = typename conditional<is_integral<T>::value ||
+                                           is_same<T, float>::value ||
+                                           is_same<T, cfloat>::value,
+                                       float, double>::type;
+    using cT    = typename conditional<is_same<convT, float>::value, cfloat,
+                                    cdouble>::type;
+
+    const dim4& sDims = signal.dims();
+    const dim4& fDims = filter.dims();
+
     dim4 oDims(1);
+    if (expand) {
+        for (int d = 0; d < AF_MAX_DIMS; ++d) {
+            if (kind == AF_BATCH_NONE || kind == AF_BATCH_RHS) {
+                oDims[d] = sDims[d] + fDims[d] - 1;
+            } else {
+                oDims[d] = (d < rank ? sDims[d] + fDims[d] - 1 : sDims[d]);
+            }
+        }
+    } else {
+        oDims = sDims;
+        if (kind == AF_BATCH_RHS) {
+            for (int i = rank; i < AF_MAX_DIMS; ++i) { oDims[i] = fDims[i]; }
+        }
+    }
+
+    const dim4 pDims = calcPackedSize<T>(signal, filter, rank);
+    Array<cT> packed = createEmptyArray<cT>(pDims);
+
+    kernel::packDataHelper<cT, T>(packed, signal, filter, rank, kind);
+    kernel::padDataHelper<cT, T>(packed, signal, filter, rank, kind);
+
+    fft_inplace<cT>(packed, rank, true);
+
+    kernel::complexMultiplyHelper<cT, T>(packed, signal, filter, rank, kind);
+
+    // Compute inverse FFT only on complex-multiplied data
+    if (kind == AF_BATCH_RHS) {
+        vector<af_seq> seqs;
+        for (int k = 0; k < AF_MAX_DIMS; k++) {
+            if (k < rank) {
+                seqs.push_back({0., static_cast<double>(pDims[k] - 1), 1.});
+            } else if (k == rank) {
+                seqs.push_back({1., static_cast<double>(pDims[k] - 1), 1.});
+            } else {
+                seqs.push_back({0., 0., 1.});
+            }
+        }
+
+        Array<cT> subPacked = createSubArray<cT>(packed, seqs);
+        fft_inplace<cT>(subPacked, rank, false);
+    } else {
+        vector<af_seq> seqs;
+        for (int k = 0; k < AF_MAX_DIMS; k++) {
+            if (k < rank) {
+                seqs.push_back({0., static_cast<double>(pDims[k]) - 1, 1.});
+            } else if (k == rank) {
+                seqs.push_back({0., static_cast<double>(pDims[k] - 2), 1.});
+            } else {
+                seqs.push_back({0., 0., 1.});
+            }
+        }
+
+        Array<cT> subPacked = createSubArray<cT>(packed, seqs);
+        fft_inplace<cT>(subPacked, rank, false);
+    }
+
     Array<T> out = createEmptyArray<T>(oDims);
+
+    kernel::reorderOutputHelper<T, cT>(out, packed, signal, filter, rank, kind,
+                                       expand);
+
     return out;
 }
 
diff --git a/src/backend/oneapi/kernel/fftconvolve_common.hpp b/src/backend/oneapi/kernel/fftconvolve_common.hpp
new file mode 100644
index 0000000000..6caf9923d2
--- /dev/null
+++ b/src/backend/oneapi/kernel/fftconvolve_common.hpp
@@ -0,0 +1,74 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
+#include <af/defines.h>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+constexpr int THREADS = 256;
+
+template<typename T, typename convT>
+void calcParamSizes(Param<T>& sig_tmp, Param<T>& filter_tmp,
+                    Param<convT>& packed, Param<T>& sig, Param<T>& filter,
+                    const int rank, AF_BATCH_KIND kind) {
+    sig_tmp.info.dims[0] = filter_tmp.info.dims[0] = packed.info.dims[0];
+    sig_tmp.info.strides[0] = filter_tmp.info.strides[0] = 1;
+
+    for (int k = 1; k < 4; k++) {
+        if (k < rank) {
+            sig_tmp.info.dims[k]    = packed.info.dims[k];
+            filter_tmp.info.dims[k] = packed.info.dims[k];
+        } else {
+            sig_tmp.info.dims[k]    = sig.info.dims[k];
+            filter_tmp.info.dims[k] = filter.info.dims[k];
+        }
+
+        sig_tmp.info.strides[k] =
+            sig_tmp.info.strides[k - 1] * sig_tmp.info.dims[k - 1];
+        filter_tmp.info.strides[k] =
+            filter_tmp.info.strides[k - 1] * filter_tmp.info.dims[k - 1];
+    }
+
+    // NOTE: The OpenCL implementation on which this oneAPI port is
+    // based treated the incoming `packed` buffer as a string of real
+    // scalars instead of complex numbers. OpenCL accomplished this
+    // with the hack depicted in the trailing two lines. This note
+    // remains here in an explanation of SYCL buffer reinterpret's in
+    // fftconvolve kernel invocations.
+
+    // sig_tmp.data    = packed.data;
+    // filter_tmp.data = packed.data;
+
+    // Calculate memory offsets for packed signal and filter
+    if (kind == AF_BATCH_RHS) {
+        filter_tmp.info.offset = 0;
+        sig_tmp.info.offset =
+            filter_tmp.info.strides[3] * filter_tmp.info.dims[3] * 2;
+    } else {
+        sig_tmp.info.offset = 0;
+        filter_tmp.info.offset =
+            sig_tmp.info.strides[3] * sig_tmp.info.dims[3] * 2;
+    }
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/fftconvolve_multiply.hpp b/src/backend/oneapi/kernel/fftconvolve_multiply.hpp
new file mode 100644
index 0000000000..e8968f6d0d
--- /dev/null
+++ b/src/backend/oneapi/kernel/fftconvolve_multiply.hpp
@@ -0,0 +1,155 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <af/defines.h>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+class fftconvolve_multiplyCreateKernel {
+   public:
+    fftconvolve_multiplyCreateKernel(write_accessor<T> d_out, KParam oInfo,
+                                     read_accessor<T> d_in1, KParam i1Info,
+                                     read_accessor<T> d_in2, KParam i2Info,
+                                     const int nelem, const int kind)
+        : d_out_(d_out)
+        , oInfo_(oInfo)
+        , d_in1_(d_in1)
+        , i1Info_(i1Info)
+        , d_in2_(d_in2)
+        , i2Info_(i2Info)
+        , nelem_(nelem)
+        , kind_(kind) {}
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+
+        const int t = it.get_global_id(0);
+
+        if (t >= nelem_) return;
+
+        if (kind_ == AF_BATCH_NONE || kind_ == AF_BATCH_SAME) {
+            // Complex multiply each signal to equivalent filter
+            const int ridx = t * 2;
+            const int iidx = t * 2 + 1;
+
+            T a = d_in1_[i1Info_.offset + ridx];
+            T b = d_in1_[i1Info_.offset + iidx];
+            T c = d_in2_[i2Info_.offset + ridx];
+            T d = d_in2_[i2Info_.offset + iidx];
+
+            d_out_[oInfo_.offset + ridx] = a * c - b * d;
+            d_out_[oInfo_.offset + iidx] = a * d + b * c;
+        } else if (kind_ == AF_BATCH_LHS) {
+            // Complex multiply all signals to filter
+            const int ridx1 = t * 2;
+            const int iidx1 = t * 2 + 1;
+
+            // Treating complex output array as real-only array,
+            // thus, multiply strides by 2
+            const int ridx2 =
+                ridx1 % (i2Info_.strides[3] * i2Info_.dims[3] * 2);
+            const int iidx2 =
+                iidx1 % (i2Info_.strides[3] * i2Info_.dims[3] * 2);
+
+            T a = d_in1_[i1Info_.offset + ridx1];
+            T b = d_in1_[i1Info_.offset + iidx1];
+            T c = d_in2_[i2Info_.offset + ridx2];
+            T d = d_in2_[i2Info_.offset + iidx2];
+
+            d_out_[oInfo_.offset + ridx1] = a * c - b * d;
+            d_out_[oInfo_.offset + iidx1] = a * d + b * c;
+        } else if (kind_ == AF_BATCH_RHS) {
+            // Complex multiply signal to all filters
+            const int ridx2 = t * 2;
+            const int iidx2 = t * 2 + 1;
+
+            // Treating complex output array as real-only array,
+            // thus, multiply strides by 2
+            const int ridx1 =
+                ridx2 % (i1Info_.strides[3] * i1Info_.dims[3] * 2);
+            const int iidx1 =
+                iidx2 % (i1Info_.strides[3] * i1Info_.dims[3] * 2);
+
+            T a = d_in1_[i1Info_.offset + ridx1];
+            T b = d_in1_[i1Info_.offset + iidx1];
+            T c = d_in2_[i2Info_.offset + ridx2];
+            T d = d_in2_[i2Info_.offset + iidx2];
+
+            d_out_[oInfo_.offset + ridx2] = a * c - b * d;
+            d_out_[oInfo_.offset + iidx2] = a * d + b * c;
+        }
+    }
+
+   private:
+    write_accessor<T> d_out_;
+    KParam oInfo_;
+    read_accessor<T> d_in1_;
+    KParam i1Info_;
+    read_accessor<T> d_in2_;
+    KParam i2Info_;
+    const int nelem_;
+    const int kind_;
+};
+
+template<typename convT, typename T>
+void complexMultiplyHelper(Param<convT> packed, Param<T> sig, Param<T> filter,
+                           const int rank, AF_BATCH_KIND kind) {
+    Param<T> sig_tmp, filter_tmp;
+    calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
+
+    int sig_packed_elem = sig_tmp.info.strides[3] * sig_tmp.info.dims[3];
+    int filter_packed_elem =
+        filter_tmp.info.strides[3] * filter_tmp.info.dims[3];
+    int mul_elem = (sig_packed_elem < filter_packed_elem) ? filter_packed_elem
+                                                          : sig_packed_elem;
+    int blocks   = divup(mul_elem, THREADS);
+
+    auto local  = sycl::range(THREADS);
+    auto global = sycl::range(blocks * THREADS);
+
+    // Treat complex output as an array of scalars
+    using convScalarT      = typename convT::value_type;
+    auto packed_num_elem   = (*packed.data).get_range().size();
+    auto packed_tmp_buffer = (*packed.data)
+                                 .template reinterpret<convScalarT>(
+                                     sycl::range<1>{packed_num_elem * 2});
+    auto sig_tmp_buffer = (*packed.data)
+                              .template reinterpret<convScalarT>(
+                                  sycl::range<1>{packed_num_elem * 2});
+    auto filter_tmp_buffer = (*packed.data)
+                                 .template reinterpret<convScalarT>(
+                                     sycl::range<1>{packed_num_elem * 2});
+
+    getQueue().submit([&](auto &h) {
+        write_accessor<convScalarT> d_packed    = {packed_tmp_buffer, h};
+        read_accessor<convScalarT> d_sig_tmp    = {sig_tmp_buffer, h};
+        read_accessor<convScalarT> d_filter_tmp = {filter_tmp_buffer, h};
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            fftconvolve_multiplyCreateKernel<typename convT::value_type>(
+                d_packed, packed.info, d_sig_tmp, sig_tmp.info, d_filter_tmp,
+                filter_tmp.info, mul_elem, (int)kind));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/fftconvolve_pack.hpp b/src/backend/oneapi/kernel/fftconvolve_pack.hpp
new file mode 100644
index 0000000000..c6b04d5a43
--- /dev/null
+++ b/src/backend/oneapi/kernel/fftconvolve_pack.hpp
@@ -0,0 +1,146 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <af/defines.h>
+
+#include <string>
+#include <vector>
+
+#include <iostream>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename inputType, typename outputType>
+class fftconvolve_packCreateKernel {
+   public:
+    fftconvolve_packCreateKernel(write_accessor<outputType> d_out, KParam oInfo,
+                                 read_accessor<inputType> d_in, KParam iInfo,
+                                 const int di0_half, const int odd_di0)
+        : d_out_(d_out)
+        , oInfo_(oInfo)
+        , d_in_(d_in)
+        , iInfo_(iInfo)
+        , di0_half_(di0_half)
+        , odd_di0_(odd_di0) {}
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+
+        const int t = it.get_global_id(0);
+
+        const int tMax = oInfo_.strides[3] * oInfo_.dims[3];
+
+        if (t >= tMax) return;
+
+        const int do0 = oInfo_.dims[0];
+        const int do1 = oInfo_.dims[1];
+        const int do2 = oInfo_.dims[2];
+
+        const int so1 = oInfo_.strides[1];
+        const int so2 = oInfo_.strides[2];
+        const int so3 = oInfo_.strides[3];
+
+        const int to0 = t % so1;
+        const int to1 = (t / so1) % do1;
+        const int to2 = (t / so2) % do2;
+        const int to3 = t / so3;
+
+        const int di0 = iInfo_.dims[0];
+        const int di1 = iInfo_.dims[1];
+        const int di2 = iInfo_.dims[2];
+
+        const int si1 = iInfo_.strides[1];
+        const int si2 = iInfo_.strides[2];
+        const int si3 = iInfo_.strides[3];
+
+        const int ti0 = to0;
+        const int ti1 = to1 * si1;
+        const int ti2 = to2 * si2;
+        const int ti3 = to3 * si3;
+
+        const int iidx1 = iInfo_.offset + ti3 + ti2 + ti1 + ti0;
+        const int iidx2 = iidx1 + di0_half_;
+
+        // Treating complex output array as real-only array,
+        // thus, multiply strides by 2
+        const int oidx1 = oInfo_.offset + to3 * so3 * 2 + to2 * so2 * 2 +
+                          to1 * so1 * 2 + to0 * 2;
+        const int oidx2 = oidx1 + 1;
+
+        if (to0 < di0_half_ && to1 < di1 && to2 < di2) {
+            d_out_[oidx1] = (outputType)d_in_[iidx1];
+            if (ti0 == di0_half_ - 1 && odd_di0_ == 1)
+                d_out_[oidx2] = (outputType)0;
+            else
+                d_out_[oidx2] = (outputType)d_in_[iidx2];
+        } else {
+            // Pad remaining elements with 0s
+            d_out_[oidx1] = (outputType)0;
+            d_out_[oidx2] = (outputType)0;
+        }
+    }
+
+   private:
+    write_accessor<outputType> d_out_;
+    KParam oInfo_;
+    read_accessor<inputType> d_in_;
+    KParam iInfo_;
+    const int di0_half_;
+    const int odd_di0_;
+};
+
+template<typename convT, typename T>
+void packDataHelper(Param<convT> packed, Param<T> sig, Param<T> filter,
+                    const int rank, AF_BATCH_KIND kind) {
+    Param<T> sig_tmp, filter_tmp;
+    calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
+
+    int sig_packed_elem = sig_tmp.info.strides[3] * sig_tmp.info.dims[3];
+    int filter_packed_elem =
+        filter_tmp.info.strides[3] * filter_tmp.info.dims[3];
+
+    // Number of packed complex elements in dimension 0
+    int sig_half_d0     = divup(sig.info.dims[0], 2);
+    int sig_half_d0_odd = sig.info.dims[0] % 2;
+
+    int blocks = divup(sig_packed_elem, THREADS);
+
+    // Locate features kernel sizes
+    auto local  = sycl::range(THREADS);
+    auto global = sycl::range(blocks * THREADS);
+
+    // Treat complex output as an array of scalars
+    using convScalarT    = typename convT::value_type;
+    auto packed_num_elem = (*packed.data).get_range().size();
+    auto sig_tmp_buffer  = (*packed.data)
+                              .template reinterpret<convScalarT>(
+                                  sycl::range<1>{packed_num_elem * 2});
+
+    getQueue().submit([&](auto &h) {
+        read_accessor<T> d_sig                = {*sig.data, h};
+        write_accessor<convScalarT> d_sig_tmp = {sig_tmp_buffer, h};
+        h.parallel_for(sycl::nd_range{global, local},
+                       fftconvolve_packCreateKernel<T, convScalarT>(
+                           d_sig_tmp, sig_tmp.info, d_sig, sig.info,
+                           sig_half_d0, sig_half_d0_odd));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/fftconvolve_pad.hpp b/src/backend/oneapi/kernel/fftconvolve_pad.hpp
new file mode 100644
index 0000000000..6276b1da72
--- /dev/null
+++ b/src/backend/oneapi/kernel/fftconvolve_pad.hpp
@@ -0,0 +1,129 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <af/defines.h>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename inputType, typename outputType>
+class fftconvolve_padCreateKernel {
+   public:
+    fftconvolve_padCreateKernel(write_accessor<outputType> d_out, KParam oInfo,
+                                read_accessor<inputType> d_in, KParam iInfo)
+        : d_out_(d_out), oInfo_(oInfo), d_in_(d_in), iInfo_(iInfo) {}
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+
+        const int t = it.get_global_id(0);
+
+        const int tMax = oInfo_.strides[3] * oInfo_.dims[3];
+
+        if (t >= tMax) return;
+
+        const int do0 = oInfo_.dims[0];
+        const int do1 = oInfo_.dims[1];
+        const int do2 = oInfo_.dims[2];
+
+        const int so1 = oInfo_.strides[1];
+        const int so2 = oInfo_.strides[2];
+        const int so3 = oInfo_.strides[3];
+
+        const int to0 = t % so1;
+        const int to1 = (t / so1) % do1;
+        const int to2 = (t / so2) % do2;
+        const int to3 = (t / so3);
+
+        const int di0 = iInfo_.dims[0];
+        const int di1 = iInfo_.dims[1];
+        const int di2 = iInfo_.dims[2];
+        const int di3 = iInfo_.dims[3];
+
+        const int si1 = iInfo_.strides[1];
+        const int si2 = iInfo_.strides[2];
+        const int si3 = iInfo_.strides[3];
+
+        const int ti0 = to0;
+        const int ti1 = to1 * si1;
+        const int ti2 = to2 * si2;
+        const int ti3 = to3 * si3;
+
+        const int iidx = iInfo_.offset + ti3 + ti2 + ti1 + ti0;
+
+        const int oidx = oInfo_.offset + t * 2;
+
+        if (to0 < di0 && to1 < di1 && to2 < di2 && to3 < di3) {
+            // Copy input elements to real elements, set imaginary elements to 0
+            d_out_[oidx]     = (outputType)d_in_[iidx];
+            d_out_[oidx + 1] = (outputType)0;
+        } else {
+            // Pad remaining of the matrix to 0s
+            d_out_[oidx]     = (outputType)0;
+            d_out_[oidx + 1] = (outputType)0;
+        }
+    }
+
+   private:
+    write_accessor<outputType> d_out_;
+    KParam oInfo_;
+    read_accessor<inputType> d_in_;
+    KParam iInfo_;
+};
+
+template<typename convT, typename T>
+void padDataHelper(Param<convT> packed, Param<T> sig, Param<T> filter,
+                   const int rank, AF_BATCH_KIND kind) {
+    Param<T> sig_tmp, filter_tmp;
+    calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
+
+    int sig_packed_elem = sig_tmp.info.strides[3] * sig_tmp.info.dims[3];
+    int filter_packed_elem =
+        filter_tmp.info.strides[3] * filter_tmp.info.dims[3];
+
+    // Number of packed complex elements in dimension 0
+    int sig_half_d0     = divup(sig.info.dims[0], 2);
+    int sig_half_d0_odd = sig.info.dims[0] % 2;
+
+    int blocks = divup(filter_packed_elem, THREADS);
+
+    // Locate features kernel sizes
+    auto local  = sycl::range(THREADS);
+    auto global = sycl::range(blocks * THREADS);
+
+    // Treat complex output as an array of scalars
+    using convScalarT      = typename convT::value_type;
+    auto packed_num_elem   = (*packed.data).get_range().size();
+    auto filter_tmp_buffer = (*packed.data)
+                                 .template reinterpret<convScalarT>(
+                                     sycl::range<1>{packed_num_elem * 2});
+
+    getQueue().submit([&](auto &h) {
+        read_accessor<T> d_filter = {*filter.data, h, sycl::read_only};
+        write_accessor<convScalarT> d_filter_tmp = {filter_tmp_buffer, h};
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            fftconvolve_padCreateKernel<T, convScalarT>(
+                d_filter_tmp, filter_tmp.info, d_filter, filter.info));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/fftconvolve_reorder.hpp b/src/backend/oneapi/kernel/fftconvolve_reorder.hpp
new file mode 100644
index 0000000000..ec71b43bae
--- /dev/null
+++ b/src/backend/oneapi/kernel/fftconvolve_reorder.hpp
@@ -0,0 +1,193 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <af/defines.h>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T, typename convScalarT>
+class fftconvolve_reorderCreateKernel {
+   public:
+    fftconvolve_reorderCreateKernel(write_accessor<T> d_out, KParam oInfo,
+                                    read_accessor<convScalarT> d_in,
+                                    KParam iInfo, KParam fInfo,
+                                    const int half_di0, const int baseDim,
+                                    const int fftScale, const bool EXPAND,
+                                    const bool ROUND_OUT)
+        : d_out_(d_out)
+        , oInfo_(oInfo)
+        , d_in_(d_in)
+        , iInfo_(iInfo)
+        , fInfo_(fInfo)
+        , half_di0_(half_di0)
+        , baseDim_(baseDim)
+        , fftScale_(fftScale)
+        , EXPAND_(EXPAND)
+        , ROUND_OUT_(ROUND_OUT) {}
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+
+        const int t = it.get_global_id(0);
+
+        const int tMax = oInfo_.strides[3] * oInfo_.dims[3];
+
+        if (t >= tMax) return;
+
+        const int do0 = oInfo_.dims[0];
+        const int do1 = oInfo_.dims[1];
+        const int do2 = oInfo_.dims[2];
+
+        const int so1 = oInfo_.strides[1];
+        const int so2 = oInfo_.strides[2];
+        const int so3 = oInfo_.strides[3];
+
+        // Treating complex input array as real-only array,
+        // thus, multiply dimension 0 and strides by 2
+        const int di0 = iInfo_.dims[0] * 2;
+        const int di1 = iInfo_.dims[1];
+        const int di2 = iInfo_.dims[2];
+
+        const int si1 = iInfo_.strides[1] * 2;
+        const int si2 = iInfo_.strides[2] * 2;
+        const int si3 = iInfo_.strides[3] * 2;
+
+        const int to0 = t % so1;
+        const int to1 = (t / so1) % do1;
+        const int to2 = (t / so2) % do2;
+        const int to3 = (t / so3);
+
+        int oidx = to3 * so3 + to2 * so2 + to1 * so1 + to0;
+
+        int ti0, ti1, ti2, ti3;
+        if (EXPAND_) {
+            ti0 = to0;
+            ti1 = to1 * si1;
+            ti2 = to2 * si2;
+            ti3 = to3 * si3;
+        } else {
+            ti0 = to0 + fInfo_.dims[0] / 2;
+            ti1 = (to1 + (baseDim_ > 1) * (fInfo_.dims[1] / 2)) * si1;
+            ti2 = (to2 + (baseDim_ > 2) * (fInfo_.dims[2] / 2)) * si2;
+            ti3 = to3 * si3;
+        }
+
+        // Divide output elements to cuFFT resulting scale, round result if
+        // output type is single or double precision floating-point
+        if (ti0 < half_di0_) {
+            // Copy top elements
+            int iidx = iInfo_.offset + ti3 + ti2 + ti1 + ti0 * 2;
+            if (ROUND_OUT_)
+                d_out_[oidx] = (T)round(d_in_[iidx] / fftScale_);
+            else
+                d_out_[oidx] = (T)(d_in_[iidx] / fftScale_);
+        } else if (ti0 < half_di0_ + fInfo_.dims[0] - 1) {
+            // Add central elements
+            int iidx1 = iInfo_.offset + ti3 + ti2 + ti1 + ti0 * 2;
+            int iidx2 =
+                iInfo_.offset + ti3 + ti2 + ti1 + (ti0 - half_di0_) * 2 + 1;
+            if (ROUND_OUT_)
+                d_out_[oidx] =
+                    (T)round((d_in_[iidx1] + d_in_[iidx2]) / fftScale_);
+            else
+                d_out_[oidx] = (T)((d_in_[iidx1] + d_in_[iidx2]) / fftScale_);
+        } else {
+            // Copy bottom elements
+            const int iidx =
+                iInfo_.offset + ti3 + ti2 + ti1 + (ti0 - half_di0_) * 2 + 1;
+            if (ROUND_OUT_)
+                d_out_[oidx] = (T)round(d_in_[iidx] / fftScale_);
+            else
+                d_out_[oidx] = (T)(d_in_[iidx] / fftScale_);
+        }
+    }
+
+   private:
+    write_accessor<T> d_out_;
+    KParam oInfo_;
+    read_accessor<convScalarT> d_in_;
+    KParam iInfo_;
+    KParam fInfo_;
+    const int half_di0_;
+    const int baseDim_;
+    const int fftScale_;
+    const bool EXPAND_;
+    const bool ROUND_OUT_;
+};
+
+template<typename T, typename convT>
+void reorderOutputHelper(Param<T> out, Param<convT> packed, Param<T> sig,
+                         Param<T> filter, const int rank, AF_BATCH_KIND kind,
+                         bool expand) {
+    int fftScale = 1;
+
+    // Calculate the scale by which to divide clFFT results
+    for (int k = 0; k < rank; k++) fftScale *= packed.info.dims[k];
+
+    Param<T> sig_tmp, filter_tmp;
+    calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
+
+    // Number of packed complex elements in dimension 0
+    int sig_half_d0 = divup(sig.info.dims[0], 2);
+
+    int blocks = divup(out.info.strides[3] * out.info.dims[3], THREADS);
+
+    constexpr bool round_out = std::is_integral<T>::value;
+
+    auto local  = sycl::range(THREADS);
+    auto global = sycl::range(blocks * THREADS);
+
+    using convScalarT = typename convT::value_type;
+
+    if (kind == AF_BATCH_RHS) {
+        auto packed_num_elem   = (*packed.data).get_range().size();
+        auto filter_tmp_buffer = (*packed.data)
+                                     .template reinterpret<convScalarT>(
+                                         sycl::range<1>{packed_num_elem * 2});
+        getQueue().submit([&](auto &h) {
+            read_accessor<convScalarT> d_filter_tmp = {filter_tmp_buffer, h};
+            write_accessor<T> d_out = {*out.data, h, sycl::write_only};
+            h.parallel_for(
+                sycl::nd_range{global, local},
+                fftconvolve_reorderCreateKernel<T, convScalarT>(
+                    d_out, out.info, d_filter_tmp, filter_tmp.info, filter.info,
+                    sig_half_d0, rank, fftScale, expand, round_out));
+        });
+    } else {
+        auto packed_num_elem = (*packed.data).get_range().size();
+        auto sig_tmp_buffer  = (*packed.data)
+                                  .template reinterpret<convScalarT>(
+                                      sycl::range<1>{packed_num_elem * 2});
+        getQueue().submit([&](auto &h) {
+            read_accessor<convScalarT> d_sig_tmp = {sig_tmp_buffer, h,
+                                                    sycl::read_only};
+            write_accessor<T> d_out              = {*out.data, h};
+            h.parallel_for(
+                sycl::nd_range{global, local},
+                fftconvolve_reorderCreateKernel<T, convScalarT>(
+                    d_out, out.info, d_sig_tmp, sig_tmp.info, filter.info,
+                    sig_half_d0, rank, fftScale, expand, round_out));
+        });
+    }
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire

From 8d84b47c0275757ea2da4ccb7dbf78a9717f6649 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 1 Jun 2023 14:43:36 -0400
Subject: [PATCH 310/473] reduce local memory usage in iir (#3440)

* reduces local memory requirements for oneapi iir

* move local memory error handling to right place
---
 src/backend/oneapi/iir.cpp        | 13 +++++++++++++
 src/backend/oneapi/kernel/iir.hpp | 26 +++++++++++++++-----------
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/src/backend/oneapi/iir.cpp b/src/backend/oneapi/iir.cpp
index f60db52e8e..4a7654bd38 100644
--- a/src/backend/oneapi/iir.cpp
+++ b/src/backend/oneapi/iir.cpp
@@ -37,6 +37,19 @@ Array<T> iir(const Array<T> &b, const Array<T> &a, const Array<T> &x) {
 
     if (num_a == 1) { return c; }
 
+    size_t local_bytes_req = (num_a * 2 + 1) * sizeof(T);
+    if (local_bytes_req >
+        getDevice().get_info<sycl::info::device::local_mem_size>()) {
+        char errMessage[256];
+        snprintf(errMessage, sizeof(errMessage),
+                 "\ncurrent OneAPI device does not have sufficient local "
+                 "memory,\n"
+                 "for iir kernel, %zu(required) > %zu(available)\n",
+                 local_bytes_req,
+                 getDevice().get_info<sycl::info::device::local_mem_size>());
+        AF_ERROR(errMessage, AF_ERR_RUNTIME);
+    }
+
     dim4 ydims = c.dims();
     Array<T> y = createEmptyArray<T>(ydims);
 
diff --git a/src/backend/oneapi/kernel/iir.hpp b/src/backend/oneapi/kernel/iir.hpp
index 38769ad46a..938202f32f 100644
--- a/src/backend/oneapi/kernel/iir.hpp
+++ b/src/backend/oneapi/kernel/iir.hpp
@@ -21,8 +21,6 @@ namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-constexpr int MAX_A_SIZE = 1024;
-
 template<typename T, bool batch_a>
 class iirKernel {
    public:
@@ -67,10 +65,9 @@ class iirKernel {
         const int repeat =
             (num_a + g.get_local_range(0) - 1) / g.get_local_range(0);
 
-        for (int ii = 0; ii < MAX_A_SIZE / g.get_local_range(0); ii++) {
-            int id   = ii * g.get_local_range(0) + tx;
-            s_z_[id] = scalar<T>(0);
-            s_a_[id] = (id < num_a) ? d_a[id] : scalar<T>(0);
+        for (int ii = tx; ii < num_a; ii += g.get_local_range(0)) {
+            s_z_[ii] = scalar<T>(0);
+            s_a_[ii] = (ii < num_a) ? d_a[ii] : scalar<T>(0);
         }
         group_barrier(g);
 
@@ -81,14 +78,19 @@ class iirKernel {
             }
             group_barrier(g);
 
-#pragma unroll
             for (int ii = 0; ii < repeat; ii++) {
                 int id = ii * g.get_local_range(0) + tx + 1;
 
-                T z = s_z_[id] - s_a_[id] * s_y_[0];
+                T z;
+
+                if (id < num_a) {
+                    z = s_z_[id] - s_a_[id] * s_y_[0];
+                } else {
+                    z = scalar<T>(0);
+                }
                 group_barrier(g);
 
-                s_z_[id - 1] = z;
+                if ((id - 1) < num_a) { s_z_[id - 1] = z; }
                 group_barrier(g);
             }
         }
@@ -124,8 +126,10 @@ void iir(Param<T> y, Param<T> c, Param<T> a) {
         read_accessor<T> cAcc{*c.data, h};
         read_accessor<T> aAcc{*a.data, h};
 
-        auto s_z = sycl::local_accessor<T>(MAX_A_SIZE, h);
-        auto s_a = sycl::local_accessor<T>(MAX_A_SIZE, h);
+        unsigned num_a = a.info.dims[0];
+
+        auto s_z = sycl::local_accessor<T>(num_a, h);
+        auto s_a = sycl::local_accessor<T>(num_a, h);
         auto s_y = sycl::local_accessor<T>(1, h);
 
         if (batch_a) {

From 4f291f3f28f7aca0359469e370538b1bf54b6be0 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Sat, 6 May 2023 18:01:53 -0400
Subject: [PATCH 311/473] new method to get array with offset

---
 src/backend/oneapi/Array.hpp | 26 +++++++++++
 src/backend/oneapi/blas.cpp  | 89 +++++++++++++++++++++++++-----------
 2 files changed, 88 insertions(+), 27 deletions(-)

diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index a6ca6c402c..249192db14 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -23,6 +23,7 @@
 #include <algorithm>
 #include <cstdlib>
 #include <memory>
+#include <type_traits>
 #include <vector>
 
 enum class kJITHeuristics;
@@ -258,6 +259,31 @@ class Array {
         return data.get();
     }
 
+    template<typename outT>
+    sycl::buffer<outT> getBufferWithOffset() const {
+        dim_t sz_remaining = data_dims.elements() - getOffset();
+        printf("dd%d, elements %d offset %d\n",data_dims.elements(), elements(), getOffset());
+        if constexpr(std::is_same_v<outT, T>) {
+            if(getOffset() == 0) {
+                printf("off0--noreint\n");
+                *data.get();
+            }
+            return sycl::buffer<outT, 1>(
+                *data.get(),
+                sycl::id<1>(getOffset()),
+                sycl::range<1>(sz_remaining));
+        } else {
+            if(getOffset() == 0) {
+                printf("off0--reint\n");
+                data.get()->template reinterpret<outT, 1>();
+            }
+            return sycl::buffer<T, 1>(
+                *data.get(),
+                sycl::id<1>(getOffset()),
+                sycl::range<1>(sz_remaining)).template reinterpret<outT, 1>();
+        }
+    }
+
     int useCount() const { return data.use_count(); }
 
     dim_t getOffset() const { return info.getOffset(); }
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
index 73dbadfcfd..0579e1421f 100644
--- a/src/backend/oneapi/blas.cpp
+++ b/src/backend/oneapi/blas.cpp
@@ -43,19 +43,26 @@ static oneapi::mkl::transpose toBlasTranspose(af_mat_prop opt) {
 }
 
 template<typename T>
-static void gemvDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts, int M,
-                         int N, const T *alpha,
+static void gemvDispatch(sycl::queue queue,
+                         oneapi::mkl::transpose lOpts,
+                         oneapi::mkl::transpose rOpts,
+                         int M, int N, const T *alpha,
                          const arrayfire::oneapi::Array<T> &lhs, dim_t lStride,
                          const arrayfire::oneapi::Array<T> &x, dim_t incx,
                          const T *beta, arrayfire::oneapi::Array<T> &out,
                          dim_t oInc) {
     using Dt                   = arrayfire::oneapi::data_t<T>;
-    sycl::buffer<Dt, 1> lhsBuf = lhs.get()->template reinterpret<Dt, 1>();
-    sycl::buffer<Dt, 1> xBuf   = x.get()->template reinterpret<Dt, 1>();
-    sycl::buffer<Dt, 1> outBuf = out.get()->template reinterpret<Dt, 1>();
-    ::oneapi::mkl::blas::gemv(queue, lOpts, (int64_t)M, (int64_t)N, (T)*alpha,
-                              lhsBuf, (int64_t)lStride, xBuf, (int64_t)incx,
-                              (T)*beta, outBuf, (int64_t)oInc);
+    const af::dim4 lStrides = lhs.strides();
+    const af::dim4 xStrides = x.strides();
+    const af::dim4 oStrides = out.strides();
+    sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
+    sycl::buffer<Dt, 1> xBuf   =   x.template getBufferWithOffset<Dt>();
+    sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
+    if constexpr(!std::is_same_v<T, arrayfire::common::half>) {
+        ::oneapi::mkl::blas::gemv(queue, lOpts, (int64_t)M, (int64_t)N, (T)*alpha,
+                                lhsBuf, (int64_t)lStride, xBuf, (int64_t)incx,
+                                (T)*beta, outBuf, (int64_t)oInc);
+    }
 }
 
 template<typename T>
@@ -65,13 +72,21 @@ static void gemmDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts,
                          dim_t lStride, const arrayfire::oneapi::Array<T> &rhs,
                          dim_t rStride, const T *beta,
                          arrayfire::oneapi::Array<T> &out, dim_t oleading) {
-    using Dt                   = arrayfire::oneapi::data_t<T>;
-    sycl::buffer<Dt, 1> lhsBuf = lhs.get()->template reinterpret<Dt, 1>();
-    sycl::buffer<Dt, 1> rhsBuf = rhs.get()->template reinterpret<Dt, 1>();
-    sycl::buffer<Dt, 1> outBuf = out.get()->template reinterpret<Dt, 1>();
+    using Dt                = arrayfire::oneapi::data_t<T>;
+    const af::dim4 lStrides = lhs.strides();
+    const af::dim4 rStrides = rhs.strides();
+    const af::dim4 oStrides = out.strides();
+    sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
+    sycl::buffer<Dt, 1> rhsBuf = rhs.template getBufferWithOffset<Dt>();
+    sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
+    try {
     ::oneapi::mkl::blas::gemm(queue, lOpts, rOpts, M, N, K, *alpha, lhsBuf,
                               lStride, rhsBuf, rStride, *beta, outBuf,
                               oleading);
+    queue.wait_and_throw();
+    } catch(sycl::exception &e)  {
+        std::cout << e.what() << std::endl;
+    }
 }
 
 namespace arrayfire {
@@ -103,13 +118,21 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
     const dim4 &lStrides = lhs.strides();
     const dim4 &rStrides = rhs.strides();
     const dim4 oStrides  = out.strides();
+    try{
 
     if (oDims.ndims() <= 2) {  // if non-batched
         if (rhs.dims()[bColDim] == 1) {
-            dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
-            gemvDispatch<T>(getQueue(), lOpts, lDims[0], lDims[1], alpha, lhs,
-                            lStrides[1], rhs, incr, beta, out, oStrides[0]);
+            if constexpr(!std::is_same_v<T, arrayfire::common::half>) {
+                dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
+                gemvDispatch<T>(getQueue(), lOpts, rOpts, lDims[0], lDims[1], alpha, lhs,
+                                lStrides[1], rhs, incr, beta, out, oStrides[0]);
+            } else {
+                gemmDispatch<T>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
+                                lStrides[1], rhs, rStrides[1], beta, out,
+                                oStrides[1]);
+            }
         } else {
+            printf("%d %d %d, l%d R%d o%d\n",  M, N, K, lStrides[1], rStrides[1], oStrides[1]);
             gemmDispatch<T>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
                             lStrides[1], rhs, rStrides[1], beta, out,
                             oStrides[1]);
@@ -117,9 +140,9 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
     } else {  // if batched
         using Dt = arrayfire::oneapi::data_t<T>;
 
-        sycl::buffer<Dt, 1> lhsBuf = lhs.get()->template reinterpret<Dt, 1>();
-        sycl::buffer<Dt, 1> rhsBuf = rhs.get()->template reinterpret<Dt, 1>();
-        sycl::buffer<Dt, 1> outBuf = out.get()->template reinterpret<Dt, 1>();
+        sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
+        sycl::buffer<Dt, 1> rhsBuf = rhs.template getBufferWithOffset<Dt>();
+        sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
 
         const int64_t lda = lStrides[1];
         const int64_t ldb = rStrides[1];
@@ -127,18 +150,36 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
 
         int64_t batchSize = static_cast<int64_t>(oDims[2] * oDims[3]);
 
+        bool is_l_d2_batched = (oDims[2] == lDims[2]) && lDims[2] != 1;
+        bool is_l_d3_batched = (oDims[3] == lDims[3]) && lDims[3] != 1;
+        bool is_r_d2_batched = (oDims[2] == rDims[2]) && rDims[2] != 1;
+        bool is_r_d3_batched = (oDims[3] == rDims[3]) && rDims[3] != 1;
+
+        std::cout << lStrides << std::endl;
+        std::cout << rStrides << std::endl;
+
         const bool not_l_batched =
             (oDims[2] != lDims[2] && oDims[3] != lDims[3]);
         const bool not_r_batched =
             (oDims[2] != rDims[2] && oDims[3] != rDims[3]);
 
+        //dim_t lstride = !not_l_batched ? 0 : (is_l_d2_batched) ? lStrides[2] : lStrides[3];
+        //dim_t rstride = !not_r_batched ? 0 : (is_r_d2_batched) ? rStrides[2] : rStrides[3];
+        dim_t lstride = (is_l_d2_batched) ? lStrides[2] : is_l_d3_batched ? lStrides[3] : 0;
+        dim_t rstride = (is_r_d2_batched) ? rStrides[2] : is_r_d3_batched ? rStrides[3] : 0;
+
         ::oneapi::mkl::blas::gemm_batch(
             getQueue(), lOpts, rOpts, M, N, K, *alpha, lhsBuf, lda,
-            not_l_batched ? 0 : lStrides[2], rhsBuf, ldb,
-            not_r_batched ? 0 : rStrides[2], *beta, outBuf, ldc, oStrides[2],
+            lstride, rhsBuf, ldb,
+            rstride, *beta, outBuf, ldc, oStrides[2],
             batchSize);
     }
 
+        getQueue().wait_and_throw();
+    } catch(sycl::exception &e)  {
+        std::cout << e.what() << std::endl;
+    }
+
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 
@@ -161,13 +202,7 @@ INSTANTIATE_GEMM(float)
 INSTANTIATE_GEMM(cfloat)
 INSTANTIATE_GEMM(double)
 INSTANTIATE_GEMM(cdouble)
-// INSTANTIATE_GEMM(half)
-template<>
-void gemm(Array<half> &out, af_mat_prop optLhs, af_mat_prop optRhs,
-          const half *alpha, const Array<half> &lhs, const Array<half> &rhs,
-          const half *beta) {
-    ONEAPI_NOT_SUPPORTED("");
-}
+INSTANTIATE_GEMM(half)
 
 #define INSTANTIATE_DOT(TYPE)                                                  \
     template Array<TYPE> dot<TYPE>(const Array<TYPE> &lhs,                     \

From 50fef60640488fb964f96a40758429bc8c2ca830 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 30 May 2023 21:27:05 -0400
Subject: [PATCH 312/473] corrects batching for reordered output

---
 src/backend/oneapi/Array.hpp | 24 +++++-----
 src/backend/oneapi/blas.cpp  | 88 +++++++++++++++++++++---------------
 2 files changed, 62 insertions(+), 50 deletions(-)

diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index 249192db14..a47e32dbee 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -260,26 +260,24 @@ class Array {
     }
 
     template<typename outT>
-    sycl::buffer<outT> getBufferWithOffset() const {
-        dim_t sz_remaining = data_dims.elements() - getOffset();
-        printf("dd%d, elements %d offset %d\n",data_dims.elements(), elements(), getOffset());
+    sycl::buffer<outT> getBufferWithOffset(dim_t offset=-1) const {
+        offset = (offset == -1) ? getOffset() : offset;
+        dim_t sz_remaining = data_dims.elements() - offset;
         if constexpr(std::is_same_v<outT, T>) {
-            if(getOffset() == 0) {
-                printf("off0--noreint\n");
-                *data.get();
+            if(offset == 0) {
+                return *get();
             }
             return sycl::buffer<outT, 1>(
-                *data.get(),
-                sycl::id<1>(getOffset()),
+                *get(),
+                sycl::id<1>(offset),
                 sycl::range<1>(sz_remaining));
         } else {
-            if(getOffset() == 0) {
-                printf("off0--reint\n");
-                data.get()->template reinterpret<outT, 1>();
+            if(offset == 0) {
+                return get()->template reinterpret<outT, 1>();
             }
             return sycl::buffer<T, 1>(
-                *data.get(),
-                sycl::id<1>(getOffset()),
+                *get(),
+                sycl::id<1>(offset),
                 sycl::range<1>(sz_remaining)).template reinterpret<outT, 1>();
         }
     }
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
index 0579e1421f..9df9dd05f3 100644
--- a/src/backend/oneapi/blas.cpp
+++ b/src/backend/oneapi/blas.cpp
@@ -74,19 +74,15 @@ static void gemmDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts,
                          arrayfire::oneapi::Array<T> &out, dim_t oleading) {
     using Dt                = arrayfire::oneapi::data_t<T>;
     const af::dim4 lStrides = lhs.strides();
+
     const af::dim4 rStrides = rhs.strides();
     const af::dim4 oStrides = out.strides();
     sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
     sycl::buffer<Dt, 1> rhsBuf = rhs.template getBufferWithOffset<Dt>();
     sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
-    try {
     ::oneapi::mkl::blas::gemm(queue, lOpts, rOpts, M, N, K, *alpha, lhsBuf,
-                              lStride, rhsBuf, rStride, *beta, outBuf,
-                              oleading);
-    queue.wait_and_throw();
-    } catch(sycl::exception &e)  {
-        std::cout << e.what() << std::endl;
-    }
+                            lStride, rhsBuf, rStride, *beta, outBuf,
+                            oleading);
 }
 
 namespace arrayfire {
@@ -98,6 +94,10 @@ void initBlas() { /*gpu_blas_init();*/
 void deInitBlas() { /*gpu_blas_deinit();*/
 }
 
+bool checkMonotonicDim4(const af::dim4 &dim) {
+    return (dim[0] <= dim[1]) && (dim[1] <= dim[2]) && (dim[2] <= dim[3]);
+}
+
 template<typename T>
 void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
           const Array<T> &lhs, const Array<T> &rhs, const T *beta) {
@@ -118,7 +118,6 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
     const dim4 &lStrides = lhs.strides();
     const dim4 &rStrides = rhs.strides();
     const dim4 oStrides  = out.strides();
-    try{
 
     if (oDims.ndims() <= 2) {  // if non-batched
         if (rhs.dims()[bColDim] == 1) {
@@ -132,7 +131,6 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
                                 oStrides[1]);
             }
         } else {
-            printf("%d %d %d, l%d R%d o%d\n",  M, N, K, lStrides[1], rStrides[1], oStrides[1]);
             gemmDispatch<T>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
                             lStrides[1], rhs, rStrides[1], beta, out,
                             oStrides[1]);
@@ -140,14 +138,6 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
     } else {  // if batched
         using Dt = arrayfire::oneapi::data_t<T>;
 
-        sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
-        sycl::buffer<Dt, 1> rhsBuf = rhs.template getBufferWithOffset<Dt>();
-        sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
-
-        const int64_t lda = lStrides[1];
-        const int64_t ldb = rStrides[1];
-        const int64_t ldc = oStrides[1];
-
         int64_t batchSize = static_cast<int64_t>(oDims[2] * oDims[3]);
 
         bool is_l_d2_batched = (oDims[2] == lDims[2]) && lDims[2] != 1;
@@ -155,31 +145,55 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
         bool is_r_d2_batched = (oDims[2] == rDims[2]) && rDims[2] != 1;
         bool is_r_d3_batched = (oDims[3] == rDims[3]) && rDims[3] != 1;
 
-        std::cout << lStrides << std::endl;
-        std::cout << rStrides << std::endl;
+        bool canBatchMKL = checkMonotonicDim4(oStrides);
+        if(canBatchMKL) {
+            sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
+            sycl::buffer<Dt, 1> rhsBuf = rhs.template getBufferWithOffset<Dt>();
+            sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
 
-        const bool not_l_batched =
-            (oDims[2] != lDims[2] && oDims[3] != lDims[3]);
-        const bool not_r_batched =
-            (oDims[2] != rDims[2] && oDims[3] != rDims[3]);
+            const int64_t lda = lStrides[1];
+            const int64_t ldb = rStrides[1];
+            const int64_t ldc = oStrides[1];
 
-        //dim_t lstride = !not_l_batched ? 0 : (is_l_d2_batched) ? lStrides[2] : lStrides[3];
-        //dim_t rstride = !not_r_batched ? 0 : (is_r_d2_batched) ? rStrides[2] : rStrides[3];
-        dim_t lstride = (is_l_d2_batched) ? lStrides[2] : is_l_d3_batched ? lStrides[3] : 0;
-        dim_t rstride = (is_r_d2_batched) ? rStrides[2] : is_r_d3_batched ? rStrides[3] : 0;
+            dim_t lstride = (is_l_d2_batched) ? lStrides[2] : is_l_d3_batched ? lStrides[3] : 0;
+            dim_t rstride = (is_r_d2_batched) ? rStrides[2] : is_r_d3_batched ? rStrides[3] : 0;
 
-        ::oneapi::mkl::blas::gemm_batch(
-            getQueue(), lOpts, rOpts, M, N, K, *alpha, lhsBuf, lda,
-            lstride, rhsBuf, ldb,
-            rstride, *beta, outBuf, ldc, oStrides[2],
-            batchSize);
-    }
+            ::oneapi::mkl::blas::gemm_batch(
+                getQueue(), lOpts, rOpts, M, N, K, *alpha, lhsBuf, lda,
+                lstride, rhsBuf, ldb,
+                rstride, *beta, outBuf, ldc, oStrides[2],
+                batchSize);
+        } else {
+            std::vector<sycl::buffer<Dt>> lptrs;
+            std::vector<sycl::buffer<Dt>> rptrs;
+            std::vector<sycl::buffer<Dt>> optrs;
+
+            lptrs.reserve(batchSize);
+            rptrs.reserve(batchSize);
+            optrs.reserve(batchSize);
+
+            for (int n = 0; n < batchSize; n++) {
+                ptrdiff_t w = n / oDims[2];
+                ptrdiff_t z = n - w * oDims[2];
+
+                ptrdiff_t loff = z * (is_l_d2_batched * lStrides[2]) +
+                                 w * (is_l_d3_batched * lStrides[3]);
+                ptrdiff_t roff = z * (is_r_d2_batched * rStrides[2]) +
+                                 w * (is_r_d3_batched * rStrides[3]);
+                ptrdiff_t zoff = z * oStrides[2] + w * oStrides[3];
+
+                lptrs.emplace_back(lhs.template getBufferWithOffset<Dt>(loff));
+                rptrs.emplace_back(rhs.template getBufferWithOffset<Dt>(roff));
+                optrs.emplace_back(out.template getBufferWithOffset<Dt>(zoff));
+            }
 
-        getQueue().wait_and_throw();
-    } catch(sycl::exception &e)  {
-        std::cout << e.what() << std::endl;
+            for (int n = 0; n < batchSize; n++) {
+                ::oneapi::mkl::blas::gemm(getQueue(), lOpts, rOpts, M, N, K,
+                    *alpha, lptrs[n], lStrides[1], rptrs[n], rStrides[1], *beta,
+                    optrs[n], oStrides[1]);
+            }
+        }
     }
-
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 

From f505951aa3dd7e2d00776e55e549ea8fdb1a56c2 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 30 May 2023 21:27:55 -0400
Subject: [PATCH 313/473] adds mkl exceptions to err_common

---
 src/backend/common/err_common.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index 9e2b2e8a2f..92df5beb27 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -26,6 +26,7 @@
 #include <platform.hpp>
 #elif defined(AF_ONEAPI)
 #include <sycl/sycl.hpp>
+#include <oneapi/mkl/exceptions.hpp>
 #endif
 
 using boost::stacktrace::stacktrace;
@@ -169,6 +170,12 @@ af_err processException() {
         snprintf(oneapi_err_msg, sizeof(oneapi_err_msg),
                  "oneAPI Error (%d): %s", ex.code().value(), ex.what());
 
+        err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL);
+    } catch (const oneapi::mkl::exception &ex) {
+        char oneapi_err_msg[1024];
+        snprintf(oneapi_err_msg, sizeof(oneapi_err_msg),
+                 "MKL Error: %s", ex.what());
+
         err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL);
 #endif
 #ifdef AF_OPENCL
@@ -184,6 +191,7 @@ af_err processException() {
             err = set_global_error_string(opencl_err_msg, AF_ERR_INTERNAL);
         }
 #endif
+    } catch (const std::exception &ex) { err = set_global_error_string(ex.what(), AF_ERR_UNKNOWN);
     } catch (...) { err = set_global_error_string(ss.str(), AF_ERR_UNKNOWN); }
 
     return err;

From 37eb8acea82e317aa0e9e39ba6c3f6ec44198686 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 30 May 2023 21:33:08 -0400
Subject: [PATCH 314/473] blas reorder clang-format

---
 src/backend/common/err_common.cpp |  9 ++--
 src/backend/oneapi/Array.hpp      | 27 ++++------
 src/backend/oneapi/blas.cpp       | 85 +++++++++++++++++--------------
 3 files changed, 62 insertions(+), 59 deletions(-)

diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index 92df5beb27..885aa8d5f5 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -25,8 +25,8 @@
 #include <errorcodes.hpp>
 #include <platform.hpp>
 #elif defined(AF_ONEAPI)
-#include <sycl/sycl.hpp>
 #include <oneapi/mkl/exceptions.hpp>
+#include <sycl/sycl.hpp>
 #endif
 
 using boost::stacktrace::stacktrace;
@@ -173,8 +173,8 @@ af_err processException() {
         err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL);
     } catch (const oneapi::mkl::exception &ex) {
         char oneapi_err_msg[1024];
-        snprintf(oneapi_err_msg, sizeof(oneapi_err_msg),
-                 "MKL Error: %s", ex.what());
+        snprintf(oneapi_err_msg, sizeof(oneapi_err_msg), "MKL Error: %s",
+                 ex.what());
 
         err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL);
 #endif
@@ -191,7 +191,8 @@ af_err processException() {
             err = set_global_error_string(opencl_err_msg, AF_ERR_INTERNAL);
         }
 #endif
-    } catch (const std::exception &ex) { err = set_global_error_string(ex.what(), AF_ERR_UNKNOWN);
+    } catch (const std::exception &ex) {
+        err = set_global_error_string(ex.what(), AF_ERR_UNKNOWN);
     } catch (...) { err = set_global_error_string(ss.str(), AF_ERR_UNKNOWN); }
 
     return err;
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index a47e32dbee..e0b0962222 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -260,25 +260,18 @@ class Array {
     }
 
     template<typename outT>
-    sycl::buffer<outT> getBufferWithOffset(dim_t offset=-1) const {
-        offset = (offset == -1) ? getOffset() : offset;
+    sycl::buffer<outT> getBufferWithOffset(dim_t offset = -1) const {
+        offset             = (offset == -1) ? getOffset() : offset;
         dim_t sz_remaining = data_dims.elements() - offset;
-        if constexpr(std::is_same_v<outT, T>) {
-            if(offset == 0) {
-                return *get();
-            }
-            return sycl::buffer<outT, 1>(
-                *get(),
-                sycl::id<1>(offset),
-                sycl::range<1>(sz_remaining));
+        if constexpr (std::is_same_v<outT, T>) {
+            if (offset == 0) { return *get(); }
+            return sycl::buffer<outT, 1>(*get(), sycl::id<1>(offset),
+                                         sycl::range<1>(sz_remaining));
         } else {
-            if(offset == 0) {
-                return get()->template reinterpret<outT, 1>();
-            }
-            return sycl::buffer<T, 1>(
-                *get(),
-                sycl::id<1>(offset),
-                sycl::range<1>(sz_remaining)).template reinterpret<outT, 1>();
+            if (offset == 0) { return get()->template reinterpret<outT, 1>(); }
+            return sycl::buffer<T, 1>(*get(), sycl::id<1>(offset),
+                                      sycl::range<1>(sz_remaining))
+                .template reinterpret<outT, 1>();
         }
     }
 
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
index 9df9dd05f3..37495957e9 100644
--- a/src/backend/oneapi/blas.cpp
+++ b/src/backend/oneapi/blas.cpp
@@ -43,25 +43,24 @@ static oneapi::mkl::transpose toBlasTranspose(af_mat_prop opt) {
 }
 
 template<typename T>
-static void gemvDispatch(sycl::queue queue,
-                         oneapi::mkl::transpose lOpts,
-                         oneapi::mkl::transpose rOpts,
-                         int M, int N, const T *alpha,
-                         const arrayfire::oneapi::Array<T> &lhs, dim_t lStride,
-                         const arrayfire::oneapi::Array<T> &x, dim_t incx,
-                         const T *beta, arrayfire::oneapi::Array<T> &out,
-                         dim_t oInc) {
+static void gemvDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts,
+                         oneapi::mkl::transpose rOpts, int M, int N,
+                         const T *alpha, const arrayfire::oneapi::Array<T> &lhs,
+                         dim_t lStride, const arrayfire::oneapi::Array<T> &x,
+                         dim_t incx, const T *beta,
+                         arrayfire::oneapi::Array<T> &out, dim_t oInc) {
     using Dt                   = arrayfire::oneapi::data_t<T>;
-    const af::dim4 lStrides = lhs.strides();
-    const af::dim4 xStrides = x.strides();
-    const af::dim4 oStrides = out.strides();
+    const af::dim4 lStrides    = lhs.strides();
+    const af::dim4 xStrides    = x.strides();
+    const af::dim4 oStrides    = out.strides();
     sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
-    sycl::buffer<Dt, 1> xBuf   =   x.template getBufferWithOffset<Dt>();
+    sycl::buffer<Dt, 1> xBuf   = x.template getBufferWithOffset<Dt>();
     sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
-    if constexpr(!std::is_same_v<T, arrayfire::common::half>) {
-        ::oneapi::mkl::blas::gemv(queue, lOpts, (int64_t)M, (int64_t)N, (T)*alpha,
-                                lhsBuf, (int64_t)lStride, xBuf, (int64_t)incx,
-                                (T)*beta, outBuf, (int64_t)oInc);
+    if constexpr (!std::is_same_v<T, arrayfire::common::half>) {
+        ::oneapi::mkl::blas::gemv(queue, lOpts, (int64_t)M, (int64_t)N,
+                                  (T)*alpha, lhsBuf, (int64_t)lStride, xBuf,
+                                  (int64_t)incx, (T)*beta, outBuf,
+                                  (int64_t)oInc);
     }
 }
 
@@ -75,14 +74,14 @@ static void gemmDispatch(sycl::queue queue, oneapi::mkl::transpose lOpts,
     using Dt                = arrayfire::oneapi::data_t<T>;
     const af::dim4 lStrides = lhs.strides();
 
-    const af::dim4 rStrides = rhs.strides();
-    const af::dim4 oStrides = out.strides();
+    const af::dim4 rStrides    = rhs.strides();
+    const af::dim4 oStrides    = out.strides();
     sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
     sycl::buffer<Dt, 1> rhsBuf = rhs.template getBufferWithOffset<Dt>();
     sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
     ::oneapi::mkl::blas::gemm(queue, lOpts, rOpts, M, N, K, *alpha, lhsBuf,
-                            lStride, rhsBuf, rStride, *beta, outBuf,
-                            oleading);
+                              lStride, rhsBuf, rStride, *beta, outBuf,
+                              oleading);
 }
 
 namespace arrayfire {
@@ -94,7 +93,7 @@ void initBlas() { /*gpu_blas_init();*/
 void deInitBlas() { /*gpu_blas_deinit();*/
 }
 
-bool checkMonotonicDim4(const af::dim4 &dim) {
+bool isStrideMonotonic(const af::dim4 &dim) {
     return (dim[0] <= dim[1]) && (dim[1] <= dim[2]) && (dim[2] <= dim[3]);
 }
 
@@ -121,14 +120,17 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
 
     if (oDims.ndims() <= 2) {  // if non-batched
         if (rhs.dims()[bColDim] == 1) {
-            if constexpr(!std::is_same_v<T, arrayfire::common::half>) {
-                dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
-                gemvDispatch<T>(getQueue(), lOpts, rOpts, lDims[0], lDims[1], alpha, lhs,
-                                lStrides[1], rhs, incr, beta, out, oStrides[0]);
-            } else {
+            if constexpr (std::is_same_v<T, arrayfire::common::half>) {
+                // currently no half support for gemv, use gemm instead
                 gemmDispatch<T>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
                                 lStrides[1], rhs, rStrides[1], beta, out,
                                 oStrides[1]);
+            } else {
+                dim_t incr =
+                    (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
+                gemvDispatch<T>(getQueue(), lOpts, rOpts, lDims[0], lDims[1],
+                                alpha, lhs, lStrides[1], rhs, incr, beta, out,
+                                oStrides[0]);
             }
         } else {
             gemmDispatch<T>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
@@ -145,8 +147,11 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
         bool is_r_d2_batched = (oDims[2] == rDims[2]) && rDims[2] != 1;
         bool is_r_d3_batched = (oDims[3] == rDims[3]) && rDims[3] != 1;
 
-        bool canBatchMKL = checkMonotonicDim4(oStrides);
-        if(canBatchMKL) {
+        // MKL requires stridec >= ldc * n, which may not be true with reordered
+        // outputs if the stride is monotonic, then MKL requirements for
+        // batching can be met
+        bool canBatchMKL = isStrideMonotonic(oStrides);
+        if (canBatchMKL) {
             sycl::buffer<Dt, 1> lhsBuf = lhs.template getBufferWithOffset<Dt>();
             sycl::buffer<Dt, 1> rhsBuf = rhs.template getBufferWithOffset<Dt>();
             sycl::buffer<Dt, 1> outBuf = out.template getBufferWithOffset<Dt>();
@@ -155,14 +160,17 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
             const int64_t ldb = rStrides[1];
             const int64_t ldc = oStrides[1];
 
-            dim_t lstride = (is_l_d2_batched) ? lStrides[2] : is_l_d3_batched ? lStrides[3] : 0;
-            dim_t rstride = (is_r_d2_batched) ? rStrides[2] : is_r_d3_batched ? rStrides[3] : 0;
-
-            ::oneapi::mkl::blas::gemm_batch(
-                getQueue(), lOpts, rOpts, M, N, K, *alpha, lhsBuf, lda,
-                lstride, rhsBuf, ldb,
-                rstride, *beta, outBuf, ldc, oStrides[2],
-                batchSize);
+            dim_t lstride = (is_l_d2_batched) ? lStrides[2]
+                            : is_l_d3_batched ? lStrides[3]
+                                              : 0;
+            dim_t rstride = (is_r_d2_batched) ? rStrides[2]
+                            : is_r_d3_batched ? rStrides[3]
+                                              : 0;
+
+            ::oneapi::mkl::blas::gemm_batch(getQueue(), lOpts, rOpts, M, N, K,
+                                            *alpha, lhsBuf, lda, lstride,
+                                            rhsBuf, ldb, rstride, *beta, outBuf,
+                                            ldc, oStrides[2], batchSize);
         } else {
             std::vector<sycl::buffer<Dt>> lptrs;
             std::vector<sycl::buffer<Dt>> rptrs;
@@ -189,8 +197,9 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
 
             for (int n = 0; n < batchSize; n++) {
                 ::oneapi::mkl::blas::gemm(getQueue(), lOpts, rOpts, M, N, K,
-                    *alpha, lptrs[n], lStrides[1], rptrs[n], rStrides[1], *beta,
-                    optrs[n], oStrides[1]);
+                                          *alpha, lptrs[n], lStrides[1],
+                                          rptrs[n], rStrides[1], *beta,
+                                          optrs[n], oStrides[1]);
             }
         }
     }

From 57dcc4f038b7f15938ef06247edda944b6fff41d Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 1 Jun 2023 20:38:02 -0400
Subject: [PATCH 315/473] fix mismatching number of elements in copies from
 managed buffers

---
 src/backend/oneapi/Array.cpp                   | 17 ++++++++++-------
 src/backend/oneapi/kernel/sort_by_key_impl.hpp |  9 ++++++---
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 3a9fbff3be..6f506ec2ba 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -125,10 +125,10 @@ Array<T>::Array(const dim4 &dims, const T *const in_data)
     static_assert(
         offsetof(Array<T>, info) == 0,
         "Array<T>::info must be the first member variable of Array<T>");
-    // getQueue().enqueueWriteBuffer(*data.get(), CL_TRUE, 0,
-    // sizeof(T) * info.elements(), in_data);
     getQueue()
-        .submit([&](sycl::handler &h) { h.copy(in_data, data->get_access(h)); })
+        .submit([&](sycl::handler &h) {
+            h.copy(in_data, data->get_access(h, sycl::range(info.elements())));
+        })
         .wait();
 }
 
@@ -145,7 +145,8 @@ Array<T>::Array(const af::dim4 &dims, buffer<T> *const mem, size_t offset,
     if (copy) {
         getQueue()
             .submit([&](sycl::handler &h) {
-                h.copy(mem->get_access(h), data->get_access(h));
+                h.copy(mem->get_access(h, sycl::range(info.elements())),
+                       data->get_access(h));
             })
             .wait();
     }
@@ -193,8 +194,10 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
     } else {
         data = memAlloc<T>(info.elements());
         getQueue()
-            .submit(
-                [&](sycl::handler &h) { h.copy(in_data, data->get_access(h)); })
+            .submit([&](sycl::handler &h) {
+                h.copy(in_data,
+                       data->get_access(h, sycl::range(info.elements())));
+            })
             .wait();
     }
 }
@@ -486,7 +489,7 @@ void writeHostDataArray(Array<T> &arr, const T *const data,
             buffer<T> &buf = *arr.get();
             // auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>)
             // TODO: offset accessor
-            auto offset_acc = buf.get_access(h);
+            auto offset_acc = buf.get_access(h, sycl::range(arr.elements()));
             h.copy(data, offset_acc);
         })
         .wait();
diff --git a/src/backend/oneapi/kernel/sort_by_key_impl.hpp b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
index c0c57d8eff..ad3b3c8a80 100644
--- a/src/backend/oneapi/kernel/sort_by_key_impl.hpp
+++ b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
@@ -108,8 +108,10 @@ void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
 
     auto cKey = memAlloc<Tk>(elements);
     getQueue().submit([&](sycl::handler &h) {
-        h.copy(pKey.data->template reinterpret<compute_t<Tk>>().get_access(),
-               cKey.get()->template reinterpret<compute_t<Tk>>().get_access());
+        h.copy(pKey.data->template reinterpret<compute_t<Tk>>().get_access(
+                   h, elements),
+               cKey.get()->template reinterpret<compute_t<Tk>>().get_access(
+                   h, elements));
     });
     auto ckey_begin =
         ::oneapi::dpl::begin(cKey.get()->template reinterpret<compute_t<Tk>>());
@@ -144,7 +146,8 @@ void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
 
     auto cSeq = memAlloc<uint>(elements);
     getQueue().submit([&](sycl::handler &h) {
-        h.copy(Seq.get()->get_access(), cSeq.get()->get_access());
+        h.copy(Seq.get()->get_access(h, elements),
+               cSeq.get()->get_access(h, elements));
     });
     auto cseq_begin = ::oneapi::dpl::begin(*cSeq.get());
     auto cseq_end   = ::oneapi::dpl::end(*cSeq.get());

From 50ca60117e11024fd5c79521df7b67dff003c171 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 1 Jun 2023 23:27:23 -0400
Subject: [PATCH 316/473] use exactly sized subbuffers for scratch space

---
 src/backend/oneapi/cholesky.cpp | 11 ++++++-----
 src/backend/oneapi/lu.cpp       | 14 +++++++++-----
 src/backend/oneapi/memory.cpp   |  1 +
 src/backend/oneapi/svd.cpp      | 16 ++++++++++++----
 4 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/src/backend/oneapi/cholesky.cpp b/src/backend/oneapi/cholesky.cpp
index 4fb0e08c58..1b81be7f03 100644
--- a/src/backend/oneapi/cholesky.cpp
+++ b/src/backend/oneapi/cholesky.cpp
@@ -14,8 +14,9 @@
 #include <platform.hpp>
 
 #if defined(WITH_LINEAR_ALGEBRA)
+#include <memory.hpp>
+#include <oneapi/mkl/lapack.hpp>
 #include <triangle.hpp>
-#include "oneapi/mkl/lapack.hpp"
 
 namespace arrayfire {
 namespace oneapi {
@@ -35,12 +36,12 @@ int cholesky_inplace(Array<T> &in, const bool is_upper) {
     lwork = ::oneapi::mkl::lapack::potrf_scratchpad_size<T>(getQueue(), uplo, N,
                                                             LDA);
 
-    Array<T> workspace = createEmptyArray<T>(af::dim4(lwork));
-    Array<int> d_info  = createEmptyArray<int>(af::dim4(1));
+    auto workspaceMem  = memAlloc<compute_t<T>>(lwork);
+    sycl::buffer<compute_t<T>> in_buffer = in.template getBufferWithOffset<compute_t<T>>();
 
     try {
-        ::oneapi::mkl::lapack::potrf(getQueue(), uplo, N, *in.get(), LDA,
-                                     *workspace.get(), lwork);
+        ::oneapi::mkl::lapack::potrf(getQueue(), uplo, N, in_buffer, LDA,
+                                     *workspaceMem, lwork);
     } catch (::oneapi::mkl::lapack::exception const &e) {
         AF_ERROR(
             "Unexpected exception caught during synchronous\
diff --git a/src/backend/oneapi/lu.cpp b/src/backend/oneapi/lu.cpp
index 200b85d23b..20c44f2529 100644
--- a/src/backend/oneapi/lu.cpp
+++ b/src/backend/oneapi/lu.cpp
@@ -11,11 +11,12 @@
 #include <lu.hpp>
 
 #if defined(WITH_LINEAR_ALGEBRA)
+#include <oneapi/mkl/lapack.hpp>
 #include <blas.hpp>
 #include <copy.hpp>
 #include <kernel/lu_split.hpp>
+#include <memory.hpp>
 #include <platform.hpp>
-#include "oneapi/mkl/lapack.hpp"
 
 namespace arrayfire {
 namespace oneapi {
@@ -76,11 +77,14 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
     std::int64_t scratchpad_size =
         ::oneapi::mkl::lapack::getrf_scratchpad_size<T>(getQueue(), M, N, LDA);
 
-    sycl::buffer<int64_t, 1> ipiv{sycl::range<1>(MN)};
-    Array<T> scratch = createEmptyArray<T>(af::dim4(scratchpad_size));
+    auto ipivMem    = memAlloc<int64_t>(MN);
+    auto scratchMem = memAlloc<compute_t<T>>(scratchpad_size);
+    sycl::buffer<int64_t> ipiv(*ipivMem, 0, MN);
+    sycl::buffer<compute_t<T>> scratchpad(*scratchMem, 0, scratchpad_size);
 
-    ::oneapi::mkl::lapack::getrf(getQueue(), M, N, *in.get(), LDA, ipiv,
-                                 *scratch.get(), scratchpad_size);
+    sycl::buffer<compute_t<T>> in_buffer = in.template getBufferWithOffset<compute_t<T>>();
+    ::oneapi::mkl::lapack::getrf(getQueue(), M, N, in_buffer,  LDA, ipiv,
+                                 scratchpad, scratchpad_size);
 
     Array<int> pivot = convertPivot(ipiv, M, convert_pivot);
     return pivot;
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index 2b383b9520..f2cbab094c 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -165,6 +165,7 @@ INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
 INSTANTIATE(arrayfire::common::half)
+INSTANTIATE(int64_t)
 
 template<>
 void *pinnedAlloc<void>(const size_t &elements) {
diff --git a/src/backend/oneapi/svd.cpp b/src/backend/oneapi/svd.cpp
index 2c9b751d15..ccea706ceb 100644
--- a/src/backend/oneapi/svd.cpp
+++ b/src/backend/oneapi/svd.cpp
@@ -38,15 +38,23 @@ void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in) {
     int64_t LDU   = uStrides[1];
     int64_t LDVt  = vStrides[1];
 
-    int64_t scratch_size = ::oneapi::mkl::lapack::gesvd_scratchpad_size<T>(
+    int64_t scratch_size = ::oneapi::mkl::lapack::gesvd_scratchpad_size<compute_t<T>>(
         getQueue(), ::oneapi::mkl::jobsvd::vectors,
         ::oneapi::mkl::jobsvd::vectors, M, N, LDA, LDU, LDVt);
-    Array<T> scratchpad = createEmptyArray<T>(af::dim4(scratch_size));
+
+    auto scratchpadMem  = memAlloc<compute_t<T>>(scratch_size);
+    sycl::buffer<compute_t<T>> scratchpad(*scratchpadMem, 0, scratch_size);
+
+    sycl::buffer<compute_t<T>> in_buffer = in.template getBufferWithOffset<compute_t<T>>();
+
+    sycl::buffer<compute_t<Tr>> sBuf  = s.template getBufferWithOffset<compute_t<Tr>>();
+    sycl::buffer<compute_t<T>> uBuf  = u.template getBufferWithOffset<compute_t<T>>();
+    sycl::buffer<compute_t<T>> vtBuf = vt.template getBufferWithOffset<compute_t<T>>();
 
     ::oneapi::mkl::lapack::gesvd(
         getQueue(), ::oneapi::mkl::jobsvd::vectors,
-        ::oneapi::mkl::jobsvd::vectors, M, N, *in.get(), LDA, *s.get(),
-        *u.get(), LDU, *vt.get(), LDVt, *scratchpad.get(), scratch_size);
+        ::oneapi::mkl::jobsvd::vectors, M, N, in_buffer, LDA, sBuf,
+        uBuf, LDU, vtBuf, LDVt, scratchpad, scratch_size);
 }
 
 template<typename T, typename Tr>

From 6602a4b85f5a04748f82845eca9de30a9d3e5402 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 2 Jun 2023 07:31:33 -0400
Subject: [PATCH 317/473] change oneMKL scratch size to exact sycl::buffer
 instead of memAlloc

---
 src/backend/oneapi/cholesky.cpp | 11 ++++++---
 src/backend/oneapi/lu.cpp       | 17 +++++++------
 src/backend/oneapi/svd.cpp      | 42 ++++++++++++++++++++-------------
 3 files changed, 43 insertions(+), 27 deletions(-)

diff --git a/src/backend/oneapi/cholesky.cpp b/src/backend/oneapi/cholesky.cpp
index 1b81be7f03..882f915d15 100644
--- a/src/backend/oneapi/cholesky.cpp
+++ b/src/backend/oneapi/cholesky.cpp
@@ -36,12 +36,17 @@ int cholesky_inplace(Array<T> &in, const bool is_upper) {
     lwork = ::oneapi::mkl::lapack::potrf_scratchpad_size<T>(getQueue(), uplo, N,
                                                             LDA);
 
-    auto workspaceMem  = memAlloc<compute_t<T>>(lwork);
-    sycl::buffer<compute_t<T>> in_buffer = in.template getBufferWithOffset<compute_t<T>>();
+    // MKL is finicky about exact scratch space size so we'll need to
+    // create sycl::buffer of exact size. if we use memAlloc, this might
+    // require a sub-buffer of a sub-buffer returned by memAlloc which is
+    // currently illegal in sycl
+    sycl::buffer<compute_t<T>> workspace(lwork);
+    sycl::buffer<compute_t<T>> in_buffer =
+        in.template getBufferWithOffset<compute_t<T>>();
 
     try {
         ::oneapi::mkl::lapack::potrf(getQueue(), uplo, N, in_buffer, LDA,
-                                     *workspaceMem, lwork);
+                                     workspace, lwork);
     } catch (::oneapi::mkl::lapack::exception const &e) {
         AF_ERROR(
             "Unexpected exception caught during synchronous\
diff --git a/src/backend/oneapi/lu.cpp b/src/backend/oneapi/lu.cpp
index 20c44f2529..1ae473650b 100644
--- a/src/backend/oneapi/lu.cpp
+++ b/src/backend/oneapi/lu.cpp
@@ -11,11 +11,11 @@
 #include <lu.hpp>
 
 #if defined(WITH_LINEAR_ALGEBRA)
-#include <oneapi/mkl/lapack.hpp>
 #include <blas.hpp>
 #include <copy.hpp>
 #include <kernel/lu_split.hpp>
 #include <memory.hpp>
+#include <oneapi/mkl/lapack.hpp>
 #include <platform.hpp>
 
 namespace arrayfire {
@@ -77,13 +77,16 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
     std::int64_t scratchpad_size =
         ::oneapi::mkl::lapack::getrf_scratchpad_size<T>(getQueue(), M, N, LDA);
 
-    auto ipivMem    = memAlloc<int64_t>(MN);
-    auto scratchMem = memAlloc<compute_t<T>>(scratchpad_size);
-    sycl::buffer<int64_t> ipiv(*ipivMem, 0, MN);
-    sycl::buffer<compute_t<T>> scratchpad(*scratchMem, 0, scratchpad_size);
+    // MKL is finicky about exact scratch space size so we'll need to
+    // create sycl::buffer of exact size. if we use memAlloc, this might
+    // require a sub-buffer of a sub-buffer returned by memAlloc which is
+    // currently illegal in sycl
+    sycl::buffer<int64_t> ipiv(MN);
+    sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
 
-    sycl::buffer<compute_t<T>> in_buffer = in.template getBufferWithOffset<compute_t<T>>();
-    ::oneapi::mkl::lapack::getrf(getQueue(), M, N, in_buffer,  LDA, ipiv,
+    sycl::buffer<compute_t<T>> in_buffer =
+        in.template getBufferWithOffset<compute_t<T>>();
+    ::oneapi::mkl::lapack::getrf(getQueue(), M, N, in_buffer, LDA, ipiv,
                                  scratchpad, scratchpad_size);
 
     Array<int> pivot = convertPivot(ipiv, M, convert_pivot);
diff --git a/src/backend/oneapi/svd.cpp b/src/backend/oneapi/svd.cpp
index ccea706ceb..97b5f3a468 100644
--- a/src/backend/oneapi/svd.cpp
+++ b/src/backend/oneapi/svd.cpp
@@ -38,23 +38,31 @@ void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in) {
     int64_t LDU   = uStrides[1];
     int64_t LDVt  = vStrides[1];
 
-    int64_t scratch_size = ::oneapi::mkl::lapack::gesvd_scratchpad_size<compute_t<T>>(
-        getQueue(), ::oneapi::mkl::jobsvd::vectors,
-        ::oneapi::mkl::jobsvd::vectors, M, N, LDA, LDU, LDVt);
-
-    auto scratchpadMem  = memAlloc<compute_t<T>>(scratch_size);
-    sycl::buffer<compute_t<T>> scratchpad(*scratchpadMem, 0, scratch_size);
-
-    sycl::buffer<compute_t<T>> in_buffer = in.template getBufferWithOffset<compute_t<T>>();
-
-    sycl::buffer<compute_t<Tr>> sBuf  = s.template getBufferWithOffset<compute_t<Tr>>();
-    sycl::buffer<compute_t<T>> uBuf  = u.template getBufferWithOffset<compute_t<T>>();
-    sycl::buffer<compute_t<T>> vtBuf = vt.template getBufferWithOffset<compute_t<T>>();
-
-    ::oneapi::mkl::lapack::gesvd(
-        getQueue(), ::oneapi::mkl::jobsvd::vectors,
-        ::oneapi::mkl::jobsvd::vectors, M, N, in_buffer, LDA, sBuf,
-        uBuf, LDU, vtBuf, LDVt, scratchpad, scratch_size);
+    // MKL is finicky about exact scratch space size so we'll need to
+    // create sycl::buffer of exact size. if we use memAlloc, this might
+    // require a sub-buffer of a sub-buffer returned by memAlloc which is
+    // currently illegal in sycl
+    int64_t scratch_size =
+        ::oneapi::mkl::lapack::gesvd_scratchpad_size<compute_t<T>>(
+            getQueue(), ::oneapi::mkl::jobsvd::vectors,
+            ::oneapi::mkl::jobsvd::vectors, M, N, LDA, LDU, LDVt);
+
+    sycl::buffer<compute_t<T>> scratchpad(scratch_size);
+
+    sycl::buffer<compute_t<T>> in_buffer =
+        in.template getBufferWithOffset<compute_t<T>>();
+
+    sycl::buffer<compute_t<Tr>> sBuf =
+        s.template getBufferWithOffset<compute_t<Tr>>();
+    sycl::buffer<compute_t<T>> uBuf =
+        u.template getBufferWithOffset<compute_t<T>>();
+    sycl::buffer<compute_t<T>> vtBuf =
+        vt.template getBufferWithOffset<compute_t<T>>();
+
+    ::oneapi::mkl::lapack::gesvd(getQueue(), ::oneapi::mkl::jobsvd::vectors,
+                                 ::oneapi::mkl::jobsvd::vectors, M, N,
+                                 in_buffer, LDA, sBuf, uBuf, LDU, vtBuf, LDVt,
+                                 scratchpad, scratch_size);
 }
 
 template<typename T, typename Tr>

From 2f6e5e3e877cba145f99bc7550e10f4dbd35cfb5 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 2 Jun 2023 07:32:02 -0400
Subject: [PATCH 318/473] implement solve, pinverse in oneapi backend

---
 src/backend/oneapi/inverse.cpp |   3 +-
 src/backend/oneapi/solve.cpp   | 453 ++++++++++++++++++---------------
 2 files changed, 247 insertions(+), 209 deletions(-)

diff --git a/src/backend/oneapi/inverse.cpp b/src/backend/oneapi/inverse.cpp
index 97d91f4db4..2779393906 100644
--- a/src/backend/oneapi/inverse.cpp
+++ b/src/backend/oneapi/inverse.cpp
@@ -19,9 +19,8 @@ namespace oneapi {
 
 template<typename T>
 Array<T> inverse(const Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("");
     Array<T> I = identity<T>(in.dims());
-    return I;
+    return solve<T>(in, I);
 }
 
 #define INSTANTIATE(T) template Array<T> inverse<T>(const Array<T> &in);
diff --git a/src/backend/oneapi/solve.cpp b/src/backend/oneapi/solve.cpp
index a4082c0d1f..07c4f3b171 100644
--- a/src/backend/oneapi/solve.cpp
+++ b/src/backend/oneapi/solve.cpp
@@ -11,111 +11,152 @@
 
 #include <err_oneapi.hpp>
 
-#if defined(WITH_LINEAR_ALGEBRA) && !defined(AF_ONEAPI)
+#if defined(WITH_LINEAR_ALGEBRA)
+#include <Array.hpp>
 #include <blas.hpp>
+#include <common/cast.hpp>
 #include <copy.hpp>
-#include <cpu/cpu_solve.hpp>
 #include <lu.hpp>
-#include <magma/magma.h>
-#include <magma/magma_blas.h>
-#include <magma/magma_data.h>
-#include <magma/magma_helper.h>
 #include <math.hpp>
+#include <memory.hpp>
+#include <oneapi/mkl/blas.hpp>
+#include <oneapi/mkl/lapack.hpp>
 #include <platform.hpp>
 #include <transpose.hpp>
-#include <af/opencl.h>
 
 #include <algorithm>
 #include <vector>
 
-using cl::Buffer;
+using arrayfire::common::cast;
 using std::min;
 using std::vector;
 
 namespace arrayfire {
 namespace oneapi {
 
+static ::oneapi::mkl::transpose toMKLTranspose(af_mat_prop opt) {
+    switch (opt) {
+        case AF_MAT_NONE: return ::oneapi::mkl::transpose::nontrans;
+        case AF_MAT_TRANS: return ::oneapi::mkl::transpose::trans;
+        case AF_MAT_CTRANS: return ::oneapi::mkl::transpose::conjtrans;
+        default: AF_ERROR("INVALID af_mat_prop", AF_ERR_ARG);
+    }
+}
+
 template<typename T>
 Array<T> solveLU(const Array<T> &A, const Array<int> &pivot, const Array<T> &b,
                  const af_mat_prop options) {
-    ONEAPI_NOT_SUPPORTED("solveLU Not supported");
+    const int64_t N    = A.dims()[0];
+    const int64_t NRHS = b.dims()[1];
+    const int64_t LDA  = A.strides()[1];
+    const int64_t LDB  = b.strides()[1];
+
+    ::oneapi::mkl::transpose opts = toMKLTranspose(options);
+    // see comments in core lapack functions about MKL scratch space
+    // avoiding memAlloc, since this may require a sub-buffer of a sub-buffer
+    // returned by memAlloc which is currently illegal in sycl
+    std::int64_t scratchpad_size =
+        ::oneapi::mkl::lapack::getrs_scratchpad_size<compute_t<T>>(
+            getQueue(), opts, N, NRHS, LDA, LDB);
+
+    // TODO: which one?
+    Array<intl> ipiv              = cast<intl, int>(pivot);
+    sycl::buffer<int64_t> ipivBuf = ipiv.get()->reinterpret<int64_t, 1>();
+    sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
+
+    /*
+    sycl::buffer<int64_t> ipivBuf(pivot.elements());
+    getQueue()
+        .submit([&](sycl::handler &h) {
+            auto ipivIn =
+                pivot.get()->template get_access<sycl::access_mode::read>(h);
+            auto ipivOut =
+                ipivBuf.get_access<sycl::access_mode::write>(h);
+            h.parallel_for(pivot.elements(),
+                    [=](sycl::id<1> i) {
+                        ipivOut[i] = static_cast<int64_t>(ipivIn[i]);
+                    });
+
+        });
+    */
+
+    Array<compute_t<T>> B = copyArray<compute_t<T>>(b);
+    sycl::buffer<compute_t<T>> aBuf =
+        A.template getBufferWithOffset<compute_t<T>>();
+    sycl::buffer<compute_t<T>> bBuf =
+        B.template getBufferWithOffset<compute_t<T>>();
+
+    ::oneapi::mkl::lapack::getrs(getQueue(), opts, N, NRHS, aBuf, LDA, ipivBuf,
+                                 bBuf, LDB, scratchpad, scratchpad_size);
+    return B;
+}
+
+template<typename T>
+Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
+    int batches = a.dims()[2] * a.dims()[3];
+
+    dim4 aDims = a.dims();
+    dim4 bDims = b.dims();
+    int M      = aDims[0];
+    int N      = aDims[1];
+    int K      = bDims[1];
+    int MN     = std::min(M, N);
+
+    int lda     = a.strides()[1];
+    int astride = a.strides()[2];
 
-    if (OpenCLCPUOffload()) { return cpu::solveLU(A, pivot, b, options); }
+    sycl::buffer<int64_t> ipiv(MN * batches);
+    int ipivstride = MN;
 
-    int N    = A.dims()[0];
-    int NRHS = b.dims()[1];
+    int ldb     = b.strides()[1];
+    int bstride = b.strides()[2];
 
-    vector<int> ipiv(N);
-    copyData(&ipiv[0], pivot);
+    vector<int> info(batches, 0);
 
+    Array<T> A = copyArray<T>(a);
     Array<T> B = copyArray<T>(b);
 
-    const Buffer *A_buf = A.get();
-    Buffer *B_buf       = B.get();
+    std::int64_t scratchpad_size =
+        ::oneapi::mkl::lapack::getrf_batch_scratchpad_size<compute_t<T>>(
+            getQueue(), M, N, lda, astride, ipivstride, batches);
 
-    int info = 0;
-    magma_getrs_gpu<T>(MagmaNoTrans, N, NRHS, (*A_buf)(), A.getOffset(),
-                       A.strides()[1], &ipiv[0], (*B_buf)(), B.getOffset(),
-                       B.strides()[1], getQueue()(), &info);
-    return B;
-}
+    sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
 
-template<typename T>
-Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
-    ONEAPI_NOT_SUPPORTED("generalSolve Not supported");
+    sycl::buffer<compute_t<T>> aBuf =
+        A.template getBufferWithOffset<compute_t<T>>();
+    sycl::buffer<compute_t<T>> bBuf =
+        B.template getBufferWithOffset<compute_t<T>>();
+    ::oneapi::mkl::lapack::getrf_batch(getQueue(), M, N, aBuf, lda, astride,
+                                       ipiv, ipivstride, batches, scratchpad,
+                                       scratchpad_size);
 
-    // dim4 aDims = a.dims();
-    // int batchz = aDims[2];
-    // int batchw = aDims[3];
+    scratchpad_size =
+        ::oneapi::mkl::lapack::getrs_batch_scratchpad_size<compute_t<T>>(
+            getQueue(), ::oneapi::mkl::transpose::nontrans, N, K, lda, astride,
+            ipivstride, ldb, bstride, batches);
 
-    // Array<T> A = copyArray<T>(a);
-    Array<T> B = copyArray<T>(b);
+    // TODO: reuse? or single large scratchpad?
+    sycl::buffer<compute_t<T>> scratchpad_rs(scratchpad_size);
+
+    ::oneapi::mkl::lapack::getrs_batch(
+        getQueue(), ::oneapi::mkl::transpose::nontrans, N, K, aBuf, lda,
+        astride, ipiv, ipivstride, bBuf, ldb, bstride, batches, scratchpad_rs,
+        scratchpad_size);
 
-    // for (int i = 0; i < batchw; i++) {
-    //     for (int j = 0; j < batchz; j++) {
-    //         int M  = aDims[0];
-    //         int N  = aDims[1];
-    //         int MN = min(M, N);
-    //         vector<int> ipiv(MN);
-
-    //         Buffer *A_buf      = A.get();
-    //         int info           = 0;
-    //         cl_command_queue q = getQueue()();
-    //         auto aoffset =
-    //             A.getOffset() + j * A.strides()[2] + i * A.strides()[3];
-    //         magma_getrf_gpu<T>(M, N, (*A_buf)(), aoffset, A.strides()[1],
-    //                            &ipiv[0], q, &info);
-
-    //         Buffer *B_buf = B.get();
-    //         int K         = B.dims()[1];
-
-    //         auto boffset =
-    //             B.getOffset() + j * B.strides()[2] + i * B.strides()[3];
-    //         magma_getrs_gpu<T>(MagmaNoTrans, M, K, (*A_buf)(), aoffset,
-    //                            A.strides()[1], &ipiv[0], (*B_buf)(), boffset,
-    //                            B.strides()[1], q, &info);
-    //     }
-    // }
     return B;
 }
 
 template<typename T>
 Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
-    ONEAPI_NOT_SUPPORTED("leastSquares Not supported");
-
-    int M  = a.dims()[0];
-    int N  = a.dims()[1];
-    int K  = b.dims()[1];
-    int MN = min(M, N);
+    int64_t M  = a.dims()[0];
+    int64_t N  = a.dims()[1];
+    int64_t K  = b.dims()[1];
+    int64_t MN = min(M, N);
 
     Array<T> B = createEmptyArray<T>(dim4());
-    gpu_blas_trsm_func<T> gpu_blas_trsm;
-
-    cl_event event;
-    cl_command_queue queue = getQueue()();
 
     if (M < N) {
-#define UNMQR 0  // FIXME: UNMQR == 1 should be faster but does not work
+        const dim4 NullShape(0, 0, 0, 0);
 
         // Least squres for this case is solved using the following
         // solve(A, B) == matmul(Q, Xpad);
@@ -127,71 +168,81 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
 
         // QR is performed on the transpose of A
         Array<T> A = transpose<T>(a, true);
-
-#if UNMQR
-        const dim4 NullShape(0, 0, 0, 0);
         dim4 endPadding(N - b.dims()[0], K - b.dims()[1], 0, 0);
         B = (endPadding == NullShape
                  ? copyArray(b)
                  : padArrayBorders(b, NullShape, endPadding, AF_PAD_ZERO));
-        B.resetDims(dim4(M, K));
-#else
-        B = copyArray<T>(b);
-#endif
-
-        int NB       = magma_get_geqrf_nb<T>(A.dims()[1]);
-        int NUM      = (2 * MN + ((M + 31) / 32) * 32) * NB;
-        Array<T> tmp = createEmptyArray<T>(dim4(NUM));
 
-        vector<T> h_tau(MN);
+        // Get workspace needed for QR
+        std::int64_t scratchpad_size =
+            ::oneapi::mkl::lapack::geqrf_scratchpad_size<compute_t<T>>(
+                getQueue(), A.dims()[0], A.dims()[1], A.strides()[1]);
 
-        int info   = 0;
-        Buffer *dA = A.get();
-        Buffer *dT = tmp.get();
-        Buffer *dB = B.get();
+        sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
+        Array<compute_t<T>> t = createEmptyArray<T>(af::dim4(MN, 1, 1, 1));
 
-        magma_geqrf3_gpu<T>(A.dims()[0], A.dims()[1], (*dA)(), A.getOffset(),
-                            A.strides()[1], &h_tau[0], (*dT)(), tmp.getOffset(),
-                            getQueue()(), &info);
+        sycl::buffer<compute_t<T>> aBuf =
+            A.template getBufferWithOffset<compute_t<T>>();
+        sycl::buffer<compute_t<T>> tBuf =
+            t.template getBufferWithOffset<compute_t<T>>();
+        // In place Perform in place QR
+        ::oneapi::mkl::lapack::geqrf(getQueue(), A.dims()[0], A.dims()[1], aBuf,
+                                     A.strides()[1], tBuf, scratchpad,
+                                     scratchpad_size);
 
+        // R1 = R(seq(M), seq(M));
         A.resetDims(dim4(M, M));
 
-        magmablas_swapdblk<T>(MN - 1, NB, (*dA)(), A.getOffset(),
-                              A.strides()[1], 1, (*dT)(),
-                              tmp.getOffset() + MN * NB, NB, 0, queue);
+        // Bt = tri_solve(R1, B);
+        B.resetDims(dim4(M, K));
 
-        OPENCL_BLAS_CHECK(
-            gpu_blas_trsm(OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_UPPER,
-                          OPENCL_BLAS_CONJ_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL,
-                          B.dims()[0], B.dims()[1], scalar<T>(1), (*dA)(),
-                          A.getOffset(), A.strides()[1], (*dB)(), B.getOffset(),
-                          B.strides()[1], 1, &queue, 0, nullptr, &event));
+        sycl::buffer<compute_t<T>> bBuf =
+            B.template getBufferWithOffset<compute_t<T>>();
+        // TODO: move to helper? trsm<T>(A, B, AF_MAT_CTRANS, true, true,
+        // false);
+        compute_t<T> alpha = scalar<compute_t<T>>(1);
+        ::oneapi::mkl::blas::trsm(
+            getQueue(), ::oneapi::mkl::side::left, ::oneapi::mkl::uplo::upper,
+            ::oneapi::mkl::transpose::conjtrans, ::oneapi::mkl::diag::nonunit,
+            B.dims()[0], B.dims()[1], alpha, aBuf, A.strides()[1], bBuf,
+            B.strides()[1]);
+
+        // Bpad = pad(Bt, ..)
+        B.resetDims(dim4(N, K));
 
-        magmablas_swapdblk<T>(MN - 1, NB, (*dT)(), tmp.getOffset() + MN * NB,
-                              NB, 0, (*dA)(), A.getOffset(), A.strides()[1], 1,
-                              queue);
+        // matmul(Q, Bpad)
+        if constexpr (std::is_same_v<compute_t<T>, float> ||
+                      std::is_same_v<compute_t<T>, double>) {
+            std::int64_t scratchpad_size =
+                ::oneapi::mkl::lapack::ormqr_scratchpad_size<compute_t<T>>(
+                    getQueue(), ::oneapi::mkl::side::left,
+                    ::oneapi::mkl::transpose::nontrans, B.dims()[0],
+                    B.dims()[1], A.dims()[0], A.strides()[1], B.strides()[1]);
+
+            sycl::buffer<compute_t<T>> scratchpad_ormqr(scratchpad_size);
+            ::oneapi::mkl::lapack::ormqr(
+                getQueue(), ::oneapi::mkl::side::left,
+                ::oneapi::mkl::transpose::nontrans, B.dims()[0], B.dims()[1],
+                A.dims()[0], aBuf, A.strides()[1], tBuf, bBuf, B.strides()[1],
+                scratchpad_ormqr, scratchpad_size);
+        } else if constexpr (std::is_same_v<compute_t<T>,
+                                            std::complex<float>> ||
+                             std::is_same_v<compute_t<T>,
+                                            std::complex<double>>) {
+            std::int64_t scratchpad_size =
+                ::oneapi::mkl::lapack::unmqr_scratchpad_size<compute_t<T>>(
+                    getQueue(), ::oneapi::mkl::side::left,
+                    ::oneapi::mkl::transpose::nontrans, B.dims()[0],
+                    B.dims()[1], A.dims()[0], A.strides()[1], B.strides()[1]);
+
+            sycl::buffer<compute_t<T>> scratchpad_unmqr(scratchpad_size);
+            ::oneapi::mkl::lapack::unmqr(
+                getQueue(), ::oneapi::mkl::side::left,
+                ::oneapi::mkl::transpose::nontrans, B.dims()[0], B.dims()[1],
+                A.dims()[0], aBuf, A.strides()[1], tBuf, bBuf, B.strides()[1],
+                scratchpad_unmqr, scratchpad_size);
+        }
 
-#if UNMQR
-        int lwork = (B.dims()[0] - A.dims()[0] + NB) * (B.dims()[1] + 2 * NB);
-        vector<T> h_work(lwork);
-        B.resetDims(dim4(N, K));
-        magma_unmqr_gpu<T>(MagmaLeft, MagmaNoTrans, B.dims()[0], B.dims()[1],
-                           A.dims()[0], (*dA)(), A.getOffset(), A.strides()[1],
-                           &h_tau[0], (*dB)(), B.getOffset(), B.strides()[1],
-                           &h_work[0], lwork, (*dT)(), tmp.getOffset(), NB,
-                           queue, &info);
-#else
-        A.resetDims(dim4(N, M));
-        magma_ungqr_gpu<T>(A.dims()[0], A.dims()[1], min(M, N), (*dA)(),
-                           A.getOffset(), A.strides()[1], &h_tau[0], (*dT)(),
-                           tmp.getOffset(), NB, queue, &info);
-
-        Array<T> B_new = createEmptyArray<T>(dim4(A.dims()[0], B.dims()[1]));
-        T alpha        = scalar<T>(1.0);
-        T beta         = scalar<T>(0.0);
-        gemm<T>(B_new, AF_MAT_NONE, AF_MAT_NONE, &alpha, A, B, &beta);
-        B = B_new;
-#endif
     } else if (M > N) {
         // Least squres for this case is solved using the following
         // solve(A, B) == tri_solve(R1, Bt);
@@ -204,56 +255,65 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
         Array<T> A = copyArray<T>(a);
         B          = copyArray(b);
 
-        int MN = min(M, N);
-        int NB = magma_get_geqrf_nb<T>(M);
-
-        int NUM      = (2 * MN + ((N + 31) / 32) * 32) * NB;
-        Array<T> tmp = createEmptyArray<T>(dim4(NUM));
-
-        vector<T> h_tau(NUM);
-
-        int info      = 0;
-        Buffer *A_buf = A.get();
-        Buffer *B_buf = B.get();
-        Buffer *dT    = tmp.get();
-
-        magma_geqrf3_gpu<T>(M, N, (*A_buf)(), A.getOffset(), A.strides()[1],
-                            &h_tau[0], (*dT)(), tmp.getOffset(), getQueue()(),
-                            &info);
-
-        int NRHS   = B.dims()[1];
-        int lhwork = (M - N + NB) * (NRHS + NB) + NRHS * NB;
-
-        vector<T> h_work(lhwork);
-        h_work[0] = scalar<T>(lhwork);
-
-        magma_unmqr_gpu<T>(MagmaLeft, MagmaConjTrans, M, NRHS, N, (*A_buf)(),
-                           A.getOffset(), A.strides()[1], &h_tau[0], (*B_buf)(),
-                           B.getOffset(), B.strides()[1], &h_work[0], lhwork,
-                           (*dT)(), tmp.getOffset(), NB, queue, &info);
-
-        magmablas_swapdblk<T>(MN - 1, NB, (*A_buf)(), A.getOffset(),
-                              A.strides()[1], 1, (*dT)(),
-                              tmp.getOffset() + NB * MN, NB, 0, queue);
-
-        if (getActivePlatform() == AFCL_PLATFORM_NVIDIA) {
-            Array<T> AT    = transpose<T>(A, true);
-            Buffer *AT_buf = AT.get();
-            OPENCL_BLAS_CHECK(gpu_blas_trsm(
-                OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_LOWER,
-                OPENCL_BLAS_CONJ_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL, N, NRHS,
-                scalar<T>(1), (*AT_buf)(), AT.getOffset(), AT.strides()[1],
-                (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0,
-                nullptr, &event));
-        } else {
-            OPENCL_BLAS_CHECK(gpu_blas_trsm(
-                OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_UPPER,
-                OPENCL_BLAS_NO_TRANS, OPENCL_BLAS_NON_UNIT_DIAGONAL, N, NRHS,
-                scalar<T>(1), (*A_buf)(), A.getOffset(), A.strides()[1],
-                (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0,
-                nullptr, &event));
+        // Get workspace needed for QR
+        std::int64_t scratchpad_size =
+            ::oneapi::mkl::lapack::geqrf_scratchpad_size<compute_t<T>>(
+                getQueue(), M, N, A.strides()[1]);
+
+        sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
+        Array<compute_t<T>> t = createEmptyArray<T>(af::dim4(MN, 1, 1, 1));
+
+        sycl::buffer<compute_t<T>> aBuf =
+            A.template getBufferWithOffset<compute_t<T>>();
+        sycl::buffer<compute_t<T>> tBuf =
+            t.template getBufferWithOffset<compute_t<T>>();
+        // In place Perform in place QR
+        ::oneapi::mkl::lapack::geqrf(getQueue(), M, N, aBuf, A.strides()[1],
+                                     tBuf, scratchpad, scratchpad_size);
+
+        // matmul(Q1, B)
+        sycl::buffer<compute_t<T>> bBuf =
+            B.template getBufferWithOffset<compute_t<T>>();
+        if constexpr (std::is_same_v<compute_t<T>, float> ||
+                      std::is_same_v<compute_t<T>, double>) {
+            std::int64_t scratchpad_size =
+                ::oneapi::mkl::lapack::ormqr_scratchpad_size<compute_t<T>>(
+                    getQueue(), ::oneapi::mkl::side::left,
+                    ::oneapi::mkl::transpose::trans, M, K, N, A.strides()[1],
+                    b.strides()[1]);
+
+            sycl::buffer<compute_t<T>> scratchpad_ormqr(scratchpad_size);
+            ::oneapi::mkl::lapack::ormqr(
+                getQueue(), ::oneapi::mkl::side::left,
+                ::oneapi::mkl::transpose::trans, M, K, N, aBuf, A.strides()[1],
+                tBuf, bBuf, b.strides()[1], scratchpad_ormqr, scratchpad_size);
+        } else if constexpr (std::is_same_v<compute_t<T>,
+                                            std::complex<float>> ||
+                             std::is_same_v<compute_t<T>,
+                                            std::complex<double>>) {
+            std::int64_t scratchpad_size =
+                ::oneapi::mkl::lapack::unmqr_scratchpad_size<compute_t<T>>(
+                    getQueue(), ::oneapi::mkl::side::left,
+                    ::oneapi::mkl::transpose::conjtrans, M, K, N,
+                    A.strides()[1], b.strides()[1]);
+
+            sycl::buffer<compute_t<T>> scratchpad_unmqr(scratchpad_size);
+            ::oneapi::mkl::lapack::unmqr(getQueue(), ::oneapi::mkl::side::left,
+                                         ::oneapi::mkl::transpose::conjtrans, M,
+                                         K, N, aBuf, A.strides()[1], tBuf, bBuf,
+                                         b.strides()[1], scratchpad_unmqr,
+                                         scratchpad_size);
         }
+
+        // tri_solve(R1, Bt)
+        A.resetDims(dim4(N, N));
         B.resetDims(dim4(N, K));
+
+        compute_t<T> alpha = scalar<compute_t<T>>(1);
+        ::oneapi::mkl::blas::trsm(
+            getQueue(), ::oneapi::mkl::side::left, ::oneapi::mkl::uplo::upper,
+            ::oneapi::mkl::transpose::nontrans, ::oneapi::mkl::diag::nonunit, N,
+            K, alpha, aBuf, A.strides()[1], bBuf, B.strides()[1]);
     }
 
     return B;
@@ -262,53 +322,32 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
 template<typename T>
 Array<T> triangleSolve(const Array<T> &A, const Array<T> &b,
                        const af_mat_prop options) {
-    gpu_blas_trsm_func<T> gpu_blas_trsm;
-
-    Array<T> B = copyArray<T>(b);
-
-    int N    = B.dims()[0];
-    int NRHS = B.dims()[1];
-
-    const Buffer *A_buf = A.get();
-    Buffer *B_buf       = B.get();
-
-    cl_event event         = 0;
-    cl_command_queue queue = getQueue()();
-
-    if (getActivePlatform() == AFCL_PLATFORM_NVIDIA &&
-        (options & AF_MAT_UPPER)) {
-        Array<T> AT = transpose<T>(A, true);
-
-        cl::Buffer *AT_buf = AT.get();
-        OPENCL_BLAS_CHECK(gpu_blas_trsm(
-            OPENCL_BLAS_SIDE_LEFT, OPENCL_BLAS_TRIANGLE_LOWER,
-            OPENCL_BLAS_CONJ_TRANS,
-            options & AF_MAT_DIAG_UNIT ? OPENCL_BLAS_UNIT_DIAGONAL
-                                       : OPENCL_BLAS_NON_UNIT_DIAGONAL,
-            N, NRHS, scalar<T>(1), (*AT_buf)(), AT.getOffset(), AT.strides()[1],
-            (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0, nullptr,
-            &event));
-    } else {
-        OPENCL_BLAS_CHECK(gpu_blas_trsm(
-            OPENCL_BLAS_SIDE_LEFT,
-            options & AF_MAT_LOWER ? OPENCL_BLAS_TRIANGLE_LOWER
-                                   : OPENCL_BLAS_TRIANGLE_UPPER,
-            OPENCL_BLAS_NO_TRANS,
-            options & AF_MAT_DIAG_UNIT ? OPENCL_BLAS_UNIT_DIAGONAL
-                                       : OPENCL_BLAS_NON_UNIT_DIAGONAL,
-            N, NRHS, scalar<T>(1), (*A_buf)(), A.getOffset(), A.strides()[1],
-            (*B_buf)(), B.getOffset(), B.strides()[1], 1, &queue, 0, nullptr,
-            &event));
-    }
-
+    Array<compute_t<T>> B = copyArray<T>(b);
+
+    compute_t<T> alpha       = scalar<compute_t<T>>(1);
+    ::oneapi::mkl::uplo uplo = (options & AF_MAT_UPPER)
+                                   ? ::oneapi::mkl::uplo::upper
+                                   : ::oneapi::mkl::uplo::lower;
+
+    ::oneapi::mkl::diag unitdiag = (options & AF_MAT_DIAG_UNIT)
+                                       ? ::oneapi::mkl::diag::unit
+                                       : ::oneapi::mkl::diag::nonunit;
+
+    sycl::buffer<compute_t<T>> aBuf =
+        A.template getBufferWithOffset<compute_t<T>>();
+    sycl::buffer<compute_t<T>> bBuf =
+        B.template getBufferWithOffset<compute_t<T>>();
+
+    ::oneapi::mkl::blas::trsm(getQueue(), ::oneapi::mkl::side::left, uplo,
+                              ::oneapi::mkl::transpose::nontrans, unitdiag,
+                              B.dims()[0], B.dims()[1], alpha, aBuf,
+                              A.strides()[1], bBuf, B.strides()[1]);
     return B;
 }
 
 template<typename T>
 Array<T> solve(const Array<T> &a, const Array<T> &b,
                const af_mat_prop options) {
-    if (OpenCLCPUOffload()) { return cpu::solve(a, b, options); }
-
     if (options & AF_MAT_UPPER || options & AF_MAT_LOWER) {
         return triangleSolve<T>(a, b, options);
     }

From 05303ad8b34593997a9ef681b96ecd9c7e0824f3 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 2 Jun 2023 17:00:48 -0400
Subject: [PATCH 319/473] minor cleanup, replaces repeated std::is_same_v with
 is_any_of

---
 src/backend/oneapi/solve.cpp | 105 +++++++++++++----------------------
 src/backend/oneapi/solve.hpp |  10 ++++
 2 files changed, 48 insertions(+), 67 deletions(-)

diff --git a/src/backend/oneapi/solve.cpp b/src/backend/oneapi/solve.cpp
index 07c4f3b171..d234b5920c 100644
--- a/src/backend/oneapi/solve.cpp
+++ b/src/backend/oneapi/solve.cpp
@@ -30,6 +30,7 @@
 using arrayfire::common::cast;
 using std::min;
 using std::vector;
+using sycl::buffer;
 
 namespace arrayfire {
 namespace oneapi {
@@ -59,32 +60,13 @@ Array<T> solveLU(const Array<T> &A, const Array<int> &pivot, const Array<T> &b,
         ::oneapi::mkl::lapack::getrs_scratchpad_size<compute_t<T>>(
             getQueue(), opts, N, NRHS, LDA, LDB);
 
-    // TODO: which one?
-    Array<intl> ipiv              = cast<intl, int>(pivot);
-    sycl::buffer<int64_t> ipivBuf = ipiv.get()->reinterpret<int64_t, 1>();
-    sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
-
-    /*
-    sycl::buffer<int64_t> ipivBuf(pivot.elements());
-    getQueue()
-        .submit([&](sycl::handler &h) {
-            auto ipivIn =
-                pivot.get()->template get_access<sycl::access_mode::read>(h);
-            auto ipivOut =
-                ipivBuf.get_access<sycl::access_mode::write>(h);
-            h.parallel_for(pivot.elements(),
-                    [=](sycl::id<1> i) {
-                        ipivOut[i] = static_cast<int64_t>(ipivIn[i]);
-                    });
-
-        });
-    */
-
-    Array<compute_t<T>> B = copyArray<compute_t<T>>(b);
-    sycl::buffer<compute_t<T>> aBuf =
-        A.template getBufferWithOffset<compute_t<T>>();
-    sycl::buffer<compute_t<T>> bBuf =
-        B.template getBufferWithOffset<compute_t<T>>();
+    Array<intl> ipiv        = cast<intl, int>(pivot);
+    buffer<int64_t> ipivBuf = ipiv.get()->reinterpret<int64_t, 1>();
+    buffer<compute_t<T>> scratchpad(scratchpad_size);
+
+    Array<compute_t<T>> B     = copyArray<compute_t<T>>(b);
+    buffer<compute_t<T>> aBuf = A.template getBufferWithOffset<compute_t<T>>();
+    buffer<compute_t<T>> bBuf = B.template getBufferWithOffset<compute_t<T>>();
 
     ::oneapi::mkl::lapack::getrs(getQueue(), opts, N, NRHS, aBuf, LDA, ipivBuf,
                                  bBuf, LDB, scratchpad, scratchpad_size);
@@ -102,10 +84,10 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
     int K      = bDims[1];
     int MN     = std::min(M, N);
 
-    int lda     = a.strides()[1];
-    int astride = a.strides()[2];
-
-    sycl::buffer<int64_t> ipiv(MN * batches);
+    int lda      = a.strides()[1];
+    int astride  = a.strides()[2];
+    auto ipivMem = memAlloc<int64_t>(MN * batches);
+    buffer<int64_t> ipiv(*ipivMem, 0, MN * batches);
     int ipivstride = MN;
 
     int ldb     = b.strides()[1];
@@ -113,19 +95,17 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
 
     vector<int> info(batches, 0);
 
-    Array<T> A = copyArray<T>(a);
-    Array<T> B = copyArray<T>(b);
+    Array<T> A = copyArray<T>(a);  // A will be overwritten by L,U
+    Array<T> B = copyArray<T>(b);  // will be overwritten with solution
 
     std::int64_t scratchpad_size =
         ::oneapi::mkl::lapack::getrf_batch_scratchpad_size<compute_t<T>>(
             getQueue(), M, N, lda, astride, ipivstride, batches);
 
-    sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
+    buffer<compute_t<T>> scratchpad(scratchpad_size);
 
-    sycl::buffer<compute_t<T>> aBuf =
-        A.template getBufferWithOffset<compute_t<T>>();
-    sycl::buffer<compute_t<T>> bBuf =
-        B.template getBufferWithOffset<compute_t<T>>();
+    buffer<compute_t<T>> aBuf = A.template getBufferWithOffset<compute_t<T>>();
+    buffer<compute_t<T>> bBuf = B.template getBufferWithOffset<compute_t<T>>();
     ::oneapi::mkl::lapack::getrf_batch(getQueue(), M, N, aBuf, lda, astride,
                                        ipiv, ipivstride, batches, scratchpad,
                                        scratchpad_size);
@@ -135,8 +115,7 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
             getQueue(), ::oneapi::mkl::transpose::nontrans, N, K, lda, astride,
             ipivstride, ldb, bstride, batches);
 
-    // TODO: reuse? or single large scratchpad?
-    sycl::buffer<compute_t<T>> scratchpad_rs(scratchpad_size);
+    buffer<compute_t<T>> scratchpad_rs(scratchpad_size);
 
     ::oneapi::mkl::lapack::getrs_batch(
         getQueue(), ::oneapi::mkl::transpose::nontrans, N, K, aBuf, lda,
@@ -178,12 +157,12 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
             ::oneapi::mkl::lapack::geqrf_scratchpad_size<compute_t<T>>(
                 getQueue(), A.dims()[0], A.dims()[1], A.strides()[1]);
 
-        sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
+        buffer<compute_t<T>> scratchpad(scratchpad_size);
         Array<compute_t<T>> t = createEmptyArray<T>(af::dim4(MN, 1, 1, 1));
 
-        sycl::buffer<compute_t<T>> aBuf =
+        buffer<compute_t<T>> aBuf =
             A.template getBufferWithOffset<compute_t<T>>();
-        sycl::buffer<compute_t<T>> tBuf =
+        buffer<compute_t<T>> tBuf =
             t.template getBufferWithOffset<compute_t<T>>();
         // In place Perform in place QR
         ::oneapi::mkl::lapack::geqrf(getQueue(), A.dims()[0], A.dims()[1], aBuf,
@@ -196,7 +175,7 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
         // Bt = tri_solve(R1, B);
         B.resetDims(dim4(M, K));
 
-        sycl::buffer<compute_t<T>> bBuf =
+        buffer<compute_t<T>> bBuf =
             B.template getBufferWithOffset<compute_t<T>>();
         // TODO: move to helper? trsm<T>(A, B, AF_MAT_CTRANS, true, true,
         // false);
@@ -211,31 +190,28 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
         B.resetDims(dim4(N, K));
 
         // matmul(Q, Bpad)
-        if constexpr (std::is_same_v<compute_t<T>, float> ||
-                      std::is_same_v<compute_t<T>, double>) {
+        if constexpr (is_any_of<compute_t<T>, float, double>()) {
             std::int64_t scratchpad_size =
                 ::oneapi::mkl::lapack::ormqr_scratchpad_size<compute_t<T>>(
                     getQueue(), ::oneapi::mkl::side::left,
                     ::oneapi::mkl::transpose::nontrans, B.dims()[0],
                     B.dims()[1], A.dims()[0], A.strides()[1], B.strides()[1]);
 
-            sycl::buffer<compute_t<T>> scratchpad_ormqr(scratchpad_size);
+            buffer<compute_t<T>> scratchpad_ormqr(scratchpad_size);
             ::oneapi::mkl::lapack::ormqr(
                 getQueue(), ::oneapi::mkl::side::left,
                 ::oneapi::mkl::transpose::nontrans, B.dims()[0], B.dims()[1],
                 A.dims()[0], aBuf, A.strides()[1], tBuf, bBuf, B.strides()[1],
                 scratchpad_ormqr, scratchpad_size);
-        } else if constexpr (std::is_same_v<compute_t<T>,
-                                            std::complex<float>> ||
-                             std::is_same_v<compute_t<T>,
-                                            std::complex<double>>) {
+        } else if constexpr (is_any_of<compute_t<T>, std::complex<float>,
+                                       std::complex<double>>()) {
             std::int64_t scratchpad_size =
                 ::oneapi::mkl::lapack::unmqr_scratchpad_size<compute_t<T>>(
                     getQueue(), ::oneapi::mkl::side::left,
                     ::oneapi::mkl::transpose::nontrans, B.dims()[0],
                     B.dims()[1], A.dims()[0], A.strides()[1], B.strides()[1]);
 
-            sycl::buffer<compute_t<T>> scratchpad_unmqr(scratchpad_size);
+            buffer<compute_t<T>> scratchpad_unmqr(scratchpad_size);
             ::oneapi::mkl::lapack::unmqr(
                 getQueue(), ::oneapi::mkl::side::left,
                 ::oneapi::mkl::transpose::nontrans, B.dims()[0], B.dims()[1],
@@ -260,44 +236,41 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
             ::oneapi::mkl::lapack::geqrf_scratchpad_size<compute_t<T>>(
                 getQueue(), M, N, A.strides()[1]);
 
-        sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
+        buffer<compute_t<T>> scratchpad(scratchpad_size);
         Array<compute_t<T>> t = createEmptyArray<T>(af::dim4(MN, 1, 1, 1));
 
-        sycl::buffer<compute_t<T>> aBuf =
+        buffer<compute_t<T>> aBuf =
             A.template getBufferWithOffset<compute_t<T>>();
-        sycl::buffer<compute_t<T>> tBuf =
+        buffer<compute_t<T>> tBuf =
             t.template getBufferWithOffset<compute_t<T>>();
         // In place Perform in place QR
         ::oneapi::mkl::lapack::geqrf(getQueue(), M, N, aBuf, A.strides()[1],
                                      tBuf, scratchpad, scratchpad_size);
 
         // matmul(Q1, B)
-        sycl::buffer<compute_t<T>> bBuf =
+        buffer<compute_t<T>> bBuf =
             B.template getBufferWithOffset<compute_t<T>>();
-        if constexpr (std::is_same_v<compute_t<T>, float> ||
-                      std::is_same_v<compute_t<T>, double>) {
+        if constexpr (is_any_of<compute_t<T>, float, double>()) {
             std::int64_t scratchpad_size =
                 ::oneapi::mkl::lapack::ormqr_scratchpad_size<compute_t<T>>(
                     getQueue(), ::oneapi::mkl::side::left,
                     ::oneapi::mkl::transpose::trans, M, K, N, A.strides()[1],
                     b.strides()[1]);
 
-            sycl::buffer<compute_t<T>> scratchpad_ormqr(scratchpad_size);
+            buffer<compute_t<T>> scratchpad_ormqr(scratchpad_size);
             ::oneapi::mkl::lapack::ormqr(
                 getQueue(), ::oneapi::mkl::side::left,
                 ::oneapi::mkl::transpose::trans, M, K, N, aBuf, A.strides()[1],
                 tBuf, bBuf, b.strides()[1], scratchpad_ormqr, scratchpad_size);
-        } else if constexpr (std::is_same_v<compute_t<T>,
-                                            std::complex<float>> ||
-                             std::is_same_v<compute_t<T>,
-                                            std::complex<double>>) {
+        } else if constexpr (is_any_of<compute_t<T>, std::complex<float>,
+                                       std::complex<double>>()) {
             std::int64_t scratchpad_size =
                 ::oneapi::mkl::lapack::unmqr_scratchpad_size<compute_t<T>>(
                     getQueue(), ::oneapi::mkl::side::left,
                     ::oneapi::mkl::transpose::conjtrans, M, K, N,
                     A.strides()[1], b.strides()[1]);
 
-            sycl::buffer<compute_t<T>> scratchpad_unmqr(scratchpad_size);
+            buffer<compute_t<T>> scratchpad_unmqr(scratchpad_size);
             ::oneapi::mkl::lapack::unmqr(getQueue(), ::oneapi::mkl::side::left,
                                          ::oneapi::mkl::transpose::conjtrans, M,
                                          K, N, aBuf, A.strides()[1], tBuf, bBuf,
@@ -333,10 +306,8 @@ Array<T> triangleSolve(const Array<T> &A, const Array<T> &b,
                                        ? ::oneapi::mkl::diag::unit
                                        : ::oneapi::mkl::diag::nonunit;
 
-    sycl::buffer<compute_t<T>> aBuf =
-        A.template getBufferWithOffset<compute_t<T>>();
-    sycl::buffer<compute_t<T>> bBuf =
-        B.template getBufferWithOffset<compute_t<T>>();
+    buffer<compute_t<T>> aBuf = A.template getBufferWithOffset<compute_t<T>>();
+    buffer<compute_t<T>> bBuf = B.template getBufferWithOffset<compute_t<T>>();
 
     ::oneapi::mkl::blas::trsm(getQueue(), ::oneapi::mkl::side::left, uplo,
                               ::oneapi::mkl::transpose::nontrans, unitdiag,
diff --git a/src/backend/oneapi/solve.hpp b/src/backend/oneapi/solve.hpp
index acea9327b4..819c0ced35 100644
--- a/src/backend/oneapi/solve.hpp
+++ b/src/backend/oneapi/solve.hpp
@@ -11,6 +11,16 @@
 
 namespace arrayfire {
 namespace oneapi {
+
+template<typename T, typename U, typename... Args>
+static inline constexpr bool is_any_of() {
+    if constexpr (!sizeof...(Args)) {
+        return std::is_same_v<T, U>;
+    } else {
+        return std::is_same_v<T, U> || is_any_of<T, Args...>();
+    }
+}
+
 template<typename T>
 Array<T> solve(const Array<T> &a, const Array<T> &b,
                const af_mat_prop options = AF_MAT_NONE);

From dbe4ee16594f862fcf30e0a7af1427b823b11ec2 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Tue, 6 Jun 2023 19:42:00 -0400
Subject: [PATCH 320/473] match workspace size to actual size of workspace
 buffer

---
 src/api/c/solve.cpp             |  8 ++-
 src/backend/common/traits.hpp   |  9 ++++
 src/backend/oneapi/cholesky.cpp | 13 ++---
 src/backend/oneapi/lu.cpp       | 26 ++++------
 src/backend/oneapi/solve.cpp    | 91 ++++++++++++++++-----------------
 src/backend/oneapi/solve.hpp    |  9 ----
 src/backend/oneapi/svd.cpp      |  8 +--
 7 files changed, 75 insertions(+), 89 deletions(-)

diff --git a/src/api/c/solve.cpp b/src/api/c/solve.cpp
index ec17aafaba..31c1489484 100644
--- a/src/api/c/solve.cpp
+++ b/src/api/c/solve.cpp
@@ -95,8 +95,9 @@ static inline af_array solve_lu(const af_array a, const af_array pivot,
 af_err af_solve_lu(af_array* out, const af_array a, const af_array piv,
                    const af_array b, const af_mat_prop options) {
     try {
-        const ArrayInfo& a_info = getInfo(a);
-        const ArrayInfo& b_info = getInfo(b);
+        const ArrayInfo& a_info   = getInfo(a);
+        const ArrayInfo& b_info   = getInfo(b);
+        const ArrayInfo& piv_info = getInfo(piv);
 
         if (a_info.ndims() > 2 || b_info.ndims() > 2) {
             AF_ERROR("solveLU can not be used in batch mode", AF_ERR_BATCH);
@@ -116,6 +117,9 @@ af_err af_solve_lu(af_array* out, const af_array a, const af_array piv,
 
         TYPE_ASSERT(a_type == b_type);
 
+        af_dtype piv_type = piv_info.getType();
+        TYPE_ASSERT(piv_type == s32);  // TODO: add support for 64 bit types
+
         DIM_ASSERT(1, adims[0] == adims[1]);
         DIM_ASSERT(1, bdims[0] == adims[0]);
         DIM_ASSERT(1, bdims[2] == adims[2]);
diff --git a/src/backend/common/traits.hpp b/src/backend/common/traits.hpp
index 2b9090727c..7798c070c2 100644
--- a/src/backend/common/traits.hpp
+++ b/src/backend/common/traits.hpp
@@ -68,6 +68,15 @@ constexpr bool isFloating(af::dtype type) {
     return (!isInteger(type) && !isBool(type));
 }
 
+template<typename T, typename U, typename... Args>
+constexpr bool is_any_of() {
+    if constexpr (!sizeof...(Args)) {
+        return std::is_same_v<T, U>;
+    } else {
+        return std::is_same_v<T, U> || is_any_of<T, Args...>();
+    }
+}
+
 }  // namespace
 }  // namespace common
 }  // namespace arrayfire
diff --git a/src/backend/oneapi/cholesky.cpp b/src/backend/oneapi/cholesky.cpp
index 882f915d15..d399034383 100644
--- a/src/backend/oneapi/cholesky.cpp
+++ b/src/backend/oneapi/cholesky.cpp
@@ -17,6 +17,7 @@
 #include <memory.hpp>
 #include <oneapi/mkl/lapack.hpp>
 #include <triangle.hpp>
+#include <algorithm>
 
 namespace arrayfire {
 namespace oneapi {
@@ -33,20 +34,16 @@ int cholesky_inplace(Array<T> &in, const bool is_upper) {
     ::oneapi::mkl::uplo uplo = ::oneapi::mkl::uplo::lower;
     if (is_upper) { uplo = ::oneapi::mkl::uplo::upper; }
 
-    lwork = ::oneapi::mkl::lapack::potrf_scratchpad_size<T>(getQueue(), uplo, N,
-                                                            LDA);
+    lwork = ::oneapi::mkl::lapack::potrf_scratchpad_size<compute_t<T>>(
+        getQueue(), uplo, N, LDA);
 
-    // MKL is finicky about exact scratch space size so we'll need to
-    // create sycl::buffer of exact size. if we use memAlloc, this might
-    // require a sub-buffer of a sub-buffer returned by memAlloc which is
-    // currently illegal in sycl
-    sycl::buffer<compute_t<T>> workspace(lwork);
+    auto workspace = memAlloc<compute_t<T>>(std::max<int64_t>(lwork, 1));
     sycl::buffer<compute_t<T>> in_buffer =
         in.template getBufferWithOffset<compute_t<T>>();
 
     try {
         ::oneapi::mkl::lapack::potrf(getQueue(), uplo, N, in_buffer, LDA,
-                                     workspace, lwork);
+                                     *workspace, workspace->size());
     } catch (::oneapi::mkl::lapack::exception const &e) {
         AF_ERROR(
             "Unexpected exception caught during synchronous\
diff --git a/src/backend/oneapi/lu.cpp b/src/backend/oneapi/lu.cpp
index 1ae473650b..27e6bd4bf3 100644
--- a/src/backend/oneapi/lu.cpp
+++ b/src/backend/oneapi/lu.cpp
@@ -21,17 +21,15 @@
 namespace arrayfire {
 namespace oneapi {
 
-Array<int> convertPivot(sycl::buffer<int64_t> &pivot, int out_sz,
+Array<int> convertPivot(sycl::buffer<int64_t> &pivot, int in_sz, int out_sz,
                         bool convert_pivot) {
-    dim_t d0 = pivot.get_range()[0];
-
     std::vector<int> d_po(out_sz);
     for (int i = 0; i < out_sz; i++) { d_po[i] = i; }
 
     auto d_pi = pivot.get_host_access();
 
     if (convert_pivot) {
-        for (int j = 0; j < d0; j++) {
+        for (int j = 0; j < in_sz; j++) {
             // 1 indexed in pivot
             std::swap(d_po[j], d_po[d_pi[j] - 1]);
         }
@@ -39,10 +37,10 @@ Array<int> convertPivot(sycl::buffer<int64_t> &pivot, int out_sz,
         Array<int> res = createHostDataArray(dim4(out_sz), &d_po[0]);
         return res;
     } else {
-        d_po.resize(d0);
-        for (int j = 0; j < d0; j++) { d_po[j] = static_cast<int>(d_pi[j]); }
+        d_po.resize(in_sz);
+        for (int j = 0; j < in_sz; j++) { d_po[j] = static_cast<int>(d_pi[j]); }
     }
-    Array<int> res = createHostDataArray(dim4(d0), &d_po[0]);
+    Array<int> res = createHostDataArray(dim4(in_sz), &d_po[0]);
     return res;
 }
 
@@ -77,19 +75,15 @@ Array<int> lu_inplace(Array<T> &in, const bool convert_pivot) {
     std::int64_t scratchpad_size =
         ::oneapi::mkl::lapack::getrf_scratchpad_size<T>(getQueue(), M, N, LDA);
 
-    // MKL is finicky about exact scratch space size so we'll need to
-    // create sycl::buffer of exact size. if we use memAlloc, this might
-    // require a sub-buffer of a sub-buffer returned by memAlloc which is
-    // currently illegal in sycl
-    sycl::buffer<int64_t> ipiv(MN);
-    sycl::buffer<compute_t<T>> scratchpad(scratchpad_size);
+    auto ipiv       = memAlloc<int64_t>(MN);
+    auto scratchpad = memAlloc<compute_t<T>>(scratchpad_size);
 
     sycl::buffer<compute_t<T>> in_buffer =
         in.template getBufferWithOffset<compute_t<T>>();
-    ::oneapi::mkl::lapack::getrf(getQueue(), M, N, in_buffer, LDA, ipiv,
-                                 scratchpad, scratchpad_size);
+    ::oneapi::mkl::lapack::getrf(getQueue(), M, N, in_buffer, LDA, *ipiv,
+                                 *scratchpad, scratchpad->size());
 
-    Array<int> pivot = convertPivot(ipiv, M, convert_pivot);
+    Array<int> pivot = convertPivot(*ipiv, MN, M, convert_pivot);
     return pivot;
 }
 
diff --git a/src/backend/oneapi/solve.cpp b/src/backend/oneapi/solve.cpp
index d234b5920c..4d213d25ae 100644
--- a/src/backend/oneapi/solve.cpp
+++ b/src/backend/oneapi/solve.cpp
@@ -24,7 +24,9 @@
 #include <platform.hpp>
 #include <transpose.hpp>
 
+#include <common/traits.hpp>
 #include <algorithm>
+#include <type_traits>
 #include <vector>
 
 using arrayfire::common::cast;
@@ -53,23 +55,20 @@ Array<T> solveLU(const Array<T> &A, const Array<int> &pivot, const Array<T> &b,
     const int64_t LDB  = b.strides()[1];
 
     ::oneapi::mkl::transpose opts = toMKLTranspose(options);
-    // see comments in core lapack functions about MKL scratch space
-    // avoiding memAlloc, since this may require a sub-buffer of a sub-buffer
-    // returned by memAlloc which is currently illegal in sycl
     std::int64_t scratchpad_size =
         ::oneapi::mkl::lapack::getrs_scratchpad_size<compute_t<T>>(
             getQueue(), opts, N, NRHS, LDA, LDB);
 
     Array<intl> ipiv        = cast<intl, int>(pivot);
     buffer<int64_t> ipivBuf = ipiv.get()->reinterpret<int64_t, 1>();
-    buffer<compute_t<T>> scratchpad(scratchpad_size);
+    auto scratchpad         = memAlloc<compute_t<T>>(scratchpad_size);
 
     Array<compute_t<T>> B     = copyArray<compute_t<T>>(b);
     buffer<compute_t<T>> aBuf = A.template getBufferWithOffset<compute_t<T>>();
     buffer<compute_t<T>> bBuf = B.template getBufferWithOffset<compute_t<T>>();
 
     ::oneapi::mkl::lapack::getrs(getQueue(), opts, N, NRHS, aBuf, LDA, ipivBuf,
-                                 bBuf, LDB, scratchpad, scratchpad_size);
+                                 bBuf, LDB, *scratchpad, scratchpad->size());
     return B;
 }
 
@@ -84,10 +83,9 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
     int K      = bDims[1];
     int MN     = std::min(M, N);
 
-    int lda      = a.strides()[1];
-    int astride  = a.strides()[2];
-    auto ipivMem = memAlloc<int64_t>(MN * batches);
-    buffer<int64_t> ipiv(*ipivMem, 0, MN * batches);
+    int lda        = a.strides()[1];
+    int astride    = a.strides()[2];
+    auto ipiv      = memAlloc<int64_t>(MN * batches);
     int ipivstride = MN;
 
     int ldb     = b.strides()[1];
@@ -102,25 +100,25 @@ Array<T> generalSolve(const Array<T> &a, const Array<T> &b) {
         ::oneapi::mkl::lapack::getrf_batch_scratchpad_size<compute_t<T>>(
             getQueue(), M, N, lda, astride, ipivstride, batches);
 
-    buffer<compute_t<T>> scratchpad(scratchpad_size);
+    auto scratchpad = memAlloc<compute_t<T>>(scratchpad_size);
 
     buffer<compute_t<T>> aBuf = A.template getBufferWithOffset<compute_t<T>>();
     buffer<compute_t<T>> bBuf = B.template getBufferWithOffset<compute_t<T>>();
     ::oneapi::mkl::lapack::getrf_batch(getQueue(), M, N, aBuf, lda, astride,
-                                       ipiv, ipivstride, batches, scratchpad,
-                                       scratchpad_size);
+                                       *ipiv, ipivstride, batches, *scratchpad,
+                                       scratchpad->size());
 
     scratchpad_size =
         ::oneapi::mkl::lapack::getrs_batch_scratchpad_size<compute_t<T>>(
             getQueue(), ::oneapi::mkl::transpose::nontrans, N, K, lda, astride,
             ipivstride, ldb, bstride, batches);
 
-    buffer<compute_t<T>> scratchpad_rs(scratchpad_size);
+    auto scratchpad_rs = memAlloc<compute_t<T>>(scratchpad_size);
 
     ::oneapi::mkl::lapack::getrs_batch(
         getQueue(), ::oneapi::mkl::transpose::nontrans, N, K, aBuf, lda,
-        astride, ipiv, ipivstride, bBuf, ldb, bstride, batches, scratchpad_rs,
-        scratchpad_size);
+        astride, *ipiv, ipivstride, bBuf, ldb, bstride, batches, *scratchpad_rs,
+        scratchpad_rs->size());
 
     return B;
 }
@@ -157,17 +155,15 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
             ::oneapi::mkl::lapack::geqrf_scratchpad_size<compute_t<T>>(
                 getQueue(), A.dims()[0], A.dims()[1], A.strides()[1]);
 
-        buffer<compute_t<T>> scratchpad(scratchpad_size);
-        Array<compute_t<T>> t = createEmptyArray<T>(af::dim4(MN, 1, 1, 1));
+        auto scratchpad = memAlloc<compute_t<T>>(scratchpad_size);
+        auto t          = memAlloc<compute_t<T>>(MN);
 
         buffer<compute_t<T>> aBuf =
             A.template getBufferWithOffset<compute_t<T>>();
-        buffer<compute_t<T>> tBuf =
-            t.template getBufferWithOffset<compute_t<T>>();
         // In place Perform in place QR
         ::oneapi::mkl::lapack::geqrf(getQueue(), A.dims()[0], A.dims()[1], aBuf,
-                                     A.strides()[1], tBuf, scratchpad,
-                                     scratchpad_size);
+                                     A.strides()[1], *t, *scratchpad,
+                                     scratchpad->size());
 
         // R1 = R(seq(M), seq(M));
         A.resetDims(dim4(M, M));
@@ -190,33 +186,33 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
         B.resetDims(dim4(N, K));
 
         // matmul(Q, Bpad)
-        if constexpr (is_any_of<compute_t<T>, float, double>()) {
+        if constexpr (std::is_floating_point<compute_t<T>>()) {
             std::int64_t scratchpad_size =
                 ::oneapi::mkl::lapack::ormqr_scratchpad_size<compute_t<T>>(
                     getQueue(), ::oneapi::mkl::side::left,
                     ::oneapi::mkl::transpose::nontrans, B.dims()[0],
                     B.dims()[1], A.dims()[0], A.strides()[1], B.strides()[1]);
 
-            buffer<compute_t<T>> scratchpad_ormqr(scratchpad_size);
+            auto scratchpad_ormqr = memAlloc<compute_t<T>>(scratchpad_size);
             ::oneapi::mkl::lapack::ormqr(
                 getQueue(), ::oneapi::mkl::side::left,
                 ::oneapi::mkl::transpose::nontrans, B.dims()[0], B.dims()[1],
-                A.dims()[0], aBuf, A.strides()[1], tBuf, bBuf, B.strides()[1],
-                scratchpad_ormqr, scratchpad_size);
-        } else if constexpr (is_any_of<compute_t<T>, std::complex<float>,
-                                       std::complex<double>>()) {
+                A.dims()[0], aBuf, A.strides()[1], *t, bBuf, B.strides()[1],
+                *scratchpad_ormqr, scratchpad_ormqr->size());
+        } else if constexpr (common::isComplex(static_cast<af::dtype>(
+                                 dtype_traits<compute_t<T>>::af_type))) {
             std::int64_t scratchpad_size =
                 ::oneapi::mkl::lapack::unmqr_scratchpad_size<compute_t<T>>(
                     getQueue(), ::oneapi::mkl::side::left,
                     ::oneapi::mkl::transpose::nontrans, B.dims()[0],
                     B.dims()[1], A.dims()[0], A.strides()[1], B.strides()[1]);
 
-            buffer<compute_t<T>> scratchpad_unmqr(scratchpad_size);
+            auto scratchpad_unmqr = memAlloc<compute_t<T>>(scratchpad_size);
             ::oneapi::mkl::lapack::unmqr(
                 getQueue(), ::oneapi::mkl::side::left,
                 ::oneapi::mkl::transpose::nontrans, B.dims()[0], B.dims()[1],
-                A.dims()[0], aBuf, A.strides()[1], tBuf, bBuf, B.strides()[1],
-                scratchpad_unmqr, scratchpad_size);
+                A.dims()[0], aBuf, A.strides()[1], *t, bBuf, B.strides()[1],
+                *scratchpad_unmqr, scratchpad_unmqr->size());
         }
 
     } else if (M > N) {
@@ -236,46 +232,45 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
             ::oneapi::mkl::lapack::geqrf_scratchpad_size<compute_t<T>>(
                 getQueue(), M, N, A.strides()[1]);
 
-        buffer<compute_t<T>> scratchpad(scratchpad_size);
-        Array<compute_t<T>> t = createEmptyArray<T>(af::dim4(MN, 1, 1, 1));
+        auto scratchpad = memAlloc<compute_t<T>>(scratchpad_size);
+        auto t          = memAlloc<compute_t<T>>(MN);
 
         buffer<compute_t<T>> aBuf =
             A.template getBufferWithOffset<compute_t<T>>();
-        buffer<compute_t<T>> tBuf =
-            t.template getBufferWithOffset<compute_t<T>>();
         // In place Perform in place QR
-        ::oneapi::mkl::lapack::geqrf(getQueue(), M, N, aBuf, A.strides()[1],
-                                     tBuf, scratchpad, scratchpad_size);
+        ::oneapi::mkl::lapack::geqrf(getQueue(), M, N, aBuf, A.strides()[1], *t,
+                                     *scratchpad, scratchpad->size());
 
         // matmul(Q1, B)
         buffer<compute_t<T>> bBuf =
             B.template getBufferWithOffset<compute_t<T>>();
-        if constexpr (is_any_of<compute_t<T>, float, double>()) {
+        if constexpr (std::is_floating_point<compute_t<T>>()) {
             std::int64_t scratchpad_size =
                 ::oneapi::mkl::lapack::ormqr_scratchpad_size<compute_t<T>>(
                     getQueue(), ::oneapi::mkl::side::left,
                     ::oneapi::mkl::transpose::trans, M, K, N, A.strides()[1],
                     b.strides()[1]);
 
-            buffer<compute_t<T>> scratchpad_ormqr(scratchpad_size);
-            ::oneapi::mkl::lapack::ormqr(
-                getQueue(), ::oneapi::mkl::side::left,
-                ::oneapi::mkl::transpose::trans, M, K, N, aBuf, A.strides()[1],
-                tBuf, bBuf, b.strides()[1], scratchpad_ormqr, scratchpad_size);
-        } else if constexpr (is_any_of<compute_t<T>, std::complex<float>,
-                                       std::complex<double>>()) {
+            auto scratchpad_ormqr = memAlloc<compute_t<T>>(scratchpad_size);
+            ::oneapi::mkl::lapack::ormqr(getQueue(), ::oneapi::mkl::side::left,
+                                         ::oneapi::mkl::transpose::trans, M, K,
+                                         N, aBuf, A.strides()[1], *t, bBuf,
+                                         b.strides()[1], *scratchpad_ormqr,
+                                         scratchpad_ormqr->size());
+        } else if constexpr (common::isComplex(static_cast<af::dtype>(
+                                 dtype_traits<compute_t<T>>::af_type))) {
             std::int64_t scratchpad_size =
                 ::oneapi::mkl::lapack::unmqr_scratchpad_size<compute_t<T>>(
                     getQueue(), ::oneapi::mkl::side::left,
                     ::oneapi::mkl::transpose::conjtrans, M, K, N,
                     A.strides()[1], b.strides()[1]);
 
-            buffer<compute_t<T>> scratchpad_unmqr(scratchpad_size);
+            auto scratchpad_unmqr = memAlloc<compute_t<T>>(scratchpad_size);
             ::oneapi::mkl::lapack::unmqr(getQueue(), ::oneapi::mkl::side::left,
                                          ::oneapi::mkl::transpose::conjtrans, M,
-                                         K, N, aBuf, A.strides()[1], tBuf, bBuf,
-                                         b.strides()[1], scratchpad_unmqr,
-                                         scratchpad_size);
+                                         K, N, aBuf, A.strides()[1], *t, bBuf,
+                                         b.strides()[1], *scratchpad_unmqr,
+                                         scratchpad_unmqr->size());
         }
 
         // tri_solve(R1, Bt)
diff --git a/src/backend/oneapi/solve.hpp b/src/backend/oneapi/solve.hpp
index 819c0ced35..a0c8924fa9 100644
--- a/src/backend/oneapi/solve.hpp
+++ b/src/backend/oneapi/solve.hpp
@@ -12,15 +12,6 @@
 namespace arrayfire {
 namespace oneapi {
 
-template<typename T, typename U, typename... Args>
-static inline constexpr bool is_any_of() {
-    if constexpr (!sizeof...(Args)) {
-        return std::is_same_v<T, U>;
-    } else {
-        return std::is_same_v<T, U> || is_any_of<T, Args...>();
-    }
-}
-
 template<typename T>
 Array<T> solve(const Array<T> &a, const Array<T> &b,
                const af_mat_prop options = AF_MAT_NONE);
diff --git a/src/backend/oneapi/svd.cpp b/src/backend/oneapi/svd.cpp
index 97b5f3a468..7255226e1b 100644
--- a/src/backend/oneapi/svd.cpp
+++ b/src/backend/oneapi/svd.cpp
@@ -38,16 +38,12 @@ void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in) {
     int64_t LDU   = uStrides[1];
     int64_t LDVt  = vStrides[1];
 
-    // MKL is finicky about exact scratch space size so we'll need to
-    // create sycl::buffer of exact size. if we use memAlloc, this might
-    // require a sub-buffer of a sub-buffer returned by memAlloc which is
-    // currently illegal in sycl
     int64_t scratch_size =
         ::oneapi::mkl::lapack::gesvd_scratchpad_size<compute_t<T>>(
             getQueue(), ::oneapi::mkl::jobsvd::vectors,
             ::oneapi::mkl::jobsvd::vectors, M, N, LDA, LDU, LDVt);
 
-    sycl::buffer<compute_t<T>> scratchpad(scratch_size);
+    auto scratchpad = memAlloc<compute_t<T>>(scratch_size);
 
     sycl::buffer<compute_t<T>> in_buffer =
         in.template getBufferWithOffset<compute_t<T>>();
@@ -62,7 +58,7 @@ void svdInPlace(Array<Tr> &s, Array<T> &u, Array<T> &vt, Array<T> &in) {
     ::oneapi::mkl::lapack::gesvd(getQueue(), ::oneapi::mkl::jobsvd::vectors,
                                  ::oneapi::mkl::jobsvd::vectors, M, N,
                                  in_buffer, LDA, sBuf, uBuf, LDU, vtBuf, LDVt,
-                                 scratchpad, scratch_size);
+                                 *scratchpad, scratchpad->size());
 }
 
 template<typename T, typename Tr>

From a644b1048ba04cbd5e6e56ca6b6891d847418024 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 15 Jun 2023 16:28:52 -0400
Subject: [PATCH 321/473] Hash using kernel name and device name for oneAPI JIT
 (#3444)

Hash using kernel name and device name for oneAPI JIT
---
 src/backend/oneapi/jit.cpp | 42 +++++++++++++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 5 deletions(-)

diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 794bb7796f..546ca233b6 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -34,6 +34,7 @@
 #include <array>
 #include <cstdio>
 #include <functional>
+#include <mutex>
 #include <sstream>
 #include <stdexcept>
 #include <string>
@@ -286,6 +287,11 @@ __kernel void )JIT";
 //     return common::getKernel("", "", true).get();
 // }
 
+static unordered_map<cl_device_id, std::string> device_name_map;
+static std::mutex device_name_map_mutex;
+static unordered_map<std::string, cl_kernel> kernel_map;
+static std::mutex kernel_map_mutex;
+
 template<typename T>
 cl_kernel getKernel(
     std::string funcName, cl_context ctx, cl_device_id dev, cl_command_queue q,
@@ -293,10 +299,36 @@ cl_kernel getKernel(
     nonstd::span<Node_ids const> full_ids, nonstd::span<int const> output_ids,
     nonstd::span<oneapi::AParam<T, sycl::access_mode::write> const> ap,
     bool is_linear) {
-    static unordered_map<std::string, cl_kernel> kernel_map;
+    std::string devName;
+    {
+        std::lock_guard<std::mutex> lock(device_name_map_mutex);
+
+        auto devNameIt = device_name_map.find(dev);
+        if (devNameIt == device_name_map.end()) {
+            size_t devNameSz;
+            CL_CHECK(
+                clGetDeviceInfo(dev, CL_DEVICE_NAME, 0, nullptr, &devNameSz));
+            string newDevName(devNameSz, '\0');
+            CL_CHECK(clGetDeviceInfo(dev, CL_DEVICE_NAME, devNameSz,
+                                     newDevName.data(), nullptr));
+            device_name_map[dev] = newDevName;
+            devName              = newDevName;
+        } else {
+            devName = devNameIt->second;
+        }
+    }
 
     vector<cl_kernel> kernels(10);
-    if (kernel_map.find(funcName) == end(kernel_map)) {
+    bool kernel_found;
+    string kernelHash = funcName + devName;
+    {
+        std::lock_guard<std::mutex> lock(kernel_map_mutex);
+        kernel_found = !(kernel_map.find(kernelHash) == end(kernel_map));
+    }
+    if (kernel_found) {
+        std::lock_guard<std::mutex> lock(kernel_map_mutex);
+        kernels[0] = kernel_map[kernelHash];
+    } else {
         string jitstr = arrayfire::opencl::getKernelString(
             funcName, full_nodes, full_ids, output_ids, is_linear, false, false,
             ap[0].dims[2] > 1);
@@ -320,10 +352,10 @@ cl_kernel getKernel(
         cl_uint ret_kernels = 0;
         CL_CHECK(
             clCreateKernelsInProgram(prog, 1, kernels.data(), &ret_kernels));
-        kernel_map[funcName] = kernels[0];
+
+        std::lock_guard<std::mutex> lock(kernel_map_mutex);
+        kernel_map[kernelHash] = kernels[0];
         CL_CHECK(clReleaseProgram(prog));
-    } else {
-        kernels[0] = kernel_map[funcName];
     }
     return kernels[0];
 }

From a8a0f83f0213a25e6df99e69d5b829ea6e5bf6f2 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 14 Jun 2023 21:53:14 -0400
Subject: [PATCH 322/473] fixes out of bounds iterators for sorts

---
 src/backend/oneapi/kernel/sort.hpp            |  4 +-
 .../oneapi/kernel/sort_by_key_impl.hpp        | 13 +++---
 src/backend/oneapi/sort.cpp                   | 44 +++++++++----------
 3 files changed, 29 insertions(+), 32 deletions(-)

diff --git a/src/backend/oneapi/kernel/sort.hpp b/src/backend/oneapi/kernel/sort.hpp
index 1789887b82..71bedd1f50 100644
--- a/src/backend/oneapi/kernel/sort.hpp
+++ b/src/backend/oneapi/kernel/sort.hpp
@@ -80,9 +80,9 @@ void sortBatched(Param<T> pVal, int dim, bool isAscending) {
     auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
 
     auto key_begin    = ::oneapi::dpl::begin(*pKey.get());
-    auto key_end      = ::oneapi::dpl::end(*pKey.get());
+    auto key_end      = key_begin + pKey.dims()[0];
     auto val_begin    = ::oneapi::dpl::begin(*pVal.data);
-    auto val_end      = ::oneapi::dpl::end(*pVal.data);
+    auto val_end      = val_begin + pVal.info.dims[0];
     auto zipped_begin = dpl::make_zip_iterator(key_begin, val_begin);
     auto zipped_end   = dpl::make_zip_iterator(key_end, val_end);
 
diff --git a/src/backend/oneapi/kernel/sort_by_key_impl.hpp b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
index ad3b3c8a80..9a6348a3ad 100644
--- a/src/backend/oneapi/kernel/sort_by_key_impl.hpp
+++ b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
@@ -98,13 +98,13 @@ void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
 
     // set up iterators for seq, key, val, and new cKey
     auto seq_begin = ::oneapi::dpl::begin(*Seq.get());
-    auto seq_end   = ::oneapi::dpl::end(*Seq.get());
+    auto seq_end   = seq_begin + elements;
     auto key_begin =
         ::oneapi::dpl::begin(pKey.data->template reinterpret<compute_t<Tk>>());
-    auto key_end =
-        ::oneapi::dpl::end(pKey.data->template reinterpret<compute_t<Tk>>());
+    auto key_end = key_begin + elements;
+
     auto val_begin = ::oneapi::dpl::begin(*pVal.data);
-    auto val_end   = ::oneapi::dpl::end(*pVal.data);
+    auto val_end   = val_begin + elements;
 
     auto cKey = memAlloc<Tk>(elements);
     getQueue().submit([&](sycl::handler &h) {
@@ -115,8 +115,7 @@ void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
     });
     auto ckey_begin =
         ::oneapi::dpl::begin(cKey.get()->template reinterpret<compute_t<Tk>>());
-    auto ckey_end =
-        ::oneapi::dpl::end(cKey.get()->template reinterpret<compute_t<Tk>>());
+    auto ckey_end = ckey_begin + elements;
 
     {
         auto zipped_begin_KV  = dpl::make_zip_iterator(key_begin, val_begin);
@@ -150,7 +149,7 @@ void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
                cSeq.get()->get_access(h, elements));
     });
     auto cseq_begin = ::oneapi::dpl::begin(*cSeq.get());
-    auto cseq_end   = ::oneapi::dpl::end(*cSeq.get());
+    auto cseq_end   = cseq_begin + elements;
 
     {
         auto zipped_begin_SV  = dpl::make_zip_iterator(seq_begin, val_begin);
diff --git a/src/backend/oneapi/sort.cpp b/src/backend/oneapi/sort.cpp
index 002385a320..a16ccadc55 100644
--- a/src/backend/oneapi/sort.cpp
+++ b/src/backend/oneapi/sort.cpp
@@ -22,31 +22,29 @@ namespace oneapi {
 
 template<typename T>
 Array<T> sort(const Array<T> &in, const unsigned dim, bool isAscending) {
-    try {
-        Array<T> out = copyArray<T>(in);
-        switch (dim) {
-            case 0: kernel::sort0<T>(out, isAscending); break;
-            case 1: kernel::sortBatched<T>(out, 1, isAscending); break;
-            case 2: kernel::sortBatched<T>(out, 2, isAscending); break;
-            case 3: kernel::sortBatched<T>(out, 3, isAscending); break;
-            default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
-        }
-
-        if (dim != 0) {
-            af::dim4 preorderDims = out.dims();
-            af::dim4 reorderDims(0, 1, 2, 3);
-            reorderDims[dim] = 0;
-            preorderDims[0]  = out.dims()[dim];
-            for (int i = 1; i <= static_cast<int>(dim); i++) {
-                reorderDims[i - 1] = i;
-                preorderDims[i]    = out.dims()[i - 1];
-            }
+    Array<T> out = copyArray<T>(in);
+    switch (dim) {
+        case 0: kernel::sort0<T>(out, isAscending); break;
+        case 1: kernel::sortBatched<T>(out, 1, isAscending); break;
+        case 2: kernel::sortBatched<T>(out, 2, isAscending); break;
+        case 3: kernel::sortBatched<T>(out, 3, isAscending); break;
+        default: AF_ERROR("Not Supported", AF_ERR_NOT_SUPPORTED);
+    }
 
-            out.setDataDims(preorderDims);
-            out = reorder<T>(out, reorderDims);
+    if (dim != 0) {
+        af::dim4 preorderDims = out.dims();
+        af::dim4 reorderDims(0, 1, 2, 3);
+        reorderDims[dim] = 0;
+        preorderDims[0]  = out.dims()[dim];
+        for (int i = 1; i <= static_cast<int>(dim); i++) {
+            reorderDims[i - 1] = i;
+            preorderDims[i]    = out.dims()[i - 1];
         }
-        return out;
-    } catch (std::exception &ex) { AF_ERROR(ex.what(), AF_ERR_INTERNAL); }
+
+        out.setDataDims(preorderDims);
+        out = reorder<T>(out, reorderDims);
+    }
+    return out;
 }
 
 #define INSTANTIATE(T)                                                \

From f103b8abff4158b048291c8a69d5b5462ef333b0 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 16 Jun 2023 03:54:57 -0400
Subject: [PATCH 323/473] adds reduce by key to oneapi backend

---
 src/backend/oneapi/CMakeLists.txt           |  11 +-
 src/backend/oneapi/kernel/reduce_by_key.hpp | 704 ++++++++++++++++++++
 src/backend/oneapi/reduce_impl.hpp          | 559 +++++++++++++++-
 3 files changed, 1267 insertions(+), 7 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/reduce_by_key.hpp

diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index b13de94f95..1c8f789806 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -1,9 +1,9 @@
-# Copyright (c) 2022, ArrayFire
-# All rights reserved.
+#Copyright(c) 2022, ArrayFire
+#All rights reserved.
 #
-# This file is distributed under 3-clause BSD license.
-# The complete license agreement can be obtained at:
-# http://arrayfire.com/licenses/BSD-3-Clause
+#This file is distributed under 3 - clause BSD license.
+#The complete license agreement can be obtained at:
+#http:  // arrayfire.com/licenses/BSD-3-Clause
 
 include(InternalUtils)
 include(build_cl2hpp)
@@ -241,6 +241,7 @@ target_sources(afoneapi
     kernel/range.hpp
     kernel/reduce.hpp
     kernel/reduce_all.hpp
+    kernel/reduce_by_key.hpp
     kernel/reduce_first.hpp
     kernel/reduce_dim.hpp
     kernel/reorder.hpp
diff --git a/src/backend/oneapi/kernel/reduce_by_key.hpp b/src/backend/oneapi/kernel/reduce_by_key.hpp
new file mode 100644
index 0000000000..1da17ca5cc
--- /dev/null
+++ b/src/backend/oneapi/kernel/reduce_by_key.hpp
@@ -0,0 +1,704 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <Param.hpp>
+#include <backend.hpp>
+#include <common/Binary.hpp>
+#include <common/Transform.hpp>
+#include <common/dispatch.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
+#include <kernel/reduce_config.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+#include <type_traits>
+
+using std::unique_ptr;
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+// Reduces keys across block boundaries
+template<typename Tk, typename To, af_op_t op>
+class finalBoundaryReduceKernel {
+   public:
+    finalBoundaryReduceKernel(write_accessor<int> reduced_block_sizes,
+                              read_accessor<Tk> iKeys, KParam iKInfo,
+                              sycl::accessor<To> oVals, KParam oVInfo,
+                              const int n)
+        : reduced_block_sizes_(reduced_block_sizes)
+        , iKeys_(iKeys)
+        , iKInfo_(iKInfo)
+        , oVals_(oVals)
+        , oVInfo_(oVInfo)
+        , n_(n) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g  = it.get_group();
+        const uint lid = it.get_local_id(0);
+        const uint gid = it.get_global_id(0);
+        const uint bid = g.get_group_id(0);
+
+        common::Binary<compute_t<To>, op> binOp;
+        if (gid == ((bid + 1) * it.get_local_range(0)) - 1 &&
+            bid < g.get_group_range(0) - 1) {
+            Tk k0 = iKeys_[gid];
+            Tk k1 = iKeys_[gid + 1];
+
+            if (k0 == k1) {
+                compute_t<To> v0          = compute_t<To>(oVals_[gid]);
+                compute_t<To> v1          = compute_t<To>(oVals_[gid + 1]);
+                oVals_[gid + 1]           = binOp(v0, v1);
+                reduced_block_sizes_[bid] = it.get_local_range(0) - 1;
+            } else {
+                reduced_block_sizes_[bid] = it.get_local_range(0);
+            }
+        }
+
+        // if last block, set block size to difference between n and block
+        // boundary
+        if (lid == 0 && bid == g.get_group_range(0) - 1) {
+            reduced_block_sizes_[bid] = n_ - (bid * it.get_local_range(0));
+        }
+    }
+
+   protected:
+    write_accessor<int> reduced_block_sizes_;
+    read_accessor<Tk> iKeys_;
+    KParam iKInfo_;
+    sycl::accessor<To> oVals_;
+    KParam oVInfo_;
+    int n_;
+};
+
+template<typename Tk, typename To, af_op_t op>
+class finalBoundaryReduceDimKernel {
+   public:
+    finalBoundaryReduceDimKernel(write_accessor<int> reduced_block_sizes,
+                                 read_accessor<Tk> iKeys, KParam iKInfo,
+                                 sycl::accessor<To> oVals, KParam oVInfo,
+                                 const int n, const int nGroupsZ)
+        : reduced_block_sizes_(reduced_block_sizes)
+        , iKeys_(iKeys)
+        , iKInfo_(iKInfo)
+        , oVals_(oVals)
+        , oVInfo_(oVInfo)
+        , n_(n)
+        , nGroupsZ_(nGroupsZ) {}
+
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g  = it.get_group();
+        const uint lid = it.get_local_id(0);
+        const uint gid = it.get_global_id(0);
+        const uint bid = g.get_group_id(0);
+
+        const int bidy = g.get_group_id(1);
+        const int bidz = g.get_group_id(2) % nGroupsZ_;
+        const int bidw = g.get_group_id(2) / nGroupsZ_;
+
+        common::Binary<compute_t<To>, op> binOp;
+        if (gid == ((bid + 1) * it.get_local_range(0)) - 1 &&
+            bid < g.get_group_range(0) - 1) {
+            Tk k0 = iKeys_[gid];
+            Tk k1 = iKeys_[gid + 1];
+
+            if (k0 == k1) {
+                compute_t<To> v0          = compute_t<To>(oVals_[gid]);
+                compute_t<To> v1          = compute_t<To>(oVals_[gid + 1]);
+                oVals_[gid + 1]           = binOp(v0, v1);
+                reduced_block_sizes_[bid] = it.get_local_range(0) - 1;
+            } else {
+                reduced_block_sizes_[bid] = it.get_local_range(0);
+            }
+        }
+
+        // if last block, set block size to difference between n and block
+        // boundary
+        if (lid == 0 && bid == g.get_group_range(0) - 1) {
+            reduced_block_sizes_[bid] = n_ - (bid * it.get_local_range(0));
+        }
+    }
+
+   protected:
+    write_accessor<int> reduced_block_sizes_;
+    read_accessor<Tk> iKeys_;
+    KParam iKInfo_;
+    sycl::accessor<To> oVals_;
+    KParam oVInfo_;
+    int n_;
+    int nGroupsZ_;
+};
+
+template<typename T>
+using global_atomic_ref =
+    sycl::atomic_ref<T, sycl::memory_order::relaxed, sycl::memory_scope::system,
+                     sycl::access::address_space::global_space>;
+
+// Tests if data needs further reduction, including across block boundaries
+template<typename Tk>
+class testNeedsReductionKernel {
+   public:
+    testNeedsReductionKernel(sycl::accessor<int> needs_another_reduction,
+                             sycl::accessor<int> needs_block_boundary_reduced,
+                             read_accessor<Tk> iKeys, KParam iKInfo,
+                             const int n, const int DIMX,
+                             sycl::local_accessor<Tk> l_keys)
+        : needs_another_reduction_(needs_another_reduction)
+        , needs_block_boundary_reduced_(needs_block_boundary_reduced)
+        , iKeys_(iKeys)
+        , iKInfo_(iKInfo)
+        , n_(n)
+        , DIMX_(DIMX)
+        , l_keys_(l_keys) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g  = it.get_group();
+        const uint lid = it.get_local_id(0);
+        const uint gid = it.get_global_id(0);
+        const uint bid = g.get_group_id(0);
+
+        Tk k;
+        if (gid < n_) { k = iKeys_[gid]; }
+
+        l_keys_[lid] = k;
+        it.barrier();
+
+        int update_key =
+            (lid < DIMX_ - 2) && (k == l_keys_[lid + 1]) && (gid < (n_ - 1));
+
+        if (update_key) {
+            global_atomic_ref<int>(needs_another_reduction_[0]) |= update_key;
+        }
+
+        it.barrier();
+
+        // last thread in each block checks if any inter-block keys need further
+        // reduction
+        if (gid == ((bid + 1) * DIMX_) - 1 &&
+            bid < (g.get_group_range(0) - 1)) {
+            int k0 = iKeys_[gid];
+            int k1 = iKeys_[gid + 1];
+            if (k0 == k1) {
+                global_atomic_ref<int>(needs_block_boundary_reduced_[0]) |= 1;
+            }
+        }
+    }
+
+   protected:
+    sycl::accessor<int> needs_another_reduction_;
+    sycl::accessor<int> needs_block_boundary_reduced_;
+    read_accessor<Tk> iKeys_;
+    KParam iKInfo_;
+    int n_;
+    int DIMX_;
+    sycl::local_accessor<Tk> l_keys_;
+};
+
+// Compacts "incomplete" block-sized chunks of data in global memory
+template<typename Tk, typename To>
+class compactKernel {
+   public:
+    compactKernel(read_accessor<int> reduced_block_sizes,
+                  write_accessor<Tk> oKeys, KParam oKInfo,
+                  write_accessor<To> oVals, KParam oVInfo,
+                  read_accessor<Tk> iKeys, KParam iKInfo,
+                  read_accessor<To> iVals, KParam iVInfo, int nGroupsZ)
+        : reduced_block_sizes_(reduced_block_sizes)
+        , oKeys_(oKeys)
+        , oKInfo_(oKInfo)
+        , oVals_(oVals)
+        , oVInfo_(oVInfo)
+        , iKeys_(iKeys)
+        , iKInfo_(iKInfo)
+        , iVals_(iVals)
+        , iVInfo_(iVInfo)
+        , nGroupsZ_(nGroupsZ) {}
+
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g  = it.get_group();
+        const uint lid = it.get_local_id(0);
+        const uint bid = g.get_group_id(0);
+        const uint gid = it.get_global_id(0);
+
+        const int bidy = g.get_group_id(1);
+        const int bidz = g.get_group_id(2) % nGroupsZ_;
+        const int bidw = g.get_group_id(2) / nGroupsZ_;
+
+        Tk k;
+        To v;
+
+        const int bOffset = bidw * oVInfo_.strides[3] +
+                            bidz * oVInfo_.strides[2] +
+                            bidy * oVInfo_.strides[1];
+
+        // reduced_block_sizes should have inclusive sum of block sizes
+        int nwrite =
+            (bid == 0)
+                ? reduced_block_sizes_[0]
+                : (reduced_block_sizes_[bid] - reduced_block_sizes_[bid - 1]);
+        int writeloc = (bid == 0) ? 0 : reduced_block_sizes_[bid - 1];
+
+        k = iKeys_[gid];
+        v = iVals_[bOffset + gid];
+
+        if (lid < nwrite) {
+            oKeys_[writeloc + lid]           = k;
+            oVals_[bOffset + writeloc + lid] = v;
+        }
+    }
+
+   protected:
+    read_accessor<int> reduced_block_sizes_;
+    write_accessor<Tk> oKeys_;
+    KParam oKInfo_;
+    write_accessor<To> oVals_;
+    KParam oVInfo_;
+    read_accessor<Tk> iKeys_;
+    KParam iKInfo_;
+    read_accessor<To> iVals_;
+    KParam iVInfo_;
+    int nGroupsZ_;
+};
+
+// Compacts "incomplete" block-sized chunks of data in global memory
+template<typename Tk, typename To>
+class compactDimKernel {
+   public:
+    compactDimKernel(read_accessor<int> reduced_block_sizes,
+                     write_accessor<Tk> oKeys, KParam oKInfo,
+                     write_accessor<To> oVals, KParam oVInfo,
+                     read_accessor<Tk> iKeys, KParam iKInfo,
+                     read_accessor<To> iVals, KParam iVInfo, int nGroupsZ,
+                     int DIM)
+        : reduced_block_sizes_(reduced_block_sizes)
+        , oKeys_(oKeys)
+        , oKInfo_(oKInfo)
+        , oVals_(oVals)
+        , oVInfo_(oVInfo)
+        , iKeys_(iKeys)
+        , iKInfo_(iKInfo)
+        , iVals_(iVals)
+        , iVInfo_(iVInfo)
+        , nGroupsZ_(nGroupsZ)
+        , DIM_(DIM) {}
+
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g = it.get_group();
+
+        const uint lid  = it.get_local_id(0);
+        const uint gidx = it.get_global_id(0);
+        const uint bid  = g.get_group_id(0);
+
+        const int bidy = g.get_group_id(1);
+        const int bidz = g.get_group_id(2) % nGroupsZ_;
+        const int bidw = g.get_group_id(2) / nGroupsZ_;
+
+        int dims_ordering[4];
+        dims_ordering[0] = DIM_;
+        int d            = 1;
+        for (int i = 0; i < 4; ++i) {
+            if (i != DIM_) dims_ordering[d++] = i;
+        }
+
+        Tk k;
+        To v;
+
+        // reduced_block_sizes should have inclusive sum of block sizes
+        int nwrite =
+            (bid == 0)
+                ? reduced_block_sizes_[0]
+                : (reduced_block_sizes_[bid] - reduced_block_sizes_[bid - 1]);
+        int writeloc = (bid == 0) ? 0 : reduced_block_sizes_[bid - 1];
+
+        const int tid = bidw * iVInfo_.strides[dims_ordering[3]] +
+                        bidz * iVInfo_.strides[dims_ordering[2]] +
+                        bidy * iVInfo_.strides[dims_ordering[1]] +
+                        gidx * iVInfo_.strides[DIM_];
+        k = iKeys_[gidx];
+        v = iVals_[tid];
+
+        if (lid < nwrite) {
+            oKeys_[writeloc + lid] = k;
+            const int bOffset      = bidw * oVInfo_.strides[dims_ordering[3]] +
+                                bidz * oVInfo_.strides[dims_ordering[2]] +
+                                bidy * oVInfo_.strides[dims_ordering[1]];
+            oVals_[bOffset + (writeloc + lid) * oVInfo_.strides[DIM_]] = v;
+        }
+    }
+
+   protected:
+    read_accessor<int> reduced_block_sizes_;
+    write_accessor<Tk> oKeys_;
+    KParam oKInfo_;
+    write_accessor<To> oVals_;
+    KParam oVInfo_;
+    read_accessor<Tk> iKeys_;
+    KParam iKInfo_;
+    read_accessor<To> iVals_;
+    KParam iVInfo_;
+    int nGroupsZ_;
+    int DIM_;
+};
+
+// Reduces each block by key
+template<typename Ti, typename Tk, typename To, af_op_t op>
+class reduceBlocksByKeyKernel {
+   public:
+    reduceBlocksByKeyKernel(sycl::accessor<int> reduced_block_sizes,
+                            write_accessor<Tk> oKeys, KParam oKInfo,
+                            write_accessor<To> oVals, KParam oVInfo,
+                            read_accessor<Tk> iKeys, KParam iKInfo,
+                            read_accessor<Ti> iVals, KParam iVInfo,
+                            int change_nan, To nanval, int n, int nGroupsZ,
+                            int DIMX, sycl::local_accessor<Tk> l_keys,
+                            sycl::local_accessor<compute_t<To>> l_vals,
+                            sycl::local_accessor<Tk> l_reduced_keys,
+                            sycl::local_accessor<compute_t<To>> l_reduced_vals,
+                            sycl::local_accessor<int> l_unique_ids,
+                            sycl::local_accessor<int> l_wg_temp,
+                            sycl::local_accessor<int> l_unique_flags,
+                            sycl::local_accessor<int> l_reduced_block_size)
+        : reduced_block_sizes_(reduced_block_sizes)
+        , oKeys_(oKeys)
+        , oKInfo_(oKInfo)
+        , oVals_(oVals)
+        , oVInfo_(oVInfo)
+        , iKeys_(iKeys)
+        , iKInfo_(iKInfo)
+        , iVals_(iVals)
+        , iVInfo_(iVInfo)
+        , change_nan_(change_nan)
+        , nanval_(nanval)
+        , n_(n)
+        , nGroupsZ_(nGroupsZ)
+        , DIMX_(DIMX)
+        , l_keys_(l_keys)
+        , l_vals_(l_vals)
+        , l_reduced_keys_(l_reduced_keys)
+        , l_reduced_vals_(l_reduced_vals)
+        , l_unique_ids_(l_unique_ids)
+        , l_wg_temp_(l_wg_temp)
+        , l_unique_flags_(l_unique_flags)
+        , l_reduced_block_size_(l_reduced_block_size) {}
+
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g  = it.get_group();
+        const uint lid = it.get_local_id(0);
+        const uint gid = it.get_global_id(0);
+
+        const int bidy = g.get_group_id(1);
+        const int bidz = g.get_group_id(2) % nGroupsZ_;
+        const int bidw = g.get_group_id(2) / nGroupsZ_;
+
+        const compute_t<To> init_val =
+            common::Binary<compute_t<To>, op>::init();
+        common::Binary<compute_t<To>, op> binOp;
+        common::Transform<Ti, compute_t<To>, op> transform;
+
+        if (lid == 0) { l_reduced_block_size_[0] = 0; }
+
+        // load keys and values to threads
+        Tk k;
+        compute_t<To> v;
+        if (gid < n_) {
+            k                 = iKeys_[gid];
+            const int bOffset = bidw * iVInfo_.strides[3] +
+                                bidz * iVInfo_.strides[2] +
+                                bidy * iVInfo_.strides[1];
+            v = transform(iVals_[bOffset + gid]);
+            if (change_nan_) v = IS_NAN(v) ? nanval_ : v;
+        } else {
+            v = init_val;
+        }
+
+        l_keys_[lid] = k;
+        l_vals_[lid] = v;
+
+        l_reduced_keys_[lid] = k;
+        it.barrier();
+
+        // mark threads containing unique keys
+        int eq_check    = (lid > 0) ? (k != l_reduced_keys_[lid - 1]) : 0;
+        int unique_flag = (eq_check || (lid == 0)) && (gid < n_);
+
+        l_unique_flags_[lid] = unique_flag;
+        int unique_id =
+            work_group_scan_inclusive_add(it, l_wg_temp_, l_unique_flags_);
+
+        l_unique_ids_[lid] = unique_id;
+
+        if (lid == DIMX_ - 1) l_reduced_block_size_[0] = unique_id;
+
+        for (int off = 1; off < DIMX_; off *= 2) {
+            it.barrier();
+            int test_unique_id =
+                (lid + off < DIMX_) ? l_unique_ids_[lid + off] : ~unique_id;
+            eq_check = (unique_id == test_unique_id);
+            int update_key =
+                eq_check && (lid < (DIMX_ - off)) &&
+                ((gid + off) <
+                 n_);  // checks if this thread should perform a reduction
+            compute_t<To> uval = (update_key) ? l_vals_[lid + off] : init_val;
+            it.barrier();
+            l_vals_[lid] =
+                binOp(l_vals_[lid], uval);  // update if thread requires it
+        }
+
+        if (unique_flag) {
+            l_reduced_keys_[unique_id - 1] = k;
+            l_reduced_vals_[unique_id - 1] = l_vals_[lid];
+        }
+        it.barrier();
+
+        const int bid = g.get_group_id(0);
+        if (lid < l_reduced_block_size_[0]) {
+            const int bOffset = bidw * oVInfo_.strides[3] +
+                                bidz * oVInfo_.strides[2] +
+                                bidy * oVInfo_.strides[1];
+            oKeys_[bid * DIMX_ + lid]               = l_reduced_keys_[lid];
+            oVals_[bOffset + ((bid * DIMX_) + lid)] = l_reduced_vals_[lid];
+        }
+
+        reduced_block_sizes_[bid] = l_reduced_block_size_[0];
+    }
+
+    int work_group_scan_inclusive_add(sycl::nd_item<3> it,
+                                      sycl::local_accessor<int> wg_temp,
+                                      sycl::local_accessor<int> arr) const {
+        const uint lid = it.get_local_id(0);
+        int *active_buf;
+
+        int val    = arr[lid];
+        active_buf = arr.get_pointer();
+
+        bool swap_buffer = false;
+        for (int off = 1; off <= DIMX_; off *= 2) {
+            it.barrier();
+            if (lid >= off) { val = val + active_buf[lid - off]; }
+            swap_buffer = !swap_buffer;
+            active_buf =
+                swap_buffer ? wg_temp.get_pointer() : arr.get_pointer();
+            active_buf[lid] = val;
+        }
+
+        int res = active_buf[lid];
+        return res;
+    }
+
+   protected:
+    sycl::accessor<int> reduced_block_sizes_;
+    write_accessor<Tk> oKeys_;
+    KParam oKInfo_;
+    write_accessor<To> oVals_;
+    KParam oVInfo_;
+    read_accessor<Tk> iKeys_;
+    KParam iKInfo_;
+    read_accessor<Ti> iVals_;
+    KParam iVInfo_;
+    int change_nan_;
+    To nanval_;
+    int n_;
+    int nGroupsZ_;
+    int DIMX_;
+    sycl::local_accessor<Tk> l_keys_;
+    sycl::local_accessor<compute_t<To>> l_vals_;
+    sycl::local_accessor<Tk> l_reduced_keys_;
+    sycl::local_accessor<compute_t<To>> l_reduced_vals_;
+    sycl::local_accessor<int> l_unique_ids_;
+    sycl::local_accessor<int> l_wg_temp_;
+    sycl::local_accessor<int> l_unique_flags_;
+    sycl::local_accessor<int> l_reduced_block_size_;
+};
+
+// Reduces each block by key
+template<typename Ti, typename Tk, typename To, af_op_t op>
+class reduceBlocksByKeyDimKernel {
+   public:
+    reduceBlocksByKeyDimKernel(
+        sycl::accessor<int> reduced_block_sizes, write_accessor<Tk> oKeys,
+        KParam oKInfo, write_accessor<To> oVals, KParam oVInfo,
+        read_accessor<Tk> iKeys, KParam iKInfo, read_accessor<Ti> iVals,
+        KParam iVInfo, int change_nan, To nanval, int n, int nGroupsZ, int DIMX,
+        int DIM, sycl::local_accessor<Tk> l_keys,
+        sycl::local_accessor<compute_t<To>> l_vals,
+        sycl::local_accessor<Tk> l_reduced_keys,
+        sycl::local_accessor<compute_t<To>> l_reduced_vals,
+        sycl::local_accessor<int> l_unique_ids,
+        sycl::local_accessor<int> l_wg_temp,
+        sycl::local_accessor<int> l_unique_flags,
+        sycl::local_accessor<int> l_reduced_block_size)
+        : reduced_block_sizes_(reduced_block_sizes)
+        , oKeys_(oKeys)
+        , oKInfo_(oKInfo)
+        , oVals_(oVals)
+        , oVInfo_(oVInfo)
+        , iKeys_(iKeys)
+        , iKInfo_(iKInfo)
+        , iVals_(iVals)
+        , iVInfo_(iVInfo)
+        , change_nan_(change_nan)
+        , nanval_(nanval)
+        , n_(n)
+        , nGroupsZ_(nGroupsZ)
+        , DIMX_(DIMX)
+        , DIM_(DIM)
+        , l_keys_(l_keys)
+        , l_vals_(l_vals)
+        , l_reduced_keys_(l_reduced_keys)
+        , l_reduced_vals_(l_reduced_vals)
+        , l_unique_ids_(l_unique_ids)
+        , l_wg_temp_(l_wg_temp)
+        , l_unique_flags_(l_unique_flags)
+        , l_reduced_block_size_(l_reduced_block_size) {}
+
+    void operator()(sycl::nd_item<3> it) const {
+        sycl::group g  = it.get_group();
+        const uint lid = it.get_local_id(0);
+        const uint gid = it.get_global_id(0);
+
+        const int bidy = g.get_group_id(1);
+        const int bidz = g.get_group_id(2) % nGroupsZ_;
+        const int bidw = g.get_group_id(2) / nGroupsZ_;
+
+        const compute_t<To> init_val =
+            common::Binary<compute_t<To>, op>::init();
+        common::Binary<compute_t<To>, op> binOp;
+        common::Transform<Ti, compute_t<To>, op> transform;
+
+        if (lid == 0) { l_reduced_block_size_[0] = 0; }
+
+        int dims_ordering[4];
+        dims_ordering[0] = DIM_;
+        int d            = 1;
+        for (int i = 0; i < 4; ++i) {
+            if (i != DIM_) dims_ordering[d++] = i;
+        }
+        it.barrier();
+
+        // load keys and values to threads
+        Tk k;
+        compute_t<To> v;
+        if (gid < n_) {
+            k                 = iKeys_[gid];
+            const int bOffset = bidw * iVInfo_.strides[dims_ordering[3]] +
+                                bidz * iVInfo_.strides[dims_ordering[2]] +
+                                bidy * iVInfo_.strides[dims_ordering[1]];
+            v = transform(iVals_[bOffset + gid * iVInfo_.strides[DIM_]]);
+            if (change_nan_) v = IS_NAN(v) ? nanval_ : v;
+        } else {
+            v = init_val;
+        }
+
+        l_keys_[lid] = k;
+        l_vals_[lid] = v;
+
+        l_reduced_keys_[lid] = k;
+        it.barrier();
+
+        // mark threads containing unique keys
+        int eq_check    = (lid > 0) ? (k != l_reduced_keys_[lid - 1]) : 0;
+        int unique_flag = (eq_check || (lid == 0)) && (gid < n_);
+
+        l_unique_flags_[lid] = unique_flag;
+        int unique_id =
+            work_group_scan_inclusive_add(it, l_wg_temp_, l_unique_flags_);
+
+        l_unique_ids_[lid] = unique_id;
+
+        if (lid == DIMX_ - 1) l_reduced_block_size_[0] = unique_id;
+
+        for (int off = 1; off < DIMX_; off *= 2) {
+            it.barrier();
+            int test_unique_id =
+                (lid + off < DIMX_) ? l_unique_ids_[lid + off] : ~unique_id;
+            eq_check = (unique_id == test_unique_id);
+            int update_key =
+                eq_check && (lid < (DIMX_ - off)) &&
+                ((gid + off) <
+                 n_);  // checks if this thread should perform a reduction
+            compute_t<To> uval = (update_key) ? l_vals_[lid + off] : init_val;
+            it.barrier();
+            l_vals_[lid] =
+                binOp(l_vals_[lid], uval);  // update if thread requires it
+        }
+
+        if (unique_flag) {
+            l_reduced_keys_[unique_id - 1] = k;
+            l_reduced_vals_[unique_id - 1] = l_vals_[lid];
+        }
+        it.barrier();
+
+        const int bid = g.get_group_id(0);
+        if (lid < l_reduced_block_size_[0]) {
+            const int bOffset = bidw * oVInfo_.strides[dims_ordering[3]] +
+                                bidz * oVInfo_.strides[dims_ordering[2]] +
+                                bidy * oVInfo_.strides[dims_ordering[1]];
+            oKeys_[gid] = l_reduced_keys_[lid];
+            oVals_[bOffset + (gid)*oVInfo_.strides[DIM_]] =
+                l_reduced_vals_[lid];
+        }
+
+        reduced_block_sizes_[bid] = l_reduced_block_size_[0];
+    }
+
+    int work_group_scan_inclusive_add(sycl::nd_item<3> it,
+                                      sycl::local_accessor<int> wg_temp,
+                                      sycl::local_accessor<int> arr) const {
+        const uint lid = it.get_local_id(0);
+        int *active_buf;
+
+        int val    = arr[lid];
+        active_buf = arr.get_pointer();
+
+        bool swap_buffer = false;
+        for (int off = 1; off <= DIMX_; off *= 2) {
+            it.barrier();
+            if (lid >= off) { val = val + active_buf[lid - off]; }
+            swap_buffer = !swap_buffer;
+            active_buf =
+                swap_buffer ? wg_temp.get_pointer() : arr.get_pointer();
+            active_buf[lid] = val;
+        }
+
+        int res = active_buf[lid];
+        return res;
+    }
+
+   protected:
+    sycl::accessor<int> reduced_block_sizes_;
+    write_accessor<Tk> oKeys_;
+    KParam oKInfo_;
+    write_accessor<To> oVals_;
+    KParam oVInfo_;
+    read_accessor<Tk> iKeys_;
+    KParam iKInfo_;
+    read_accessor<Ti> iVals_;
+    KParam iVInfo_;
+    int change_nan_;
+    To nanval_;
+    int n_;
+    int nGroupsZ_;
+    int DIMX_;
+    int DIM_;
+    sycl::local_accessor<Tk> l_keys_;
+    sycl::local_accessor<compute_t<To>> l_vals_;
+    sycl::local_accessor<Tk> l_reduced_keys_;
+    sycl::local_accessor<compute_t<To>> l_reduced_vals_;
+    sycl::local_accessor<int> l_unique_ids_;
+    sycl::local_accessor<int> l_wg_temp_;
+    sycl::local_accessor<int> l_unique_flags_;
+    sycl::local_accessor<int> l_reduced_block_size_;
+};
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp
index 14b5a9e269..efada203e1 100644
--- a/src/backend/oneapi/reduce_impl.hpp
+++ b/src/backend/oneapi/reduce_impl.hpp
@@ -6,11 +6,19 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
+#pragma once
+
+// oneDPL headers should be included before standard headers
+#define ONEDPL_USE_PREDEFINED_POLICIES 0
+#include <oneapi/dpl/execution>
+#include <oneapi/dpl/iterator>
+#include <oneapi/dpl/numeric>
 
 #include <Array.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
 #include <kernel/reduce.hpp>
-// #include <kernel/reduce_by_key.hpp>
+#include <kernel/reduce_by_key.hpp>
 #include <reduce.hpp>
 #include <af/dim4.hpp>
 #include <complex>
@@ -31,11 +39,558 @@ Array<To> reduce(const Array<Ti> &in, const int dim, bool change_nan,
     return out;
 }
 
+template<typename Ti, typename Tk, typename To, af_op_t op>
+void reduceBlocksByKey(sycl::buffer<int> &reduced_block_sizes,
+                       Array<Tk> keys_out, Array<To> vals_out,
+                       const Array<Tk> keys, const Array<Ti> vals,
+                       int change_nan, double nanval, const int n,
+                       const int threads_x) {
+    int numBlocks = divup(n, threads_x);
+
+    sycl::range<3> local(threads_x, 1, 1);
+    sycl::range<3> global(local[0] * numBlocks, vals_out.dims()[1],
+                          vals_out.dims()[2] * vals_out.dims()[3]);
+
+    getQueue().submit([&](sycl::handler &h) {
+        sycl::accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
+        write_accessor<Tk> keys_out_acc{*keys_out.get(), h};
+        write_accessor<To> vals_out_acc{*vals_out.get(), h};
+        read_accessor<Tk> keys_acc{*keys.get(), h};
+        read_accessor<Ti> vals_acc{*vals.get(), h};
+
+        auto l_keys         = sycl::local_accessor<Tk>(threads_x, h);
+        auto l_vals         = sycl::local_accessor<compute_t<To>>(threads_x, h);
+        auto l_reduced_keys = sycl::local_accessor<Tk>(threads_x, h);
+        auto l_reduced_vals = sycl::local_accessor<compute_t<To>>(threads_x, h);
+        auto l_unique_ids   = sycl::local_accessor<int>(threads_x, h);
+        auto l_wq_temp      = sycl::local_accessor<int>(threads_x, h);
+        auto l_unique_flags = sycl::local_accessor<int>(threads_x, h);
+        auto l_reduced_block_size = sycl::local_accessor<int>(1, h);
+
+        h.parallel_for(
+            sycl::nd_range<3>(global, local),
+            kernel::reduceBlocksByKeyKernel<Ti, Tk, To, op>(
+                reduced_block_sizes_acc, keys_out_acc, keys_out, vals_out_acc,
+                vals_out, keys_acc, keys, vals_acc, vals, change_nan,
+                scalar<To>(nanval), n, static_cast<int>(vals_out.dims()[2]),
+                threads_x, l_keys, l_vals, l_reduced_keys, l_reduced_vals,
+                l_unique_ids, l_wq_temp, l_unique_flags, l_reduced_block_size));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Ti, typename Tk, typename To, af_op_t op>
+void reduceBlocksByKeyDim(sycl::buffer<int> &reduced_block_sizes,
+                          Array<Tk> keys_out, Array<To> vals_out,
+                          const Array<Tk> keys, const Array<Ti> vals,
+                          int change_nan, double nanval, const int n,
+                          const int threads_x, const int dim,
+                          std::vector<int> dim_ordering) {
+    int numBlocks = divup(n, threads_x);
+
+    sycl::range<3> local(threads_x, 1, 1);
+    sycl::range<3> global(
+        local[0] * numBlocks, vals_out.dims()[dim_ordering[1]],
+        vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]);
+
+    getQueue().submit([&](sycl::handler &h) {
+        sycl::accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
+        write_accessor<Tk> keys_out_acc{*keys_out.get(), h};
+        write_accessor<To> vals_out_acc{*vals_out.get(), h};
+        read_accessor<Tk> keys_acc{*keys.get(), h};
+        read_accessor<Ti> vals_acc{*vals.get(), h};
+
+        auto l_keys         = sycl::local_accessor<Tk>(threads_x, h);
+        auto l_vals         = sycl::local_accessor<compute_t<To>>(threads_x, h);
+        auto l_reduced_keys = sycl::local_accessor<Tk>(threads_x, h);
+        auto l_reduced_vals = sycl::local_accessor<compute_t<To>>(threads_x, h);
+        auto l_unique_ids   = sycl::local_accessor<int>(threads_x, h);
+        auto l_wq_temp      = sycl::local_accessor<int>(threads_x, h);
+        auto l_unique_flags = sycl::local_accessor<int>(threads_x, h);
+        auto l_reduced_block_size = sycl::local_accessor<int>(1, h);
+
+        h.parallel_for(
+            sycl::nd_range<3>(global, local),
+            kernel::reduceBlocksByKeyDimKernel<Ti, Tk, To, op>(
+                reduced_block_sizes_acc, keys_out_acc, keys_out, vals_out_acc,
+                vals_out, keys_acc, keys, vals_acc, vals, change_nan,
+                scalar<To>(nanval), n, static_cast<int>(vals_out.dims()[2]),
+                threads_x, dim, l_keys, l_vals, l_reduced_keys, l_reduced_vals,
+                l_unique_ids, l_wq_temp, l_unique_flags, l_reduced_block_size));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Tk, typename To, af_op_t op>
+void finalBoundaryReduce(sycl::buffer<int> &reduced_block_sizes, Array<Tk> keys,
+                         Array<To> vals_out, const int n, const int numBlocks,
+                         const int threads_x) {
+    sycl::range<1> local(threads_x);
+    sycl::range<1> global(local[0] * numBlocks);
+
+    getQueue().submit([&](sycl::handler &h) {
+        write_accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
+        read_accessor<Tk> keys_acc{*keys.get(), h};
+        sycl::accessor<To> vals_out_acc{*vals_out.get(), h};
+
+        h.parallel_for(sycl::nd_range<1>(global, local),
+                       kernel::finalBoundaryReduceKernel<Tk, To, op>(
+                           reduced_block_sizes_acc, keys_acc, keys,
+                           vals_out_acc, vals_out, n));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Tk, typename To, af_op_t op>
+void finalBoundaryReduceDim(sycl::buffer<int> &reduced_block_sizes,
+                            Array<Tk> keys, Array<To> vals_out, const int n,
+                            const int numBlocks, const int threads_x,
+                            const int dim, std::vector<int> dim_ordering) {
+    sycl::range<3> local(threads_x, 1, 1);
+    sycl::range<3> global(
+        local[0] * numBlocks, vals_out.dims()[dim_ordering[1]],
+        vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]);
+
+    getQueue().submit([&](sycl::handler &h) {
+        write_accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
+        read_accessor<Tk> keys_acc{*keys.get(), h};
+        sycl::accessor<To> vals_out_acc{*vals_out.get(), h};
+
+        // TODO: fold 3,4 dimensions
+        h.parallel_for(
+            sycl::nd_range<3>(global, local),
+            kernel::finalBoundaryReduceDimKernel<Tk, To, op>(
+                reduced_block_sizes_acc, keys_acc, keys, vals_out_acc, vals_out,
+                n, vals_out.dims()[dim_ordering[2]]));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Tk, typename To>
+void compact(sycl::buffer<int> reduced_block_sizes, Array<Tk> &keys_out,
+             Array<To> &vals_out, const Array<Tk> &keys, const Array<To> &vals,
+             const int numBlocks, const int threads_x) {
+    sycl::range<3> local(threads_x, 1, 1);
+    sycl::range<3> global(local[0] * numBlocks, vals_out.dims()[1],
+                          vals_out.dims()[2] * vals_out.dims()[3]);
+
+    getQueue().submit([&](sycl::handler &h) {
+        read_accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
+        write_accessor<Tk> keys_out_acc{*keys_out.get(), h};
+        write_accessor<To> vals_out_acc{*vals_out.get(), h};
+        read_accessor<Tk> keys_acc{*keys.get(), h};
+        read_accessor<To> vals_acc{*vals.get(), h};
+
+        h.parallel_for(sycl::nd_range<3>(global, local),
+                       kernel::compactKernel<Tk, To>(
+                           reduced_block_sizes_acc, keys_out_acc, keys_out,
+                           vals_out_acc, vals_out, keys_acc, keys, vals_acc,
+                           vals, static_cast<int>(vals_out.dims()[2])));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Tk, typename To>
+void compactDim(sycl::buffer<int> &reduced_block_sizes, Array<Tk> &keys_out,
+                Array<To> &vals_out, const Array<Tk> &keys,
+                const Array<To> &vals, const int numBlocks, const int threads_x,
+                const int dim, std::vector<int> dim_ordering) {
+    sycl::range<3> local(threads_x, 1, 1);
+    sycl::range<3> global(
+        local[0] * numBlocks, vals_out.dims()[dim_ordering[1]],
+        vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]);
+
+    getQueue().submit([&](sycl::handler &h) {
+        read_accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
+        write_accessor<Tk> keys_out_acc{*keys_out.get(), h};
+        write_accessor<To> vals_out_acc{*vals_out.get(), h};
+        read_accessor<Tk> keys_acc{*keys.get(), h};
+        read_accessor<To> vals_acc{*vals.get(), h};
+
+        h.parallel_for(
+            sycl::nd_range<3>(global, local),
+            kernel::compactDimKernel<Tk, To>(
+                reduced_block_sizes_acc, keys_out_acc, keys_out, vals_out_acc,
+                vals_out, keys_acc, keys, vals_acc, vals,
+                static_cast<int>(vals_out.dims()[dim_ordering[2]]), dim));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename Tk>
+void testNeedsReduction(sycl::buffer<int> needs_reduction,
+                        sycl::buffer<int> needs_boundary, const Array<Tk> &keys,
+                        const int n, const int numBlocks, const int threads_x) {
+    sycl::range<1> local(threads_x);
+    sycl::range<1> global(local[0] * numBlocks);
+
+    getQueue().submit([&](sycl::handler &h) {
+        sycl::accessor<int> needs_reduction_acc{needs_reduction, h};
+        sycl::accessor<int> needs_boundary_acc{needs_boundary, h};
+        read_accessor<Tk> keys_acc{*keys.get(), h};
+        auto l_keys = sycl::local_accessor<Tk>(threads_x, h);
+
+        h.parallel_for(sycl::nd_range<1>(global, local),
+                       kernel::testNeedsReductionKernel<Tk>(
+                           needs_reduction_acc, needs_boundary_acc, keys_acc,
+                           keys, n, threads_x, l_keys));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<af_op_t op, typename Ti, typename Tk, typename To>
+int reduce_by_key_first(Array<Tk> &keys_out, Array<To> &vals_out,
+                        const Array<Tk> &keys, const Array<Ti> &vals,
+                        bool change_nan, double nanval) {
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
+
+    dim4 kdims = keys.dims();
+    dim4 odims = vals.dims();
+
+    Array<Tk> reduced_keys   = createEmptyArray<Tk>(kdims);
+    Array<To> reduced_vals   = createEmptyArray<To>(odims);
+    Array<Tk> t_reduced_keys = createEmptyArray<Tk>(kdims);
+    Array<To> t_reduced_vals = createEmptyArray<To>(odims);
+
+    // flags determining more reduction is necessary
+    auto needs_another_reduction        = memAlloc<int>(1);
+    auto needs_block_boundary_reduction = memAlloc<int>(1);
+
+    // reset flags
+    getQueue().submit([&](sycl::handler &h) {
+        auto wacc =
+            needs_another_reduction->get_access<sycl::access_mode::write>(h);
+        h.fill(wacc, 0);
+    });
+    getQueue().submit([&](sycl::handler &h) {
+        auto wacc = needs_block_boundary_reduction
+                        ->get_access<sycl::access_mode::write>(h);
+        h.fill(wacc, 0);
+    });
+
+    size_t nelems = kdims[0];
+
+    const unsigned int numThreads = 128;
+    int numBlocksD0               = divup(nelems, numThreads);
+    auto reduced_block_sizes      = memAlloc<int>(numBlocksD0);
+
+    int n_reduced_host = nelems;
+
+    int needs_another_reduction_host        = 0;
+    int needs_block_boundary_reduction_host = 0;
+
+    bool first_pass = true;
+    do {
+        numBlocksD0 = divup(n_reduced_host, numThreads);
+
+        if (first_pass) {
+            reduceBlocksByKey<Ti, Tk, To, op>(
+                *reduced_block_sizes.get(), reduced_keys, reduced_vals, keys,
+                vals, change_nan, nanval, n_reduced_host, numThreads);
+            first_pass = false;
+        } else {
+            constexpr af_op_t op2 = (op == af_notzero_t) ? af_add_t : op;
+            reduceBlocksByKey<To, Tk, To, op2>(
+                *reduced_block_sizes.get(), reduced_keys, reduced_vals,
+                t_reduced_keys, t_reduced_vals, change_nan, nanval,
+                n_reduced_host, numThreads);
+        }
+
+        auto val_buf_begin = ::oneapi::dpl::begin(*reduced_block_sizes.get());
+        auto val_buf_end   = val_buf_begin + numBlocksD0;
+        std::inclusive_scan(dpl_policy, val_buf_begin, val_buf_end,
+                            val_buf_begin);
+
+        compact<Tk, To>(*reduced_block_sizes.get(), t_reduced_keys,
+                        t_reduced_vals, reduced_keys, reduced_vals, numBlocksD0,
+                        numThreads);
+
+        sycl::event reduce_host_event =
+            getQueue().submit([&](sycl::handler &h) {
+                sycl::range rr(1);
+                sycl::id offset_id(numBlocksD0 - 1);
+                auto offset_acc =
+                    reduced_block_sizes
+                        ->template get_access<sycl::access_mode::read>(
+                            h, rr, offset_id);
+                h.copy(offset_acc, &n_reduced_host);
+            });
+
+        // reset flags
+        getQueue().submit([&](sycl::handler &h) {
+            auto wacc =
+                needs_another_reduction->get_access<sycl::access_mode::write>(
+                    h);
+            h.fill(wacc, 0);
+        });
+        getQueue().submit([&](sycl::handler &h) {
+            auto wacc = needs_block_boundary_reduction
+                            ->get_access<sycl::access_mode::write>(h);
+            h.fill(wacc, 0);
+        });
+
+        reduce_host_event.wait();
+
+        numBlocksD0 = divup(n_reduced_host, numThreads);
+
+        testNeedsReduction<Tk>(*needs_another_reduction.get(),
+                               *needs_block_boundary_reduction.get(),
+                               t_reduced_keys, n_reduced_host, numBlocksD0,
+                               numThreads);
+
+        sycl::event host_flag0_event = getQueue().submit([&](sycl::handler &h) {
+            sycl::range rr(1);
+            auto acc =
+                needs_another_reduction
+                    ->template get_access<sycl::access_mode::read>(h, rr);
+            h.copy(acc, &needs_another_reduction_host);
+        });
+        sycl::event host_flag1_event = getQueue().submit([&](sycl::handler &h) {
+            sycl::range rr(1);
+            auto acc =
+                needs_block_boundary_reduction
+                    ->template get_access<sycl::access_mode::read>(h, rr);
+            h.copy(acc, &needs_block_boundary_reduction_host);
+        });
+
+        host_flag1_event.wait();
+        host_flag0_event.wait();
+
+        if (needs_block_boundary_reduction_host &&
+            !needs_another_reduction_host) {
+            finalBoundaryReduce<Tk, To, op>(
+                *reduced_block_sizes.get(), t_reduced_keys, t_reduced_vals,
+                n_reduced_host, numBlocksD0, numThreads);
+
+            auto val_buf_begin =
+                ::oneapi::dpl::begin(*reduced_block_sizes.get());
+            auto val_buf_end = val_buf_begin + numBlocksD0;
+            std::inclusive_scan(dpl_policy, val_buf_begin, val_buf_end,
+                                val_buf_begin);
+
+            sycl::event reduce_host_event =
+                getQueue().submit([&](sycl::handler &h) {
+                    sycl::range rr(1);
+                    sycl::id offset_id(numBlocksD0 - 1);
+                    auto offset_acc =
+                        reduced_block_sizes
+                            ->template get_access<sycl::access_mode::read>(
+                                h, rr, offset_id);
+                    h.copy(offset_acc, &n_reduced_host);
+                });
+
+            compact<Tk, To>(*reduced_block_sizes.get(), reduced_keys,
+                            reduced_vals, t_reduced_keys, t_reduced_vals,
+                            numBlocksD0, numThreads);
+
+            std::swap(t_reduced_keys, reduced_keys);
+            std::swap(t_reduced_vals, reduced_vals);
+            reduce_host_event.wait();
+        }
+    } while (needs_another_reduction_host ||
+             needs_block_boundary_reduction_host);
+
+    keys_out = t_reduced_keys;
+    vals_out = t_reduced_vals;
+    return n_reduced_host;
+}
+
+template<af_op_t op, typename Ti, typename Tk, typename To>
+int reduce_by_key_dim(Array<Tk> &keys_out, Array<To> &vals_out,
+                      const Array<Tk> &keys, const Array<Ti> &vals,
+                      bool change_nan, double nanval, const int dim) {
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
+
+    std::vector<int> dim_ordering = {dim};
+    for (int i = 0; i < 4; ++i) {
+        if (i != dim) { dim_ordering.push_back(i); }
+    }
+
+    dim4 kdims = keys.dims();
+    dim4 odims = vals.dims();
+
+    Array<Tk> reduced_keys   = createEmptyArray<Tk>(kdims);
+    Array<To> reduced_vals   = createEmptyArray<To>(odims);
+    Array<Tk> t_reduced_keys = createEmptyArray<Tk>(kdims);
+    Array<To> t_reduced_vals = createEmptyArray<To>(odims);
+
+    // flags determining more reduction is necessary
+    auto needs_another_reduction        = memAlloc<int>(1);
+    auto needs_block_boundary_reduction = memAlloc<int>(1);
+
+    // reset flags
+    getQueue().submit([&](sycl::handler &h) {
+        auto wacc =
+            needs_another_reduction->get_access<sycl::access_mode::write>(h);
+        h.fill(wacc, 0);
+    });
+    getQueue().submit([&](sycl::handler &h) {
+        auto wacc = needs_block_boundary_reduction
+                        ->get_access<sycl::access_mode::write>(h);
+        h.fill(wacc, 0);
+    });
+
+    int nelems = kdims[0];
+
+    const unsigned int numThreads = 128;
+    int numBlocksD0               = divup(nelems, numThreads);
+    auto reduced_block_sizes      = memAlloc<int>(numBlocksD0);
+
+    int n_reduced_host = nelems;
+
+    int needs_another_reduction_host        = 0;
+    int needs_block_boundary_reduction_host = 0;
+
+    bool first_pass = true;
+    do {
+        numBlocksD0 = divup(n_reduced_host, numThreads);
+
+        if (first_pass) {
+            reduceBlocksByKeyDim<Ti, Tk, To, op>(
+                *reduced_block_sizes.get(), reduced_keys, reduced_vals, keys,
+                vals, change_nan, nanval, n_reduced_host, numThreads, dim,
+                dim_ordering);
+            first_pass = false;
+        } else {
+            constexpr af_op_t op2 = op == af_notzero_t ? af_add_t : op;
+            reduceBlocksByKeyDim<To, Tk, To, op2>(
+                *reduced_block_sizes.get(), reduced_keys, reduced_vals,
+                t_reduced_keys, t_reduced_vals, change_nan, nanval,
+                n_reduced_host, numThreads, dim, dim_ordering);
+        }
+
+        auto val_buf_begin = ::oneapi::dpl::begin(*reduced_block_sizes.get());
+        auto val_buf_end   = val_buf_begin + numBlocksD0;
+        std::inclusive_scan(dpl_policy, val_buf_begin, val_buf_end,
+                            val_buf_begin);
+
+        compactDim<Tk, To>(*reduced_block_sizes.get(), t_reduced_keys,
+                           t_reduced_vals, reduced_keys, reduced_vals,
+                           numBlocksD0, numThreads, dim, dim_ordering);
+
+        sycl::event reduce_host_event =
+            getQueue().submit([&](sycl::handler &h) {
+                sycl::range rr(1);
+                sycl::id offset_id(numBlocksD0 - 1);
+                auto offset_acc =
+                    reduced_block_sizes
+                        ->template get_access<sycl::access_mode::read>(
+                            h, rr, offset_id);
+                h.copy(offset_acc, &n_reduced_host);
+            });
+
+        // reset flags
+        getQueue().submit([&](sycl::handler &h) {
+            auto wacc =
+                needs_another_reduction->get_access<sycl::access_mode::write>(
+                    h);
+            h.fill(wacc, 0);
+        });
+        getQueue().submit([&](sycl::handler &h) {
+            auto wacc = needs_block_boundary_reduction
+                            ->get_access<sycl::access_mode::write>(h);
+            h.fill(wacc, 0);
+        });
+
+        reduce_host_event.wait();
+
+        numBlocksD0 = divup(n_reduced_host, numThreads);
+
+        testNeedsReduction<Tk>(*needs_another_reduction.get(),
+                               *needs_block_boundary_reduction.get(),
+                               t_reduced_keys, n_reduced_host, numBlocksD0,
+                               numThreads);
+
+        sycl::event host_flag0_event = getQueue().submit([&](sycl::handler &h) {
+            sycl::range rr(1);
+            auto acc =
+                needs_another_reduction
+                    ->template get_access<sycl::access_mode::read>(h, rr);
+            h.copy(acc, &needs_another_reduction_host);
+        });
+        sycl::event host_flag1_event = getQueue().submit([&](sycl::handler &h) {
+            sycl::range rr(1);
+            auto acc =
+                needs_block_boundary_reduction
+                    ->template get_access<sycl::access_mode::read>(h, rr);
+            h.copy(acc, &needs_block_boundary_reduction_host);
+        });
+
+        host_flag1_event.wait();
+        host_flag0_event.wait();
+
+        if (needs_block_boundary_reduction_host &&
+            !needs_another_reduction_host) {
+            finalBoundaryReduceDim<Tk, To, op>(
+                *reduced_block_sizes.get(), t_reduced_keys, t_reduced_vals,
+                n_reduced_host, numBlocksD0, numThreads, dim, dim_ordering);
+
+            auto val_buf_begin =
+                ::oneapi::dpl::begin(*reduced_block_sizes.get());
+            auto val_buf_end = val_buf_begin + numBlocksD0;
+            std::inclusive_scan(dpl_policy, val_buf_begin, val_buf_end,
+                                val_buf_begin);
+
+            sycl::event reduce_host_event =
+                getQueue().submit([&](sycl::handler &h) {
+                    sycl::range rr(1);
+                    sycl::id offset_id(numBlocksD0 - 1);
+                    auto offset_acc =
+                        reduced_block_sizes
+                            ->template get_access<sycl::access_mode::read>(
+                                h, rr, offset_id);
+                    h.copy(offset_acc, &n_reduced_host);
+                });
+
+            compactDim<Tk, To>(*reduced_block_sizes.get(), reduced_keys,
+                               reduced_vals, t_reduced_keys, t_reduced_vals,
+                               numBlocksD0, numThreads, dim, dim_ordering);
+
+            std::swap(t_reduced_keys, reduced_keys);
+            std::swap(t_reduced_vals, reduced_vals);
+            reduce_host_event.wait();
+        }
+    } while (needs_another_reduction_host ||
+             needs_block_boundary_reduction_host);
+
+    keys_out = t_reduced_keys;
+    vals_out = t_reduced_vals;
+
+    return n_reduced_host;
+}
+
 template<af_op_t op, typename Ti, typename Tk, typename To>
 void reduce_by_key(Array<Tk> &keys_out, Array<To> &vals_out,
                    const Array<Tk> &keys, const Array<Ti> &vals, const int dim,
                    bool change_nan, double nanval) {
-    ONEAPI_NOT_SUPPORTED("");
+    dim4 kdims = keys.dims();
+    dim4 odims = vals.dims();
+
+    // prepare output arrays
+    Array<Tk> reduced_keys = createEmptyArray<Tk>(dim4());
+    Array<To> reduced_vals = createEmptyArray<To>(dim4());
+
+    size_t n_reduced = 0;
+    if (dim == 0) {
+        n_reduced = reduce_by_key_first<op, Ti, Tk, To>(
+            reduced_keys, reduced_vals, keys, vals, change_nan, nanval);
+    } else {
+        n_reduced = reduce_by_key_dim<op, Ti, Tk, To>(
+            reduced_keys, reduced_vals, keys, vals, change_nan, nanval, dim);
+    }
+
+    kdims[0]   = n_reduced;
+    odims[dim] = n_reduced;
+    std::vector<af_seq> kindex, vindex;
+    for (int i = 0; i < odims.ndims(); ++i) {
+        af_seq sk = {0.0, (double)kdims[i] - 1, 1.0};
+        af_seq sv = {0.0, (double)odims[i] - 1, 1.0};
+        kindex.push_back(sk);
+        vindex.push_back(sv);
+    }
+
+    keys_out = createSubArray<Tk>(reduced_keys, kindex, true);
+    vals_out = createSubArray<To>(reduced_vals, vindex, true);
 }
 
 template<af_op_t op, typename Ti, typename To>

From b9cdc1941ed4a0991082fe8dbec4e425fc8277e3 Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Tue, 20 Jun 2023 16:57:03 -0400
Subject: [PATCH 324/473] Fix FFT errors in oneAPI backend because of
 descriptor parameters (#3449)

---------

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
Co-authored-by: syurkevi <stefan@arrayfire.com>
Co-authored-by: Umar Arshad <umar@arrayfire.com>
---
 src/backend/oneapi/fft.cpp | 75 ++++++++++++++++++++++++++++----------
 1 file changed, 56 insertions(+), 19 deletions(-)

diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
index b32c801423..5c3621c5e1 100644
--- a/src/backend/oneapi/fft.cpp
+++ b/src/backend/oneapi/fft.cpp
@@ -50,11 +50,22 @@ void fft_inplace(Array<T> &in, const int rank, const bool direction) {
 
     auto desc = [rank, &idims]() {
         if (rank == 1) return desc_ty(idims[0]);
-        if (rank == 2) return desc_ty({idims[1], idims[0]});
-        if (rank == 3) return desc_ty({idims[2], idims[1], idims[0]});
-        return desc_ty({idims[3], idims[2], idims[1], idims[0]});
+        if (rank == 2) return desc_ty({idims[0], idims[1]});
+        if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]});
+        return desc_ty({idims[0], idims[1], idims[2], idims[3]});
     }();
 
+    if (rank > 1) {
+        std::int64_t fft_input_strides[5];
+        fft_input_strides[0] = in.getOffset();
+        fft_input_strides[1] = istrides[0];
+        fft_input_strides[2] = istrides[1];
+        fft_input_strides[3] = istrides[2];
+        fft_input_strides[4] = istrides[3];
+        desc.set_value(::oneapi::mkl::dft::config_param::INPUT_STRIDES,
+                       fft_input_strides);
+    }
+
     desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT, DFTI_INPLACE);
 
     int batch = 1;
@@ -96,6 +107,25 @@ Array<Tc> fft_r2c(const Array<Tr> &in, const int rank) {
         if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]});
         return desc_ty({idims[0], idims[1], idims[2], idims[3]});
     }();
+    if (rank > 1) {
+        std::int64_t fft_input_strides[5];
+        fft_input_strides[0] = in.getOffset();
+        fft_input_strides[1] = istrides[0];
+        fft_input_strides[2] = istrides[1];
+        fft_input_strides[3] = istrides[2];
+        fft_input_strides[4] = istrides[3];
+        desc.set_value(::oneapi::mkl::dft::config_param::INPUT_STRIDES,
+                       fft_input_strides);
+
+        std::int64_t fft_output_strides[5];
+        fft_output_strides[0] = out.getOffset();
+        fft_output_strides[1] = ostrides[0];
+        fft_output_strides[2] = ostrides[1];
+        fft_output_strides[3] = ostrides[2];
+        fft_output_strides[4] = ostrides[3];
+        desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
+                       fft_output_strides);
+    }
 
     desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
                    DFTI_NOT_INPLACE);
@@ -110,12 +140,6 @@ Array<Tc> fft_r2c(const Array<Tr> &in, const int rank) {
     desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE,
                    istrides[rank]);
 
-    const std::int64_t fft_output_strides[5] = {
-        0, ostrides[(rank == 2) ? 1 : 0], ostrides[(rank == 2) ? 0 : 1],
-        ostrides[2], ostrides[3]};
-    desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
-                   fft_output_strides, rank);
-
     desc.commit(getQueue());
     ::oneapi::mkl::dft::compute_forward(desc, *in.get(), *out.get());
 
@@ -139,16 +163,35 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank) {
 
     auto desc = [rank, &odims]() {
         if (rank == 1) return desc_ty(odims[0]);
-        if (rank == 2) return desc_ty({odims[1], odims[0]});
-        if (rank == 3) return desc_ty({odims[2], odims[1], odims[0]});
-        return desc_ty({odims[3], odims[2], odims[1], odims[0]});
+        if (rank == 2) return desc_ty({odims[0], odims[1]});
+        if (rank == 3) return desc_ty({odims[0], odims[1], odims[2]});
+        return desc_ty({odims[0], odims[1], odims[2], odims[3]});
     }();
+    if (rank > 1) {
+        std::int64_t fft_input_strides[5];
+        fft_input_strides[0] = in.getOffset();
+        fft_input_strides[1] = istrides[0];
+        fft_input_strides[2] = istrides[1];
+        fft_input_strides[3] = istrides[2];
+        fft_input_strides[4] = istrides[3];
+        desc.set_value(::oneapi::mkl::dft::config_param::INPUT_STRIDES,
+                       fft_input_strides);
+
+        std::int64_t fft_output_strides[5];
+        fft_output_strides[0] = out.getOffset();
+        fft_output_strides[1] = ostrides[0];
+        fft_output_strides[2] = ostrides[1];
+        fft_output_strides[3] = ostrides[2];
+        fft_output_strides[4] = ostrides[3];
+        desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
+                       fft_output_strides);
+    }
 
     desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
                    DFTI_NOT_INPLACE);
 
     int batch = 1;
-    for (int i = rank; i < 4; i++) { batch *= idims[i]; }
+    for (int i = rank; i < 4; i++) { batch *= odims[i]; }
     desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
                    (int64_t)batch);
 
@@ -157,12 +200,6 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank) {
     desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE,
                    ostrides[rank]);
 
-    const std::int64_t fft_output_strides[5] = {
-        0, ostrides[(rank == 2) ? 1 : 0], ostrides[(rank == 2) ? 0 : 1],
-        ostrides[2], ostrides[3]};
-    desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
-                   fft_output_strides, rank);
-
     desc.commit(getQueue());
     ::oneapi::mkl::dft::compute_backward(desc, *in.get(), *out.get());
     return out;

From 159b744ee041a415685ce00f1c8dd25add023481 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 21 Jun 2023 17:26:54 -0400
Subject: [PATCH 325/473] Update processException to return NO_MEM for out of
 memory exceptions

---
 src/backend/common/err_common.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index 885aa8d5f5..60fc207a63 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -170,7 +170,11 @@ af_err processException() {
         snprintf(oneapi_err_msg, sizeof(oneapi_err_msg),
                  "oneAPI Error (%d): %s", ex.code().value(), ex.what());
 
-        err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL);
+        if (ex.code() == sycl::errc::memory_allocation) {
+            err = set_global_error_string(oneapi_err_msg, AF_ERR_NO_MEM);
+        } else {
+            err = set_global_error_string(oneapi_err_msg, AF_ERR_INTERNAL);
+        }
     } catch (const oneapi::mkl::exception &ex) {
         char oneapi_err_msg[1024];
         snprintf(oneapi_err_msg, sizeof(oneapi_err_msg), "MKL Error: %s",

From 4e5cc2ef717119b81b5371f2f3b795c10f0236c4 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 21 Jun 2023 17:31:03 -0400
Subject: [PATCH 326/473] Fix all memAlloc and memorymanager tests

---
 src/backend/oneapi/memory.cpp   | 25 ++++++++-----------------
 src/backend/oneapi/platform.cpp |  2 +-
 2 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index f2cbab094c..f94b6df5a4 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -62,7 +62,7 @@ template<typename T>
 std::unique_ptr<sycl::buffer<T>, std::function<void(sycl::buffer<T> *)>>
 memAlloc(const size_t &elements) {
     if (elements) {
-        dim4 dims(elements * sizeof(T));
+        dim4 dims(elements);
 
         // The alloc function returns a pointer to a buffer<std::byte> object.
         // We need to reinterpret that object into buffer<T> while keeping the
@@ -71,7 +71,7 @@ memAlloc(const size_t &elements) {
         // This would delete the buffer<std::byte> object and replace it with
         // the buffer<T> object. We do the reverse in the memFree function
         auto *ptr = static_cast<sycl::buffer<std::byte> *>(
-            memoryManager().alloc(false, 1, dims.get(), 1));
+            memoryManager().alloc(false, 1, dims.get(), sizeof(T)));
         sycl::buffer<T> *optr = static_cast<sycl::buffer<T> *>((void *)ptr);
         size_t bytes          = ptr->byte_size();
 
@@ -104,14 +104,7 @@ void memFree(sycl::buffer<T> *ptr) {
     }
 }
 
-void memFreeUser(void *ptr) {
-    ONEAPI_NOT_SUPPORTED("memFreeUser Not supported");
-
-    // cl::Buffer *buf = static_cast<cl::Buffer *>(ptr);
-    // cl_mem mem      = (*buf)();
-    // delete buf;
-    memoryManager().unlock(ptr, true);
-}
+void memFreeUser(void *ptr) { memoryManager().unlock(ptr, true); }
 
 template<typename T>
 void memLock(const sycl::buffer<T> *ptr) {
@@ -169,17 +162,15 @@ INSTANTIATE(int64_t)
 
 template<>
 void *pinnedAlloc<void>(const size_t &elements) {
-    ONEAPI_NOT_SUPPORTED("pinnedAlloc Not supported");
-
-    // // TODO: make pinnedAlloc aware of array shapes
-    // dim4 dims(elements);
-    // void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), sizeof(T));
-    return static_cast<void *>(nullptr);
+    // TODO: make pinnedAlloc aware of array shapes
+    dim4 dims(elements);
+    void *ptr = pinnedMemoryManager().alloc(false, 1, dims.get(), 1);
+    return ptr;
 }
 
 Allocator::Allocator() { logger = common::loggerFactory("mem"); }
 
-void Allocator::shutdown() {}
+void Allocator::shutdown() { shutdownMemoryManager(); }
 
 int Allocator::getActiveDeviceId() { return oneapi::getActiveDeviceId(); }
 
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index d9b6f1d832..a3f6a490e8 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -605,7 +605,7 @@ void setMemoryManager(unique_ptr<MemoryManagerBase> mgr) {
 }
 
 void resetMemoryManager() {
-    return DeviceManager::getInstance().resetMemoryManagerPinned();
+    return DeviceManager::getInstance().resetMemoryManager();
 }
 
 void setMemoryManagerPinned(unique_ptr<MemoryManagerBase> mgr) {

From c56ec51a7cde38cbd8a683a90aa5124388982ff1 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 7 Jun 2023 18:23:04 -0400
Subject: [PATCH 327/473] adds write functions to oneapi backend

---
 src/backend/oneapi/Array.cpp | 32 +++++++++++++-------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 6f506ec2ba..5845d95ecc 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -393,10 +393,6 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     return kJITHeuristics::Pass;
 }
 
-// Doesn't make sense with sycl::buffer
-// TODO: accessors? or return sycl::buffer?
-// TODO: return accessor.get_pointer() for access::target::global_buffer or
-// (host_buffer?)
 template<typename T>
 void *getDevicePtr(const Array<T> &arr) {
     const buffer<T> *buf = arr.device();
@@ -486,15 +482,12 @@ void writeHostDataArray(Array<T> &arr, const T *const data,
     if (!arr.isOwner()) { arr = copyArray<T>(arr); }
     getQueue()
         .submit([&](sycl::handler &h) {
-            buffer<T> &buf = *arr.get();
-            // auto offset_acc = buf.get_access(h, sycl::range, sycl::id<>)
-            // TODO: offset accessor
-            auto offset_acc = buf.get_access(h, sycl::range(arr.elements()));
-            h.copy(data, offset_acc);
+            auto host_acc =
+                arr.get()->template get_access<sycl::access_mode::write>(
+                    h, sycl::range(bytes / sizeof(T)), arr.getOffset());
+            h.copy(data, host_acc);
         })
         .wait();
-    // getQueue().enqueueWriteBuffer(*arr.get(), CL_TRUE, arr.getOffset(),
-    // bytes, data);
 }
 
 template<typename T>
@@ -502,14 +495,15 @@ void writeDeviceDataArray(Array<T> &arr, const void *const data,
                           const size_t bytes) {
     if (!arr.isOwner()) { arr = copyArray<T>(arr); }
 
-    // clRetainMemObject(
-    //    reinterpret_cast<buffer<T> *>(const_cast<void *>(data)));
-    // buffer<T> data_buf =
-    //  buffer<T>(reinterpret_cast<buffer<T>*>(const_cast<void *>(data)));
-
-    ONEAPI_NOT_SUPPORTED("writeDeviceDataArray not supported");
-    // getQueue().enqueueCopyBuffer(data_buf, buf, 0,
-    // static_cast<size_t>(arr.getOffset()), bytes);
+    sycl::buffer<T> *dataptr =
+        static_cast<sycl::buffer<T> *>(const_cast<void *>(data));
+    getQueue().submit([&](sycl::handler &h) {
+        auto src_acc = dataptr->template get_access<sycl::access_mode::read>(
+            h, sycl::range(bytes / sizeof(T)));
+        auto dst_acc = arr.get()->template get_access<sycl::access_mode::write>(
+            h, sycl::range(bytes / sizeof(T)), arr.getOffset());
+        h.copy(src_acc, dst_acc);
+    });
 }
 
 template<typename T>

From aca7f01ecc195ea8f14bd3547a2f169ded4113c8 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 24 Jun 2023 18:31:38 -0400
Subject: [PATCH 328/473] Introduce kNodeType enum to differentiate nodes

---
 src/backend/common/jit/BufferNodeBase.hpp |  6 +++---
 src/backend/common/jit/NaryNode.hpp       |  3 ++-
 src/backend/common/jit/Node.hpp           | 24 +++++++++++++++++++----
 src/backend/common/jit/ScalarNode.hpp     |  6 ++----
 src/backend/common/jit/ShiftNodeBase.hpp  |  4 +++-
 src/backend/cpu/jit/BinaryNode.hpp        |  2 +-
 src/backend/cpu/jit/BufferNode.hpp        |  4 +---
 src/backend/cpu/jit/Node.hpp              |  5 +++--
 src/backend/cpu/jit/ScalarNode.hpp        |  4 +---
 src/backend/cpu/jit/UnaryNode.hpp         |  3 ++-
 src/backend/oneapi/jit.cpp                |  3 +++
 11 files changed, 41 insertions(+), 23 deletions(-)

diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index 061aa37a8c..2e6d29c6d1 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -30,9 +30,9 @@ class BufferNodeBase : public common::Node {
    public:
     ParamType m_param;
     BufferNodeBase(af::dtype type)
-        : Node(type, 0, {}), m_bytes(0), m_linear_buffer(true) {}
-
-    bool isBuffer() const final { return true; }
+        : Node(type, 0, {}, kNodeType::Buffer)
+        , m_bytes(0)
+        , m_linear_buffer(true) {}
 
     std::unique_ptr<Node> clone() final {
         return std::make_unique<BufferNodeBase>(*this);
diff --git a/src/backend/common/jit/NaryNode.hpp b/src/backend/common/jit/NaryNode.hpp
index 0d78b9e86c..5f1e91a570 100644
--- a/src/backend/common/jit/NaryNode.hpp
+++ b/src/backend/common/jit/NaryNode.hpp
@@ -40,7 +40,8 @@ class NaryNode : public Node {
               type, height,
               std::forward<
                   const std::array<common::Node_ptr, Node::kMaxChildren>>(
-                  children))
+                  children),
+              kNodeType::Nary)
         , m_num_children(num_children)
         , m_op_str(op_str)
         , m_op(op) {
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 8a262e0734..3106172dae 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -32,6 +32,15 @@ enum class kJITHeuristics {
 
 namespace arrayfire {
 namespace common {
+
+enum class kNodeType {
+    Generic = 0,
+    Scalar  = 1,
+    Buffer  = 2,
+    Nary    = 3,
+    Shift   = 4,
+};
+
 class Node;
 }  // namespace common
 }  // namespace arrayfire
@@ -122,13 +131,17 @@ class Node {
     std::array<Node_ptr, kMaxChildren> m_children;
     af::dtype m_type;
     int m_height;
+    kNodeType m_node_type = kNodeType::Generic;
 
     template<typename T>
     friend class NodeIterator;
     Node() = default;
     Node(const af::dtype type, const int height,
-         const std::array<Node_ptr, kMaxChildren> children)
-        : m_children(children), m_type(type), m_height(height) {
+         const std::array<Node_ptr, kMaxChildren> children, kNodeType node_type)
+        : m_children(children)
+        , m_type(type)
+        , m_height(height)
+        , m_node_type(node_type) {
         static_assert(std::is_nothrow_move_assignable<Node>::value,
                       "Node is not move assignable");
     }
@@ -249,14 +262,17 @@ class Node {
     virtual size_t getBytes() const { return 0; }
 
     // Returns true if this node is a Buffer
-    virtual bool isBuffer() const { return false; }
+    bool isBuffer() const { return m_node_type == kNodeType::Buffer; }
 
     // Returns true if this node is a Scalar
-    virtual bool isScalar() const { return false; }
+    bool isScalar() const { return m_node_type == kNodeType::Scalar; }
 
     /// Returns true if the buffer is linear
     virtual bool isLinear(const dim_t dims[4]) const;
 
+    /// Returns the node type
+    kNodeType getNodeType() const { return m_node_type; }
+
     /// Returns the type
     af::dtype getType() const { return m_type; }
 
diff --git a/src/backend/common/jit/ScalarNode.hpp b/src/backend/common/jit/ScalarNode.hpp
index 3a530a6911..3dbc98df5d 100644
--- a/src/backend/common/jit/ScalarNode.hpp
+++ b/src/backend/common/jit/ScalarNode.hpp
@@ -26,7 +26,8 @@ class ScalarNode : public common::Node {
 
    public:
     ScalarNode(T val)
-        : Node(static_cast<af::dtype>(af::dtype_traits<T>::af_type), 0, {})
+        : Node(static_cast<af::dtype>(af::dtype_traits<T>::af_type), 0, {},
+               kNodeType::Scalar)
         , m_val(val) {
         static_assert(std::is_nothrow_move_assignable<ScalarNode>::value,
                       "ScalarNode is not move assignable");
@@ -85,9 +86,6 @@ class ScalarNode : public common::Node {
                   << ";\n";
     }
 
-    // Returns true if this node is a Buffer
-    virtual bool isScalar() const { return false; }
-
     std::string getNameStr() const final { return detail::shortname<T>(false); }
 
     // Return the info for the params and the size of the buffers
diff --git a/src/backend/common/jit/ShiftNodeBase.hpp b/src/backend/common/jit/ShiftNodeBase.hpp
index bbc0f5863f..13cd8cb0ac 100644
--- a/src/backend/common/jit/ShiftNodeBase.hpp
+++ b/src/backend/common/jit/ShiftNodeBase.hpp
@@ -32,7 +32,9 @@ class ShiftNodeBase : public Node {
    public:
     ShiftNodeBase(const af::dtype type, std::shared_ptr<BufferNode> buffer_node,
                   const std::array<int, 4> shifts)
-        : Node(type, 0, {}), m_buffer_node(buffer_node), m_shifts(shifts) {
+        : Node(type, 0, {}, kNodeType::Shift)
+        , m_buffer_node(buffer_node)
+        , m_shifts(shifts) {
         static_assert(std::is_nothrow_move_assignable<ShiftNodeBase>::value,
                       "ShiftNode is not move assignable");
         static_assert(std::is_nothrow_move_constructible<ShiftNodeBase>::value,
diff --git a/src/backend/cpu/jit/BinaryNode.hpp b/src/backend/cpu/jit/BinaryNode.hpp
index 8c1cc39d68..d715d15f44 100644
--- a/src/backend/cpu/jit/BinaryNode.hpp
+++ b/src/backend/cpu/jit/BinaryNode.hpp
@@ -32,7 +32,7 @@ class BinaryNode : public TNode<compute_t<To>> {
     BinaryNode(common::Node_ptr lhs, common::Node_ptr rhs)
         : TNode<compute_t<To>>(compute_t<To>(0),
                                std::max(lhs->getHeight(), rhs->getHeight()) + 1,
-                               {{lhs, rhs}}) {}
+                               {{lhs, rhs}}, common::kNodeType::Nary) {}
 
     std::unique_ptr<common::Node> clone() final {
         return std::make_unique<BinaryNode>(*this);
diff --git a/src/backend/cpu/jit/BufferNode.hpp b/src/backend/cpu/jit/BufferNode.hpp
index e6be492b7f..2d53a52486 100644
--- a/src/backend/cpu/jit/BufferNode.hpp
+++ b/src/backend/cpu/jit/BufferNode.hpp
@@ -35,7 +35,7 @@ class BufferNode : public TNode<T> {
 
    public:
     BufferNode()
-        : TNode<T>(T(0), 0, {})
+        : TNode<T>(T(0), 0, {}, common::kNodeType::Buffer)
         , m_bytes(0)
         , m_strides{0, 0, 0, 0}
         , m_dims{0, 0, 0, 0}
@@ -145,8 +145,6 @@ class BufferNode : public TNode<T> {
                dims[3] == m_dims[3];
     }
 
-    bool isBuffer() const final { return true; }
-
     size_t getHash() const noexcept final {
         std::hash<const void *> ptr_hash;
         std::hash<af::dtype> aftype_hash;
diff --git a/src/backend/cpu/jit/Node.hpp b/src/backend/cpu/jit/Node.hpp
index b3914cbc70..c40b0adf92 100644
--- a/src/backend/cpu/jit/Node.hpp
+++ b/src/backend/cpu/jit/Node.hpp
@@ -43,9 +43,10 @@ class TNode : public common::Node {
 
    public:
     TNode(T val, const int height,
-          const std::array<common::Node_ptr, kMaxChildren> &&children)
+          const std::array<common::Node_ptr, kMaxChildren> &&children,
+          common::kNodeType node_type)
         : Node(static_cast<af::dtype>(af::dtype_traits<T>::af_type), height,
-               move(children)) {
+               move(children), node_type) {
         using namespace common;
         m_val.fill(static_cast<compute_t<T>>(val));
     }
diff --git a/src/backend/cpu/jit/ScalarNode.hpp b/src/backend/cpu/jit/ScalarNode.hpp
index a6d7eff5df..05c13cd386 100644
--- a/src/backend/cpu/jit/ScalarNode.hpp
+++ b/src/backend/cpu/jit/ScalarNode.hpp
@@ -20,7 +20,7 @@ namespace jit {
 template<typename T>
 class ScalarNode : public TNode<T> {
    public:
-    ScalarNode(T val) : TNode<T>(val, 0, {}) {}
+    ScalarNode(T val) : TNode<T>(val, 0, {}, common::kNodeType::Scalar) {}
 
     std::unique_ptr<common::Node> clone() final {
         return std::make_unique<ScalarNode>(*this);
@@ -59,8 +59,6 @@ class ScalarNode : public TNode<T> {
         UNUSED(kerStream);
         UNUSED(ids);
     }
-
-    bool isScalar() const final { return true; }
 };
 }  // namespace jit
 }  // namespace cpu
diff --git a/src/backend/cpu/jit/UnaryNode.hpp b/src/backend/cpu/jit/UnaryNode.hpp
index 9ae8e0aa94..5ca37ca8f4 100644
--- a/src/backend/cpu/jit/UnaryNode.hpp
+++ b/src/backend/cpu/jit/UnaryNode.hpp
@@ -34,7 +34,8 @@ class UnaryNode : public TNode<To> {
 
    public:
     UnaryNode(common::Node_ptr child)
-        : TNode<To>(To(0), child->getHeight() + 1, {{child}}) {}
+        : TNode<To>(To(0), child->getHeight() + 1, {{child}},
+                    common::kNodeType::Nary) {}
 
     std::unique_ptr<common::Node> clone() final {
         return std::make_unique<UnaryNode>(*this);
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 546ca233b6..b6a1a5c6d2 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -19,6 +19,7 @@
 #include <common/jit/ModdimNode.hpp>
 #include <common/jit/Node.hpp>
 #include <common/jit/NodeIterator.hpp>
+#include <common/jit/ShiftNodeBase.hpp>
 #include <common/util.hpp>
 #include <copy.hpp>
 #include <device_manager.hpp>
@@ -43,12 +44,14 @@
 
 using arrayfire::common::getFuncName;
 using arrayfire::common::half;
+using arrayfire::common::kNodeType;
 using arrayfire::common::ModdimNode;
 using arrayfire::common::Node;
 using arrayfire::common::Node_ids;
 using arrayfire::common::Node_map_t;
 using arrayfire::common::Node_ptr;
 using arrayfire::common::NodeIterator;
+using arrayfire::common::ShiftNodeBase;
 using arrayfire::oneapi::getActiveDeviceBaseBuildFlags;
 using arrayfire::oneapi::jit::BufferNode;
 

From d50195f7aee92ad2ea3f0908593b0ad89e59a2e6 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 24 Jun 2023 18:35:00 -0400
Subject: [PATCH 329/473] Add is_buffer paramenter to the setArgs functor

---
 src/backend/common/jit/BufferNodeBase.hpp    |  7 ++--
 src/backend/common/jit/Node.hpp              |  8 ++---
 src/backend/common/jit/ScalarNode.hpp        |  5 +--
 src/backend/common/jit/ShiftNodeBase.hpp     |  6 ++--
 src/backend/cpu/jit/BinaryNode.hpp           |  3 +-
 src/backend/cpu/jit/BufferNode.hpp           |  3 +-
 src/backend/cpu/jit/ScalarNode.hpp           |  3 +-
 src/backend/cuda/jit.cpp                     |  3 +-
 src/backend/cuda/jit/kernel_generators.hpp   | 10 +++---
 src/backend/oneapi/jit.cpp                   | 37 ++++++++------------
 src/backend/oneapi/jit/kernel_generators.hpp |  8 +++--
 src/backend/opencl/jit.cpp                   |  9 ++---
 src/backend/opencl/jit/kernel_generators.hpp | 12 ++++---
 13 files changed, 61 insertions(+), 53 deletions(-)

diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index 2e6d29c6d1..fd63e89932 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -71,10 +71,11 @@ class BufferNodeBase : public common::Node {
     }
 
     int setArgs(int start_id, bool is_linear,
-                std::function<void(int id, const void *ptr, size_t arg_size)>
+                std::function<void(int id, const void *ptr, size_t arg_size,
+                                   bool is_buffer)>
                     setArg) const override {
-        return detail::setKernelArguments(start_id, is_linear, setArg, m_data,
-                                          m_param);
+        return detail::setBufferKernelArguments(start_id, is_linear, setArg,
+                                                m_data, m_param);
     }
 
     void genOffsets(std::stringstream &kerStream, int id,
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 3106172dae..42da5a09d3 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -238,10 +238,10 @@ class Node {
     ///
     /// \returns the next index that will need to be set in the kernl. This
     ///          is usually start_id + the number of times setArg is called
-    virtual int setArgs(
-        int start_id, bool is_linear,
-        std::function<void(int id, const void *ptr, size_t arg_size)> setArg)
-        const {
+    virtual int setArgs(int start_id, bool is_linear,
+                        std::function<void(int id, const void *ptr,
+                                           size_t arg_size, bool is_buffer)>
+                            setArg) const {
         UNUSED(is_linear);
         UNUSED(setArg);
         return start_id;
diff --git a/src/backend/common/jit/ScalarNode.hpp b/src/backend/common/jit/ScalarNode.hpp
index 3dbc98df5d..4236ec4725 100644
--- a/src/backend/common/jit/ScalarNode.hpp
+++ b/src/backend/common/jit/ScalarNode.hpp
@@ -73,10 +73,11 @@ class ScalarNode : public common::Node {
     }
 
     int setArgs(int start_id, bool is_linear,
-                std::function<void(int id, const void* ptr, size_t arg_size)>
+                std::function<void(int id, const void* ptr, size_t arg_size,
+                                   bool is_buffer)>
                     setArg) const final {
         UNUSED(is_linear);
-        setArg(start_id, static_cast<const void*>(&m_val), sizeof(T));
+        setArg(start_id, static_cast<const void*>(&m_val), sizeof(T), false);
         return start_id + 1;
     }
 
diff --git a/src/backend/common/jit/ShiftNodeBase.hpp b/src/backend/common/jit/ShiftNodeBase.hpp
index 13cd8cb0ac..9f03e2a5ad 100644
--- a/src/backend/common/jit/ShiftNodeBase.hpp
+++ b/src/backend/common/jit/ShiftNodeBase.hpp
@@ -87,12 +87,14 @@ class ShiftNodeBase : public Node {
     }
 
     int setArgs(int start_id, bool is_linear,
-                std::function<void(int id, const void *ptr, size_t arg_size)>
+                std::function<void(int id, const void *ptr, size_t arg_size,
+                                   bool is_buffer)>
                     setArg) const {
         int curr_id = m_buffer_node->setArgs(start_id, is_linear, setArg);
         for (int i = 0; i < 4; i++) {
             const int &d = m_shifts[i];
-            setArg(curr_id + i, static_cast<const void *>(&d), sizeof(int));
+            setArg(curr_id + i, static_cast<const void *>(&d), sizeof(int),
+                   false);
         }
         return curr_id + 4;
     }
diff --git a/src/backend/cpu/jit/BinaryNode.hpp b/src/backend/cpu/jit/BinaryNode.hpp
index d715d15f44..424e37a63f 100644
--- a/src/backend/cpu/jit/BinaryNode.hpp
+++ b/src/backend/cpu/jit/BinaryNode.hpp
@@ -71,7 +71,8 @@ class BinaryNode : public TNode<compute_t<To>> {
     }
 
     int setArgs(int start_id, bool is_linear,
-                std::function<void(int id, const void *ptr, size_t arg_size)>
+                std::function<void(int id, const void *ptr, size_t arg_size,
+                                   bool is_buffer)>
                     setArg) const override {
         UNUSED(is_linear);
         UNUSED(setArg);
diff --git a/src/backend/cpu/jit/BufferNode.hpp b/src/backend/cpu/jit/BufferNode.hpp
index 2d53a52486..32a94b2a74 100644
--- a/src/backend/cpu/jit/BufferNode.hpp
+++ b/src/backend/cpu/jit/BufferNode.hpp
@@ -119,7 +119,8 @@ class BufferNode : public TNode<T> {
     }
 
     int setArgs(int start_id, bool is_linear,
-                std::function<void(int id, const void *ptr, size_t arg_size)>
+                std::function<void(int id, const void *ptr, size_t arg_size,
+                                   bool is_buffer)>
                     setArg) const override {
         UNUSED(is_linear);
         UNUSED(setArg);
diff --git a/src/backend/cpu/jit/ScalarNode.hpp b/src/backend/cpu/jit/ScalarNode.hpp
index 05c13cd386..0b119deb82 100644
--- a/src/backend/cpu/jit/ScalarNode.hpp
+++ b/src/backend/cpu/jit/ScalarNode.hpp
@@ -40,7 +40,8 @@ class ScalarNode : public TNode<T> {
     }
 
     int setArgs(int start_id, bool is_linear,
-                std::function<void(int id, const void *ptr, size_t arg_size)>
+                std::function<void(int id, const void *ptr, size_t arg_size,
+                                   bool is_buffer)>
                     setArg) const override {
         UNUSED(is_linear);
         UNUSED(setArg);
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 33a80adb50..903c47fe9f 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -508,7 +508,8 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
         vector<void*> args;
         for (const Node* node : full_nodes) {
             node->setArgs(0, is_linear,
-                          [&](int /*id*/, const void* ptr, size_t /*size*/) {
+                          [&](int /*id*/, const void* ptr, size_t /*size*/,
+                              bool /*is_buffer*/) {
                               args.push_back(const_cast<void*>(ptr));
                           });
         }
diff --git a/src/backend/cuda/jit/kernel_generators.hpp b/src/backend/cuda/jit/kernel_generators.hpp
index f675faf4b4..02f58f432d 100644
--- a/src/backend/cuda/jit/kernel_generators.hpp
+++ b/src/backend/cuda/jit/kernel_generators.hpp
@@ -33,15 +33,17 @@ void generateParamDeclaration(std::stringstream& kerStream, int id,
 
 /// Calls the setArg function to set the arguments for a kernel call
 template<typename T>
-int setKernelArguments(
+int setBufferKernelArguments(
     int start_id, bool is_linear,
-    std::function<void(int id, const void* ptr, size_t arg_size)>& setArg,
+    std::function<void(int id, const void* ptr, size_t arg_size,
+                       bool is_buffer)>& setArg,
     const std::shared_ptr<T>& ptr, const Param<T>& info) {
     UNUSED(ptr);
     if (is_linear) {
-        setArg(start_id, static_cast<const void*>(&info.ptr), sizeof(T*));
+        setArg(start_id, static_cast<const void*>(&info.ptr), sizeof(T*), true);
     } else {
-        setArg(start_id, static_cast<const void*>(&info), sizeof(Param<T>));
+        setArg(start_id, static_cast<const void*>(&info), sizeof(Param<T>),
+               true);
     }
     return start_id + 1;
 }
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index b6a1a5c6d2..a0793ff6d3 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -426,12 +426,10 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                                                        : 4);
     }
 
-    // for (auto* node : full_nodes) SHOW(*node);
     //  Keep in global scope, so that the nodes remain active for later
     //  referral in case moddims operations or column elimination have to
-    //  take place
-    //  Avoid all cloning/copying when no moddims node is present (high
-    //  chance)
+    //  take place Avoid all cloning/copying when no moddims node is present
+    //  (high chance)
     if (moddimsFound || emptyColumnsFound) {
         for (const Node_ids& ids : full_ids) {
             auto& children{node_clones[ids.id]->m_children};
@@ -524,15 +522,15 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                         ap, is_linear);
                     int nargs{0};
                     for (Node* node : full_nodes) {
-                        if (node->isBuffer()) {
-                            nargs = node->setArgs(
-                                nargs, is_linear,
-                                [&kernel, &hh, &is_linear](
-                                    int id, const void* ptr, size_t arg_size) {
-                                    AParam<T, sycl::access_mode::read>* info =
-                                        static_cast<AParam<
-                                            T, sycl::access_mode::read>*>(
-                                            const_cast<void*>(ptr));
+                        nargs = node->setArgs(
+                            nargs, is_linear,
+                            [&kernel, &hh, &is_linear](int id, const void* ptr,
+                                                       size_t arg_size,
+                                                       bool is_buffer) {
+                                if (is_buffer) {
+                                    auto* info = static_cast<
+                                        AParam<T, sycl::access_mode::read>*>(
+                                        const_cast<void*>(ptr));
                                     vector<cl_mem> mem =
                                         hh.get_native_mem<backend::opencl>(
                                             info->data);
@@ -552,16 +550,12 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                                                                 sizeof(KParam),
                                                                 &ooo));
                                     }
-                                });
-                        } else {
-                            nargs = node->setArgs(
-                                nargs, is_linear,
-                                [&kernel](int id, const void* ptr,
-                                          size_t arg_size) {
+
+                                } else {
                                     CL_CHECK(clSetKernelArg(kernel, id,
                                                             arg_size, ptr));
-                                });
-                        }
+                                }
+                            });
                     }
 
                     // Set output parameters
@@ -589,7 +583,6 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                                   (size_t)ap[0].dims[2]};
                         ndims  = 3;
                     }
-                    // SHOW(global);
                     cl_event kernel_event;
                     CL_CHECK(clEnqueueNDRangeKernel(
                         q, kernel, ndims, offset.data(), global.data(), nullptr,
diff --git a/src/backend/oneapi/jit/kernel_generators.hpp b/src/backend/oneapi/jit/kernel_generators.hpp
index 5a3321d0a0..9ca9cd984e 100644
--- a/src/backend/oneapi/jit/kernel_generators.hpp
+++ b/src/backend/oneapi/jit/kernel_generators.hpp
@@ -38,12 +38,14 @@ inline void generateParamDeclaration(std::stringstream& kerStream, int id,
 
 /// Calls the setArg function to set the arguments for a kernel call
 template<typename T>
-inline int setKernelArguments(
+inline int setBufferKernelArguments(
     int start_id, bool is_linear,
-    std::function<void(int id, const void* ptr, size_t arg_size)>& setArg,
+    std::function<void(int id, const void* ptr, size_t arg_size,
+                       bool is_buffer)>& setArg,
     const std::shared_ptr<sycl::buffer<T>>& ptr,
     const AParam<T, sycl::access_mode::read>& info) {
-    setArg(start_id + 0, static_cast<const void*>(&info), sizeof(Param<T>));
+    setArg(start_id + 0, static_cast<const void*>(&info),
+           sizeof(AParam<T, sycl::access_mode::read>), true);
     return start_id + 2;
 }
 
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index f7ba973032..727724cc85 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -448,10 +448,11 @@ void evalNodes(vector<Param>& outputs, const vector<Node*>& output_nodes) {
 
     int nargs{0};
     for (const Node* node : full_nodes) {
-        nargs = node->setArgs(nargs, is_linear,
-                              [&ker](int id, const void* ptr, size_t arg_size) {
-                                  ker.setArg(id, arg_size, ptr);
-                              });
+        nargs = node->setArgs(
+            nargs, is_linear,
+            [&ker](int id, const void* ptr, size_t arg_size, bool is_buffer) {
+                ker.setArg(id, arg_size, ptr);
+            });
     }
 
     // Set output parameters
diff --git a/src/backend/opencl/jit/kernel_generators.hpp b/src/backend/opencl/jit/kernel_generators.hpp
index d4700260c4..0228e7173f 100644
--- a/src/backend/opencl/jit/kernel_generators.hpp
+++ b/src/backend/opencl/jit/kernel_generators.hpp
@@ -30,17 +30,19 @@ inline void generateParamDeclaration(std::stringstream& kerStream, int id,
 }
 
 /// Calls the setArg function to set the arguments for a kernel call
-inline int setKernelArguments(
+inline int setBufferKernelArguments(
     int start_id, bool is_linear,
-    std::function<void(int id, const void* ptr, size_t arg_size)>& setArg,
+    std::function<void(int id, const void* ptr, size_t arg_size,
+                       bool is_buffer)>& setArg,
     const std::shared_ptr<cl::Buffer>& ptr, const KParam& info) {
     setArg(start_id + 0, static_cast<const void*>(&ptr.get()->operator()()),
-           sizeof(cl_mem));
+           sizeof(cl_mem), true);
     if (is_linear) {
         setArg(start_id + 1, static_cast<const void*>(&info.offset),
-               sizeof(dim_t));
+               sizeof(dim_t), true);
     } else {
-        setArg(start_id + 1, static_cast<const void*>(&info), sizeof(KParam));
+        setArg(start_id + 1, static_cast<const void*>(&info), sizeof(KParam),
+               true);
     }
     return start_id + 2;
 }

From 86e28ae5f09471b845a02ad9ad59e7e822297522 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 22 Jun 2023 14:25:46 -0400
Subject: [PATCH 330/473] Add support for shift kernel.

---
 src/backend/common/jit/ShiftNodeBase.hpp |  2 ++
 src/backend/oneapi/Param.hpp             |  2 +-
 src/backend/oneapi/jit.cpp               | 14 +++++++++++---
 src/backend/oneapi/shift.cpp             | 11 ++++-------
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/backend/common/jit/ShiftNodeBase.hpp b/src/backend/common/jit/ShiftNodeBase.hpp
index 9f03e2a5ad..106040f693 100644
--- a/src/backend/common/jit/ShiftNodeBase.hpp
+++ b/src/backend/common/jit/ShiftNodeBase.hpp
@@ -65,6 +65,8 @@ class ShiftNodeBase : public Node {
         swap(m_shifts, other.m_shifts);
     }
 
+    const BufferNode &getBufferNode() const { return *m_buffer_node; }
+
     bool isLinear(const dim_t dims[4]) const final {
         UNUSED(dims);
         return false;
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index 7df0a73f85..752a6f7039 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -72,7 +72,7 @@ struct AParam {
         return *data;
     }
 
-    void require(sycl::handler& h) { h.require(data); }
+    void require(sycl::handler& h) const { h.require(data); }
 
     operator KParam() const {
         return KParam{{dims[0], dims[1], dims[2], dims[3]},
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index a0793ff6d3..31c2a0b881 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -494,9 +494,17 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
 
     getQueue().submit([&](sycl::handler& h) {
         for (Node* node : full_nodes) {
-            if (node->isBuffer()) {
-                BufferNode<T>* n = static_cast<BufferNode<T>*>(node);
-                n->m_param.require(h);
+            switch (node->getNodeType()) {
+                case kNodeType::Buffer: {
+                    BufferNode<T>* n = static_cast<BufferNode<T>*>(node);
+                    n->m_param.require(h);
+                } break;
+                case kNodeType::Shift: {
+                    ShiftNodeBase<jit::BufferNode<T>>* sn =
+                        static_cast<ShiftNodeBase<jit::BufferNode<T>>*>(node);
+                    sn->getBufferNode().m_param.require(h);
+                } break;
+                default: break;
             }
         }
         vector<AParam<T, sycl::access_mode::write>> ap;
diff --git a/src/backend/oneapi/shift.cpp b/src/backend/oneapi/shift.cpp
index d72477c770..8a12eb81a8 100644
--- a/src/backend/oneapi/shift.cpp
+++ b/src/backend/oneapi/shift.cpp
@@ -23,13 +23,11 @@ using std::string;
 
 namespace arrayfire {
 namespace oneapi {
+template<typename T>
+using ShiftNode = ShiftNodeBase<jit::BufferNode<T>>;
 
 template<typename T>
 Array<T> shift(const Array<T> &in, const int sdims[4]) {
-    ONEAPI_NOT_SUPPORTED("");
-    Array<T> o = createEmptyArray<T>(dim4(1));
-    return o;
-    /*
     // Shift should only be the first node in the JIT tree.
     // Force input to be evaluated so that in is always a buffer.
     in.eval();
@@ -49,11 +47,10 @@ Array<T> shift(const Array<T> &in, const int sdims[4]) {
         assert(shifts[i] >= 0 && shifts[i] <= oDims[i]);
     }
 
-    auto node = make_shared<ShiftNode>(
+    auto node = make_shared<ShiftNode<T>>(
         static_cast<af::dtype>(dtype_traits<T>::af_type),
-        static_pointer_cast<BufferNode>(in.getNode()), shifts);
+        static_pointer_cast<jit::BufferNode<T>>(in.getNode()), shifts);
     return createNodeArray<T>(oDims, common::Node_ptr(node));
-    */
 }
 
 #define INSTANTIATE(T) \

From 787d92780d30a4b8ccccb4d7bfdc6bd312899d5f Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Mon, 26 Jun 2023 13:03:46 -0400
Subject: [PATCH 331/473] fix: rotate did not pass INTERP_ORDER to Interp2
 class (#3452)

* fix: rotate did not pass INTERP_ORDER to Interp2 class

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
---
 src/backend/oneapi/kernel/rotate.hpp | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/backend/oneapi/kernel/rotate.hpp b/src/backend/oneapi/kernel/rotate.hpp
index a6d255d369..2bb945f9a2 100644
--- a/src/backend/oneapi/kernel/rotate.hpp
+++ b/src/backend/oneapi/kernel/rotate.hpp
@@ -53,8 +53,7 @@ class rotateCreateKernel {
         , batches_(batches)
         , blocksXPerImage_(blocksXPerImage)
         , blocksYPerImage_(blocksYPerImage)
-        , method_(method)
-        , INTERP_ORDER_(INTERP_ORDER) {}
+        , method_(method) {}
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g = it.get_group();
 
@@ -72,7 +71,8 @@ class rotateCreateKernel {
         const int limages =
             std::min((int)out_.dims[2] - setId * nimages_, nimages_);
 
-        if (xido >= out_.dims[0] || yido >= out_.dims[1]) return;
+        if (xido >= (unsigned)out_.dims[0] || yido >= (unsigned)out_.dims[1])
+            return;
 
         InterpPosTy xidi = xido * t_.tmat[0] + yido * t_.tmat[1] + t_.tmat[2];
         InterpPosTy yidi = xido * t_.tmat[3] + yido * t_.tmat[4] + t_.tmat[5];
@@ -85,7 +85,7 @@ class rotateCreateKernel {
         const int loco = outoff + (yido * out_.strides[1] + xido);
 
         InterpInTy zero = (InterpInTy)0;
-        if (INTERP_ORDER_ > 1) {
+        if constexpr (INTERP_ORDER > 1) {
             // Special conditions to deal with boundaries for bilinear and
             // bicubic
             // FIXME: Ideally this condition should be removed or be present for
@@ -102,8 +102,8 @@ class rotateCreateKernel {
 
         // FIXME: Nearest and lower do not do clamping, but other methods do
         // Make it consistent
-        const bool doclamp = INTERP_ORDER_ != 1;
-        Interp2<T, InterpPosTy, 1> interp2;  // INTERP_ORDER> interp2;
+        constexpr bool doclamp = INTERP_ORDER != 1;
+        Interp2<T, InterpPosTy, INTERP_ORDER> interp2;
         interp2(d_out_, out_, loco, d_in_, in_, inoff, xidi, yidi, 0, 1,
                 method_, limages, doclamp, 2);
     }
@@ -119,7 +119,6 @@ class rotateCreateKernel {
     const int blocksXPerImage_;
     const int blocksYPerImage_;
     af::interpType method_;
-    const int INTERP_ORDER_;
 };
 
 template<typename T>

From 6a5ff1f1021330d5ec751c28b3ce3fe26ab6ca01 Mon Sep 17 00:00:00 2001
From: Mike Mullen <96440448+mfzmullen@users.noreply.github.com>
Date: Tue, 4 Jul 2023 19:41:16 -0500
Subject: [PATCH 332/473] Fix cuda_fp16 not finding vector_functions.h (#3461)

* fix cuda_fp16 not finding vector_functions.h

---------

Co-authored-by: Michael Mullen <michael@adialante.com>
---
 src/backend/cuda/CMakeLists.txt     | 1 +
 src/backend/cuda/compile_module.cpp | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 0dc208fd8b..b0b0841b54 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -142,6 +142,7 @@ set(nvrtc_src
   ${CUDA_INCLUDE_DIRS}/cuda_fp16.hpp
   ${CUDA_TOOLKIT_ROOT_DIR}/include/cuComplex.h
   ${CUDA_TOOLKIT_ROOT_DIR}/include/math_constants.h
+  ${CUDA_TOOLKIT_ROOT_DIR}/include/vector_functions.h
 
   ${PROJECT_SOURCE_DIR}/src/api/c/optypes.hpp
   ${PROJECT_SOURCE_DIR}/include/af/defines.h
diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp
index 06dfd0f377..d1d988e66f 100644
--- a/src/backend/cuda/compile_module.cpp
+++ b/src/backend/cuda/compile_module.cpp
@@ -39,6 +39,7 @@
 #include <nvrtc_kernel_headers/traits_hpp.hpp>
 #include <nvrtc_kernel_headers/types_hpp.hpp>
 #include <nvrtc_kernel_headers/utility_hpp.hpp>
+#include <nvrtc_kernel_headers/vector_functions_h.hpp>
 #include <nvrtc_kernel_headers/version_h.hpp>
 #include <optypes.hpp>
 #include <platform.hpp>
@@ -201,6 +202,7 @@ Module compileModule(const string &moduleKey, span<const string> sources,
             "dims_param.hpp",
             "common/internal_enums.hpp",
             "minmax_op.hpp",
+            "vector_functions.h",
         };
 
         constexpr size_t numHeaders = extent<decltype(includeNames)>::value;
@@ -234,6 +236,7 @@ Module compileModule(const string &moduleKey, span<const string> sources,
             string(dims_param_hpp, dims_param_hpp_len),
             string(internal_enums_hpp, internal_enums_hpp_len),
             string(minmax_op_hpp, minmax_op_hpp_len),
+            string(vector_functions_h, vector_functions_h_len),
         }};
 
         static const char *headers[] = {
@@ -251,7 +254,7 @@ Module compileModule(const string &moduleKey, span<const string> sources,
             sourceStrings[22].c_str(), sourceStrings[23].c_str(),
             sourceStrings[24].c_str(), sourceStrings[25].c_str(),
             sourceStrings[26].c_str(), sourceStrings[27].c_str(),
-            sourceStrings[28].c_str()};
+            sourceStrings[28].c_str(), sourceStrings[29].c_str()};
         static_assert(extent<decltype(headers)>::value == numHeaders,
                       "headers array contains fewer sources than includeNames");
         NVRTC_CHECK(nvrtcCreateProgram(&prog, sources[0].c_str(),

From 66d858e37110413b38dc5855de16f72a5b86e951 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 3 Jul 2023 20:17:19 -0400
Subject: [PATCH 333/473] adds fft plan caching, corrects descriptor strides

---
 src/backend/oneapi/fft.cpp      | 284 +++++++++++++++++++-------------
 src/backend/oneapi/fft.hpp      |   1 +
 src/backend/oneapi/onefft.hpp   |  39 +++++
 src/backend/oneapi/platform.cpp |  11 ++
 src/backend/oneapi/platform.hpp |   2 +
 5 files changed, 227 insertions(+), 110 deletions(-)
 create mode 100644 src/backend/oneapi/onefft.hpp

diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
index 5c3621c5e1..3bf15acf0a 100644
--- a/src/backend/oneapi/fft.cpp
+++ b/src/backend/oneapi/fft.cpp
@@ -7,32 +7,164 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-#include <common/dispatch.hpp>
-
 #include <fft.hpp>
 
+#include <common/dispatch.hpp>
 #include <copy.hpp>
 #include <err_oneapi.hpp>
 #include <math.hpp>
 #include <memory.hpp>
+#include <onefft.hpp>
+#include <platform.hpp>
 #include <af/dim4.hpp>
 
-#include <array>
-using std::array;
+#include <oneapi/mkl/dfti.hpp>
+#include <oneapi/mkl/exceptions.hpp>
 
-using af::dim4;
+#include <cstdint>
+#include <memory>
 
-#include <oneapi/mkl/dfti.hpp>
+using std::make_shared;
+
+using af::dim4;
 
 namespace arrayfire {
 namespace oneapi {
 
 void setFFTPlanCacheSize(size_t numPlans) {}
 
-inline array<int, AF_MAX_DIMS> computeDims(const int rank, const dim4 &idims) {
-    array<int, AF_MAX_DIMS> retVal = {};
-    for (int i = 0; i < rank; i++) { retVal[i] = idims[(rank - 1) - i]; }
-    return retVal;
+std::string genPlanHashStr(int rank, ::oneapi::mkl::dft::precision precision,
+                           ::oneapi::mkl::dft::domain domain,
+                           const bool isInPlace, const dim_t *n,
+                           std::int64_t *istrides, int ibatch,
+                           std::int64_t *ostrides, int obatch, int nbatch) {
+    // create the key string
+    char key_str_temp[64];
+    sprintf(key_str_temp, "%d:", rank);
+
+    std::string key_string(key_str_temp);
+
+    if (precision == ::oneapi::mkl::dft::precision::SINGLE) {
+        key_string.append("S:");
+    } else if (precision == ::oneapi::mkl::dft::precision::DOUBLE) {
+        key_string.append("D:");
+    }
+    if (domain == ::oneapi::mkl::dft::domain::REAL) {
+        key_string.append("R:");
+    } else if (domain == ::oneapi::mkl::dft::domain::COMPLEX) {
+        key_string.append("C:");
+    }
+    if (isInPlace) {
+        key_string.append("IIP:");
+    } else {
+        key_string.append("OOP:");
+    }
+
+    for (int r = 0; r < rank; ++r) {
+        sprintf(key_str_temp, "%lld:", n[r]);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    if (istrides != nullptr) {
+        for (int r = 0; r < rank + 1; ++r) {
+            sprintf(key_str_temp, "%ld:", istrides[r]);
+            key_string.append(std::string(key_str_temp));
+        }
+        sprintf(key_str_temp, "%d:", ibatch);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    if (ostrides != nullptr) {
+        for (int r = 0; r < rank + 1; ++r) {
+            sprintf(key_str_temp, "%ld:", ostrides[r]);
+            key_string.append(std::string(key_str_temp));
+        }
+        sprintf(key_str_temp, "%d:", obatch);
+        key_string.append(std::string(key_str_temp));
+    }
+
+    sprintf(key_str_temp, "%d", nbatch);
+    key_string.append(std::string(key_str_temp));
+
+    return key_string;
+}
+
+std::vector<std::int64_t> computeStrides(const int rank, const dim4 istrides,
+                                         const dim_t offset) {
+    if (rank == 2) return {offset, istrides[1], istrides[0]};
+    if (rank == 3) return {offset, istrides[2], istrides[1], istrides[0]};
+    if (rank == 4)
+        return {offset, istrides[3], istrides[2], istrides[1], istrides[0]};
+    return {offset};
+}
+
+template<::oneapi::mkl::dft::precision precision,
+         ::oneapi::mkl::dft::domain domain>
+PlanType findPlan(int rank, const bool isInPlace, const dim_t *idims,
+                  std::int64_t *istrides, int ibatch, std::int64_t *ostrides,
+                  int obatch, int nbatch) {
+    using desc_ty = ::oneapi::mkl::dft::descriptor<precision, domain>;
+
+    std::string key_string =
+        genPlanHashStr(rank, precision, domain, isInPlace, idims, istrides,
+                       ibatch, ostrides, obatch, nbatch);
+
+    PlanCache &planner               = arrayfire::oneapi::fftManager();
+    std::shared_ptr<PlanType> retVal = (planner.find(key_string));
+    if (retVal) { return *retVal; }
+
+    desc_ty *desc = [rank, &idims]() {
+        if (rank == 1) return new desc_ty(static_cast<int64_t>(idims[0]));
+        if (rank == 2) return new desc_ty({idims[1], idims[0]});
+        if (rank == 3) return new desc_ty({idims[2], idims[1], idims[0]});
+        return new desc_ty({idims[3], idims[2], idims[1], idims[0]});
+    }();
+
+    if (rank > 1) {
+        desc->set_value(::oneapi::mkl::dft::config_param::INPUT_STRIDES,
+                        istrides);
+        desc->set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
+                        ostrides);
+    }
+
+    if (isInPlace) {
+        desc->set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
+                        DFTI_INPLACE);
+    } else {
+        desc->set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
+                        DFTI_NOT_INPLACE);
+    }
+
+    desc->set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
+                    (int64_t)nbatch);
+
+    desc->set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE, ibatch);
+    desc->set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE, obatch);
+
+    if constexpr (domain == ::oneapi::mkl::dft::domain::COMPLEX) {
+        desc->set_value(::oneapi::mkl::dft::config_param::COMPLEX_STORAGE,
+                        DFTI_COMPLEX_COMPLEX);
+    } else {
+        desc->set_value(
+            ::oneapi::mkl::dft::config_param::CONJUGATE_EVEN_STORAGE,
+            DFTI_COMPLEX_COMPLEX);
+        desc->set_value(::oneapi::mkl::dft::config_param::PACKED_FORMAT,
+                        DFTI_CCE_FORMAT);
+    }
+
+    try {
+        desc->commit(getQueue());
+    } catch (::oneapi::mkl::device_bad_alloc &e) {
+        // If plan creation fails, clean up the memory we hold on to and try
+        // again
+        arrayfire::oneapi::signalMemoryCleanup();
+        desc->commit(getQueue());
+    }
+
+    // push the plan into plan cache
+    std::shared_ptr<void> ptr(desc);
+    planner.push(key_string, make_shared<PlanType>(ptr));
+    return ptr;
 }
 
 template<typename T>
@@ -48,41 +180,23 @@ void fft_inplace(Array<T> &in, const int rank, const bool direction) {
         ::oneapi::mkl::dft::descriptor<precision,
                                        ::oneapi::mkl::dft::domain::COMPLEX>;
 
-    auto desc = [rank, &idims]() {
-        if (rank == 1) return desc_ty(idims[0]);
-        if (rank == 2) return desc_ty({idims[0], idims[1]});
-        if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]});
-        return desc_ty({idims[0], idims[1], idims[2], idims[3]});
-    }();
-
-    if (rank > 1) {
-        std::int64_t fft_input_strides[5];
-        fft_input_strides[0] = in.getOffset();
-        fft_input_strides[1] = istrides[0];
-        fft_input_strides[2] = istrides[1];
-        fft_input_strides[3] = istrides[2];
-        fft_input_strides[4] = istrides[3];
-        desc.set_value(::oneapi::mkl::dft::config_param::INPUT_STRIDES,
-                       fft_input_strides);
-    }
-
-    desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT, DFTI_INPLACE);
+    std::vector<std::int64_t> fft_input_strides =
+        computeStrides(rank, istrides, in.getOffset());
 
     int batch = 1;
     for (int i = rank; i < 4; i++) { batch *= idims[i]; }
-    desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
-                   (int64_t)batch);
 
-    desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE,
-                   istrides[rank]);
-    desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE,
-                   istrides[rank]);
+    const bool isInPlace = true;
+    PlanType descP = findPlan<precision, ::oneapi::mkl::dft::domain::COMPLEX>(
+        rank, isInPlace, idims.get(), fft_input_strides.data(), istrides[rank],
+        fft_input_strides.data(), istrides[rank], batch);
+
+    desc_ty *desc = (desc_ty *)descP.get();
 
-    desc.commit(getQueue());
     if (direction)
-        ::oneapi::mkl::dft::compute_forward(desc, *in.get());
+        ::oneapi::mkl::dft::compute_forward(*desc, *in.get());
     else
-        ::oneapi::mkl::dft::compute_backward(desc, *in.get());
+        ::oneapi::mkl::dft::compute_backward(*desc, *in.get());
 }
 
 template<typename Tc, typename Tr>
@@ -101,47 +215,22 @@ Array<Tc> fft_r2c(const Array<Tr> &in, const int rank) {
         ::oneapi::mkl::dft::descriptor<precision,
                                        ::oneapi::mkl::dft::domain::REAL>;
 
-    auto desc = [rank, &idims]() {
-        if (rank == 1) return desc_ty(idims[0]);
-        if (rank == 2) return desc_ty({idims[0], idims[1]});
-        if (rank == 3) return desc_ty({idims[0], idims[1], idims[2]});
-        return desc_ty({idims[0], idims[1], idims[2], idims[3]});
-    }();
-    if (rank > 1) {
-        std::int64_t fft_input_strides[5];
-        fft_input_strides[0] = in.getOffset();
-        fft_input_strides[1] = istrides[0];
-        fft_input_strides[2] = istrides[1];
-        fft_input_strides[3] = istrides[2];
-        fft_input_strides[4] = istrides[3];
-        desc.set_value(::oneapi::mkl::dft::config_param::INPUT_STRIDES,
-                       fft_input_strides);
-
-        std::int64_t fft_output_strides[5];
-        fft_output_strides[0] = out.getOffset();
-        fft_output_strides[1] = ostrides[0];
-        fft_output_strides[2] = ostrides[1];
-        fft_output_strides[3] = ostrides[2];
-        fft_output_strides[4] = ostrides[3];
-        desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
-                       fft_output_strides);
-    }
-
-    desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
-                   DFTI_NOT_INPLACE);
+    std::vector<std::int64_t> fft_input_strides =
+        computeStrides(rank, istrides, in.getOffset());
+    std::vector<std::int64_t> fft_output_strides =
+        computeStrides(rank, ostrides, out.getOffset());
 
     int batch = 1;
     for (int i = rank; i < 4; i++) { batch *= idims[i]; }
-    desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
-                   (int64_t)batch);
 
-    desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE,
-                   ostrides[rank]);
-    desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE,
-                   istrides[rank]);
+    const bool isInPlace = false;
+    PlanType descP = findPlan<precision, ::oneapi::mkl::dft::domain::REAL>(
+        rank, isInPlace, idims.get(), fft_input_strides.data(), istrides[rank],
+        fft_output_strides.data(), ostrides[rank], batch);
 
-    desc.commit(getQueue());
-    ::oneapi::mkl::dft::compute_forward(desc, *in.get(), *out.get());
+    desc_ty *desc = (desc_ty *)descP.get();
+
+    ::oneapi::mkl::dft::compute_forward(*desc, *in.get(), *out.get());
 
     return out;
 }
@@ -161,47 +250,22 @@ Array<Tr> fft_c2r(const Array<Tc> &in, const dim4 &odims, const int rank) {
         ::oneapi::mkl::dft::descriptor<precision,
                                        ::oneapi::mkl::dft::domain::REAL>;
 
-    auto desc = [rank, &odims]() {
-        if (rank == 1) return desc_ty(odims[0]);
-        if (rank == 2) return desc_ty({odims[0], odims[1]});
-        if (rank == 3) return desc_ty({odims[0], odims[1], odims[2]});
-        return desc_ty({odims[0], odims[1], odims[2], odims[3]});
-    }();
-    if (rank > 1) {
-        std::int64_t fft_input_strides[5];
-        fft_input_strides[0] = in.getOffset();
-        fft_input_strides[1] = istrides[0];
-        fft_input_strides[2] = istrides[1];
-        fft_input_strides[3] = istrides[2];
-        fft_input_strides[4] = istrides[3];
-        desc.set_value(::oneapi::mkl::dft::config_param::INPUT_STRIDES,
-                       fft_input_strides);
-
-        std::int64_t fft_output_strides[5];
-        fft_output_strides[0] = out.getOffset();
-        fft_output_strides[1] = ostrides[0];
-        fft_output_strides[2] = ostrides[1];
-        fft_output_strides[3] = ostrides[2];
-        fft_output_strides[4] = ostrides[3];
-        desc.set_value(::oneapi::mkl::dft::config_param::OUTPUT_STRIDES,
-                       fft_output_strides);
-    }
-
-    desc.set_value(::oneapi::mkl::dft::config_param::PLACEMENT,
-                   DFTI_NOT_INPLACE);
+    std::vector<std::int64_t> fft_input_strides =
+        computeStrides(rank, istrides, in.getOffset());
+    std::vector<std::int64_t> fft_output_strides =
+        computeStrides(rank, ostrides, out.getOffset());
 
     int batch = 1;
     for (int i = rank; i < 4; i++) { batch *= odims[i]; }
-    desc.set_value(::oneapi::mkl::dft::config_param::NUMBER_OF_TRANSFORMS,
-                   (int64_t)batch);
 
-    desc.set_value(::oneapi::mkl::dft::config_param::BWD_DISTANCE,
-                   istrides[rank]);
-    desc.set_value(::oneapi::mkl::dft::config_param::FWD_DISTANCE,
-                   ostrides[rank]);
+    const bool isInPlace = false;
+    PlanType descP = findPlan<precision, ::oneapi::mkl::dft::domain::REAL>(
+        rank, isInPlace, odims.get(), fft_input_strides.data(), ostrides[rank],
+        fft_output_strides.data(), istrides[rank], batch);
+
+    desc_ty *desc = (desc_ty *)descP.get();
 
-    desc.commit(getQueue());
-    ::oneapi::mkl::dft::compute_backward(desc, *in.get(), *out.get());
+    ::oneapi::mkl::dft::compute_backward(*desc, *in.get(), *out.get());
     return out;
 }
 
diff --git a/src/backend/oneapi/fft.hpp b/src/backend/oneapi/fft.hpp
index 0138970ba9..ca82f06118 100644
--- a/src/backend/oneapi/fft.hpp
+++ b/src/backend/oneapi/fft.hpp
@@ -6,6 +6,7 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
+#pragma once
 
 #include <Array.hpp>
 
diff --git a/src/backend/oneapi/onefft.hpp b/src/backend/oneapi/onefft.hpp
new file mode 100644
index 0000000000..a31a91d1e1
--- /dev/null
+++ b/src/backend/oneapi/onefft.hpp
@@ -0,0 +1,39 @@
+/*******************************************************
+ * Copyright (c) 2016, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <common/FFTPlanCache.hpp>
+#include <memory.hpp>
+#include <oneapi/mkl/dfti.hpp>
+
+#include <cstdint>
+
+namespace arrayfire {
+namespace oneapi {
+
+using ::oneapi::mkl::dft::domain;
+using ::oneapi::mkl::dft::precision;
+
+using PlanType   = std::shared_ptr<void>;
+using SharedPlan = std::shared_ptr<PlanType>;
+
+template<precision p, domain d>
+PlanType findPlan(int rank, const bool isInPlace, int *n,
+                  std::int64_t *istrides, int ibatch, std::int64_t *ostrides,
+                  int obatch, int nbatch);
+
+class PlanCache : public common::FFTPlanCache<PlanCache, PlanType> {
+    template<precision p, domain d>
+    friend PlanType findPlan(int rank, const bool isInPlace, int *n,
+                             std::int64_t *istrides, int ibatch,
+                             std::int64_t *ostrides, int obatch, int nbatch);
+};
+
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index a3f6a490e8..91e307d56c 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -21,6 +21,7 @@
 #include <err_oneapi.hpp>
 #include <errorcodes.hpp>
 #include <memory.hpp>
+#include <onefft.hpp>
 #include <af/oneapi.h>
 #include <af/version.h>
 
@@ -634,6 +635,16 @@ GraphicsResourceManager& interopManager() {
     return *(inst.gfxManagers[id].get());
 }
 
+unique_ptr<PlanCache>& oneFFTManager(const int deviceId) {
+    thread_local unique_ptr<PlanCache> caches[DeviceManager::MAX_DEVICES];
+    thread_local once_flag initFlags[DeviceManager::MAX_DEVICES];
+    call_once(initFlags[deviceId],
+              [&] { caches[deviceId] = make_unique<PlanCache>(); });
+    return caches[deviceId];
+}
+
+PlanCache& fftManager() { return *oneFFTManager(getActiveDeviceId()); }
+
 }  // namespace oneapi
 }  // namespace arrayfire
 
diff --git a/src/backend/oneapi/platform.hpp b/src/backend/oneapi/platform.hpp
index 86439a685c..bceb1e5db6 100644
--- a/src/backend/oneapi/platform.hpp
+++ b/src/backend/oneapi/platform.hpp
@@ -131,6 +131,8 @@ arrayfire::common::ForgeManager& forgeManager();
 
 GraphicsResourceManager& interopManager();
 
+PlanCache& fftManager();
+
 // afcl::platform getPlatformEnum(cl::Device dev);
 
 void setActiveContext(int device);

From d29ed794442ff190d057f087f59cd182f1b02a90 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 4 Jul 2023 16:40:09 -0400
Subject: [PATCH 334/473] Fix missing try/catch in C API layers

---
 src/api/c/binary.cpp      | 140 +++++++++++++++++++++-----------------
 src/api/c/fftconvolve.cpp |  22 +++---
 src/api/c/plot.cpp        |  34 ++++++---
 src/api/c/type_util.cpp   |   7 +-
 4 files changed, 119 insertions(+), 84 deletions(-)

diff --git a/src/api/c/binary.cpp b/src/api/c/binary.cpp
index 566a4b22b5..dc5eddf4bc 100644
--- a/src/api/c/binary.cpp
+++ b/src/api/c/binary.cpp
@@ -277,87 +277,101 @@ static af_err af_arith_sparse_dense(af_array *out, const af_array lhs,
 
 af_err af_add(af_array *out, const af_array lhs, const af_array rhs,
               const bool batchMode) {
-    // Check if inputs are sparse
-    const ArrayInfo &linfo = getInfo(lhs, false, true);
-    const ArrayInfo &rinfo = getInfo(rhs, false, true);
+    try {
+        // Check if inputs are sparse
+        const ArrayInfo &linfo = getInfo(lhs, false, true);
+        const ArrayInfo &rinfo = getInfo(rhs, false, true);
 
-    if (linfo.isSparse() && rinfo.isSparse()) {
-        return af_arith_sparse<af_add_t>(out, lhs, rhs);
-    }
-    if (linfo.isSparse() && !rinfo.isSparse()) {
-        return af_arith_sparse_dense<af_add_t>(out, lhs, rhs);
-    }
-    if (!linfo.isSparse() && rinfo.isSparse()) {
-        // second operand(Array) of af_arith call should be dense
-        return af_arith_sparse_dense<af_add_t>(out, rhs, lhs, true);
+        if (linfo.isSparse() && rinfo.isSparse()) {
+            return af_arith_sparse<af_add_t>(out, lhs, rhs);
+        }
+        if (linfo.isSparse() && !rinfo.isSparse()) {
+            return af_arith_sparse_dense<af_add_t>(out, lhs, rhs);
+        }
+        if (!linfo.isSparse() && rinfo.isSparse()) {
+            // second operand(Array) of af_arith call should be dense
+            return af_arith_sparse_dense<af_add_t>(out, rhs, lhs, true);
+        }
+        return af_arith<af_add_t>(out, lhs, rhs, batchMode);
     }
-    return af_arith<af_add_t>(out, lhs, rhs, batchMode);
+    CATCHALL;
 }
 
 af_err af_mul(af_array *out, const af_array lhs, const af_array rhs,
               const bool batchMode) {
-    // Check if inputs are sparse
-    const ArrayInfo &linfo = getInfo(lhs, false, true);
-    const ArrayInfo &rinfo = getInfo(rhs, false, true);
-
-    if (linfo.isSparse() && rinfo.isSparse()) {
-        // return af_arith_sparse<af_mul_t>(out, lhs, rhs);
-        // MKL doesn't have mul or div support yet, hence
-        // this is commented out although alternative cpu code exists
-        return AF_ERR_NOT_SUPPORTED;
-    }
-    if (linfo.isSparse() && !rinfo.isSparse()) {
-        return af_arith_sparse_dense<af_mul_t>(out, lhs, rhs);
-    }
-    if (!linfo.isSparse() && rinfo.isSparse()) {
-        return af_arith_sparse_dense<af_mul_t>(out, rhs, lhs,
-                                               true);  // dense should be rhs
+    try {
+        // Check if inputs are sparse
+        const ArrayInfo &linfo = getInfo(lhs, false, true);
+        const ArrayInfo &rinfo = getInfo(rhs, false, true);
+
+        if (linfo.isSparse() && rinfo.isSparse()) {
+            // return af_arith_sparse<af_mul_t>(out, lhs, rhs);
+            // MKL doesn't have mul or div support yet, hence
+            // this is commented out although alternative cpu code exists
+            return AF_ERR_NOT_SUPPORTED;
+        }
+        if (linfo.isSparse() && !rinfo.isSparse()) {
+            return af_arith_sparse_dense<af_mul_t>(out, lhs, rhs);
+        }
+        if (!linfo.isSparse() && rinfo.isSparse()) {
+            return af_arith_sparse_dense<af_mul_t>(
+                out, rhs, lhs,
+                true);  // dense should be rhs
+        }
+        return af_arith<af_mul_t>(out, lhs, rhs, batchMode);
     }
-    return af_arith<af_mul_t>(out, lhs, rhs, batchMode);
+    CATCHALL;
 }
 
 af_err af_sub(af_array *out, const af_array lhs, const af_array rhs,
               const bool batchMode) {
-    // Check if inputs are sparse
-    const ArrayInfo &linfo = getInfo(lhs, false, true);
-    const ArrayInfo &rinfo = getInfo(rhs, false, true);
+    try {
+        // Check if inputs are sparse
+        const ArrayInfo &linfo = getInfo(lhs, false, true);
+        const ArrayInfo &rinfo = getInfo(rhs, false, true);
 
-    if (linfo.isSparse() && rinfo.isSparse()) {
-        return af_arith_sparse<af_sub_t>(out, lhs, rhs);
-    }
-    if (linfo.isSparse() && !rinfo.isSparse()) {
-        return af_arith_sparse_dense<af_sub_t>(out, lhs, rhs);
-    }
-    if (!linfo.isSparse() && rinfo.isSparse()) {
-        return af_arith_sparse_dense<af_sub_t>(out, rhs, lhs,
-                                               true);  // dense should be rhs
+        if (linfo.isSparse() && rinfo.isSparse()) {
+            return af_arith_sparse<af_sub_t>(out, lhs, rhs);
+        }
+        if (linfo.isSparse() && !rinfo.isSparse()) {
+            return af_arith_sparse_dense<af_sub_t>(out, lhs, rhs);
+        }
+        if (!linfo.isSparse() && rinfo.isSparse()) {
+            return af_arith_sparse_dense<af_sub_t>(
+                out, rhs, lhs,
+                true);  // dense should be rhs
+        }
+        return af_arith<af_sub_t>(out, lhs, rhs, batchMode);
     }
-    return af_arith<af_sub_t>(out, lhs, rhs, batchMode);
+    CATCHALL;
 }
 
 af_err af_div(af_array *out, const af_array lhs, const af_array rhs,
               const bool batchMode) {
-    // Check if inputs are sparse
-    const ArrayInfo &linfo = getInfo(lhs, false, true);
-    const ArrayInfo &rinfo = getInfo(rhs, false, true);
-
-    if (linfo.isSparse() && rinfo.isSparse()) {
-        // return af_arith_sparse<af_div_t>(out, lhs, rhs);
-        // MKL doesn't have mul or div support yet, hence
-        // this is commented out although alternative cpu code exists
-        return AF_ERR_NOT_SUPPORTED;
-    }
-    if (linfo.isSparse() && !rinfo.isSparse()) {
-        return af_arith_sparse_dense<af_div_t>(out, lhs, rhs);
-    }
-    if (!linfo.isSparse() && rinfo.isSparse()) {
-        // Division by sparse is currently not allowed - for convinence of
-        // dealing with division by 0
-        // return af_arith_sparse_dense<af_div_t>(out, rhs, lhs, true); // dense
-        // should be rhs
-        return AF_ERR_NOT_SUPPORTED;
+    try {
+        // Check if inputs are sparse
+        const ArrayInfo &linfo = getInfo(lhs, false, true);
+        const ArrayInfo &rinfo = getInfo(rhs, false, true);
+
+        if (linfo.isSparse() && rinfo.isSparse()) {
+            // return af_arith_sparse<af_div_t>(out, lhs, rhs);
+            // MKL doesn't have mul or div support yet, hence
+            // this is commented out although alternative cpu code exists
+            return AF_ERR_NOT_SUPPORTED;
+        }
+        if (linfo.isSparse() && !rinfo.isSparse()) {
+            return af_arith_sparse_dense<af_div_t>(out, lhs, rhs);
+        }
+        if (!linfo.isSparse() && rinfo.isSparse()) {
+            // Division by sparse is currently not allowed - for convinence of
+            // dealing with division by 0
+            // return af_arith_sparse_dense<af_div_t>(out, rhs, lhs, true); //
+            // dense should be rhs
+            return AF_ERR_NOT_SUPPORTED;
+        }
+        return af_arith<af_div_t>(out, lhs, rhs, batchMode);
     }
-    return af_arith<af_div_t>(out, lhs, rhs, batchMode);
+    CATCHALL;
 }
 
 af_err af_maxof(af_array *out, const af_array lhs, const af_array rhs,
diff --git a/src/api/c/fftconvolve.cpp b/src/api/c/fftconvolve.cpp
index f92a3fc655..5e69d5d0ce 100644
--- a/src/api/c/fftconvolve.cpp
+++ b/src/api/c/fftconvolve.cpp
@@ -239,18 +239,24 @@ af_err af_fft_convolve1(af_array *out, const af_array signal,
 
 af_err af_fft_convolve2(af_array *out, const af_array signal,
                         const af_array filter, const af_conv_mode mode) {
-    if (getInfo(signal).dims().ndims() < 2 &&
-        getInfo(filter).dims().ndims() < 2) {
-        return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 1);
+    try {
+        if (getInfo(signal).dims().ndims() < 2 &&
+            getInfo(filter).dims().ndims() < 2) {
+            return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 1);
+        }
+        return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 2);
     }
-    return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 2);
+    CATCHALL;
 }
 
 af_err af_fft_convolve3(af_array *out, const af_array signal,
                         const af_array filter, const af_conv_mode mode) {
-    if (getInfo(signal).dims().ndims() < 3 &&
-        getInfo(filter).dims().ndims() < 3) {
-        return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 2);
+    try {
+        if (getInfo(signal).dims().ndims() < 3 &&
+            getInfo(filter).dims().ndims() < 3) {
+            return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 2);
+        }
+        return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 3);
     }
-    return fft_convolve(out, signal, filter, mode == AF_CONV_EXPAND, 3);
+    CATCHALL;
 }
diff --git a/src/api/c/plot.cpp b/src/api/c/plot.cpp
index 3cf03d05cf..c2d954d481 100644
--- a/src/api/c/plot.cpp
+++ b/src/api/c/plot.cpp
@@ -385,40 +385,52 @@ af_err af_draw_plot3(const af_window wind, const af_array P,
 af_err af_draw_scatter_nd(const af_window wind, const af_array in,
                           const af_marker_type af_marker,
                           const af_cell* const props) {
-    fg_marker_type fg_marker = getFGMarker(af_marker);
-    return plotWrapper(wind, in, 1, props, FG_PLOT_SCATTER, fg_marker);
+    try {
+        fg_marker_type fg_marker = getFGMarker(af_marker);
+        return plotWrapper(wind, in, 1, props, FG_PLOT_SCATTER, fg_marker);
+    }
+    CATCHALL;
 }
 
 af_err af_draw_scatter_2d(const af_window wind, const af_array X,
                           const af_array Y, const af_marker_type af_marker,
                           const af_cell* const props) {
-    fg_marker_type fg_marker = getFGMarker(af_marker);
-    return plotWrapper(wind, X, Y, props, FG_PLOT_SCATTER, fg_marker);
+    try {
+        fg_marker_type fg_marker = getFGMarker(af_marker);
+        return plotWrapper(wind, X, Y, props, FG_PLOT_SCATTER, fg_marker);
+    }
+    CATCHALL;
 }
 
 af_err af_draw_scatter_3d(const af_window wind, const af_array X,
                           const af_array Y, const af_array Z,
                           const af_marker_type af_marker,
                           const af_cell* const props) {
-    fg_marker_type fg_marker = getFGMarker(af_marker);
-    return plotWrapper(wind, X, Y, Z, props, FG_PLOT_SCATTER, fg_marker);
+    try {
+        fg_marker_type fg_marker = getFGMarker(af_marker);
+        return plotWrapper(wind, X, Y, Z, props, FG_PLOT_SCATTER, fg_marker);
+    }
+    CATCHALL;
 }
 
 // Deprecated Scatter API
 af_err af_draw_scatter(const af_window wind, const af_array X, const af_array Y,
                        const af_marker_type af_marker,
                        const af_cell* const props) {
-    fg_marker_type fg_marker = getFGMarker(af_marker);
-    return plotWrapper(wind, X, Y, props, FG_PLOT_SCATTER, fg_marker);
+    try {
+        fg_marker_type fg_marker = getFGMarker(af_marker);
+        return plotWrapper(wind, X, Y, props, FG_PLOT_SCATTER, fg_marker);
+    }
+    CATCHALL;
 }
 
 af_err af_draw_scatter3(const af_window wind, const af_array P,
                         const af_marker_type af_marker,
                         const af_cell* const props) {
-    fg_marker_type fg_marker = getFGMarker(af_marker);
     try {
-        const ArrayInfo& info = getInfo(P);
-        af::dim4 dims         = info.dims();
+        fg_marker_type fg_marker = getFGMarker(af_marker);
+        const ArrayInfo& info    = getInfo(P);
+        af::dim4 dims            = info.dims();
 
         if (dims.ndims() == 2 && dims[1] == 3) {
             return plotWrapper(wind, P, 1, props, FG_PLOT_SCATTER, fg_marker);
diff --git a/src/api/c/type_util.cpp b/src/api/c/type_util.cpp
index 4b70df3295..c78b85b1da 100644
--- a/src/api/c/type_util.cpp
+++ b/src/api/c/type_util.cpp
@@ -38,6 +38,9 @@ size_t size_of(af_dtype type) {
 }
 
 af_err af_get_size_of(size_t *size, af_dtype type) {
-    *size = size_of(type);
-    return AF_SUCCESS;
+    try {
+        *size = size_of(type);
+        return AF_SUCCESS;
+    }
+    CATCHALL;
 }

From e02bb301579760ad1f954f7944307d8e8a9694e4 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 28 Jun 2023 14:36:15 -0400
Subject: [PATCH 335/473] adds qr to oneapi backend

---
 src/backend/oneapi/qr.cpp | 156 +++++++++++++++++++++-----------------
 1 file changed, 86 insertions(+), 70 deletions(-)

diff --git a/src/backend/oneapi/qr.cpp b/src/backend/oneapi/qr.cpp
index 32bf559f4c..64884e4c24 100644
--- a/src/backend/oneapi/qr.cpp
+++ b/src/backend/oneapi/qr.cpp
@@ -11,94 +11,110 @@
 
 #include <err_oneapi.hpp>
 
-#if defined(WITH_LINEAR_ALGEBRA) && !defined(AF_ONEAPI)
+#if defined(WITH_LINEAR_ALGEBRA)
 
 #include <blas.hpp>
 #include <copy.hpp>
-#include <cpu/cpu_qr.hpp>
 #include <identity.hpp>
-// #include <kernel/triangle.hpp>
-#include <magma/magma.h>
-#include <magma/magma_data.h>
-#include <magma/magma_helper.h>
+#include <kernel/triangle.hpp>
+#include <memory.hpp>
+#include <oneapi/mkl/lapack.hpp>
 #include <platform.hpp>
 
 namespace arrayfire {
 namespace oneapi {
 
-template<typename T>
-void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &orig) {
-    if (OpenCLCPUOffload()) { return cpu::qr(q, r, t, orig); }
-
-    const dim4 NullShape(0, 0, 0, 0);
+using sycl::buffer;
 
-    dim4 iDims = orig.dims();
+template<typename T>
+void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in) {
+    dim4 iDims = in.dims();
     int M      = iDims[0];
     int N      = iDims[1];
 
-    dim4 endPadding(M - iDims[0], max(M, N) - iDims[1], 0, 0);
-    Array<T> in =
-        (endPadding == NullShape
-             ? copyArray(orig)
-             : padArrayBorders(orig, NullShape, endPadding, AF_PAD_ZERO));
-    in.resetDims(iDims);
-
-    int MN = std::min(M, N);
-    int NB = magma_get_geqrf_nb<T>(M);
-
-    int NUM      = (2 * MN + ((N + 31) / 32) * 32) * NB;
-    Array<T> tmp = createEmptyArray<T>(dim4(NUM));
-
-    std::vector<T> h_tau(MN);
-
-    int info           = 0;
-    cl::Buffer *in_buf = in.get();
-    cl::Buffer *dT     = tmp.get();
-
-    magma_geqrf3_gpu<T>(M, N, (*in_buf)(), in.getOffset(), in.strides()[1],
-                        &h_tau[0], (*dT)(), tmp.getOffset(), getQueue()(),
-                        &info);
-
-    r = createEmptyArray<T>(in.dims());
-    kernel::triangle<T>(r, in, true, false);
-
-    cl::Buffer *r_buf = r.get();
-    magmablas_swapdblk<T>(MN - 1, NB, (*r_buf)(), r.getOffset(), r.strides()[1],
-                          1, (*dT)(), tmp.getOffset() + MN * NB, NB, 0,
-                          getQueue()());
-
-    q = in;  // No need to copy
+    Array<T> in_copy = copyArray<T>(in);
+
+    // Get workspace needed for QR
+    std::int64_t scratchpad_size =
+        ::oneapi::mkl::lapack::geqrf_scratchpad_size<compute_t<T>>(
+            getQueue(), iDims[0], iDims[1], in_copy.strides()[1]);
+
+    auto scratchpad = memAlloc<compute_t<T>>(scratchpad_size);
+
+    t = createEmptyArray<T>(af::dim4(min(M, N), 1, 1, 1));
+
+    buffer<compute_t<T>> iBuf =
+        in_copy.template getBufferWithOffset<compute_t<T>>();
+    buffer<compute_t<T>> tBuf = t.template getBufferWithOffset<compute_t<T>>();
+    ::oneapi::mkl::lapack::geqrf(getQueue(), M, N, iBuf, in_copy.strides()[1],
+                                 tBuf, *scratchpad, scratchpad->size());
+    // SPLIT into q and r
+    dim4 rdims(M, N);
+    r = createEmptyArray<T>(rdims);
+
+    constexpr bool is_upper     = true;
+    constexpr bool is_unit_diag = false;
+    kernel::triangle<T>(r, in_copy, is_upper, is_unit_diag);
+
+    int mn = max(M, N);
+    dim4 qdims(M, mn);
+    q = identity<T>(qdims);
+
+    buffer<compute_t<T>> qBuf = q.template getBufferWithOffset<compute_t<T>>();
+    if constexpr (std::is_floating_point<compute_t<T>>()) {
+        std::int64_t scratchpad_size =
+            ::oneapi::mkl::lapack::ormqr_scratchpad_size<compute_t<T>>(
+                getQueue(), ::oneapi::mkl::side::left,
+                ::oneapi::mkl::transpose::nontrans, q.dims()[0], q.dims()[1],
+                min(M, N), in_copy.strides()[1], q.strides()[1]);
+
+        auto scratchpad_ormqr = memAlloc<compute_t<T>>(scratchpad_size);
+        ::oneapi::mkl::lapack::ormqr(
+            getQueue(), ::oneapi::mkl::side::left,
+            ::oneapi::mkl::transpose::nontrans, q.dims()[0], q.dims()[1],
+            min(M, N), iBuf, in_copy.strides()[1], tBuf, qBuf, q.strides()[1],
+            *scratchpad_ormqr, scratchpad_ormqr->size());
+
+    } else if constexpr (common::isComplex(static_cast<af::dtype>(
+                             dtype_traits<compute_t<T>>::af_type))) {
+        std::int64_t scratchpad_size =
+            ::oneapi::mkl::lapack::unmqr_scratchpad_size<compute_t<T>>(
+                getQueue(), ::oneapi::mkl::side::left,
+                ::oneapi::mkl::transpose::nontrans, q.dims()[0], q.dims()[1],
+                min(M, N), in_copy.strides()[1], q.strides()[1]);
+
+        auto scratchpad_ormqr = memAlloc<compute_t<T>>(scratchpad_size);
+        ::oneapi::mkl::lapack::unmqr(
+            getQueue(), ::oneapi::mkl::side::left,
+            ::oneapi::mkl::transpose::nontrans, q.dims()[0], q.dims()[1],
+            min(M, N), iBuf, in_copy.strides()[1], tBuf, qBuf, q.strides()[1],
+            *scratchpad_ormqr, scratchpad_ormqr->size());
+    }
     q.resetDims(dim4(M, M));
-    cl::Buffer *q_buf = q.get();
-
-    magma_ungqr_gpu<T>(q.dims()[0], q.dims()[1], std::min(M, N), (*q_buf)(),
-                       q.getOffset(), q.strides()[1], &h_tau[0], (*dT)(),
-                       tmp.getOffset(), NB, getQueue()(), &info);
-
-    t = createHostDataArray(dim4(MN), &h_tau[0]);
 }
 
 template<typename T>
 Array<T> qr_inplace(Array<T> &in) {
-    if (OpenCLCPUOffload()) { return cpu::qr_inplace(in); }
-
-    dim4 iDims = in.dims();
-    int M      = iDims[0];
-    int N      = iDims[1];
-    int MN     = std::min(M, N);
-
-    getQueue().finish();  // FIXME: Does this need to be here?
-    cl::CommandQueue Queue2(getContext(), getDevice());
-    cl_command_queue queues[] = {getQueue()(), Queue2()};
-
-    std::vector<T> h_tau(MN);
-    cl::Buffer *in_buf = in.get();
-
-    int info = 0;
-    magma_geqrf2_gpu<T>(M, N, (*in_buf)(), in.getOffset(), in.strides()[1],
-                        &h_tau[0], queues, &info);
-
-    Array<T> t = createHostDataArray(dim4(MN), &h_tau[0]);
+    dim4 iDims    = in.dims();
+    dim4 iStrides = in.strides();
+    int M         = iDims[0];
+    int N         = iDims[1];
+
+    Array<T> t = createEmptyArray<T>(af::dim4(min(M, N), 1, 1, 1));
+
+    // Get workspace needed for QR
+    std::int64_t scratchpad_size =
+        ::oneapi::mkl::lapack::geqrf_scratchpad_size<compute_t<T>>(
+            getQueue(), iDims[0], iDims[1], iStrides[1]);
+
+    auto scratchpad = memAlloc<compute_t<T>>(scratchpad_size);
+
+    buffer<compute_t<T>> iBuf = in.template getBufferWithOffset<compute_t<T>>();
+    buffer<compute_t<T>> tBuf = t.template getBufferWithOffset<compute_t<T>>();
+    // In place Perform in place QR
+    ::oneapi::mkl::lapack::geqrf(getQueue(), iDims[0], iDims[1], iBuf,
+                                 iStrides[1], tBuf, *scratchpad,
+                                 scratchpad->size());
     return t;
 }
 

From a4f9a8c95071c3250b9ef1da74082cc6e0af8411 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 5 Jul 2023 15:40:10 -0400
Subject: [PATCH 336/473] adds set functions to oneapi backend (#3457)

* adds set functions to oneapi backend
---
 src/backend/oneapi/set.cpp | 146 +++++++++++++++----------------------
 src/backend/oneapi/set.hpp |   1 +
 2 files changed, 61 insertions(+), 86 deletions(-)

diff --git a/src/backend/oneapi/set.cpp b/src/backend/oneapi/set.cpp
index a76363f10b..416efb4040 100644
--- a/src/backend/oneapi/set.cpp
+++ b/src/backend/oneapi/set.cpp
@@ -6,6 +6,11 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
+// oneDPL headers should be included before standard headers
+#define ONEDPL_USE_PREDEFINED_POLICIES 0
+#include <oneapi/dpl/algorithm>
+#include <oneapi/dpl/execution>
+#include <oneapi/dpl/iterator>
 
 #include <Array.hpp>
 #include <common/deprecated.hpp>
@@ -30,115 +35,84 @@ using type_t =
 
 template<typename T>
 Array<T> setUnique(const Array<T> &in, const bool is_sorted) {
-    ONEAPI_NOT_SUPPORTED("setUnique Not supported");
-    return createEmptyArray<T>(dim4(1, 1, 1, 1));
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
 
-    // try {
-    //     Array<T> out = copyArray<T>(in);
+    Array<T> out = copyArray<T>(in);
 
-    //     compute::command_queue queue(getQueue()());
+    auto out_begin = ::oneapi::dpl::begin(*out.get());
+    auto out_end   = out_begin + out.elements();
 
-    //     compute::buffer out_data((*out.get())());
+    if (!is_sorted) {
+        std::sort(dpl_policy, out_begin, out_end,
+                  [](auto lhs, auto rhs) { return lhs < rhs; });
+    }
 
-    //     compute::buffer_iterator<type_t<T>> begin(out_data, 0);
-    //     compute::buffer_iterator<type_t<T>> end(out_data, out.elements());
+    out_end = std::unique(dpl_policy, out_begin, out_end);
 
-    //     if (!is_sorted) { compute::sort(begin, end, queue); }
+    out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
 
-    //     end = compute::unique(begin, end, queue);
-
-    //     out.resetDims(dim4(std::distance(begin, end), 1, 1, 1));
-
-    //     return out;
-    // } catch (const std::exception &ex) { AF_ERROR(ex.what(),
-    // AF_ERR_INTERNAL); }
+    return out;
 }
 
 template<typename T>
 Array<T> setUnion(const Array<T> &first, const Array<T> &second,
                   const bool is_unique) {
-    ONEAPI_NOT_SUPPORTED("setUnion Not supported");
-    return createEmptyArray<T>(dim4(1, 1, 1, 1));
-
-    // try {
-    //     Array<T> unique_first  = first;
-    //     Array<T> unique_second = second;
+    Array<T> unique_first  = first;
+    Array<T> unique_second = second;
 
-    //     if (!is_unique) {
-    //         unique_first  = setUnique(first, false);
-    //         unique_second = setUnique(second, false);
-    //     }
+    if (!is_unique) {
+        unique_first  = setUnique(first, false);
+        unique_second = setUnique(second, false);
+    }
 
-    //     size_t out_size = unique_first.elements() + unique_second.elements();
-    //     Array<T> out    = createEmptyArray<T>(dim4(out_size, 1, 1, 1));
+    size_t out_size = unique_first.elements() + unique_second.elements();
+    Array<T> out    = createEmptyArray<T>(dim4(out_size, 1, 1, 1));
 
-    //     compute::command_queue queue(getQueue()());
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
 
-    //     compute::buffer first_data((*unique_first.get())());
-    //     compute::buffer second_data((*unique_second.get())());
-    //     compute::buffer out_data((*out.get())());
+    auto first_begin = ::oneapi::dpl::begin(*unique_first.get());
+    auto first_end   = first_begin + unique_first.elements();
 
-    //     compute::buffer_iterator<type_t<T>> first_begin(first_data, 0);
-    //     compute::buffer_iterator<type_t<T>> first_end(first_data,
-    //                                                   unique_first.elements());
-    //     compute::buffer_iterator<type_t<T>> second_begin(second_data, 0);
-    //     compute::buffer_iterator<type_t<T>> second_end(
-    //         second_data, unique_second.elements());
-    //     compute::buffer_iterator<type_t<T>> out_begin(out_data, 0);
+    auto second_begin = ::oneapi::dpl::begin(*unique_second.get());
+    auto second_end   = second_begin + unique_second.elements();
 
-    //     compute::buffer_iterator<type_t<T>> out_end = compute::set_union(
-    //         first_begin, first_end, second_begin, second_end, out_begin,
-    //         queue);
+    auto out_begin = ::oneapi::dpl::begin(*out.get());
 
-    //     out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
-    //     return out;
-
-    // } catch (const std::exception &ex) { AF_ERROR(ex.what(),
-    // AF_ERR_INTERNAL); }
+    auto out_end = std::set_union(dpl_policy, first_begin, first_end,
+                                  second_begin, second_end, out_begin);
+    out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
+    return out;
 }
 
 template<typename T>
 Array<T> setIntersect(const Array<T> &first, const Array<T> &second,
                       const bool is_unique) {
-    ONEAPI_NOT_SUPPORTED("setIntersect Not supported");
-    return createEmptyArray<T>(dim4(1, 1, 1, 1));
-
-    // try {
-    //     Array<T> unique_first  = first;
-    //     Array<T> unique_second = second;
-
-    //     if (!is_unique) {
-    //         unique_first  = setUnique(first, false);
-    //         unique_second = setUnique(second, false);
-    //     }
-
-    //     size_t out_size =
-    //         std::max(unique_first.elements(), unique_second.elements());
-    //     Array<T> out = createEmptyArray<T>(dim4(out_size, 1, 1, 1));
-
-    //     compute::command_queue queue(getQueue()());
-
-    //     compute::buffer first_data((*unique_first.get())());
-    //     compute::buffer second_data((*unique_second.get())());
-    //     compute::buffer out_data((*out.get())());
-
-    //     compute::buffer_iterator<type_t<T>> first_begin(first_data, 0);
-    //     compute::buffer_iterator<type_t<T>> first_end(first_data,
-    //                                                   unique_first.elements());
-    //     compute::buffer_iterator<type_t<T>> second_begin(second_data, 0);
-    //     compute::buffer_iterator<type_t<T>> second_end(
-    //         second_data, unique_second.elements());
-    //     compute::buffer_iterator<type_t<T>> out_begin(out_data, 0);
-
-    //     compute::buffer_iterator<type_t<T>> out_end =
-    //     compute::set_intersection(
-    //         first_begin, first_end, second_begin, second_end, out_begin,
-    //         queue);
-
-    //     out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
-    //     return out;
-    // } catch (const std::exception &ex) { AF_ERROR(ex.what(),
-    // AF_ERR_INTERNAL); }
+    Array<T> unique_first  = first;
+    Array<T> unique_second = second;
+
+    if (!is_unique) {
+        unique_first  = setUnique(first, false);
+        unique_second = setUnique(second, false);
+    }
+
+    size_t out_size =
+        std::max(unique_first.elements(), unique_second.elements());
+    Array<T> out = createEmptyArray<T>(dim4(out_size, 1, 1, 1));
+
+    auto dpl_policy = ::oneapi::dpl::execution::make_device_policy(getQueue());
+
+    auto first_begin = ::oneapi::dpl::begin(*unique_first.get());
+    auto first_end   = first_begin + unique_first.elements();
+
+    auto second_begin = ::oneapi::dpl::begin(*unique_second.get());
+    auto second_end   = second_begin + unique_second.elements();
+
+    auto out_begin = ::oneapi::dpl::begin(*out.get());
+
+    auto out_end = std::set_intersection(dpl_policy, first_begin, first_end,
+                                         second_begin, second_end, out_begin);
+    out.resetDims(dim4(std::distance(out_begin, out_end), 1, 1, 1));
+    return out;
 }
 
 #define INSTANTIATE(T)                                                        \
diff --git a/src/backend/oneapi/set.hpp b/src/backend/oneapi/set.hpp
index 85d3386489..beef4a44b4 100644
--- a/src/backend/oneapi/set.hpp
+++ b/src/backend/oneapi/set.hpp
@@ -6,6 +6,7 @@
  * The complete license agreement can be obtained at:
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
+#pragma once
 
 #include <Array.hpp>
 

From f0b8538b30678dd20be5e38aee344d8b33b1d554 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 5 Jul 2023 15:45:24 -0400
Subject: [PATCH 337/473] remove oneapi warnings (#3463)

* remove several oneapi warnings from fftconvolve, mean,  rbk, convolve_seperable
* temporarily suppress internal DPL warnings
---
 .../oneapi/kernel/convolve_separable.cpp      |  5 ---
 .../oneapi/kernel/fftconvolve_multiply.hpp    |  2 --
 .../oneapi/kernel/fftconvolve_pack.hpp        |  8 ++---
 src/backend/oneapi/kernel/fftconvolve_pad.hpp |  9 +-----
 .../oneapi/kernel/fftconvolve_reorder.hpp     |  8 +----
 src/backend/oneapi/kernel/mean.hpp            | 31 ++++++-------------
 src/backend/oneapi/kernel/reduce_by_key.hpp   | 25 +++++----------
 .../oneapi/kernel/sort_by_key_impl.hpp        | 12 +++++++
 src/backend/oneapi/reduce_impl.hpp            | 12 +++++++
 src/backend/oneapi/sort.cpp                   | 12 +++++++
 10 files changed, 57 insertions(+), 67 deletions(-)

diff --git a/src/backend/oneapi/kernel/convolve_separable.cpp b/src/backend/oneapi/kernel/convolve_separable.cpp
index 712570a558..45a86efb7a 100644
--- a/src/backend/oneapi/kernel/convolve_separable.cpp
+++ b/src/backend/oneapi/kernel/convolve_separable.cpp
@@ -76,7 +76,6 @@ class convolveSeparableCreateKernel {
         if (CONV_DIM_ == 0) {
             gx += (EXPAND_ ? 0 : FLEN_ >> 1);
             int endX = ((FLEN_ - 1) << 1) + g.get_local_range(0);
-#pragma unroll
             for (int lx = it.get_local_id(0), glb_x = gx; lx < endX;
                  lx += g.get_local_range(0), glb_x += g.get_local_range(0)) {
                 int i     = glb_x - radius;
@@ -90,7 +89,6 @@ class convolveSeparableCreateKernel {
         } else if (CONV_DIM_ == 1) {
             gy += (EXPAND_ ? 0 : FLEN_ >> 1);
             int endY = ((FLEN_ - 1) << 1) + g.get_local_range(1);
-#pragma unroll
             for (int ly = it.get_local_id(1), glb_y = gy; ly < endY;
                  ly += g.get_local_range(1), glb_y += g.get_local_range(1)) {
                 int i     = gx;
@@ -108,7 +106,6 @@ class convolveSeparableCreateKernel {
             // kernel compilation
             int i         = (CONV_DIM_ == 0 ? lx : ly) + radius;
             accType accum = (accType)(0);
-#pragma unroll
             for (int f = 0; f < FLEN_; ++f) {
                 accType f_val = impulse_[f];
                 // below conditional statement is based on MACRO value passed
@@ -163,8 +160,6 @@ void convSep(Param<T> out, const Param<T> signal, const Param<accType> filter,
     }
     constexpr int THREADS_X = 16;
     constexpr int THREADS_Y = 16;
-    constexpr bool IsComplex =
-        std::is_same<T, cfloat>::value || std::is_same<T, cdouble>::value;
 
     const int fLen       = filter.info.dims[0] * filter.info.dims[1];
     const size_t C0_SIZE = (THREADS_X + 2 * (fLen - 1)) * THREADS_Y;
diff --git a/src/backend/oneapi/kernel/fftconvolve_multiply.hpp b/src/backend/oneapi/kernel/fftconvolve_multiply.hpp
index e8968f6d0d..32516f4056 100644
--- a/src/backend/oneapi/kernel/fftconvolve_multiply.hpp
+++ b/src/backend/oneapi/kernel/fftconvolve_multiply.hpp
@@ -38,8 +38,6 @@ class fftconvolve_multiplyCreateKernel {
         , nelem_(nelem)
         , kind_(kind) {}
     void operator()(sycl::nd_item<1> it) const {
-        sycl::group g = it.get_group();
-
         const int t = it.get_global_id(0);
 
         if (t >= nelem_) return;
diff --git a/src/backend/oneapi/kernel/fftconvolve_pack.hpp b/src/backend/oneapi/kernel/fftconvolve_pack.hpp
index c6b04d5a43..5f8afc2b7a 100644
--- a/src/backend/oneapi/kernel/fftconvolve_pack.hpp
+++ b/src/backend/oneapi/kernel/fftconvolve_pack.hpp
@@ -37,15 +37,13 @@ class fftconvolve_packCreateKernel {
         , di0_half_(di0_half)
         , odd_di0_(odd_di0) {}
     void operator()(sycl::nd_item<1> it) const {
-        sycl::group g = it.get_group();
-
         const int t = it.get_global_id(0);
 
         const int tMax = oInfo_.strides[3] * oInfo_.dims[3];
 
         if (t >= tMax) return;
 
-        const int do0 = oInfo_.dims[0];
+        // const int do0 = oInfo_.dims[0];
         const int do1 = oInfo_.dims[1];
         const int do2 = oInfo_.dims[2];
 
@@ -58,7 +56,7 @@ class fftconvolve_packCreateKernel {
         const int to2 = (t / so2) % do2;
         const int to3 = t / so3;
 
-        const int di0 = iInfo_.dims[0];
+        // const int di0 = iInfo_.dims[0];
         const int di1 = iInfo_.dims[1];
         const int di2 = iInfo_.dims[2];
 
@@ -109,8 +107,6 @@ void packDataHelper(Param<convT> packed, Param<T> sig, Param<T> filter,
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
 
     int sig_packed_elem = sig_tmp.info.strides[3] * sig_tmp.info.dims[3];
-    int filter_packed_elem =
-        filter_tmp.info.strides[3] * filter_tmp.info.dims[3];
 
     // Number of packed complex elements in dimension 0
     int sig_half_d0     = divup(sig.info.dims[0], 2);
diff --git a/src/backend/oneapi/kernel/fftconvolve_pad.hpp b/src/backend/oneapi/kernel/fftconvolve_pad.hpp
index 6276b1da72..6d60506236 100644
--- a/src/backend/oneapi/kernel/fftconvolve_pad.hpp
+++ b/src/backend/oneapi/kernel/fftconvolve_pad.hpp
@@ -29,15 +29,13 @@ class fftconvolve_padCreateKernel {
                                 read_accessor<inputType> d_in, KParam iInfo)
         : d_out_(d_out), oInfo_(oInfo), d_in_(d_in), iInfo_(iInfo) {}
     void operator()(sycl::nd_item<1> it) const {
-        sycl::group g = it.get_group();
-
         const int t = it.get_global_id(0);
 
         const int tMax = oInfo_.strides[3] * oInfo_.dims[3];
 
         if (t >= tMax) return;
 
-        const int do0 = oInfo_.dims[0];
+        // const int do0 = oInfo_.dims[0];
         const int do1 = oInfo_.dims[1];
         const int do2 = oInfo_.dims[2];
 
@@ -92,14 +90,9 @@ void padDataHelper(Param<convT> packed, Param<T> sig, Param<T> filter,
     Param<T> sig_tmp, filter_tmp;
     calcParamSizes(sig_tmp, filter_tmp, packed, sig, filter, rank, kind);
 
-    int sig_packed_elem = sig_tmp.info.strides[3] * sig_tmp.info.dims[3];
     int filter_packed_elem =
         filter_tmp.info.strides[3] * filter_tmp.info.dims[3];
 
-    // Number of packed complex elements in dimension 0
-    int sig_half_d0     = divup(sig.info.dims[0], 2);
-    int sig_half_d0_odd = sig.info.dims[0] % 2;
-
     int blocks = divup(filter_packed_elem, THREADS);
 
     // Locate features kernel sizes
diff --git a/src/backend/oneapi/kernel/fftconvolve_reorder.hpp b/src/backend/oneapi/kernel/fftconvolve_reorder.hpp
index ec71b43bae..589242007a 100644
--- a/src/backend/oneapi/kernel/fftconvolve_reorder.hpp
+++ b/src/backend/oneapi/kernel/fftconvolve_reorder.hpp
@@ -42,15 +42,13 @@ class fftconvolve_reorderCreateKernel {
         , EXPAND_(EXPAND)
         , ROUND_OUT_(ROUND_OUT) {}
     void operator()(sycl::nd_item<1> it) const {
-        sycl::group g = it.get_group();
-
         const int t = it.get_global_id(0);
 
         const int tMax = oInfo_.strides[3] * oInfo_.dims[3];
 
         if (t >= tMax) return;
 
-        const int do0 = oInfo_.dims[0];
+        // const int do0 = oInfo_.dims[0];
         const int do1 = oInfo_.dims[1];
         const int do2 = oInfo_.dims[2];
 
@@ -60,10 +58,6 @@ class fftconvolve_reorderCreateKernel {
 
         // Treating complex input array as real-only array,
         // thus, multiply dimension 0 and strides by 2
-        const int di0 = iInfo_.dims[0] * 2;
-        const int di1 = iInfo_.dims[1];
-        const int di2 = iInfo_.dims[2];
-
         const int si1 = iInfo_.strides[1] * 2;
         const int si2 = iInfo_.strides[2] * 2;
         const int si3 = iInfo_.strides[3] * 2;
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 7c0f6f3243..695fb7b375 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -1,3 +1,4 @@
+
 /*******************************************************
  * Copyright (c) 2022, ArrayFire
  * All rights reserved.
@@ -611,13 +612,9 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         getQueue()
             .submit([&](sycl::handler &h) {
                 auto acc_in =
-                    tmpOut.get()
-                        ->template get_access<sycl::access_mode::read,
-                                              sycl::target::host_buffer>(h);
+                    tmpOut.get()->template get_host_access(h, sycl::read_only);
                 auto acc_wt =
-                    tmpWt.get()
-                        ->template get_access<sycl::access_mode::read,
-                                              sycl::target::host_buffer>(h);
+                    tmpWt.get()->template get_host_access(h, sycl::read_only);
 
                 h.host_task([acc_in, acc_wt, tmp_elements, &val] {
                     val = static_cast<compute_t<T>>(acc_in[0]);
@@ -636,13 +633,10 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         compute_t<T> val;
         getQueue()
             .submit([&](sycl::handler &h) {
-                auto acc_in =
-                    in.data->template get_access<sycl::access_mode::read,
-                                                 sycl::target::host_buffer>(
-                        h, sycl::range{in_elements});
-                auto acc_wt =
-                    iwt.data->template get_access<sycl::access_mode::read>(
-                        h, sycl::range{in_elements});
+                auto acc_in = in.data->template get_host_access(
+                    h, sycl::range{in_elements}, sycl::read_only);
+                auto acc_wt = iwt.data->template get_host_access(
+                    h, sycl::range{in_elements}, sycl::read_only);
 
                 h.host_task([acc_in, acc_wt, in_elements, &val]() {
                     val                  = acc_in[0];
@@ -702,13 +696,9 @@ To mean_all(Param<Ti> in) {
         getQueue()
             .submit([&](sycl::handler &h) {
                 auto out =
-                    tmpOut.get()
-                        ->template get_access<sycl::access_mode::read,
-                                              sycl::target::host_buffer>(h);
+                    tmpOut.get()->template get_host_access(h, sycl::read_only);
                 auto ct =
-                    tmpCt.get()
-                        ->template get_access<sycl::access_mode::read,
-                                              sycl::target::host_buffer>(h);
+                    tmpCt.get()->template get_host_access(h, sycl::read_only);
 
                 h.host_task([out, ct, tmp_elements, &val] {
                     val                  = static_cast<compute_t<To>>(out[0]);
@@ -727,8 +717,7 @@ To mean_all(Param<Ti> in) {
         getQueue()
             .submit([&](sycl::handler &h) {
                 auto acc_in =
-                    in.data->template get_access<sycl::access_mode::read,
-                                                 sycl::target::host_buffer>(h);
+                    in.data->template get_host_access(h, sycl::read_only);
                 h.host_task([acc_in, in_elements, &val]() {
                     common::Transform<Ti, compute_t<To>, af_add_t> transform;
                     compute_t<Tw> count = static_cast<compute_t<Tw>>(1);
diff --git a/src/backend/oneapi/kernel/reduce_by_key.hpp b/src/backend/oneapi/kernel/reduce_by_key.hpp
index 1da17ca5cc..3b5058a6bf 100644
--- a/src/backend/oneapi/kernel/reduce_by_key.hpp
+++ b/src/backend/oneapi/kernel/reduce_by_key.hpp
@@ -101,10 +101,6 @@ class finalBoundaryReduceDimKernel {
         const uint gid = it.get_global_id(0);
         const uint bid = g.get_group_id(0);
 
-        const int bidy = g.get_group_id(1);
-        const int bidz = g.get_group_id(2) % nGroupsZ_;
-        const int bidw = g.get_group_id(2) / nGroupsZ_;
-
         common::Binary<compute_t<To>, op> binOp;
         if (gid == ((bid + 1) * it.get_local_range(0)) - 1 &&
             bid < g.get_group_range(0) - 1) {
@@ -166,7 +162,7 @@ class testNeedsReductionKernel {
         const uint gid = it.get_global_id(0);
         const uint bid = g.get_group_id(0);
 
-        Tk k;
+        Tk k = scalar<Tk>(0);
         if (gid < n_) { k = iKeys_[gid]; }
 
         l_keys_[lid] = k;
@@ -233,9 +229,6 @@ class compactKernel {
         const int bidz = g.get_group_id(2) % nGroupsZ_;
         const int bidw = g.get_group_id(2) / nGroupsZ_;
 
-        Tk k;
-        To v;
-
         const int bOffset = bidw * oVInfo_.strides[3] +
                             bidz * oVInfo_.strides[2] +
                             bidy * oVInfo_.strides[1];
@@ -247,8 +240,8 @@ class compactKernel {
                 : (reduced_block_sizes_[bid] - reduced_block_sizes_[bid - 1]);
         int writeloc = (bid == 0) ? 0 : reduced_block_sizes_[bid - 1];
 
-        k = iKeys_[gid];
-        v = iVals_[bOffset + gid];
+        Tk k = iKeys_[gid];
+        To v = iVals_[bOffset + gid];
 
         if (lid < nwrite) {
             oKeys_[writeloc + lid]           = k;
@@ -407,8 +400,8 @@ class reduceBlocksByKeyKernel {
         if (lid == 0) { l_reduced_block_size_[0] = 0; }
 
         // load keys and values to threads
-        Tk k;
-        compute_t<To> v;
+        Tk k            = scalar<Tk>(0);
+        compute_t<To> v = init_val;
         if (gid < n_) {
             k                 = iKeys_[gid];
             const int bOffset = bidw * iVInfo_.strides[3] +
@@ -416,8 +409,6 @@ class reduceBlocksByKeyKernel {
                                 bidy * iVInfo_.strides[1];
             v = transform(iVals_[bOffset + gid]);
             if (change_nan_) v = IS_NAN(v) ? nanval_ : v;
-        } else {
-            v = init_val;
         }
 
         l_keys_[lid] = k;
@@ -585,8 +576,8 @@ class reduceBlocksByKeyDimKernel {
         it.barrier();
 
         // load keys and values to threads
-        Tk k;
-        compute_t<To> v;
+        Tk k            = scalar<Tk>(0);
+        compute_t<To> v = init_val;
         if (gid < n_) {
             k                 = iKeys_[gid];
             const int bOffset = bidw * iVInfo_.strides[dims_ordering[3]] +
@@ -594,8 +585,6 @@ class reduceBlocksByKeyDimKernel {
                                 bidy * iVInfo_.strides[dims_ordering[1]];
             v = transform(iVals_[bOffset + gid * iVInfo_.strides[DIM_]]);
             if (change_nan_) v = IS_NAN(v) ? nanval_ : v;
-        } else {
-            v = init_val;
         }
 
         l_keys_[lid] = k;
diff --git a/src/backend/oneapi/kernel/sort_by_key_impl.hpp b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
index 9a6348a3ad..5a05eac58c 100644
--- a/src/backend/oneapi/kernel/sort_by_key_impl.hpp
+++ b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
@@ -8,6 +8,13 @@
  ********************************************************/
 #pragma once
 
+#if defined(__clang__)
+#pragma clang diagnostic push
+// temporary ignores for DPL internals
+#pragma clang diagnostic ignored "-Wunused-variable"
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
 // oneDPL headers should be included before standard headers
 #define ONEDPL_USE_PREDEFINED_POLICIES 0
 #include <oneapi/dpl/algorithm>
@@ -206,3 +213,8 @@ void sort0ByKey(Param<Tk> pKey, Param<Tv> pVal, bool isAscending) {
 }  // namespace kernel
 }  // namespace oneapi
 }  // namespace arrayfire
+
+#if defined(__clang__)
+/* Clang/LLVM */
+#pragma clang diagnostic pop
+#endif
diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp
index efada203e1..698f2f1831 100644
--- a/src/backend/oneapi/reduce_impl.hpp
+++ b/src/backend/oneapi/reduce_impl.hpp
@@ -8,6 +8,13 @@
  ********************************************************/
 #pragma once
 
+#if defined(__clang__)
+#pragma clang diagnostic push
+// temporary ignores for DPL internals
+#pragma clang diagnostic ignored "-Wunused-variable"
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
 // oneDPL headers should be included before standard headers
 #define ONEDPL_USE_PREDEFINED_POLICIES 0
 #include <oneapi/dpl/execution>
@@ -614,3 +621,8 @@ Array<To> reduce_all(const Array<Ti> &in, bool change_nan, double nanval) {
         const Array<Ti> &vals, const int dim, bool change_nan, double nanval); \
     template Array<To> reduce_all<Op, Ti, To>(const Array<Ti> &in,             \
                                               bool change_nan, double nanval);
+
+#if defined(__clang__)
+/* Clang/LLVM */
+#pragma clang diagnostic pop
+#endif
diff --git a/src/backend/oneapi/sort.cpp b/src/backend/oneapi/sort.cpp
index a16ccadc55..4dc65a621c 100644
--- a/src/backend/oneapi/sort.cpp
+++ b/src/backend/oneapi/sort.cpp
@@ -7,6 +7,13 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#if defined(__clang__)
+#pragma clang diagnostic push
+// temporary ignores for DPL internals
+#pragma clang diagnostic ignored "-Wunused-variable"
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
 #include <kernel/sort.hpp>
 
 #include <Array.hpp>
@@ -64,3 +71,8 @@ INSTANTIATE(uintl)
 
 }  // namespace oneapi
 }  // namespace arrayfire
+
+#if defined(__clang__)
+/* Clang/LLVM */
+#pragma clang diagnostic pop
+#endif

From aea98356f2396b29fffbf2f7e848dd6668306d08 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 5 Jul 2023 23:12:23 -0400
Subject: [PATCH 338/473] correct -infinity for half datatype in oneapi (#3466)

---
 src/backend/oneapi/math.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp
index 359b4ae9a3..b6aba91663 100644
--- a/src/backend/oneapi/math.hpp
+++ b/src/backend/oneapi/math.hpp
@@ -156,8 +156,8 @@ inline double minval() {
     return -std::numeric_limits<double>::infinity();
 }
 template<>
-inline arrayfire::common::half minval() {
-    return -std::numeric_limits<arrayfire::common::half>::infinity();
+inline sycl::half minval() {
+    return -1 * std::numeric_limits<sycl::half>::infinity();
 }
 
 template<typename T>

From 1eb6bcaef3c7eb18e33744d93753f0afa2872bb6 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Wed, 5 Jul 2023 23:24:50 -0400
Subject: [PATCH 339/473] enable half for convolve strided oneapi (#3465)

---
 src/backend/oneapi/blas.hpp     |   5 +-
 src/backend/oneapi/convolve.cpp | 126 ++++++++++++++------------------
 2 files changed, 57 insertions(+), 74 deletions(-)

diff --git a/src/backend/oneapi/blas.hpp b/src/backend/oneapi/blas.hpp
index 194fc4e6fb..9e2381c336 100644
--- a/src/backend/oneapi/blas.hpp
+++ b/src/backend/oneapi/blas.hpp
@@ -9,6 +9,7 @@
 
 #pragma once
 #include <Array.hpp>
+#include <math.hpp>
 
 // This file contains the common interface for OneAPI BLAS
 // functions
@@ -30,8 +31,8 @@ Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
     int Ndim     = optRhs == AF_MAT_NONE ? 1 : 0;
     Array<T> res = createEmptyArray<T>(
         dim4(lhs.dims()[Mdim], rhs.dims()[Ndim], lhs.dims()[2], lhs.dims()[3]));
-    static constexpr T alpha = 1.0;
-    static constexpr T beta  = 0.0;
+    static const T alpha = scalar<T>(1.0);
+    static const T beta  = scalar<T>(0.0);
     gemm(res, optLhs, optRhs, &alpha, lhs, rhs, &beta);
     return res;
 }
diff --git a/src/backend/oneapi/convolve.cpp b/src/backend/oneapi/convolve.cpp
index 69c120569b..d2cc41c588 100644
--- a/src/backend/oneapi/convolve.cpp
+++ b/src/backend/oneapi/convolve.cpp
@@ -149,15 +149,9 @@ Array<T> convolve2_unwrap(const Array<T> &signal, const Array<T> &filter,
 template<typename T>
 Array<T> convolve2(Array<T> const &signal, Array<T> const &filter,
                    const dim4 stride, const dim4 padding, const dim4 dilation) {
-    if constexpr (!std::is_same<T, half>::value) {
-        Array<T> out =
-            convolve2_unwrap<T>(signal, filter, stride, padding, dilation);
-        return out;
-    } else {
-        ONEAPI_NOT_SUPPORTED("");
-        Array<T> out = createEmptyArray<T>(dim4(1));
-        return out;
-    }
+    Array<T> out =
+        convolve2_unwrap<T>(signal, filter, stride, padding, dilation);
+    return out;
 }
 
 #define INSTANTIATE(T)                                                        \
@@ -177,39 +171,33 @@ Array<T> conv2DataGradient(const Array<T> &incoming_gradient,
                            const Array<T> & /*convolved_output*/,
                            af::dim4 stride, af::dim4 padding,
                            af::dim4 dilation) {
-    if constexpr (!std::is_same<T, half>::value) {
-        const dim4 &cDims = incoming_gradient.dims();
-        const dim4 &sDims = original_signal.dims();
-        const dim4 &fDims = original_filter.dims();
-
-        Array<T> collapsed_filter = original_filter;
-
-        collapsed_filter = flip(collapsed_filter, {1, 1, 0, 0});
-        collapsed_filter = modDims(
-            collapsed_filter, dim4(fDims[0] * fDims[1] * fDims[2], fDims[3]));
-
-        Array<T> collapsed_gradient = incoming_gradient;
-        collapsed_gradient = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
-        collapsed_gradient = modDims(
-            collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
-
-        Array<T> res = matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE,
-                              AF_MAT_TRANS);
-        res          = modDims(res, dim4(res.dims()[0] / sDims[3], sDims[3],
-                                         fDims[0] * fDims[1], sDims[2]));
-        res          = reorder(res, dim4(0, 2, 3, 1));
-
-        const bool retCols = false;
-        res = wrap_dilated(res, sDims[0], sDims[1], fDims[0], fDims[1],
-                           stride[0], stride[1], padding[0], padding[1],
-                           dilation[0], dilation[1], retCols);
-
-        return res;
-    } else {
-        ONEAPI_NOT_SUPPORTED("");
-        Array<T> out = createEmptyArray<T>(dim4(1));
-        return out;
-    }
+    const dim4 &cDims = incoming_gradient.dims();
+    const dim4 &sDims = original_signal.dims();
+    const dim4 &fDims = original_filter.dims();
+
+    Array<T> collapsed_filter = original_filter;
+
+    collapsed_filter = flip(collapsed_filter, {1, 1, 0, 0});
+    collapsed_filter = modDims(collapsed_filter,
+                               dim4(fDims[0] * fDims[1] * fDims[2], fDims[3]));
+
+    Array<T> collapsed_gradient = incoming_gradient;
+    collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
+    collapsed_gradient          = modDims(
+        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+
+    Array<T> res =
+        matmul(collapsed_gradient, collapsed_filter, AF_MAT_NONE, AF_MAT_TRANS);
+    res = modDims(res, dim4(res.dims()[0] / sDims[3], sDims[3],
+                            fDims[0] * fDims[1], sDims[2]));
+    res = reorder(res, dim4(0, 2, 3, 1));
+
+    const bool retCols = false;
+    res = wrap_dilated(res, sDims[0], sDims[1], fDims[0], fDims[1], stride[0],
+                       stride[1], padding[0], padding[1], dilation[0],
+                       dilation[1], retCols);
+
+    return res;
 }
 
 template<typename T>
@@ -219,36 +207,30 @@ Array<T> conv2FilterGradient(const Array<T> &incoming_gradient,
                              const Array<T> & /*convolved_output*/,
                              af::dim4 stride, af::dim4 padding,
                              af::dim4 dilation) {
-    if constexpr (!std::is_same<T, half>::value) {
-        const dim4 &cDims = incoming_gradient.dims();
-        const dim4 &fDims = original_filter.dims();
-
-        const bool retCols = false;
-        Array<T> unwrapped =
-            unwrap(original_signal, fDims[0], fDims[1], stride[0], stride[1],
-                   padding[0], padding[1], dilation[0], dilation[1], retCols);
-
-        unwrapped  = reorder(unwrapped, dim4(1, 2, 0, 3));
-        dim4 uDims = unwrapped.dims();
-        unwrapped =
-            modDims(unwrapped, dim4(uDims[0] * uDims[1], uDims[2] * uDims[3]));
-
-        Array<T> collapsed_gradient = incoming_gradient;
-        collapsed_gradient = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
-        collapsed_gradient = modDims(
-            collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
-
-        Array<T> res =
-            matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE);
-        res = modDims(res, dim4(fDims[0], fDims[1], fDims[2], fDims[3]));
-
-        auto out = flip(res, {1, 1, 0, 0});
-        return out;
-    } else {
-        ONEAPI_NOT_SUPPORTED("");
-        Array<T> out = createEmptyArray<T>(dim4(1));
-        return out;
-    }
+    const dim4 &cDims = incoming_gradient.dims();
+    const dim4 &fDims = original_filter.dims();
+
+    const bool retCols = false;
+    Array<T> unwrapped =
+        unwrap(original_signal, fDims[0], fDims[1], stride[0], stride[1],
+               padding[0], padding[1], dilation[0], dilation[1], retCols);
+
+    unwrapped  = reorder(unwrapped, dim4(1, 2, 0, 3));
+    dim4 uDims = unwrapped.dims();
+    unwrapped =
+        modDims(unwrapped, dim4(uDims[0] * uDims[1], uDims[2] * uDims[3]));
+
+    Array<T> collapsed_gradient = incoming_gradient;
+    collapsed_gradient          = reorder(collapsed_gradient, dim4(0, 1, 3, 2));
+    collapsed_gradient          = modDims(
+        collapsed_gradient, dim4(cDims[0] * cDims[1] * cDims[3], cDims[2]));
+
+    Array<T> res =
+        matmul(unwrapped, collapsed_gradient, AF_MAT_NONE, AF_MAT_NONE);
+    res = modDims(res, dim4(fDims[0], fDims[1], fDims[2], fDims[3]));
+
+    auto out = flip(res, {1, 1, 0, 0});
+    return out;
 }
 
 #define INSTANTIATE(T)                                                      \

From fddead17e689b58fb81f0eeb7aea4f7ac02ec552 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Thu, 6 Jul 2023 21:08:44 -0400
Subject: [PATCH 340/473] fix fftconvolve one2many tests

---
 src/backend/oneapi/fft.cpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/backend/oneapi/fft.cpp b/src/backend/oneapi/fft.cpp
index 3bf15acf0a..03ae19efc6 100644
--- a/src/backend/oneapi/fft.cpp
+++ b/src/backend/oneapi/fft.cpp
@@ -95,7 +95,7 @@ std::vector<std::int64_t> computeStrides(const int rank, const dim4 istrides,
     if (rank == 3) return {offset, istrides[2], istrides[1], istrides[0]};
     if (rank == 4)
         return {offset, istrides[3], istrides[2], istrides[1], istrides[0]};
-    return {offset};
+    return {offset, istrides[0]};
 }
 
 template<::oneapi::mkl::dft::precision precision,
@@ -180,9 +180,14 @@ void fft_inplace(Array<T> &in, const int rank, const bool direction) {
         ::oneapi::mkl::dft::descriptor<precision,
                                        ::oneapi::mkl::dft::domain::COMPLEX>;
 
+    // TODO[STF]: WTF
+    // getOffset() for s0 throwing Invalid Descriptor when targeting gpu
+    // on CPU, results are wrong but does not throw
+    // strides not working? TODO: test standalone oneMKL
+    // perhaps in.getDataDims() needed instead of in.dims()?
     std::vector<std::int64_t> fft_input_strides =
-        computeStrides(rank, istrides, in.getOffset());
-
+        computeStrides(rank, istrides, 0);
+    // computeStrides(rank, istrides, in.getOffset()); //TODO[STF]: WTF,
     int batch = 1;
     for (int i = rank; i < 4; i++) { batch *= idims[i]; }
 

From 48cd41fdca8a10e48e0d44f3cad1dd1fab830ff0 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 17 Jun 2023 18:23:44 -0400
Subject: [PATCH 341/473] Add logging to jit heuristic checks

---
 src/backend/cuda/Array.cpp   | 11 +++++++++++
 src/backend/oneapi/Array.cpp | 20 +++++++++++++++++---
 src/backend/opencl/Array.cpp | 19 +++++++++++++++++--
 3 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index db03d1b3e5..eb71a9f7a2 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include <Array.hpp>
+#include <common/Logger.hpp>
 #include <common/half.hpp>
 #include <common/jit/NodeIterator.hpp>
 #include <copy.hpp>
@@ -253,8 +254,13 @@ Node_ptr Array<T>::getNode() const {
 template<typename T>
 kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     if (!evalFlag()) { return kJITHeuristics::Pass; }
+    static auto getLogger = [&] { return spdlog::get("jit"); };
     for (Node *n : root_nodes) {
         if (n->getHeight() > static_cast<int>(getMaxJitSize())) {
+            AF_TRACE(
+                "JIT tree evaluated because of tree height exceeds limit: {} > "
+                "{}",
+                n->getHeight(), getMaxJitSize());
             return kJITHeuristics::TreeHeight;
         }
     }
@@ -313,9 +319,14 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
         // should be checking the amount of memory available to guard
         // this eval
         if (param_size >= max_param_size) {
+            AF_TRACE(
+                "JIT tree evaluated because of kernel parameter size: {} >= {}",
+                param_size, max_param_size);
             return kJITHeuristics::KernelParameterSize;
         }
         if (jitTreeExceedsMemoryPressure(info.total_buffer_size)) {
+            AF_TRACE("JIT tree evaluated because of memory pressure: {}",
+                     info.total_buffer_size);
             return kJITHeuristics::MemoryPressure;
         }
     }
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 5845d95ecc..f227f8def3 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -10,6 +10,7 @@
 #include <Array.hpp>
 
 #include <Param.hpp>
+#include <common/Logger.hpp>
 #include <common/MemoryManagerBase.hpp>
 #include <common/half.hpp>
 #include <common/jit/NodeIterator.hpp>
@@ -312,8 +313,13 @@ Node_ptr Array<T>::getNode() const {
 template<typename T>
 kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     if (!evalFlag()) { return kJITHeuristics::Pass; }
+    static auto getLogger = [&] { return common::loggerFactory("jit"); };
     for (const Node *n : root_nodes) {
         if (n->getHeight() > static_cast<int>(getMaxJitSize())) {
+            AF_TRACE(
+                "JIT tree evaluated because of tree height exceeds limit: {} > "
+                "{}",
+                n->getHeight(), getMaxJitSize());
             return kJITHeuristics::TreeHeight;
         }
     }
@@ -386,9 +392,17 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
 
         bool isParamLimit = param_size >= max_param_size;
 
-        if (isParamLimit) { return kJITHeuristics::KernelParameterSize; }
-        // TODO(umar): check buffer limit for JIT kernel generation
-        // if (isBufferLimit) { return kJITHeuristics::MemoryPressure; }
+        if (isParamLimit) {
+            AF_TRACE(
+                "JIT tree evaluated because of kernel parameter size: {} >= {}",
+                param_size, max_param_size);
+            return kJITHeuristics::KernelParameterSize;
+        }
+        if (isBufferLimit) {
+            AF_TRACE("JIT tree evaluated because of memory pressure: {}",
+                     info.total_buffer_size);
+            return kJITHeuristics::MemoryPressure;
+        }
     }
     return kJITHeuristics::Pass;
 }
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index d479ac5752..2d3bc40e0b 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -9,6 +9,7 @@
 
 #include <Array.hpp>
 
+#include <common/Logger.hpp>
 #include <common/half.hpp>
 #include <common/jit/NodeIterator.hpp>
 #include <common/jit/ScalarNode.hpp>
@@ -301,8 +302,13 @@ Node_ptr Array<T>::getNode() const {
 template<typename T>
 kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
     if (!evalFlag()) { return kJITHeuristics::Pass; }
+    static auto getLogger = [&] { return common::loggerFactory("jit"); };
     for (const Node *n : root_nodes) {
         if (n->getHeight() > static_cast<int>(getMaxJitSize())) {
+            AF_TRACE(
+                "JIT tree evaluated because of tree height exceeds limit: {} > "
+                "{}",
+                n->getHeight(), getMaxJitSize());
             return kJITHeuristics::TreeHeight;
         }
     }
@@ -377,8 +383,17 @@ kJITHeuristics passesJitHeuristics(span<Node *> root_nodes) {
 
         bool isParamLimit = param_size >= max_param_size;
 
-        if (isParamLimit) { return kJITHeuristics::KernelParameterSize; }
-        if (isBufferLimit) { return kJITHeuristics::MemoryPressure; }
+        if (isParamLimit) {
+            AF_TRACE(
+                "JIT tree evaluated because of kernel parameter size: {} >= {}",
+                param_size, max_param_size);
+            return kJITHeuristics::KernelParameterSize;
+        }
+        if (isBufferLimit) {
+            AF_TRACE("JIT tree evaluated because of memory pressure: {}",
+                     info.total_buffer_size);
+            return kJITHeuristics::MemoryPressure;
+        }
     }
     return kJITHeuristics::Pass;
 }

From c2e31fde950baff6c22dcf839a4cf7a843e60518 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 30 Jun 2023 18:23:51 -0400
Subject: [PATCH 342/473] Add logging to oneAPI JIT kernel launches

---
 src/backend/oneapi/jit.cpp | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 31c2a0b881..3e317b68e2 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -14,6 +14,7 @@
 #include <kernel_headers/jit.hpp>
 
 #include <Array.hpp>
+#include <Kernel.hpp>
 #include <common/dispatch.hpp>
 #include <common/half.hpp>
 #include <common/jit/ModdimNode.hpp>
@@ -591,6 +592,19 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                                   (size_t)ap[0].dims[2]};
                         ndims  = 3;
                     }
+
+                    {
+                        using namespace oneapi::kernel_logger;
+                        AF_TRACE(
+                            "Launching {}: Dims: [{},{},{},{}] Global: "
+                            "[{},{},{}] threads: {}",
+                            funcName, ap[0].dims[0], ap[0].dims[1],
+                            ap[0].dims[2], ap[0].dims[3], global[0], global[1],
+                            global[2],
+                            global[0] * std::max<size_t>(1, global[1]) *
+                                std::max<size_t>(1, global[2]));
+                    }
+
                     cl_event kernel_event;
                     CL_CHECK(clEnqueueNDRangeKernel(
                         q, kernel, ndims, offset.data(), global.data(), nullptr,

From e9132c237bc0020ac1275427c3b93355979b3bd2 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 7 Jul 2023 16:53:09 -0400
Subject: [PATCH 343/473] Update JIT mem pressure heuristic for small number of
 buffers

---
 src/backend/common/DefaultMemoryManager.cpp | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/backend/common/DefaultMemoryManager.cpp b/src/backend/common/DefaultMemoryManager.cpp
index d4aae2138e..0e0694631d 100644
--- a/src/backend/common/DefaultMemoryManager.cpp
+++ b/src/backend/common/DefaultMemoryManager.cpp
@@ -140,10 +140,19 @@ float DefaultMemoryManager::getMemoryPressure() {
     }
 }
 
-bool DefaultMemoryManager::jitTreeExceedsMemoryPressure(size_t bytes) {
+bool DefaultMemoryManager::jitTreeExceedsMemoryPressure(
+    size_t jit_tree_buffer_bytes) {
     lock_guard_t lock(this->memory_mutex);
     memory_info &current = this->getCurrentMemoryInfo();
-    return 2 * bytes > current.lock_bytes;
+    if (current.lock_bytes > 0.25f * current.max_bytes) {
+        /// Evaluate JIT if half of all locked buffers are locked by this JIT
+        /// tree
+        return jit_tree_buffer_bytes > current.lock_bytes * 0.5f;
+    } else {
+        /// Evaluate if this JIT Tree accounts for 10% of total memory on the
+        /// device
+        return jit_tree_buffer_bytes > 0.10f * current.max_bytes;
+    }
 }
 
 void *DefaultMemoryManager::alloc(bool user_lock, const unsigned ndims,

From d74656f7a4257fb9bcde84638f01efe2223a76f0 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Sat, 8 Jul 2023 14:25:12 -0400
Subject: [PATCH 344/473] oneapi sparse (#3469)

* adds sparse_arith, sparse blas to oneapi backend
---
 src/backend/common/Binary.hpp              |  14 +
 src/backend/oneapi/CMakeLists.txt          |   2 +
 src/backend/oneapi/kernel/sparse.hpp       | 470 +++++++++++++++++
 src/backend/oneapi/kernel/sparse_arith.hpp | 569 +++++++++++++++++++++
 src/backend/oneapi/sparse.cpp              | 229 ++++-----
 src/backend/oneapi/sparse_arith.cpp        | 159 +++---
 src/backend/oneapi/sparse_blas.cpp         | 124 ++---
 7 files changed, 1308 insertions(+), 259 deletions(-)
 create mode 100644 src/backend/oneapi/kernel/sparse.hpp
 create mode 100644 src/backend/oneapi/kernel/sparse_arith.hpp

diff --git a/src/backend/common/Binary.hpp b/src/backend/common/Binary.hpp
index 6ad8654f83..128cf18988 100644
--- a/src/backend/common/Binary.hpp
+++ b/src/backend/common/Binary.hpp
@@ -40,6 +40,13 @@ struct Binary<T, af_add_t> {
     __DH__ T operator()(T lhs, T rhs) { return lhs + rhs; }
 };
 
+template<typename T>
+struct Binary<T, af_sub_t> {
+    static __DH__ T init() { return scalar<T>(0); }
+
+    __DH__ T operator()(T lhs, T rhs) { return lhs - rhs; }
+};
+
 template<typename T>
 struct Binary<T, af_mul_t> {
     static __DH__ T init() { return scalar<T>(1); }
@@ -47,6 +54,13 @@ struct Binary<T, af_mul_t> {
     __DH__ T operator()(T lhs, T rhs) { return lhs * rhs; }
 };
 
+template<typename T>
+struct Binary<T, af_div_t> {
+    static __DH__ T init() { return scalar<T>(1); }
+
+    __DH__ T operator()(T lhs, T rhs) { return lhs / rhs; }
+};
+
 template<typename T>
 struct Binary<T, af_or_t> {
     static __DH__ T init() { return scalar<T>(0); }
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 1c8f789806..d4c7245311 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -249,6 +249,8 @@ target_sources(afoneapi
     kernel/scan_dim.hpp
     kernel/sort.hpp
     kernel/sort_by_key.hpp
+    kernel/sparse.hpp
+    kernel/sparse_arith.hpp
     kernel/transpose.hpp
     kernel/transpose_inplace.hpp
     kernel/triangle.hpp
diff --git a/src/backend/oneapi/kernel/sparse.hpp b/src/backend/oneapi/kernel/sparse.hpp
new file mode 100644
index 0000000000..70bf051868
--- /dev/null
+++ b/src/backend/oneapi/kernel/sparse.hpp
@@ -0,0 +1,470 @@
+/*******************************************************
+ * Copyright (c) 2014, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#pragma once
+
+#include <Array.hpp>
+#include <Param.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <kernel/accessors.hpp>
+#include <kernel/reduce.hpp>
+#include <kernel/scan_dim.hpp>
+#include <kernel/scan_first.hpp>
+#include <kernel/sort_by_key.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+template<typename T>
+class coo2DenseCreateKernel {
+   public:
+    coo2DenseCreateKernel(write_accessor<T> oPtr, const KParam output,
+                          write_accessor<T> vPtr, const KParam values,
+                          read_accessor<int> rPtr, const KParam rowIdx,
+                          read_accessor<int> cPtr, const KParam colIdx)
+        : oPtr_(oPtr)
+        , output_(output)
+        , vPtr_(vPtr)
+        , values_(values)
+        , rPtr_(rPtr)
+        , rowIdx_(rowIdx)
+        , cPtr_(cPtr)
+        , colIdx_(colIdx) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        const int id = g.get_group_id(0) * g.get_local_range(0) * REPEAT +
+                       it.get_local_id(0);
+
+        if (id >= values_.dims[0]) return;
+
+        const int dimSize = g.get_local_range(0);
+
+        for (int i = it.get_local_id(0); i < REPEAT * dimSize; i += dimSize) {
+            if (i >= values_.dims[0]) return;
+
+            T v   = vPtr_[i];
+            int r = rPtr_[i];
+            int c = cPtr_[i];
+
+            int offset = r + c * output_.strides[1];
+
+            oPtr_[offset] = v;
+        }
+    }
+
+   private:
+    write_accessor<T> oPtr_;
+    const KParam output_;
+    write_accessor<T> vPtr_;
+    const KParam values_;
+    read_accessor<int> rPtr_;
+    const KParam rowIdx_;
+    read_accessor<int> cPtr_;
+    const KParam colIdx_;
+};
+
+template<typename T>
+void coo2dense(Param<T> out, const Param<T> values, const Param<int> rowIdx,
+               const Param<int> colIdx) {
+    auto local  = sycl::range(THREADS_PER_BLOCK, 1);
+    auto global = sycl::range(
+        divup(out.info.dims[0], local[0] * REPEAT) * THREADS_PER_BLOCK, 1);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only};
+        sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only};
+        sycl::accessor d_out{*out.data, h, sycl::write_only, sycl::no_init};
+        sycl::accessor d_values{*values.data, h, sycl::write_only,
+                                sycl::no_init};
+        h.parallel_for(sycl::nd_range{global, local},
+                       coo2DenseCreateKernel<T>(
+                           d_out, out.info, d_values, values.info, d_rowIdx,
+                           rowIdx.info, d_colIdx, colIdx.info));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T, int THREADS>
+class csr2DenseCreateKernel {
+   public:
+    csr2DenseCreateKernel(write_accessor<T> output, read_accessor<T> values,
+                          read_accessor<int> rowidx, read_accessor<int> colidx,
+                          const int M)
+        : output_(output)
+        , values_(values)
+        , rowidx_(rowidx)
+        , colidx_(colidx)
+        , M_(M) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        int lid = it.get_local_id(0);
+        for (int rowId = g.get_group_id(0); rowId < M_;
+             rowId += it.get_group_range(0)) {
+            int colStart = rowidx_[rowId];
+            int colEnd   = rowidx_[rowId + 1];
+            for (int colId = colStart + lid; colId < colEnd; colId += THREADS) {
+                output_[rowId + colidx_[colId] * M_] = values_[colId];
+            }
+        }
+    }
+
+   private:
+    write_accessor<T> output_;
+    read_accessor<T> values_;
+    read_accessor<int> rowidx_;
+    read_accessor<int> colidx_;
+    const int M_;
+};
+
+template<typename T>
+void csr2dense(Param<T> output, const Param<T> values, const Param<int> rowIdx,
+               const Param<int> colIdx) {
+    constexpr int MAX_GROUPS = 4096;
+    // FIXME: This needs to be based non nonzeros per row
+    constexpr int threads = 64;
+
+    const int M = rowIdx.info.dims[0] - 1;
+
+    auto local   = sycl::range(threads, 1);
+    int groups_x = std::min((int)(divup(M, local[0])), MAX_GROUPS);
+    auto global  = sycl::range(local[0] * groups_x, 1);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_values{*values.data, h, sycl::read_only};
+        sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only};
+        sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only};
+        sycl::accessor d_output{*output.data, h, sycl::write_only,
+                                sycl::no_init};
+        h.parallel_for(sycl::nd_range{global, local},
+                       csr2DenseCreateKernel<T, threads>(
+                           d_output, d_values, d_rowIdx, d_colIdx, M));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+class dense2csrCreateKernel {
+   public:
+    dense2csrCreateKernel(write_accessor<T> svalptr,
+                          write_accessor<int> scolptr, read_accessor<T> dvalptr,
+                          const KParam valinfo, read_accessor<int> dcolptr,
+                          const KParam colinfo, read_accessor<int> rowptr)
+        : svalptr_(svalptr)
+        , scolptr_(scolptr)
+        , dvalptr_(dvalptr)
+        , valinfo_(valinfo)
+        , dcolptr_(dcolptr)
+        , colinfo_(colinfo)
+        , rowptr_(rowptr) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        // sycl::group g = it.get_group();
+
+        int gidx = it.get_global_id(0);
+        int gidy = it.get_global_id(1);
+
+        if (gidx >= (unsigned)valinfo_.dims[0]) return;
+        if (gidy >= (unsigned)valinfo_.dims[1]) return;
+
+        int rowoff       = rowptr_[gidx];
+        T *svalptr_ptr   = svalptr_.get_pointer();
+        int *scolptr_ptr = scolptr_.get_pointer();
+        svalptr_ptr += rowoff;
+        scolptr_ptr += rowoff;
+
+        T *dvalptr_ptr   = dvalptr_.get_pointer();
+        int *dcolptr_ptr = dcolptr_.get_pointer();
+        dvalptr_ptr += valinfo_.offset;
+        dcolptr_ptr += colinfo_.offset;
+
+        T val = dvalptr_ptr[gidx + gidy * (unsigned)valinfo_.strides[1]];
+
+        if constexpr (std::is_same_v<decltype(val), std::complex<float>> ||
+                      std::is_same_v<decltype(val), std::complex<double>>) {
+            if (val.real() == 0 && val.imag() == 0) return;
+        } else {
+            if (val == 0) return;
+        }
+
+        int oloc              = dcolptr_ptr[gidx + gidy * colinfo_.strides[1]];
+        svalptr_ptr[oloc - 1] = val;
+        scolptr_ptr[oloc - 1] = gidy;
+    }
+
+   private:
+    write_accessor<T> svalptr_;
+    write_accessor<int> scolptr_;
+    read_accessor<T> dvalptr_;
+    const KParam valinfo_;
+    read_accessor<int> dcolptr_;
+    const KParam colinfo_;
+    read_accessor<int> rowptr_;
+};
+
+template<typename T>
+void dense2csr(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
+               const Param<T> dense) {
+    int num_rows = dense.info.dims[0];
+    int num_cols = dense.info.dims[1];
+
+    // sd1 contains output of scan along dim 1 of dense
+    Array<int> sd1 = createEmptyArray<int>(dim4(num_rows, num_cols));
+    // rd1 contains output of nonzero count along dim 1 along dense
+    Array<int> rd1 = createEmptyArray<int>(num_rows);
+
+    scan_dim<T, int, af_notzero_t, 1>(sd1, dense, true);
+    reduce_dim_default<T, int, af_notzero_t, 1>(rd1, dense, 0, 0);
+    scan_first<int, int, af_add_t>(rowIdx, rd1, false);
+
+    const int nnz = values.info.dims[0];
+
+    const sycl::id<1> fillOffset(rowIdx.info.offset +
+                                 (rowIdx.info.dims[0] - 1));
+    const sycl::range<1> fillRange(rowIdx.info.dims[0] - fillOffset[0]);
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_rowIdx{*rowIdx.data, h, fillRange, fillOffset};
+        h.fill(d_rowIdx, nnz);
+    });
+
+    auto local   = sycl::range(THREADS_X, THREADS_Y);
+    int groups_x = divup(dense.info.dims[0], local[0]);
+    int groups_y = divup(dense.info.dims[1], local[1]);
+    auto global  = sycl::range(groups_x * local[0], groups_y * local[1]);
+
+    const Param<int> sdParam = sd1;
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_dense{*dense.data, h, sycl::read_only};
+        sycl::accessor d_sdParam{*sdParam.data, h, sycl::read_only};
+        sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only};
+        sycl::accessor d_values{*values.data, h, sycl::write_only,
+                                sycl::no_init};
+        sycl::accessor d_colIdx{*colIdx.data, h, sycl::write_only,
+                                sycl::no_init};
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            dense2csrCreateKernel<T>(d_values, d_colIdx, d_dense, dense.info,
+                                     d_sdParam, sdParam.info, d_rowIdx));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+class swapIndexCreateKernel {
+   public:
+    swapIndexCreateKernel(write_accessor<T> ovalues, write_accessor<int> oindex,
+                          read_accessor<T> ivalues, read_accessor<int> iindex,
+                          read_accessor<int> swapIdx, const int nNZ)
+        : ovalues_(ovalues)
+        , oindex_(oindex)
+        , ivalues_(ivalues)
+        , iindex_(iindex)
+        , swapIdx_(swapIdx)
+        , nNZ_(nNZ) {}
+
+    void operator()(sycl::item<1> it) const {
+        int id = it.get_id(0);
+        if (id < nNZ_) {
+            int idx = swapIdx_[id];
+
+            ovalues_[id] = ivalues_[idx];
+            oindex_[id]  = iindex_[idx];
+        }
+    }
+
+   private:
+    write_accessor<T> ovalues_;
+    write_accessor<int> oindex_;
+    read_accessor<T> ivalues_;
+    read_accessor<int> iindex_;
+    read_accessor<int> swapIdx_;
+    const int nNZ_;
+};
+
+template<typename T>
+void swapIndex(Param<T> ovalues, Param<int> oindex, const Param<T> ivalues,
+               sycl::buffer<int> iindex, const Param<int> swapIdx) {
+    auto global = sycl::range(ovalues.info.dims[0]);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_ivalues{*ivalues.data, h, sycl::read_only};
+        sycl::accessor d_iindex{iindex, h, sycl::read_only};
+        sycl::accessor d_swapIdx{*swapIdx.data, h, sycl::read_only};
+        sycl::accessor d_ovalues{*ovalues.data, h, sycl::write_only,
+                                 sycl::no_init};
+        sycl::accessor d_oindex{*oindex.data, h, sycl::write_only,
+                                sycl::no_init};
+
+        h.parallel_for(global,
+                       swapIndexCreateKernel<T>(
+                           d_ovalues, d_oindex, d_ivalues, d_iindex, d_swapIdx,
+                           static_cast<int>(ovalues.info.dims[0])));
+    });
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+class csr2CooCreateKernel {
+   public:
+    csr2CooCreateKernel(write_accessor<int> orowidx,
+                        write_accessor<int> ocolidx, read_accessor<int> irowidx,
+                        read_accessor<int> icolidx, const int M)
+        : orowidx_(orowidx)
+        , ocolidx_(ocolidx)
+        , irowidx_(irowidx)
+        , icolidx_(icolidx)
+        , M_(M) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+
+        int lid = it.get_local_id(0);
+        for (int rowId = g.get_group_id(0); rowId < M_;
+             rowId += it.get_group_range(0)) {
+            int colStart = irowidx_[rowId];
+            int colEnd   = irowidx_[rowId + 1];
+            for (int colId = colStart + lid; colId < colEnd;
+                 colId += g.get_local_range(0)) {
+                orowidx_[colId] = rowId;
+                ocolidx_[colId] = icolidx_[colId];
+            }
+        }
+    }
+
+   private:
+    write_accessor<int> orowidx_;
+    write_accessor<int> ocolidx_;
+    read_accessor<int> irowidx_;
+    read_accessor<int> icolidx_;
+    const int M_;
+};
+
+template<typename T>
+void csr2coo(Param<T> ovalues, Param<int> orowIdx, Param<int> ocolIdx,
+             const Param<T> ivalues, const Param<int> irowIdx,
+             const Param<int> icolIdx, Param<int> index) {
+    const int MAX_GROUPS = 4096;
+    int M                = irowIdx.info.dims[0] - 1;
+    // FIXME: This needs to be based non nonzeros per row
+    int threads = 64;
+
+    auto scratch = memAlloc<int>(orowIdx.info.dims[0]);
+
+    auto local   = sycl::range(threads, 1);
+    int groups_x = std::min((int)(divup(M, local[0])), MAX_GROUPS);
+    auto global  = sycl::range(local[0] * groups_x, 1);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_irowIdx{*irowIdx.data, h, sycl::read_only};
+        sycl::accessor d_icolIdx{*icolIdx.data, h, sycl::read_only};
+        sycl::accessor d_scratch{*scratch, h, sycl::write_only, sycl::no_init};
+        sycl::accessor d_ocolIdx{*ocolIdx.data, h, sycl::write_only,
+                                 sycl::no_init};
+        h.parallel_for(sycl::nd_range{global, local},
+                       csr2CooCreateKernel<T>(d_scratch, d_ocolIdx, d_irowIdx,
+                                              d_icolIdx, M));
+    });
+
+    // Now we need to sort this into column major
+    kernel::sort0ByKeyIterative<int, int>(ocolIdx, index, true);
+
+    // Now use index to sort values and rows
+    kernel::swapIndex<T>(ovalues, orowIdx, ivalues, *scratch, index);
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T>
+class csrReduceKernel {
+   public:
+    csrReduceKernel(write_accessor<int> orowidx, read_accessor<int> irowidx,
+                    const int M, const int nNZ)
+        : orowidx_(orowidx), irowidx_(irowidx), M_(M), nNZ_(nNZ) {}
+
+    void operator()(sycl::item<1> it) const {
+        int id = it.get_id(0);
+
+        if (id < nNZ_) {
+            // Read COO row indices
+            int iRId  = irowidx_[id];
+            int iRId1 = 0;
+            if (id > 0) iRId1 = irowidx_[id - 1];
+
+            // If id is 0, then mark the edge cases of csrRow[0] and csrRow[M]
+            if (id == 0) {
+                orowidx_[id] = 0;
+                orowidx_[M_] = nNZ_;
+            } else if (iRId1 != iRId) {
+                // If iRId1 and iRId are not same, that means the row has
+                // incremented For example, if iRId is 5 and iRId1 is 4, that
+                // means row 4 has ended and row 5 has begun at index id. We use
+                // the for-loop because there can be any number of empty rows
+                // between iRId1 and iRId, all of which should be marked by id
+                for (int i = iRId1 + 1; i <= iRId; i++) orowidx_[i] = id;
+            }
+
+            // The last X rows are corner cases if they dont have any values
+            if (id < M_) {
+                if (id > irowidx_[nNZ_ - 1] && orowidx_[id] == 0) {
+                    orowidx_[id] = nNZ_;
+                }
+            }
+        }
+    }
+
+   private:
+    write_accessor<int> orowidx_;
+    read_accessor<int> irowidx_;
+    const int M_;
+    const int nNZ_;
+};
+
+template<typename T>
+void coo2csr(Param<T> ovalues, Param<int> orowIdx, Param<int> ocolIdx,
+             const Param<T> ivalues, const Param<int> irowIdx,
+             const Param<int> icolIdx, Param<int> index, Param<int> rowCopy,
+             const int M) {
+    // Now we need to sort this into column major
+    kernel::sort0ByKeyIterative<int, int>(rowCopy, index, true);
+
+    // Now use index to sort values and rows
+    kernel::swapIndex<T>(ovalues, ocolIdx, ivalues, *icolIdx.data, index);
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+
+    auto global = sycl::range(irowIdx.info.dims[0]);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_orowIdx{*orowIdx.data, h, sycl::write_only};
+        sycl::accessor d_rowCopy{*rowCopy.data, h, sycl::read_only};
+        h.parallel_for(
+            sycl::range{global},
+            csrReduceKernel<T>(d_orowIdx, d_rowCopy, M,
+                               static_cast<int>(ovalues.info.dims[0])));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/kernel/sparse_arith.hpp b/src/backend/oneapi/kernel/sparse_arith.hpp
new file mode 100644
index 0000000000..819af6ffce
--- /dev/null
+++ b/src/backend/oneapi/kernel/sparse_arith.hpp
@@ -0,0 +1,569 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+#pragma once
+
+#include <Array.hpp>
+#include <Param.hpp>
+#include <common/Binary.hpp>
+#include <common/complex.hpp>
+#include <common/dispatch.hpp>
+#include <common/kernel_cache.hpp>
+#include <debug_oneapi.hpp>
+#include <err_oneapi.hpp>
+#include <kernel/accessors.hpp>
+#include <math.hpp>
+#include <memory.hpp>
+#include <traits.hpp>
+
+#include <string>
+#include <vector>
+
+namespace arrayfire {
+namespace oneapi {
+namespace kernel {
+
+constexpr unsigned TX      = 32;
+constexpr unsigned TY      = 8;
+constexpr unsigned THREADS = TX * TY;
+
+template<typename T>
+using global_atomic_ref =
+    sycl::atomic_ref<T, sycl::memory_order::relaxed, sycl::memory_scope::system,
+                     sycl::access::address_space::global_space>;
+
+template<typename T, af_op_t op>
+class sparseArithCSRKernel {
+   public:
+    sparseArithCSRKernel(write_accessor<T> oPtr, const KParam out,
+                         read_accessor<T> values, read_accessor<int> rowIdx,
+                         read_accessor<int> colIdx, const int nNZ,
+                         read_accessor<T> rPtr, const KParam rhs,
+                         const int reverse)
+        : oPtr_(oPtr)
+        , out_(out)
+        , values_(values)
+        , rowIdx_(rowIdx)
+        , colIdx_(colIdx)
+        , nNZ_(nNZ)
+        , rPtr_(rPtr)
+        , rhs_(rhs)
+        , reverse_(reverse) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        common::Binary<T, op> binOP;
+
+        const int row =
+            g.get_group_id(0) * g.get_local_range(1) + it.get_local_id(1);
+
+        if (row < out_.dims[0]) {
+            const int rowStartIdx = rowIdx_[row];
+            const int rowEndIdx   = rowIdx_[row + 1];
+
+            // Repeat loop until all values in the row are computed
+            for (int idx = rowStartIdx + it.get_local_id(0); idx < rowEndIdx;
+                 idx += g.get_local_range(0)) {
+                const int col = colIdx_[idx];
+
+                if (row >= out_.dims[0] || col >= out_.dims[1])
+                    continue;  // Bad indices
+
+                // Get Values
+                const T val  = values_[idx];
+                const T rval = rPtr_[col * rhs_.strides[1] + row];
+
+                const int offset = col * out_.strides[1] + row;
+                if (reverse_)
+                    oPtr_[offset] = binOP(rval, val);
+                else
+                    oPtr_[offset] = binOP(val, rval);
+            }
+        }
+    }
+
+   private:
+    write_accessor<T> oPtr_;
+    const KParam out_;
+    read_accessor<T> values_;
+    read_accessor<int> rowIdx_;
+    read_accessor<int> colIdx_;
+    const int nNZ_;
+    read_accessor<T> rPtr_;
+    const KParam rhs_;
+    const int reverse_;
+};
+
+template<typename T, af_op_t op>
+void sparseArithOpCSR(Param<T> out, const Param<T> values,
+                      const Param<int> rowIdx, const Param<int> colIdx,
+                      const Param<T> rhs, const bool reverse) {
+    auto local  = sycl::range(TX, TY);
+    auto global = sycl::range(divup(out.info.dims[0], TY) * TX, TY);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_out{*out.data, h, sycl::write_only};
+        sycl::accessor d_values{*values.data, h, sycl::read_only};
+        sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only};
+        sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only};
+        sycl::accessor d_rhs{*rhs.data, h, sycl::read_only};
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       sparseArithCSRKernel<T, op>(
+                           d_out, out.info, d_values, d_rowIdx, d_colIdx,
+                           static_cast<int>(values.info.dims[0]), d_rhs,
+                           rhs.info, static_cast<int>(reverse)));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T, af_op_t op>
+class sparseArithCOOKernel {
+   public:
+    sparseArithCOOKernel(write_accessor<T> oPtr, const KParam out,
+                         read_accessor<T> values, read_accessor<int> rowIdx,
+                         read_accessor<int> colIdx, const int nNZ,
+                         read_accessor<T> rPtr, const KParam rhs,
+                         const int reverse)
+        : oPtr_(oPtr)
+        , out_(out)
+        , values_(values)
+        , rowIdx_(rowIdx)
+        , colIdx_(colIdx)
+        , nNZ_(nNZ)
+        , rPtr_(rPtr)
+        , rhs_(rhs)
+        , reverse_(reverse) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        common::Binary<T, op> binOP;
+
+        const int idx = it.get_global_id(0);
+
+        if (idx < nNZ_) {
+            const int row = rowIdx_[idx];
+            const int col = colIdx_[idx];
+
+            if (row >= out_.dims[0] || col >= out_.dims[1])
+                return;  // Bad indices
+
+            // Get Values
+            const T val  = values_[idx];
+            const T rval = rPtr_[col * rhs_.strides[1] + row];
+
+            const int offset = col * out_.strides[1] + row;
+            if (reverse_)
+                oPtr_[offset] = binOP(rval, val);
+            else
+                oPtr_[offset] = binOP(val, rval);
+        }
+    }
+
+   private:
+    write_accessor<T> oPtr_;
+    const KParam out_;
+    read_accessor<T> values_;
+    read_accessor<int> rowIdx_;
+    read_accessor<int> colIdx_;
+    const int nNZ_;
+    read_accessor<T> rPtr_;
+    const KParam rhs_;
+    const int reverse_;
+};
+
+template<typename T, af_op_t op>
+void sparseArithOpCOO(Param<T> out, const Param<T> values,
+                      const Param<int> rowIdx, const Param<int> colIdx,
+                      const Param<T> rhs, const bool reverse) {
+    auto local  = sycl::range(THREADS);
+    auto global = sycl::range(divup(values.info.dims[0], THREADS) * THREADS);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_out{*out.data, h, sycl::write_only};
+        sycl::accessor d_values{*values.data, h, sycl::read_only};
+        sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only};
+        sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only};
+        sycl::accessor d_rhs{*rhs.data, h, sycl::read_only};
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       sparseArithCOOKernel<T, op>(
+                           d_out, out.info, d_values, d_rowIdx, d_colIdx,
+                           static_cast<int>(values.info.dims[0]), d_rhs,
+                           rhs.info, static_cast<int>(reverse)));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T, af_op_t op>
+class sparseArithCSR2Kernel {
+   public:
+    sparseArithCSR2Kernel(sycl::accessor<T> values, read_accessor<int> rowIdx,
+                          read_accessor<int> colIdx, const int nNZ,
+                          read_accessor<T> rPtr, const KParam rhs,
+                          const int reverse)
+        : values_(values)
+        , rowIdx_(rowIdx)
+        , colIdx_(colIdx)
+        , nNZ_(nNZ)
+        , rPtr_(rPtr)
+        , rhs_(rhs)
+        , reverse_(reverse) {}
+
+    void operator()(sycl::nd_item<2> it) const {
+        sycl::group g = it.get_group();
+        common::Binary<T, op> binOP;
+
+        const int row =
+            g.get_group_id(0) * g.get_local_range(1) + it.get_local_id(1);
+
+        if (row < rhs_.dims[0]) {
+            const int rowStartIdx = rowIdx_[row];
+            const int rowEndIdx   = rowIdx_[row + 1];
+
+            // Repeat loop until all values in the row are computed
+            for (int idx = rowStartIdx + it.get_local_id(0); idx < rowEndIdx;
+                 idx += g.get_local_range(0)) {
+                const int col = colIdx_[idx];
+
+                if (row >= rhs_.dims[0] || col >= rhs_.dims[1])
+                    continue;  // Bad indices
+
+                // Get Values
+                const T val  = values_[idx];
+                const T rval = rPtr_[col * rhs_.strides[1] + row];
+
+                if (reverse_)
+                    values_[idx] = binOP(rval, val);
+                else
+                    values_[idx] = binOP(val, rval);
+            }
+        }
+    }
+
+   private:
+    sycl::accessor<T> values_;
+    read_accessor<int> rowIdx_;
+    read_accessor<int> colIdx_;
+    const int nNZ_;
+    read_accessor<T> rPtr_;
+    const KParam rhs_;
+    const int reverse_;
+};
+
+template<typename T, af_op_t op>
+void sparseArithOpCSR(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
+                      const Param<T> rhs, const bool reverse) {
+    auto local  = sycl::range(TX, TY);
+    auto global = sycl::range(divup(values.info.dims[0], TY) * TX, TY);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_values{*values.data, h, sycl::read_write};
+        sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only};
+        sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only};
+        sycl::accessor d_rhs{*rhs.data, h, sycl::read_only};
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       sparseArithCSR2Kernel<T, op>(
+                           d_values, d_rowIdx, d_colIdx,
+                           static_cast<int>(values.info.dims[0]), d_rhs,
+                           rhs.info, static_cast<int>(reverse)));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T, af_op_t op>
+class sparseArithCOO2Kernel {
+   public:
+    sparseArithCOO2Kernel(sycl::accessor<T> values, read_accessor<int> rowIdx,
+                          read_accessor<int> colIdx, const int nNZ,
+                          read_accessor<T> rPtr, const KParam rhs,
+                          const int reverse)
+        : values_(values)
+        , rowIdx_(rowIdx)
+        , colIdx_(colIdx)
+        , nNZ_(nNZ)
+        , rPtr_(rPtr)
+        , rhs_(rhs)
+        , reverse_(reverse) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        common::Binary<T, op> binOP;
+
+        const int idx = it.get_global_id(0);
+
+        if (idx < nNZ_) {
+            const int row = rowIdx_[idx];
+            const int col = colIdx_[idx];
+
+            if (row >= rhs_.dims[0] || col >= rhs_.dims[1])
+                return;  // Bad indices
+
+            // Get Values
+            const T val  = values_[idx];
+            const T rval = rPtr_[col * rhs_.strides[1] + row];
+
+            if (reverse_)
+                values_[idx] = binOP(rval, val);
+            else
+                values_[idx] = binOP(val, rval);
+        }
+    }
+
+   private:
+    sycl::accessor<T> values_;
+    read_accessor<int> rowIdx_;
+    read_accessor<int> colIdx_;
+    const int nNZ_;
+    read_accessor<T> rPtr_;
+    const KParam rhs_;
+    const int reverse_;
+};
+
+template<typename T, af_op_t op>
+void sparseArithOpCOO(Param<T> values, Param<int> rowIdx, Param<int> colIdx,
+                      const Param<T> rhs, const bool reverse) {
+    auto local  = sycl::range(THREADS);
+    auto global = sycl::range(divup(values.info.dims[0], THREADS) * THREADS);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_values{*values.data, h, sycl::read_write};
+        sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only};
+        sycl::accessor d_colIdx{*colIdx.data, h, sycl::read_only};
+        sycl::accessor d_rhs{*rhs.data, h, sycl::read_only};
+
+        h.parallel_for(sycl::nd_range{global, local},
+                       sparseArithCOO2Kernel<T, op>(
+                           d_values, d_rowIdx, d_colIdx,
+                           static_cast<int>(values.info.dims[0]), d_rhs,
+                           rhs.info, static_cast<int>(reverse)));
+    });
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+class csrCalcOutNNZKernel {
+   public:
+    csrCalcOutNNZKernel(write_accessor<unsigned> nnzc,
+                        write_accessor<int> oRowIdx, unsigned M,
+                        read_accessor<int> lRowIdx, read_accessor<int> lColIdx,
+                        read_accessor<int> rRowIdx, read_accessor<int> rColIdx,
+                        sycl::local_accessor<unsigned, 1> blkNNZ)
+        : nnzc_(nnzc)
+        , oRowIdx_(oRowIdx)
+        , M_(M)
+        , lRowIdx_(lRowIdx)
+        , lColIdx_(lColIdx)
+        , rRowIdx_(rRowIdx)
+        , rColIdx_(rColIdx)
+        , blkNNZ_(blkNNZ) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        sycl::group g = it.get_group();
+
+        const uint row = it.get_global_id(0);
+        const uint tid = it.get_local_id(0);
+
+        const bool valid = row < M_;
+
+        const uint lEnd = (valid ? lRowIdx_[row + 1] : 0);
+        const uint rEnd = (valid ? rRowIdx_[row + 1] : 0);
+
+        blkNNZ_[tid] = 0;
+        it.barrier();
+
+        uint l   = (valid ? lRowIdx_[row] : 0);
+        uint r   = (valid ? rRowIdx_[row] : 0);
+        uint nnz = 0;
+        while (l < lEnd && r < rEnd) {
+            uint lci = lColIdx_[l];
+            uint rci = rColIdx_[r];
+            l += (lci <= rci);
+            r += (lci >= rci);
+            nnz++;
+        }
+        nnz += (lEnd - l);
+        nnz += (rEnd - r);
+
+        blkNNZ_[tid] = nnz;
+        it.barrier();
+
+        if (valid) oRowIdx_[row + 1] = nnz;
+
+        for (uint s = g.get_local_range(0) / 2; s > 0; s >>= 1) {
+            if (tid < s) { blkNNZ_[tid] += blkNNZ_[tid + s]; }
+            it.barrier();
+        }
+
+        if (tid == 0) {
+            nnz = blkNNZ_[0];
+            global_atomic_ref<uint>(nnzc_[0]) += nnz;
+        }
+    }
+
+   private:
+    write_accessor<unsigned> nnzc_;
+    write_accessor<int> oRowIdx_;
+    unsigned M_;
+    read_accessor<int> lRowIdx_;
+    read_accessor<int> lColIdx_;
+    read_accessor<int> rRowIdx_;
+    read_accessor<int> rColIdx_;
+    sycl::local_accessor<unsigned, 1> blkNNZ_;
+};
+
+static void csrCalcOutNNZ(Param<int> outRowIdx, unsigned &nnzC, const uint M,
+                          const uint N, uint nnzA, const Param<int> lrowIdx,
+                          const Param<int> lcolIdx, uint nnzB,
+                          const Param<int> rrowIdx, const Param<int> rcolIdx) {
+    UNUSED(N);
+    UNUSED(nnzA);
+    UNUSED(nnzB);
+
+    auto local  = sycl::range(256);
+    auto global = sycl::range(divup(M, local[0]) * local[0]);
+
+    Array<unsigned> out = createValueArray<unsigned>(1, 0);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_out{*out.get(), h, sycl::write_only};
+        sycl::accessor d_outRowIdx{*outRowIdx.data, h, sycl::write_only};
+        sycl::accessor d_lRowIdx{*lrowIdx.data, h, sycl::read_only};
+        sycl::accessor d_lColIdx{*lcolIdx.data, h, sycl::read_only};
+        sycl::accessor d_rRowIdx{*rrowIdx.data, h, sycl::read_only};
+        sycl::accessor d_rColIdx{*rcolIdx.data, h, sycl::read_only};
+
+        auto blkNNZ = sycl::local_accessor<unsigned, 1>(local[0], h);
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            csrCalcOutNNZKernel(d_out, d_outRowIdx, M, d_lRowIdx, d_lColIdx,
+                                d_rRowIdx, d_rColIdx, blkNNZ));
+    });
+
+    {
+        sycl::host_accessor nnz_acc{*out.get(), sycl::read_only};
+        nnzC = nnz_acc[0];
+    }
+
+    ONEAPI_DEBUG_FINISH(getQueue());
+}
+
+template<typename T, af_op_t op>
+class ssarithCSRKernel {
+   public:
+    ssarithCSRKernel(write_accessor<T> oVals, write_accessor<int> oColIdx,
+                     read_accessor<int> oRowIdx, unsigned M, unsigned N,
+                     unsigned nnza, read_accessor<T> lVals,
+                     read_accessor<int> lRowIdx, read_accessor<int> lColIdx,
+                     unsigned nnzb, read_accessor<T> rVals,
+                     read_accessor<int> rRowIdx, read_accessor<int> rColIdx)
+        : oVals_(oVals)
+        , oColIdx_(oColIdx)
+        , oRowIdx_(oRowIdx)
+        , M_(M)
+        , N_(N)
+        , nnza_(nnza)
+        , lVals_(lVals)
+        , lRowIdx_(lRowIdx)
+        , lColIdx_(lColIdx)
+        , nnzb_(nnzb)
+        , rVals_(rVals)
+        , rRowIdx_(rRowIdx)
+        , rColIdx_(rColIdx) {}
+
+    void operator()(sycl::nd_item<1> it) const {
+        common::Binary<T, op> binOP;
+
+        const uint row = it.get_global_id(0);
+
+        const bool valid  = row < M_;
+        const uint lEnd   = (valid ? lRowIdx_[row + 1] : 0);
+        const uint rEnd   = (valid ? rRowIdx_[row + 1] : 0);
+        const uint offset = (valid ? oRowIdx_[row] : 0);
+
+        T *ovPtr   = oVals_.get_pointer() + offset;
+        int *ocPtr = oColIdx_.get_pointer() + offset;
+
+        uint l = (valid ? lRowIdx_[row] : 0);
+        uint r = (valid ? rRowIdx_[row] : 0);
+
+        uint nnz = 0;
+        while (l < lEnd && r < rEnd) {
+            uint lci = lColIdx_[l];
+            uint rci = rColIdx_[r];
+
+            T lhs = (lci <= rci ? lVals_[l] : common::Binary<T, op>::init());
+            T rhs = (lci >= rci ? rVals_[r] : common::Binary<T, op>::init());
+
+            ovPtr[nnz] = binOP(lhs, rhs);
+            ocPtr[nnz] = (lci <= rci) ? lci : rci;
+
+            l += (lci <= rci);
+            r += (lci >= rci);
+            nnz++;
+        }
+        while (l < lEnd) {
+            ovPtr[nnz] = binOP(lVals_[l], common::Binary<T, op>::init());
+            ocPtr[nnz] = lColIdx_[l];
+            l++;
+            nnz++;
+        }
+        while (r < rEnd) {
+            ovPtr[nnz] = binOP(common::Binary<T, op>::init(), rVals_[r]);
+            ocPtr[nnz] = rColIdx_[r];
+            r++;
+            nnz++;
+        }
+    }
+
+   private:
+    write_accessor<T> oVals_;
+    write_accessor<int> oColIdx_;
+    read_accessor<int> oRowIdx_;
+    unsigned M_, N_;
+    unsigned nnza_;
+    read_accessor<T> lVals_;
+    read_accessor<int> lRowIdx_;
+    read_accessor<int> lColIdx_;
+    unsigned nnzb_;
+    read_accessor<T> rVals_;
+    read_accessor<int> rRowIdx_;
+    read_accessor<int> rColIdx_;
+};
+
+template<typename T, af_op_t op>
+void ssArithCSR(Param<T> oVals, Param<int> oColIdx, const Param<int> oRowIdx,
+                const uint M, const uint N, unsigned nnzA, const Param<T> lVals,
+                const Param<int> lRowIdx, const Param<int> lColIdx,
+                unsigned nnzB, const Param<T> rVals, const Param<int> rRowIdx,
+                const Param<int> rColIdx) {
+    auto local  = sycl::range(256);
+    auto global = sycl::range(divup(M, local[0]) * local[0]);
+
+    getQueue().submit([&](auto &h) {
+        sycl::accessor d_oVals{*oVals.data, h, sycl::write_only};
+        sycl::accessor d_oColIdx{*oColIdx.data, h, sycl::write_only};
+        sycl::accessor d_oRowIdx{*oRowIdx.data, h, sycl::read_only};
+
+        sycl::accessor d_lVals{*lVals.data, h, sycl::read_only};
+        sycl::accessor d_lRowIdx{*lRowIdx.data, h, sycl::read_only};
+        sycl::accessor d_lColIdx{*lColIdx.data, h, sycl::read_only};
+
+        sycl::accessor d_rVals{*rVals.data, h, sycl::read_only};
+        sycl::accessor d_rRowIdx{*rRowIdx.data, h, sycl::read_only};
+        sycl::accessor d_rColIdx{*rColIdx.data, h, sycl::read_only};
+
+        h.parallel_for(
+            sycl::nd_range{global, local},
+            ssarithCSRKernel<T, op>(d_oVals, d_oColIdx, d_oRowIdx, M, N, nnzA,
+                                    d_lVals, d_lRowIdx, d_lColIdx, nnzB,
+                                    d_rVals, d_rRowIdx, d_rColIdx));
+    });
+}
+
+}  // namespace kernel
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/oneapi/sparse.cpp b/src/backend/oneapi/sparse.cpp
index 37e5826430..2e9a67213f 100644
--- a/src/backend/oneapi/sparse.cpp
+++ b/src/backend/oneapi/sparse.cpp
@@ -7,7 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-// #include <kernel/sparse.hpp>
+#include <kernel/sparse.hpp>
 #include <sparse.hpp>
 
 #include <arith.hpp>
@@ -26,154 +26,151 @@
 #include <stdexcept>
 #include <string>
 
+#include <handle.hpp>
+
 namespace arrayfire {
 namespace oneapi {
 
 using namespace common;
 
+#define P(exp) af_print_array_gen(#exp, getHandle(exp), 2)
+
 // Partial template specialization of sparseConvertDenseToStorage for COO
 // However, template specialization is not allowed
 template<typename T>
 SparseArray<T> sparseConvertDenseToCOO(const Array<T> &in) {
-    ONEAPI_NOT_SUPPORTED("sparseConvertDenseToCOO Not supported");
-    // in.eval();
+    in.eval();
 
-    // Array<uint> nonZeroIdx_ = where<T>(in);
-    // Array<int> nonZeroIdx   = cast<int, uint>(nonZeroIdx_);
+    Array<uint> nonZeroIdx_ = where<T>(in);
+    Array<int> nonZeroIdx   = cast<int, uint>(nonZeroIdx_);
+    nonZeroIdx.eval();
 
-    // dim_t nNZ = nonZeroIdx.elements();
+    dim_t nNZ = nonZeroIdx.elements();
 
-    // Array<int> constDim = createValueArray<int>(dim4(nNZ), in.dims()[0]);
-    // constDim.eval();
+    Array<int> constDim = createValueArray<int>(dim4(nNZ), in.dims()[0]);
+    constDim.eval();
 
-    // Array<int> rowIdx =
-    //     arithOp<int, af_mod_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
-    // Array<int> colIdx =
-    //     arithOp<int, af_div_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
+    Array<int> rowIdx =
+        arithOp<int, af_mod_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
+    Array<int> colIdx =
+        arithOp<int, af_div_t>(nonZeroIdx, constDim, nonZeroIdx.dims());
 
-    // Array<T> values = copyArray<T>(in);
-    // values          = modDims(values, dim4(values.elements()));
-    // values          = lookup<T, int>(values, nonZeroIdx, 0);
+    Array<T> values = copyArray<T>(in);
+    values          = modDims(values, dim4(values.elements()));
+    values          = lookup<T, int>(values, nonZeroIdx, 0);
 
-    // return createArrayDataSparseArray<T>(in.dims(), values, rowIdx, colIdx,
-    //                                      AF_STORAGE_COO);
+    return createArrayDataSparseArray<T>(in.dims(), values, rowIdx, colIdx,
+                                         AF_STORAGE_COO);
 }
 
 template<typename T, af_storage stype>
 SparseArray<T> sparseConvertDenseToStorage(const Array<T> &in_) {
-    ONEAPI_NOT_SUPPORTED("sparseConvertDenseToStorage Not supported");
-    //     in_.eval();
-    //
-    //     uint nNZ = getScalar<uint>(reduce_all<af_notzero_t, T, uint>(in_));
-    //
-    //     SparseArray<T> sparse_ = createEmptySparseArray<T>(in_.dims(), nNZ,
-    //     stype); sparse_.eval();
-    //
-    //     Array<T> &values   = sparse_.getValues();
-    //     Array<int> &rowIdx = sparse_.getRowIdx();
-    //     Array<int> &colIdx = sparse_.getColIdx();
-
-    // kernel::dense2csr<T>(values, rowIdx, colIdx, in_);
-
-    // return sparse_;
+    in_.eval();
+
+    uint nNZ = getScalar<uint>(reduce_all<af_notzero_t, T, uint>(in_));
+
+    SparseArray<T> sparse_ = createEmptySparseArray<T>(in_.dims(), nNZ, stype);
+    sparse_.eval();
+
+    Array<T> &values   = sparse_.getValues();
+    Array<int> &rowIdx = sparse_.getRowIdx();
+    Array<int> &colIdx = sparse_.getColIdx();
+
+    kernel::dense2csr<T>(values, rowIdx, colIdx, in_);
+
+    return sparse_;
 }
 
 // Partial template specialization of sparseConvertStorageToDense for COO
 // However, template specialization is not allowed
 template<typename T>
 Array<T> sparseConvertCOOToDense(const SparseArray<T> &in) {
-    ONEAPI_NOT_SUPPORTED("sparseConvertCOOToDense Not supported");
-    //    in.eval();
-    //
-    //    Array<T> dense = createValueArray<T>(in.dims(), scalar<T>(0));
-    //    dense.eval();
-    //
-    //    const Array<T> values   = in.getValues();
-    //    const Array<int> rowIdx = in.getRowIdx();
-    //    const Array<int> colIdx = in.getColIdx();
-
-    // kernel::coo2dense<T>(dense, values, rowIdx, colIdx);
-
-    // return dense;
+    in.eval();
+
+    Array<T> dense = createValueArray<T>(in.dims(), scalar<T>(0));
+    dense.eval();
+
+    const Array<T> values   = in.getValues();
+    const Array<int> rowIdx = in.getRowIdx();
+    const Array<int> colIdx = in.getColIdx();
+
+    kernel::coo2dense<T>(dense, values, rowIdx, colIdx);
+
+    return dense;
 }
 
 template<typename T, af_storage stype>
 Array<T> sparseConvertStorageToDense(const SparseArray<T> &in_) {
-    ONEAPI_NOT_SUPPORTED("sparseConvertStorageToDense Not supported");
-    //
-    //    if (stype != AF_STORAGE_CSR) {
-    //        AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
-    //                 AF_ERR_NOT_SUPPORTED);
-    //    }
-    //
-    //    in_.eval();
-    //
-    //    Array<T> dense_ = createValueArray<T>(in_.dims(), scalar<T>(0));
-    //    dense_.eval();
-    //
-    //    const Array<T> &values   = in_.getValues();
-    //    const Array<int> &rowIdx = in_.getRowIdx();
-    //    const Array<int> &colIdx = in_.getColIdx();
-    //
-    //    if (stype == AF_STORAGE_CSR) {
-    //        // kernel::csr2dense<T>(dense_, values, rowIdx, colIdx);
-    //    } else {
-    //        AF_ERROR("OpenCL Backend only supports CSR or COO to Dense",
-    //                 AF_ERR_NOT_SUPPORTED);
-    //    }
-    //
-    //    return dense_;
+    if (stype != AF_STORAGE_CSR) {
+        AF_ERROR("oneAPI Backend only supports CSR or COO to Dense",
+                 AF_ERR_NOT_SUPPORTED);
+    }
+
+    in_.eval();
+
+    Array<T> dense_ = createValueArray<T>(in_.dims(), scalar<T>(0));
+    dense_.eval();
+
+    const Array<T> &values   = in_.getValues();
+    const Array<int> &rowIdx = in_.getRowIdx();
+    const Array<int> &colIdx = in_.getColIdx();
+
+    if (stype == AF_STORAGE_CSR) {
+        kernel::csr2dense<T>(dense_, values, rowIdx, colIdx);
+    } else {
+        AF_ERROR("oneAPI Backend only supports CSR or COO to Dense",
+                 AF_ERR_NOT_SUPPORTED);
+    }
+
+    return dense_;
 }
 
 template<typename T, af_storage dest, af_storage src>
 SparseArray<T> sparseConvertStorageToStorage(const SparseArray<T> &in) {
-    ONEAPI_NOT_SUPPORTED("sparseConvertStorageToStorage Not supported");
-    // in.eval();
-
-    // SparseArray<T> converted = createEmptySparseArray<T>(
-    //     in.dims(), static_cast<int>(in.getNNZ()), dest);
-    // converted.eval();
-
-    // if (src == AF_STORAGE_CSR && dest == AF_STORAGE_COO) {
-    //     Array<int> index = range<int>(in.getNNZ(), 0);
-    //     index.eval();
-
-    //    Array<T> &ovalues         = converted.getValues();
-    //    Array<int> &orowIdx       = converted.getRowIdx();
-    //    Array<int> &ocolIdx       = converted.getColIdx();
-    //    const Array<T> &ivalues   = in.getValues();
-    //    const Array<int> &irowIdx = in.getRowIdx();
-    //    const Array<int> &icolIdx = in.getColIdx();
-
-    //    // kernel::csr2coo<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx,
-    //    // icolIdx,
-    //    //                    index);
-
-    //} else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) {
-    //    Array<int> index = range<int>(in.getNNZ(), 0);
-    //    index.eval();
-
-    //    Array<T> &ovalues         = converted.getValues();
-    //    Array<int> &orowIdx       = converted.getRowIdx();
-    //    Array<int> &ocolIdx       = converted.getColIdx();
-    //    const Array<T> &ivalues   = in.getValues();
-    //    const Array<int> &irowIdx = in.getRowIdx();
-    //    const Array<int> &icolIdx = in.getColIdx();
-
-    //    Array<int> rowCopy = copyArray<int>(irowIdx);
-    //    rowCopy.eval();
-
-    //    kernel::coo2csr<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx,
-    //    icolIdx,
-    //                       index, rowCopy, in.dims()[0]);
-
-    //} else {
-    //    // Should never come here
-    //    AF_ERROR("OpenCL Backend invalid conversion combination",
-    //             AF_ERR_NOT_SUPPORTED);
-    //}
-
-    // return converted;
+    in.eval();
+
+    SparseArray<T> converted = createEmptySparseArray<T>(
+        in.dims(), static_cast<int>(in.getNNZ()), dest);
+    converted.eval();
+
+    if (src == AF_STORAGE_CSR && dest == AF_STORAGE_COO) {
+        Array<int> index = range<int>(in.getNNZ(), 0);
+        index.eval();
+
+        Array<T> &ovalues         = converted.getValues();
+        Array<int> &orowIdx       = converted.getRowIdx();
+        Array<int> &ocolIdx       = converted.getColIdx();
+        const Array<T> &ivalues   = in.getValues();
+        const Array<int> &irowIdx = in.getRowIdx();
+        const Array<int> &icolIdx = in.getColIdx();
+
+        kernel::csr2coo<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, icolIdx,
+                           index);
+
+    } else if (src == AF_STORAGE_COO && dest == AF_STORAGE_CSR) {
+        Array<int> index = range<int>(in.getNNZ(), 0);
+        index.eval();
+
+        Array<T> &ovalues         = converted.getValues();
+        Array<int> &orowIdx       = converted.getRowIdx();
+        Array<int> &ocolIdx       = converted.getColIdx();
+        const Array<T> &ivalues   = in.getValues();
+        const Array<int> &irowIdx = in.getRowIdx();
+        const Array<int> &icolIdx = in.getColIdx();
+
+        Array<int> rowCopy = copyArray<int>(irowIdx);
+        rowCopy.eval();
+
+        kernel::coo2csr<T>(ovalues, orowIdx, ocolIdx, ivalues, irowIdx, icolIdx,
+                           index, rowCopy, in.dims()[0]);
+
+    } else {
+        // Should never come here
+        AF_ERROR("oneAPI Backend invalid conversion combination",
+                 AF_ERR_NOT_SUPPORTED);
+    }
+
+    return converted;
 }
 
 #define INSTANTIATE_TO_STORAGE(T, S)                     \
diff --git a/src/backend/oneapi/sparse_arith.cpp b/src/backend/oneapi/sparse_arith.cpp
index 856d300553..4b3e7301c4 100644
--- a/src/backend/oneapi/sparse_arith.cpp
+++ b/src/backend/oneapi/sparse_arith.cpp
@@ -7,8 +7,8 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-// #include <kernel/sparse_arith.hpp>
 #include <err_oneapi.hpp>
+#include <kernel/sparse_arith.hpp>
 #include <sparse.hpp>
 
 #include <stdexcept>
@@ -51,104 +51,101 @@ cdouble getInf() {
 template<typename T, af_op_t op>
 Array<T> arithOpD(const SparseArray<T> &lhs, const Array<T> &rhs,
                   const bool reverse) {
-    ONEAPI_NOT_SUPPORTED("arithOpD Not supported");
-    // lhs.eval();
-    // rhs.eval();
-
-    // Array<T> out  = createEmptyArray<T>(dim4(0));
-    // Array<T> zero = createValueArray<T>(rhs.dims(), scalar<T>(0));
-    // switch (op) {
-    //     case af_add_t: out = copyArray<T>(rhs); break;
-    //     case af_sub_t:
-    //         out = reverse ? copyArray<T>(rhs)
-    //                       : arithOp<T, af_sub_t>(zero, rhs, rhs.dims());
-    //         break;
-    //     default: out = copyArray<T>(rhs);
-    // }
-    // out.eval();
-    // switch (lhs.getStorage()) {
-    //     case AF_STORAGE_CSR:
-    //         kernel::sparseArithOpCSR<T, op>(out, lhs.getValues(),
-    //                                         lhs.getRowIdx(), lhs.getColIdx(),
-    //                                         rhs, reverse);
-    //         break;
-    //     case AF_STORAGE_COO:
-    //         kernel::sparseArithOpCOO<T, op>(out, lhs.getValues(),
-    //                                         lhs.getRowIdx(), lhs.getColIdx(),
-    //                                         rhs, reverse);
-    //         break;
-    //     default:
-    //         AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
-    //                  AF_ERR_NOT_SUPPORTED);
-    // }
-
-    // return out;
+    lhs.eval();
+    rhs.eval();
+
+    Array<T> out  = createEmptyArray<T>(dim4(0));
+    Array<T> zero = createValueArray<T>(rhs.dims(), scalar<T>(0));
+    switch (op) {
+        case af_add_t: out = copyArray<T>(rhs); break;
+        case af_sub_t:
+            out = reverse ? copyArray<T>(rhs)
+                          : arithOp<T, af_sub_t>(zero, rhs, rhs.dims());
+            break;
+        default: out = copyArray<T>(rhs);
+    }
+    out.eval();
+    switch (lhs.getStorage()) {
+        case AF_STORAGE_CSR:
+            kernel::sparseArithOpCSR<T, op>(out, lhs.getValues(),
+                                            lhs.getRowIdx(), lhs.getColIdx(),
+                                            rhs, reverse);
+            break;
+        case AF_STORAGE_COO:
+            kernel::sparseArithOpCOO<T, op>(out, lhs.getValues(),
+                                            lhs.getRowIdx(), lhs.getColIdx(),
+                                            rhs, reverse);
+            break;
+        default:
+            AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
+                     AF_ERR_NOT_SUPPORTED);
+    }
+
+    return out;
 }
 
 template<typename T, af_op_t op>
 SparseArray<T> arithOp(const SparseArray<T> &lhs, const Array<T> &rhs,
                        const bool reverse) {
-    ONEAPI_NOT_SUPPORTED("arithOp Not supported");
-    // lhs.eval();
-    // rhs.eval();
-
-    // SparseArray<T> out = createArrayDataSparseArray<T>(
-    //     lhs.dims(), lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(),
-    //     lhs.getStorage(), true);
-    // out.eval();
-    // switch (lhs.getStorage()) {
-    //     case AF_STORAGE_CSR:
-    //         kernel::sparseArithOpCSR<T, op>(out.getValues(), out.getRowIdx(),
-    //                                         out.getColIdx(), rhs, reverse);
-    //         break;
-    //     case AF_STORAGE_COO:
-    //         kernel::sparseArithOpCOO<T, op>(out.getValues(), out.getRowIdx(),
-    //                                         out.getColIdx(), rhs, reverse);
-    //         break;
-    //     default:
-    //         AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
-    //                  AF_ERR_NOT_SUPPORTED);
-    // }
-
-    // return out;
+    lhs.eval();
+    rhs.eval();
+
+    SparseArray<T> out = createArrayDataSparseArray<T>(
+        lhs.dims(), lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(),
+        lhs.getStorage(), true);
+    out.eval();
+    switch (lhs.getStorage()) {
+        case AF_STORAGE_CSR:
+            kernel::sparseArithOpCSR<T, op>(out.getValues(), out.getRowIdx(),
+                                            out.getColIdx(), rhs, reverse);
+            break;
+        case AF_STORAGE_COO:
+            kernel::sparseArithOpCOO<T, op>(out.getValues(), out.getRowIdx(),
+                                            out.getColIdx(), rhs, reverse);
+            break;
+        default:
+            AF_ERROR("Sparse Arithmetic only supported for CSR or COO",
+                     AF_ERR_NOT_SUPPORTED);
+    }
+
+    return out;
 }
 
 template<typename T, af_op_t op>
 SparseArray<T> arithOp(const SparseArray<T> &lhs, const SparseArray<T> &rhs) {
-    ONEAPI_NOT_SUPPORTED("arithOp Not supported");
-    // lhs.eval();
-    // rhs.eval();
-    // af::storage sfmt = lhs.getStorage();
+    lhs.eval();
+    rhs.eval();
+    af::storage sfmt = lhs.getStorage();
 
-    // const dim4 &ldims = lhs.dims();
+    const dim4 &ldims = lhs.dims();
 
-    // const uint M = ldims[0];
-    // const uint N = ldims[1];
+    const uint M = ldims[0];
+    const uint N = ldims[1];
 
-    // const dim_t nnzA = lhs.getNNZ();
-    // const dim_t nnzB = rhs.getNNZ();
+    const dim_t nnzA = lhs.getNNZ();
+    const dim_t nnzB = rhs.getNNZ();
 
-    // auto temp = createValueArray<int>(dim4(M + 1), scalar<int>(0));
-    // temp.eval();
+    auto temp = createValueArray<int>(dim4(M + 1), scalar<int>(0));
+    temp.eval();
 
-    // unsigned nnzC = 0;
-    // kernel::csrCalcOutNNZ(temp, nnzC, M, N, nnzA, lhs.getRowIdx(),
-    //                       lhs.getColIdx(), nnzB, rhs.getRowIdx(),
-    //                       rhs.getColIdx());
+    unsigned nnzC = 0;
+    kernel::csrCalcOutNNZ(temp, nnzC, M, N, nnzA, lhs.getRowIdx(),
+                          lhs.getColIdx(), nnzB, rhs.getRowIdx(),
+                          rhs.getColIdx());
 
-    // auto outRowIdx = scan<af_add_t, int, int>(temp, 0);
+    auto outRowIdx = scan<af_add_t, int, int>(temp, 0);
 
-    // auto outColIdx = createEmptyArray<int>(dim4(nnzC));
-    // auto outValues = createEmptyArray<T>(dim4(nnzC));
+    auto outColIdx = createEmptyArray<int>(dim4(nnzC));
+    auto outValues = createEmptyArray<T>(dim4(nnzC));
 
-    // kernel::ssArithCSR<T, op>(outValues, outColIdx, outRowIdx, M, N, nnzA,
-    //                           lhs.getValues(), lhs.getRowIdx(),
-    //                           lhs.getColIdx(), nnzB, rhs.getValues(),
-    //                           rhs.getRowIdx(), rhs.getColIdx());
+    kernel::ssArithCSR<T, op>(outValues, outColIdx, outRowIdx, M, N, nnzA,
+                              lhs.getValues(), lhs.getRowIdx(), lhs.getColIdx(),
+                              nnzB, rhs.getValues(), rhs.getRowIdx(),
+                              rhs.getColIdx());
 
-    // SparseArray<T> retVal = createArrayDataSparseArray(
-    //     ldims, outValues, outRowIdx, outColIdx, sfmt);
-    // return retVal;
+    SparseArray<T> retVal = createArrayDataSparseArray(
+        ldims, outValues, outRowIdx, outColIdx, sfmt);
+    return retVal;
 }
 
 #define INSTANTIATE(T)                                                         \
diff --git a/src/backend/oneapi/sparse_blas.cpp b/src/backend/oneapi/sparse_blas.cpp
index 67d7cb8352..0494a5806e 100644
--- a/src/backend/oneapi/sparse_blas.cpp
+++ b/src/backend/oneapi/sparse_blas.cpp
@@ -9,15 +9,6 @@
 
 #include <sparse_blas.hpp>
 
-// #include <kernel/cscmm.hpp>
-// #include <kernel/cscmv.hpp>
-// #include <kernel/csrmm.hpp>
-// #include <kernel/csrmv.hpp>
-
-#include <cassert>
-#include <stdexcept>
-#include <string>
-
 #include <common/err_common.hpp>
 #include <complex.hpp>
 #include <err_oneapi.hpp>
@@ -26,68 +17,77 @@
 #include <transpose.hpp>
 #include <af/dim4.hpp>
 
-#if defined(WITH_LINEAR_ALGEBRA)
-// #include <cpu/cpu_sparse_blas.hpp>
-#endif  // WITH_LINEAR_ALGEBRA
+#include <oneapi/mkl/spblas.hpp>
+
+#include <sycl/sycl.hpp>
+
+#include <cassert>
+#include <stdexcept>
+#include <string>
 
 namespace arrayfire {
 namespace oneapi {
 
 using namespace common;
 
+// Converts an af_mat_prop options to a transpose type for mkl
+static ::oneapi::mkl::transpose toBlasTranspose(af_mat_prop opt) {
+    switch (opt) {
+        case AF_MAT_NONE: return ::oneapi::mkl::transpose::nontrans;
+        case AF_MAT_TRANS: return ::oneapi::mkl::transpose::trans;
+        case AF_MAT_CTRANS: return ::oneapi::mkl::transpose::conjtrans;
+        default: AF_ERROR("INVALID af_mat_prop", AF_ERR_ARG);
+    }
+}
+
 template<typename T>
 Array<T> matmul(const common::SparseArray<T>& lhs, const Array<T>& rhsIn,
                 af_mat_prop optLhs, af_mat_prop optRhs) {
-    ONEAPI_NOT_SUPPORTED("sparse matmul Not supported");
-    // #if defined(WITH_LINEAR_ALGEBRA)
-    //     if (OpenCLCPUOffload(
-    //             false)) {  // Do not force offload gemm on OSX Intel devices
-    //         return cpu::matmul(lhs, rhsIn, optLhs, optRhs);
-    //     }
-    // #endif
-    //
-    //     int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1;
-    //     // int lColDim = (optLhs == AF_MAT_NONE) ? 1 : 0;
-    //     static const int rColDim =
-    //         1;  // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0;
-    //
-    //     dim4 lDims = lhs.dims();
-    //     dim4 rDims = rhsIn.dims();
-    //     int M      = lDims[lRowDim];
-    //     int N      = rDims[rColDim];
-    //     // int K = lDims[lColDim];
-    //
-    //     const Array<T> rhs =
-    //         (N != 1 && optLhs == AF_MAT_NONE) ? transpose(rhsIn, false) :
-    //         rhsIn;
-    //     Array<T> out = createEmptyArray<T>(af::dim4(M, N, 1, 1));
-    //
-    //     static const T alpha = scalar<T>(1.0);
-    //     static const T beta  = scalar<T>(0.0);
-    //
-    //     const Array<T>& values   = lhs.getValues();
-    //     const Array<int>& rowIdx = lhs.getRowIdx();
-    //     const Array<int>& colIdx = lhs.getColIdx();
-    //
-    //     if (optLhs == AF_MAT_NONE) {
-    //         if (N == 1) {
-    //             kernel::csrmv(out, values, rowIdx, colIdx, rhs, alpha, beta);
-    //         } else {
-    //             kernel::csrmm_nt(out, values, rowIdx, colIdx, rhs, alpha,
-    //             beta);
-    //         }
-    //     } else {
-    //         // CSR transpose is a CSC matrix
-    //         if (N == 1) {
-    //             kernel::cscmv(out, values, rowIdx, colIdx, rhs, alpha, beta,
-    //                           optLhs == AF_MAT_CTRANS);
-    //         } else {
-    //             kernel::cscmm_nn(out, values, rowIdx, colIdx, rhs, alpha,
-    //             beta,
-    //                              optLhs == AF_MAT_CTRANS);
-    //         }
-    //     }
-    //     return out;
+    int lRowDim = (optLhs == AF_MAT_NONE) ? 0 : 1;
+    static const int rColDim =
+        1;  // Unsupported : (optRhs == AF_MAT_NONE) ? 1 : 0;
+
+    dim4 lDims    = lhs.dims();
+    dim4 rDims    = rhsIn.dims();
+    dim4 rStrides = rhsIn.strides();
+    int M         = lDims[lRowDim];
+    int N         = rDims[rColDim];
+
+    Array<T> out  = createEmptyArray<T>(af::dim4(M, N, 1, 1));
+    dim4 oStrides = out.strides();
+
+    static const T alpha = scalar<T>(1.0);
+    static const T beta  = scalar<T>(0.0);
+
+    const Array<T>& values      = lhs.getValues();
+    const Array<int>& rowIdx    = lhs.getRowIdx();
+    const Array<int>& colIdx    = lhs.getColIdx();
+    sycl::buffer<T, 1> valBuf   = values.template getBufferWithOffset<T>();
+    sycl::buffer<int, 1> rowBuf = rowIdx.template getBufferWithOffset<int>();
+    sycl::buffer<int, 1> colBuf = colIdx.template getBufferWithOffset<int>();
+
+    const auto lOpts = toBlasTranspose(optLhs);
+    const auto rOpts = toBlasTranspose(optRhs);
+
+    sycl::buffer<T, 1> rhsBuf = rhsIn.template getBufferWithOffset<T>();
+    sycl::buffer<T, 1> outBuf = out.template getBufferWithOffset<T>();
+
+    ::oneapi::mkl::sparse::matrix_handle_t CSRHandle = nullptr;
+    ::oneapi::mkl::sparse::init_matrix_handle(&CSRHandle);
+    ::oneapi::mkl::sparse::set_csr_data(
+        getQueue(), CSRHandle, lDims[0], lDims[1],
+        ::oneapi::mkl::index_base::zero, rowBuf, colBuf, valBuf);
+
+    if (N == 1) {
+        ::oneapi::mkl::sparse::gemv(getQueue(), lOpts, alpha, CSRHandle, rhsBuf,
+                                    beta, outBuf);
+    } else {
+        ::oneapi::mkl::sparse::gemm(
+            getQueue(), ::oneapi::mkl::layout::col_major, lOpts, rOpts, alpha,
+            CSRHandle, rhsBuf, N, rStrides[1], beta, outBuf, oStrides[1]);
+    }
+    ::oneapi::mkl::sparse::release_matrix_handle(getQueue(), &CSRHandle);
+    return out;
 }
 
 #define INSTANTIATE_SPARSE(T)                                            \

From 886db208836b6d5ffba3442f36025cc3dd2bfca1 Mon Sep 17 00:00:00 2001
From: Mike Mullen <96440448+mfzmullen@users.noreply.github.com>
Date: Thu, 27 Jul 2023 17:56:02 -0500
Subject: [PATCH 345/473] fix finding vector_types and vector_functions (#3471)

* add to sourceIsJit

---------

Co-authored-by: Umar Arshad <umar@arrayfire.com>
---
 src/backend/cuda/CMakeLists.txt     |  1 +
 src/backend/cuda/compile_module.cpp | 10 ++++------
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index b0b0841b54..1f6e819b2f 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -142,6 +142,7 @@ set(nvrtc_src
   ${CUDA_INCLUDE_DIRS}/cuda_fp16.hpp
   ${CUDA_TOOLKIT_ROOT_DIR}/include/cuComplex.h
   ${CUDA_TOOLKIT_ROOT_DIR}/include/math_constants.h
+  ${CUDA_TOOLKIT_ROOT_DIR}/include/vector_types.h
   ${CUDA_TOOLKIT_ROOT_DIR}/include/vector_functions.h
 
   ${PROJECT_SOURCE_DIR}/src/api/c/optypes.hpp
diff --git a/src/backend/cuda/compile_module.cpp b/src/backend/cuda/compile_module.cpp
index d1d988e66f..d7ee8182bc 100644
--- a/src/backend/cuda/compile_module.cpp
+++ b/src/backend/cuda/compile_module.cpp
@@ -40,6 +40,7 @@
 #include <nvrtc_kernel_headers/types_hpp.hpp>
 #include <nvrtc_kernel_headers/utility_hpp.hpp>
 #include <nvrtc_kernel_headers/vector_functions_h.hpp>
+#include <nvrtc_kernel_headers/vector_types_h.hpp>
 #include <nvrtc_kernel_headers/version_h.hpp>
 #include <optypes.hpp>
 #include <platform.hpp>
@@ -156,15 +157,12 @@ Module compileModule(const string &moduleKey, span<const string> sources,
     using namespace arrayfire::cuda;
     if (sourceIsJIT) {
         constexpr const char *header_names[] = {
-            "utility",
-            "cuda_fp16.hpp",
-            "cuda_fp16.h",
+            "utility",        "cuda_fp16.hpp",      "cuda_fp16.h",
+            "vector_types.h", "vector_functions.h",
         };
         constexpr size_t numHeaders = extent<decltype(header_names)>::value;
         array<const char *, numHeaders> headers = {
-            "",
-            cuda_fp16_hpp,
-            cuda_fp16_h,
+            "", cuda_fp16_hpp, cuda_fp16_h, vector_types_h, vector_functions_h,
         };
         static_assert(headers.size() == numHeaders,
                       "headers array contains fewer sources than header_names");

From bcf0e54c51d5727cf602b9fa8af29da83fbf13a4 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 27 Jul 2023 18:58:54 -0400
Subject: [PATCH 346/473] Add minimum driver versions for cuda 12.2

---
 src/backend/cuda/device_manager.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 8000f2f635..c60bf35437 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -101,6 +101,7 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {12020, 9, 0, 0},
     {12010, 9, 0, 0},
     {12000, 9, 0, 0},
     {11080, 9, 0, 0},
@@ -139,6 +140,7 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
+        {12020, 525.60f, 527.41f},
         {12010, 525.60f, 527.41f},
         {12000, 525.60f, 527.41f},
         {11080, 450.80f, 452.39f},

From 0c16f7e586d1fab3c4a1562d07fc48cedfc12e46 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 27 Jul 2023 19:02:00 -0400
Subject: [PATCH 347/473] Use ::value functions and AF_IF_CONSTEXPR for cuda
 10.2 support

---
 src/backend/common/half.hpp   | 17 ++++++++---------
 src/backend/common/traits.hpp |  7 ++-----
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index ac03ea6d89..b6585dc905 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -855,25 +855,24 @@ AF_CONSTEXPR __DH__ native_half_t int2half(T value) noexcept {
 template<std::float_round_style R, bool E, typename T>
 AF_CONSTEXPR T half2int(native_half_t value) {
 #ifdef __CUDA_ARCH__
-    AF_IF_CONSTEXPR(std::is_same_v<T, short> || std::is_same_v<T, char> ||
-                    std::is_same_v<T, unsigned char>) {
+    AF_IF_CONSTEXPR(std::is_same<T, short>::value ||
+                    std::is_same<T, char>::value ||
+                    std::is_same<T, unsigned char>::value) {
         return __half2short_rn(value);
     }
-    else AF_IF_CONSTEXPR(std::is_same_v<T, unsigned short>) {
+    else AF_IF_CONSTEXPR(std::is_same<T, unsigned short>::value) {
         return __half2ushort_rn(value);
     }
-    else AF_IF_CONSTEXPR(std::is_same_v<T, long long>) {
+    else AF_IF_CONSTEXPR(std::is_same<T, long long>::value) {
         return __half2ll_rn(value);
     }
-    else AF_IF_CONSTEXPR(std::is_same_v<T, unsigned long long>) {
+    else AF_IF_CONSTEXPR(std::is_same<T, unsigned long long>::value) {
         return __half2ull_rn(value);
     }
-    else AF_IF_CONSTEXPR(std::is_same_v<T, int>) {
+    else AF_IF_CONSTEXPR(std::is_same<T, int>::value) {
         return __half2int_rn(value);
     }
-    else AF_IF_CONSTEXPR(std::is_same_v<T, unsigned>) {
-        return __half2uint_rn(value);
-    }
+    else { return __half2uint_rn(value); }
 #elif defined(AF_ONEAPI)
     return static_cast<T>(value);
 #else
diff --git a/src/backend/common/traits.hpp b/src/backend/common/traits.hpp
index 7798c070c2..3036d91dd0 100644
--- a/src/backend/common/traits.hpp
+++ b/src/backend/common/traits.hpp
@@ -70,11 +70,8 @@ constexpr bool isFloating(af::dtype type) {
 
 template<typename T, typename U, typename... Args>
 constexpr bool is_any_of() {
-    if constexpr (!sizeof...(Args)) {
-        return std::is_same_v<T, U>;
-    } else {
-        return std::is_same_v<T, U> || is_any_of<T, Args...>();
-    }
+    AF_IF_CONSTEXPR(!sizeof...(Args)) { return std::is_same<T, U>::value; }
+    else { return std::is_same<T, U>::value || is_any_of<T, Args...>(); }
 }
 
 }  // namespace

From 5d469b8e5c9a7eed475b76285919db2d1a0c6a70 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 27 Jul 2023 20:56:53 -0400
Subject: [PATCH 348/473] Fix tests that fail on devices that do not support
 double

---
 test/binary.cpp | 7 ++++++-
 test/memory.cpp | 2 ++
 test/reduce.cpp | 6 ++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/test/binary.cpp b/test/binary.cpp
index ab557f8c9a..dafc3b8bff 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -373,7 +373,12 @@ class PowPrecisionTest : public ::testing::TestWithParam<T> {
         vector<T> hres(1, 0);                                              \
         B.host(&hres[0]);                                                  \
         std::fesetround(FE_TONEAREST);                                     \
-        T gold = (T)std::rint(std::pow((double)param, 2.0));               \
+        T gold;                                                            \
+        if (!af::isDoubleAvailable(af::getDevice())) {                     \
+            gold = (T)std::rint(std::pow((float)param, 2.0f));             \
+        } else {                                                           \
+            gold = (T)std::rint(std::pow((double)param, 2.0));             \
+        }                                                                  \
         ASSERT_EQ(hres[0], gold);                                          \
     }
 
diff --git a/test/memory.cpp b/test/memory.cpp
index 37a1de87b1..991756ca0b 100644
--- a/test/memory.cpp
+++ b/test/memory.cpp
@@ -917,6 +917,7 @@ TEST_F(MemoryManagerApi, E2ETest4D) {
 }
 
 TEST_F(MemoryManagerApi, E2ETest4DComplexDouble) {
+    SUPPORTED_TYPE_CHECK(double);
     size_t aSize = 8;
 
     af::array a = af::array(aSize, aSize, aSize, aSize, af::dtype::c64);
@@ -932,6 +933,7 @@ TEST_F(MemoryManagerApi, E2ETest4DComplexDouble) {
 }
 
 TEST_F(MemoryManagerApi, E2ETestMultipleAllocations) {
+    SUPPORTED_TYPE_CHECK(double);
     size_t aSize = 8;
 
     af::array a = af::array(aSize, af::dtype::c64);
diff --git a/test/reduce.cpp b/test/reduce.cpp
index f01dafec45..0726a11791 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -559,6 +559,9 @@ TEST_P(ReduceByKeyP, SumDim0) {
     if (noHalfTests(GetParam()->kType_)) {
         GTEST_SKIP() << "Half not supported on this device";
     }
+    if (noDoubleTests(GetParam()->vType_)) {
+        GTEST_SKIP() << "Double not supported on this device";
+    }
     array keyRes, valsReduced;
     sumByKey(keyRes, valsReduced, keys, vals, 0, 0);
 
@@ -573,6 +576,9 @@ TEST_P(ReduceByKeyP, SumDim2) {
     if (noHalfTests(GetParam()->kType_)) {
         GTEST_SKIP() << "Half not supported on this device";
     }
+    if (noDoubleTests(GetParam()->vType_)) {
+        GTEST_SKIP() << "Double not supported on this device";
+    }
     const int ntile = 2;
     vals            = tile(vals, 1, ntile, 1, 1);
     vals            = reorder(vals, 1, 2, 0, 3);

From d2a66367d859cdb554f2374e29d39c88d5fff978 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 28 Jul 2023 12:49:03 -0400
Subject: [PATCH 349/473] Update vcpkg baseline

---
 .github/workflows/win_cpu_build.yml | 2 +-
 vcpkg.json                          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/win_cpu_build.yml b/.github/workflows/win_cpu_build.yml
index 8564bd03b8..d42450f103 100644
--- a/.github/workflows/win_cpu_build.yml
+++ b/.github/workflows/win_cpu_build.yml
@@ -13,7 +13,7 @@ jobs:
         name: CPU (fftw, OpenBLAS, windows-latest)
         runs-on: windows-latest
         env:
-          VCPKG_HASH: f14984af3738e69f197bf0e647a8dca12de92996
+          VCPKG_HASH: 9d47b24eacbd1cd94f139457ef6cd35e5d92cc84
           VCPKG_DEFAULT_TRIPLET: x64-windows
         steps:
             - name: Checkout Repository
diff --git a/vcpkg.json b/vcpkg.json
index 5cf6972ce0..db3318eb47 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -83,5 +83,5 @@
             ]
         }
     },
-    "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996"
+    "builtin-baseline": "9d47b24eacbd1cd94f139457ef6cd35e5d92cc84"
 }

From 171d12d73ec30536f8055ca8b1079e808d23190a Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 2 Aug 2023 21:31:08 -0400
Subject: [PATCH 350/473] Add CMake files to add support for the SYCL language

---
 CMakeModules/CMakeDetermineSYCLCompiler.cmake | 237 ++++++++++++++
 CMakeModules/CMakeSYCLCompiler.cmake.in       |  83 +++++
 CMakeModules/CMakeSYCLCompilerABI.cpp         |  31 ++
 CMakeModules/CMakeSYCLInformation.cmake       | 296 ++++++++++++++++++
 CMakeModules/CMakeTestSYCLCompiler.cmake      |  89 ++++++
 5 files changed, 736 insertions(+)
 create mode 100644 CMakeModules/CMakeDetermineSYCLCompiler.cmake
 create mode 100644 CMakeModules/CMakeSYCLCompiler.cmake.in
 create mode 100644 CMakeModules/CMakeSYCLCompilerABI.cpp
 create mode 100644 CMakeModules/CMakeSYCLInformation.cmake
 create mode 100644 CMakeModules/CMakeTestSYCLCompiler.cmake

diff --git a/CMakeModules/CMakeDetermineSYCLCompiler.cmake b/CMakeModules/CMakeDetermineSYCLCompiler.cmake
new file mode 100644
index 0000000000..c4ddf75e3f
--- /dev/null
+++ b/CMakeModules/CMakeDetermineSYCLCompiler.cmake
@@ -0,0 +1,237 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+
+# determine the compiler to use for C++ programs
+# NOTE, a generator may set CMAKE_CXX_COMPILER before
+# loading this file to force a compiler.
+# use environment variable CXX first if defined by user, next use
+# the cmake variable CMAKE_GENERATOR_CXX which can be defined by a generator
+# as a default compiler
+# If the internal cmake variable _CMAKE_TOOLCHAIN_PREFIX is set, this is used
+# as prefix for the tools (e.g. arm-elf-g++, arm-elf-ar etc.)
+#
+# Sets the following variables:
+#   CMAKE_CXX_COMPILER
+#   CMAKE_COMPILER_IS_GNUCXX
+#   CMAKE_AR
+#   CMAKE_RANLIB
+#
+# If not already set before, it also sets
+#   _CMAKE_TOOLCHAIN_PREFIX
+
+include(${CMAKE_ROOT}/Modules/CMakeDetermineCompiler.cmake)
+
+# Load system-specific compiler preferences for this language.
+include(Platform/${CMAKE_SYSTEM_NAME}-Determine-CXX OPTIONAL)
+include(Platform/${CMAKE_SYSTEM_NAME}-CXX OPTIONAL)
+if(NOT CMAKE_CXX_COMPILER_NAMES)
+  set(CMAKE_CXX_COMPILER_NAMES CC)
+endif()
+
+if(${CMAKE_GENERATOR} MATCHES "Visual Studio")
+elseif("${CMAKE_GENERATOR}" MATCHES "Green Hills MULTI")
+elseif("${CMAKE_GENERATOR}" MATCHES "Xcode")
+  set(CMAKE_CXX_COMPILER_XCODE_TYPE sourcecode.cpp.cpp)
+  _cmake_find_compiler_path(CXX)
+else()
+  if(NOT CMAKE_CXX_COMPILER)
+    set(CMAKE_CXX_COMPILER_INIT NOTFOUND)
+
+    # prefer the environment variable CXX
+    if(NOT $ENV{CXX} STREQUAL "")
+      get_filename_component(CMAKE_CXX_COMPILER_INIT $ENV{CXX} PROGRAM PROGRAM_ARGS CMAKE_CXX_FLAGS_ENV_INIT)
+      if(CMAKE_CXX_FLAGS_ENV_INIT)
+        set(CMAKE_CXX_COMPILER_ARG1 "${CMAKE_CXX_FLAGS_ENV_INIT}" CACHE STRING "Arguments to CXX compiler")
+      endif()
+      if(NOT EXISTS ${CMAKE_CXX_COMPILER_INIT})
+        message(FATAL_ERROR "Could not find compiler set in environment variable CXX:\n$ENV{CXX}.\n${CMAKE_CXX_COMPILER_INIT}")
+      endif()
+    endif()
+
+    # next prefer the generator specified compiler
+    if(CMAKE_GENERATOR_CXX)
+      if(NOT CMAKE_CXX_COMPILER_INIT)
+        set(CMAKE_CXX_COMPILER_INIT ${CMAKE_GENERATOR_CXX})
+      endif()
+    endif()
+
+    # finally list compilers to try
+    if(NOT CMAKE_CXX_COMPILER_INIT)
+      set(CMAKE_CXX_COMPILER_LIST CC ${_CMAKE_TOOLCHAIN_PREFIX}c++ ${_CMAKE_TOOLCHAIN_PREFIX}g++ aCC cl bcc xlC)
+      if(NOT CMAKE_HOST_WIN32)
+        # FIXME(#24314): Add support for the GNU-like icpx compiler driver
+        # on Windows, first introduced by Intel oneAPI 2023.0.
+        list(APPEND CMAKE_CXX_COMPILER_LIST icpx)
+      endif()
+      list(APPEND CMAKE_CXX_COMPILER_LIST icx clang++)
+    endif()
+
+    _cmake_find_compiler(CXX)
+  else()
+    _cmake_find_compiler_path(CXX)
+  endif()
+  mark_as_advanced(CMAKE_CXX_COMPILER)
+
+  # Each entry in this list is a set of extra flags to try
+  # adding to the compile line to see if it helps produce
+  # a valid identification file.
+  set(CMAKE_CXX_COMPILER_ID_TEST_FLAGS_FIRST)
+  set(CMAKE_CXX_COMPILER_ID_TEST_FLAGS
+    # Try compiling to an object file only.
+    "-c"
+    # IAR does not detect language automatically
+    "--c++"
+    "--ec++"
+
+    # ARMClang need target options
+    "--target=arm-arm-none-eabi -mcpu=cortex-m3"
+
+    # MSVC needs at least one include directory for __has_include to function,
+    # but custom toolchains may run MSVC with no INCLUDE env var and no -I flags.
+    # Also avoid linking so this works with no LIB env var.
+    "-c -I__does_not_exist__"
+    )
+endif()
+
+if(CMAKE_CXX_COMPILER_TARGET)
+  set(CMAKE_CXX_COMPILER_ID_TEST_FLAGS_FIRST "-c --target=${CMAKE_CXX_COMPILER_TARGET}")
+endif()
+
+# Build a small source file to identify the compiler.
+if(NOT CMAKE_CXX_COMPILER_ID_RUN)
+  set(CMAKE_CXX_COMPILER_ID_RUN 1)
+
+  # Try to identify the compiler.
+  set(CMAKE_CXX_COMPILER_ID)
+  set(CMAKE_CXX_PLATFORM_ID)
+  file(READ ${CMAKE_ROOT}/Modules/CMakePlatformId.h.in
+    CMAKE_CXX_COMPILER_ID_PLATFORM_CONTENT)
+
+  # The IAR compiler produces weird output.
+  # See https://gitlab.kitware.com/cmake/cmake/-/issues/10176#note_153591
+  list(APPEND CMAKE_CXX_COMPILER_ID_VENDORS IAR)
+  set(CMAKE_CXX_COMPILER_ID_VENDOR_FLAGS_IAR )
+  set(CMAKE_CXX_COMPILER_ID_VENDOR_REGEX_IAR "IAR .+ Compiler")
+
+  # Match the link line from xcodebuild output of the form
+  #  Ld ...
+  #      ...
+  #      /path/to/cc ...CompilerIdCXX/...
+  # to extract the compiler front-end for the language.
+  set(CMAKE_CXX_COMPILER_ID_TOOL_MATCH_REGEX "\nLd[^\n]*(\n[ \t]+[^\n]*)*\n[ \t]+([^ \t\r\n]+)[^\r\n]*-o[^\r\n]*CompilerIdCXX/(\\./)?(CompilerIdCXX.(framework|xctest|build/[^ \t\r\n]+)/)?CompilerIdCXX[ \t\n\\\"]")
+  set(CMAKE_CXX_COMPILER_ID_TOOL_MATCH_INDEX 2)
+
+  include(${CMAKE_ROOT}/Modules/CMakeDetermineCompilerId.cmake)
+  CMAKE_DETERMINE_COMPILER_ID(CXX CXXFLAGS CMakeCXXCompilerId.cpp)
+
+  _cmake_find_compiler_sysroot(CXX)
+
+  # Set old compiler and platform id variables.
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set(CMAKE_COMPILER_IS_GNUCXX 1)
+  endif()
+else()
+  if(NOT DEFINED CMAKE_CXX_COMPILER_FRONTEND_VARIANT)
+    # Some toolchain files set our internal CMAKE_CXX_COMPILER_ID_RUN
+    # variable but are not aware of CMAKE_CXX_COMPILER_FRONTEND_VARIANT.
+    # They pre-date our support for the GNU-like variant targeting the
+    # MSVC ABI so we do not consider that here.
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang"
+      OR "x${CMAKE_CXX_COMPILER_ID}" STREQUAL "xIntelLLVM")
+      if("x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC")
+        set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "MSVC")
+      else()
+        set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "GNU")
+      endif()
+    else()
+      set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "")
+    endif()
+  endif()
+endif()
+
+if (NOT _CMAKE_TOOLCHAIN_LOCATION)
+  get_filename_component(_CMAKE_TOOLCHAIN_LOCATION "${CMAKE_CXX_COMPILER}" PATH)
+endif ()
+
+# if we have a g++ cross compiler, they have usually some prefix, like
+# e.g. powerpc-linux-g++, arm-elf-g++ or i586-mingw32msvc-g++ , optionally
+# with a 3-component version number at the end (e.g. arm-eabi-gcc-4.5.2).
+# The other tools of the toolchain usually have the same prefix
+# NAME_WE cannot be used since then this test will fail for names like
+# "arm-unknown-nto-qnx6.3.0-gcc.exe", where BASENAME would be
+# "arm-unknown-nto-qnx6" instead of the correct "arm-unknown-nto-qnx6.3.0-"
+
+
+if (NOT _CMAKE_TOOLCHAIN_PREFIX)
+
+  if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU|Clang|QCC|LCC")
+    get_filename_component(COMPILER_BASENAME "${CMAKE_CXX_COMPILER}" NAME)
+    if (COMPILER_BASENAME MATCHES "^(.+-)?(clang\\+\\+|[gc]\\+\\+|clang-cl)(-[0-9]+(\\.[0-9]+)*)?(-[^.]+)?(\\.exe)?$")
+      set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_MATCH_1})
+      set(_CMAKE_TOOLCHAIN_SUFFIX ${CMAKE_MATCH_3})
+      set(_CMAKE_COMPILER_SUFFIX ${CMAKE_MATCH_5})
+    elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+      if(CMAKE_CXX_COMPILER_TARGET)
+        set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_CXX_COMPILER_TARGET}-)
+      endif()
+    elseif(COMPILER_BASENAME MATCHES "QCC(\\.exe)?$")
+      if(CMAKE_CXX_COMPILER_TARGET MATCHES "gcc_nto([a-z0-9]+_[0-9]+|[^_le]+)(le)")
+        set(_CMAKE_TOOLCHAIN_PREFIX nto${CMAKE_MATCH_1}-)
+      endif()
+    endif ()
+
+    # if "llvm-" is part of the prefix, remove it, since llvm doesn't have its own binutils
+    # but uses the regular ar, objcopy, etc. (instead of llvm-objcopy etc.)
+    if ("${_CMAKE_TOOLCHAIN_PREFIX}" MATCHES "(.+-)?llvm-$")
+      set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_MATCH_1})
+    endif ()
+  elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "TI")
+    # TI compilers are named e.g. cl6x, cl470 or armcl.exe
+    get_filename_component(COMPILER_BASENAME "${CMAKE_CXX_COMPILER}" NAME)
+    if (COMPILER_BASENAME MATCHES "^(.+)?cl([^.]+)?(\\.exe)?$")
+      set(_CMAKE_TOOLCHAIN_PREFIX "${CMAKE_MATCH_1}")
+      set(_CMAKE_TOOLCHAIN_SUFFIX "${CMAKE_MATCH_2}")
+    endif ()
+
+  endif()
+
+endif ()
+
+set(_CMAKE_PROCESSING_LANGUAGE "CXX")
+include(CMakeFindBinUtils)
+include(Compiler/${CMAKE_CXX_COMPILER_ID}-FindBinUtils OPTIONAL)
+unset(_CMAKE_PROCESSING_LANGUAGE)
+
+if(CMAKE_CXX_COMPILER_SYSROOT)
+  string(CONCAT _SET_CMAKE_CXX_COMPILER_SYSROOT
+    "set(CMAKE_CXX_COMPILER_SYSROOT \"${CMAKE_CXX_COMPILER_SYSROOT}\")\n"
+    "set(CMAKE_COMPILER_SYSROOT \"${CMAKE_CXX_COMPILER_SYSROOT}\")")
+else()
+  set(_SET_CMAKE_CXX_COMPILER_SYSROOT "")
+endif()
+
+if(CMAKE_CXX_COMPILER_ARCHITECTURE_ID)
+  set(_SET_CMAKE_CXX_COMPILER_ARCHITECTURE_ID
+    "set(CMAKE_CXX_COMPILER_ARCHITECTURE_ID ${CMAKE_CXX_COMPILER_ARCHITECTURE_ID})")
+else()
+  set(_SET_CMAKE_CXX_COMPILER_ARCHITECTURE_ID "")
+endif()
+
+if(MSVC_CXX_ARCHITECTURE_ID)
+  set(SET_MSVC_CXX_ARCHITECTURE_ID
+    "set(MSVC_CXX_ARCHITECTURE_ID ${MSVC_CXX_ARCHITECTURE_ID})")
+endif()
+
+if(CMAKE_CXX_XCODE_ARCHS)
+  set(SET_CMAKE_XCODE_ARCHS
+    "set(CMAKE_XCODE_ARCHS \"${CMAKE_CXX_XCODE_ARCHS}\")")
+endif()
+
+# configure all variables set in this file
+configure_file(${CMAKE_ROOT}/Modules/CMakeCXXCompiler.cmake.in
+  ${CMAKE_PLATFORM_INFO_DIR}/CMakeCXXCompiler.cmake
+  @ONLY
+  )
+
+set(CMAKE_CXX_COMPILER_ENV_VAR "CXX")
diff --git a/CMakeModules/CMakeSYCLCompiler.cmake.in b/CMakeModules/CMakeSYCLCompiler.cmake.in
new file mode 100644
index 0000000000..50edc9e474
--- /dev/null
+++ b/CMakeModules/CMakeSYCLCompiler.cmake.in
@@ -0,0 +1,83 @@
+set(CMAKE_SYCL_COMPILER "@CMAKE_SYCL_COMPILER@")
+set(CMAKE_SYCL_COMPILER_ARG1 "@CMAKE_SYCL_COMPILER_ARG1@")
+set(CMAKE_SYCL_COMPILER_ID "@CMAKE_SYCL_COMPILER_ID@")
+set(CMAKE_SYCL_COMPILER_VERSION "@CMAKE_SYCL_COMPILER_VERSION@")
+set(CMAKE_SYCL_COMPILER_VERSION_INTERNAL "@CMAKE_SYCL_COMPILER_VERSION_INTERNAL@")
+set(CMAKE_SYCL_COMPILER_WRAPPER "@CMAKE_SYCL_COMPILER_WRAPPER@")
+set(CMAKE_SYCL_STANDARD_COMPUTED_DEFAULT "@CMAKE_SYCL_STANDARD_COMPUTED_DEFAULT@")
+set(CMAKE_SYCL_EXTENSIONS_COMPUTED_DEFAULT "@CMAKE_SYCL_EXTENSIONS_COMPUTED_DEFAULT@")
+set(CMAKE_SYCL_COMPILE_FEATURES "@CMAKE_SYCL_COMPILE_FEATURES@")
+set(CMAKE_SYCL98_COMPILE_FEATURES "@CMAKE_SYCL98_COMPILE_FEATURES@")
+set(CMAKE_SYCL11_COMPILE_FEATURES "@CMAKE_SYCL11_COMPILE_FEATURES@")
+set(CMAKE_SYCL14_COMPILE_FEATURES "@CMAKE_SYCL14_COMPILE_FEATURES@")
+set(CMAKE_SYCL17_COMPILE_FEATURES "@CMAKE_SYCL17_COMPILE_FEATURES@")
+set(CMAKE_SYCL20_COMPILE_FEATURES "@CMAKE_SYCL20_COMPILE_FEATURES@")
+set(CMAKE_SYCL23_COMPILE_FEATURES "@CMAKE_SYCL23_COMPILE_FEATURES@")
+
+set(CMAKE_SYCL_PLATFORM_ID "@CMAKE_SYCL_PLATFORM_ID@")
+set(CMAKE_SYCL_SIMULATE_ID "@CMAKE_SYCL_SIMULATE_ID@")
+set(CMAKE_SYCL_COMPILER_FRONTEND_VARIANT "@CMAKE_SYCL_COMPILER_FRONTEND_VARIANT@")
+set(CMAKE_SYCL_SIMULATE_VERSION "@CMAKE_SYCL_SIMULATE_VERSION@")
+@_SET_CMAKE_SYCL_COMPILER_ARCHITECTURE_ID@
+@_SET_CMAKE_SYCL_COMPILER_SYSROOT@
+@SET_MSVC_SYCL_ARCHITECTURE_ID@
+@SET_CMAKE_XCODE_ARCHS@
+set(CMAKE_AR "@CMAKE_AR@")
+set(CMAKE_SYCL_COMPILER_AR "@CMAKE_SYCL_COMPILER_AR@")
+set(CMAKE_RANLIB "@CMAKE_RANLIB@")
+set(CMAKE_SYCL_COMPILER_RANLIB "@CMAKE_SYCL_COMPILER_RANLIB@")
+set(CMAKE_LINKER "@CMAKE_LINKER@")
+set(CMAKE_MT "@CMAKE_MT@")
+set(CMAKE_COMPILER_IS_GNUSYCL @CMAKE_COMPILER_IS_GNUSYCL@)
+set(CMAKE_SYCL_COMPILER_LOADED 1)
+set(CMAKE_SYCL_COMPILER_WORKS @CMAKE_SYCL_COMPILER_WORKS@)
+set(CMAKE_SYCL_ABI_COMPILED @CMAKE_SYCL_ABI_COMPILED@)
+
+set(CMAKE_SYCL_COMPILER_ENV_VAR "SYCL")
+
+set(CMAKE_SYCL_COMPILER_ID_RUN 1)
+set(CMAKE_SYCL_SOURCE_FILE_EXTENSIONS C;M;c++;cc;cpp;cxx;m;mm;mpp;CPP;ixx;cppm)
+set(CMAKE_SYCL_IGNORE_EXTENSIONS inl;h;hpp;HPP;H;o;O;obj;OBJ;def;DEF;rc;RC)
+
+foreach (lang C OBJC OBJSYCL)
+  if (CMAKE_${lang}_COMPILER_ID_RUN)
+    foreach(extension IN LISTS CMAKE_${lang}_SOURCE_FILE_EXTENSIONS)
+      list(REMOVE_ITEM CMAKE_SYCL_SOURCE_FILE_EXTENSIONS ${extension})
+    endforeach()
+  endif()
+endforeach()
+
+set(CMAKE_SYCL_LINKER_PREFERENCE 30)
+set(CMAKE_SYCL_LINKER_PREFERENCE_PROPAGATES 1)
+
+# Save compiler ABI information.
+set(CMAKE_SYCL_SIZEOF_DATA_PTR "@CMAKE_SYCL_SIZEOF_DATA_PTR@")
+set(CMAKE_SYCL_COMPILER_ABI "@CMAKE_SYCL_COMPILER_ABI@")
+set(CMAKE_SYCL_BYTE_ORDER "@CMAKE_SYCL_BYTE_ORDER@")
+set(CMAKE_SYCL_LIBRARY_ARCHITECTURE "@CMAKE_SYCL_LIBRARY_ARCHITECTURE@")
+
+if(CMAKE_SYCL_SIZEOF_DATA_PTR)
+  set(CMAKE_SIZEOF_VOID_P "${CMAKE_SYCL_SIZEOF_DATA_PTR}")
+endif()
+
+if(CMAKE_SYCL_COMPILER_ABI)
+  set(CMAKE_INTERNAL_PLATFORM_ABI "${CMAKE_SYCL_COMPILER_ABI}")
+endif()
+
+if(CMAKE_SYCL_LIBRARY_ARCHITECTURE)
+  set(CMAKE_LIBRARY_ARCHITECTURE "@CMAKE_SYCL_LIBRARY_ARCHITECTURE@")
+endif()
+
+set(CMAKE_SYCL_CL_SHOWINCLUDES_PREFIX "@CMAKE_SYCL_CL_SHOWINCLUDES_PREFIX@")
+if(CMAKE_SYCL_CL_SHOWINCLUDES_PREFIX)
+  set(CMAKE_CL_SHOWINCLUDES_PREFIX "${CMAKE_SYCL_CL_SHOWINCLUDES_PREFIX}")
+endif()
+
+@CMAKE_SYCL_COMPILER_CUSTOM_CODE@
+@CMAKE_SYCL_SYSROOT_FLAG_CODE@
+@CMAKE_SYCL_OSX_DEPLOYMENT_TARGET_FLAG_CODE@
+
+set(CMAKE_SYCL_IMPLICIT_INCLUDE_DIRECTORIES "@CMAKE_SYCL_IMPLICIT_INCLUDE_DIRECTORIES@")
+set(CMAKE_SYCL_IMPLICIT_LINK_LIBRARIES "@CMAKE_SYCL_IMPLICIT_LINK_LIBRARIES@")
+set(CMAKE_SYCL_IMPLICIT_LINK_DIRECTORIES "@CMAKE_SYCL_IMPLICIT_LINK_DIRECTORIES@")
+set(CMAKE_SYCL_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES "@CMAKE_SYCL_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES@")
diff --git a/CMakeModules/CMakeSYCLCompilerABI.cpp b/CMakeModules/CMakeSYCLCompilerABI.cpp
new file mode 100644
index 0000000000..fe7c926993
--- /dev/null
+++ b/CMakeModules/CMakeSYCLCompilerABI.cpp
@@ -0,0 +1,31 @@
+#ifndef __cplusplus
+#  error "A C compiler has been selected for C++."
+#endif
+
+#include "CMakeCompilerABI.h"
+#include <sycl/sycl.hpp>
+
+int main(int argc, char* argv[])
+{
+  int require = 0;
+  require += info_sizeof_dptr[argc];
+  require += info_byte_order_big_endian[argc];
+  require += info_byte_order_little_endian[argc];
+#if defined(ABI_ID)
+  require += info_abi[argc];
+#endif
+  static_cast<void>(argv);
+
+  int count = 0;
+  auto platforms = sycl::platform::get_platforms();
+  for(sycl::platform &platform : platforms) {
+    count += platform.get_devices().size();
+  }
+
+  if(count == 0) {
+    std::fprintf(stderr, "No SYCL devices found.\n");
+    return -1;
+  }
+
+  return require;
+}
diff --git a/CMakeModules/CMakeSYCLInformation.cmake b/CMakeModules/CMakeSYCLInformation.cmake
new file mode 100644
index 0000000000..53abf378d5
--- /dev/null
+++ b/CMakeModules/CMakeSYCLInformation.cmake
@@ -0,0 +1,296 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+
+# This file sets the basic flags for the C++ language in CMake.
+# It also loads the available platform file for the system-compiler
+# if it exists.
+# It also loads a system - compiler - processor (or target hardware)
+# specific file, which is mainly useful for crosscompiling and embedded systems.
+
+include(CMakeLanguageInformation)
+
+# some compilers use different extensions (e.g. sdcc uses .rel)
+# so set the extension here first so it can be overridden by the compiler specific file
+if(UNIX)
+  set(CMAKE_CXX_OUTPUT_EXTENSION .o)
+else()
+  set(CMAKE_CXX_OUTPUT_EXTENSION .obj)
+endif()
+
+set(_INCLUDED_FILE 0)
+
+# Load compiler-specific information.
+if(CMAKE_CXX_COMPILER_ID)
+  include(Compiler/${CMAKE_CXX_COMPILER_ID}-CXX OPTIONAL)
+endif()
+
+set(CMAKE_BASE_NAME)
+get_filename_component(CMAKE_BASE_NAME "${CMAKE_CXX_COMPILER}" NAME_WE)
+# since the gnu compiler has several names force g++
+if(CMAKE_COMPILER_IS_GNUCXX)
+  set(CMAKE_BASE_NAME g++)
+endif()
+
+
+# load a hardware specific file, mostly useful for embedded compilers
+if(CMAKE_SYSTEM_PROCESSOR)
+  if(CMAKE_CXX_COMPILER_ID)
+    include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}-CXX-${CMAKE_SYSTEM_PROCESSOR} OPTIONAL RESULT_VARIABLE _INCLUDED_FILE)
+  endif()
+  if (NOT _INCLUDED_FILE)
+    include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_BASE_NAME}-${CMAKE_SYSTEM_PROCESSOR} OPTIONAL)
+  endif ()
+endif()
+
+# load the system- and compiler specific files
+if(CMAKE_CXX_COMPILER_ID)
+  include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}-CXX OPTIONAL RESULT_VARIABLE _INCLUDED_FILE)
+endif()
+if (NOT _INCLUDED_FILE)
+  include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_BASE_NAME} OPTIONAL
+          RESULT_VARIABLE _INCLUDED_FILE)
+endif ()
+
+# load any compiler-wrapper specific information
+if (CMAKE_CXX_COMPILER_WRAPPER)
+  __cmake_include_compiler_wrapper(CXX)
+endif ()
+
+# We specify the compiler information in the system file for some
+# platforms, but this language may not have been enabled when the file
+# was first included.  Include it again to get the language info.
+# Remove this when all compiler info is removed from system files.
+if (NOT _INCLUDED_FILE)
+  include(Platform/${CMAKE_SYSTEM_NAME} OPTIONAL)
+endif ()
+
+if(CMAKE_CXX_SIZEOF_DATA_PTR)
+  foreach(f ${CMAKE_CXX_ABI_FILES})
+    include(${f})
+  endforeach()
+  unset(CMAKE_CXX_ABI_FILES)
+endif()
+
+# This should be included before the _INIT variables are
+# used to initialize the cache.  Since the rule variables
+# have if blocks on them, users can still define them here.
+# But, it should still be after the platform file so changes can
+# be made to those values.
+
+if(CMAKE_USER_MAKE_RULES_OVERRIDE)
+  # Save the full path of the file so try_compile can use it.
+  include(${CMAKE_USER_MAKE_RULES_OVERRIDE} RESULT_VARIABLE _override)
+  set(CMAKE_USER_MAKE_RULES_OVERRIDE "${_override}")
+endif()
+
+if(CMAKE_USER_MAKE_RULES_OVERRIDE_CXX)
+  # Save the full path of the file so try_compile can use it.
+  include(${CMAKE_USER_MAKE_RULES_OVERRIDE_CXX} RESULT_VARIABLE _override)
+  set(CMAKE_USER_MAKE_RULES_OVERRIDE_CXX "${_override}")
+endif()
+
+
+# Create a set of shared library variable specific to C++
+# For 90% of the systems, these are the same flags as the C versions
+# so if these are not set just copy the flags from the c version
+if(NOT CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS)
+  set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS})
+endif()
+
+if(NOT CMAKE_CXX_COMPILE_OPTIONS_PIC)
+  set(CMAKE_CXX_COMPILE_OPTIONS_PIC ${CMAKE_C_COMPILE_OPTIONS_PIC})
+endif()
+
+if(NOT CMAKE_CXX_COMPILE_OPTIONS_PIE)
+  set(CMAKE_CXX_COMPILE_OPTIONS_PIE ${CMAKE_C_COMPILE_OPTIONS_PIE})
+endif()
+if(NOT CMAKE_CXX_LINK_OPTIONS_PIE)
+  set(CMAKE_CXX_LINK_OPTIONS_PIE ${CMAKE_C_LINK_OPTIONS_PIE})
+endif()
+if(NOT CMAKE_CXX_LINK_OPTIONS_NO_PIE)
+  set(CMAKE_CXX_LINK_OPTIONS_NO_PIE ${CMAKE_C_LINK_OPTIONS_NO_PIE})
+endif()
+
+if(NOT CMAKE_CXX_COMPILE_OPTIONS_DLL)
+  set(CMAKE_CXX_COMPILE_OPTIONS_DLL ${CMAKE_C_COMPILE_OPTIONS_DLL})
+endif()
+
+if(NOT CMAKE_SHARED_LIBRARY_CXX_FLAGS)
+  set(CMAKE_SHARED_LIBRARY_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_C_FLAGS})
+endif()
+
+if(NOT DEFINED CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS)
+  set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_C_FLAGS})
+endif()
+
+if(NOT CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG)
+  set(CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG})
+endif()
+
+if(NOT CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP)
+  set(CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG_SEP})
+endif()
+
+if(NOT CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG)
+  set(CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG ${CMAKE_SHARED_LIBRARY_RPATH_LINK_C_FLAG})
+endif()
+
+if(NOT DEFINED CMAKE_EXE_EXPORTS_CXX_FLAG)
+  set(CMAKE_EXE_EXPORTS_CXX_FLAG ${CMAKE_EXE_EXPORTS_C_FLAG})
+endif()
+
+if(NOT DEFINED CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG)
+  set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_C_FLAG})
+endif()
+
+if(NOT CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG)
+  set(CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
+endif()
+
+if(NOT CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG_SEP)
+  set(CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
+endif()
+
+if(NOT CMAKE_EXECUTABLE_RPATH_LINK_CXX_FLAG)
+  set(CMAKE_EXECUTABLE_RPATH_LINK_CXX_FLAG ${CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG})
+endif()
+
+if(NOT DEFINED CMAKE_SHARED_LIBRARY_LINK_CXX_WITH_RUNTIME_PATH)
+  set(CMAKE_SHARED_LIBRARY_LINK_CXX_WITH_RUNTIME_PATH ${CMAKE_SHARED_LIBRARY_LINK_C_WITH_RUNTIME_PATH})
+endif()
+
+if(NOT CMAKE_INCLUDE_FLAG_CXX)
+  set(CMAKE_INCLUDE_FLAG_CXX ${CMAKE_INCLUDE_FLAG_C})
+endif()
+
+# for most systems a module is the same as a shared library
+# so unless the variable CMAKE_MODULE_EXISTS is set just
+# copy the values from the LIBRARY variables
+if(NOT CMAKE_MODULE_EXISTS)
+  set(CMAKE_SHARED_MODULE_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
+  set(CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
+endif()
+
+# repeat for modules
+if(NOT CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS)
+  set(CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS ${CMAKE_SHARED_MODULE_CREATE_C_FLAGS})
+endif()
+
+if(NOT CMAKE_SHARED_MODULE_CXX_FLAGS)
+  set(CMAKE_SHARED_MODULE_CXX_FLAGS ${CMAKE_SHARED_MODULE_C_FLAGS})
+endif()
+
+# Initialize CXX link type selection flags from C versions.
+foreach(type SHARED_LIBRARY SHARED_MODULE EXE)
+  if(NOT CMAKE_${type}_LINK_STATIC_CXX_FLAGS)
+    set(CMAKE_${type}_LINK_STATIC_CXX_FLAGS
+      ${CMAKE_${type}_LINK_STATIC_C_FLAGS})
+  endif()
+  if(NOT CMAKE_${type}_LINK_DYNAMIC_CXX_FLAGS)
+    set(CMAKE_${type}_LINK_DYNAMIC_CXX_FLAGS
+      ${CMAKE_${type}_LINK_DYNAMIC_C_FLAGS})
+  endif()
+endforeach()
+
+if(CMAKE_EXECUTABLE_FORMAT STREQUAL "ELF")
+  if(NOT DEFINED CMAKE_CXX_LINK_WHAT_YOU_USE_FLAG)
+    set(CMAKE_CXX_LINK_WHAT_YOU_USE_FLAG "LINKER:--no-as-needed")
+  endif()
+  if(NOT DEFINED CMAKE_LINK_WHAT_YOU_USE_CHECK)
+    set(CMAKE_LINK_WHAT_YOU_USE_CHECK ldd -u -r)
+  endif()
+endif()
+
+# add the flags to the cache based
+# on the initial values computed in the platform/*.cmake files
+# use _INIT variables so that this only happens the first time
+# and you can set these flags in the cmake cache
+set(CMAKE_CXX_FLAGS_INIT "$ENV{CXXFLAGS} ${CMAKE_CXX_FLAGS_INIT}")
+
+cmake_initialize_per_config_variable(CMAKE_CXX_FLAGS "Flags used by the CXX compiler")
+
+if(CMAKE_CXX_STANDARD_LIBRARIES_INIT)
+  set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES_INIT}"
+    CACHE STRING "Libraries linked by default with all C++ applications.")
+  mark_as_advanced(CMAKE_CXX_STANDARD_LIBRARIES)
+endif()
+
+if(NOT CMAKE_CXX_COMPILER_LAUNCHER AND DEFINED ENV{CMAKE_CXX_COMPILER_LAUNCHER})
+  set(CMAKE_CXX_COMPILER_LAUNCHER "$ENV{CMAKE_CXX_COMPILER_LAUNCHER}"
+    CACHE STRING "Compiler launcher for CXX.")
+endif()
+
+if(NOT CMAKE_CXX_LINKER_LAUNCHER AND DEFINED ENV{CMAKE_CXX_LINKER_LAUNCHER})
+  set(CMAKE_CXX_LINKER_LAUNCHER "$ENV{CMAKE_CXX_LINKER_LAUNCHER}"
+    CACHE STRING "Linker launcher for CXX.")
+endif()
+
+include(CMakeCommonLanguageInclude)
+
+# now define the following rules:
+# CMAKE_CXX_CREATE_SHARED_LIBRARY
+# CMAKE_CXX_CREATE_SHARED_MODULE
+# CMAKE_CXX_COMPILE_OBJECT
+# CMAKE_CXX_LINK_EXECUTABLE
+
+# variables supplied by the generator at use time
+# <TARGET>
+# <TARGET_BASE> the target without the suffix
+# <OBJECTS>
+# <OBJECT>
+# <LINK_LIBRARIES>
+# <FLAGS>
+# <LINK_FLAGS>
+
+# CXX compiler information
+# <CMAKE_CXX_COMPILER>
+# <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS>
+# <CMAKE_CXX_SHARED_MODULE_CREATE_FLAGS>
+# <CMAKE_CXX_LINK_FLAGS>
+
+# Static library tools
+# <CMAKE_AR>
+# <CMAKE_RANLIB>
+
+
+# create a shared C++ library
+if(NOT CMAKE_CXX_CREATE_SHARED_LIBRARY)
+  set(CMAKE_CXX_CREATE_SHARED_LIBRARY
+      "<CMAKE_CXX_COMPILER> <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
+endif()
+
+# create a c++ shared module copy the shared library rule by default
+if(NOT CMAKE_CXX_CREATE_SHARED_MODULE)
+  set(CMAKE_CXX_CREATE_SHARED_MODULE ${CMAKE_CXX_CREATE_SHARED_LIBRARY})
+endif()
+
+
+# Create a static archive incrementally for large object file counts.
+# If CMAKE_CXX_CREATE_STATIC_LIBRARY is set it will override these.
+if(NOT DEFINED CMAKE_CXX_ARCHIVE_CREATE)
+  set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> qc <TARGET> <LINK_FLAGS> <OBJECTS>")
+endif()
+if(NOT DEFINED CMAKE_CXX_ARCHIVE_APPEND)
+  set(CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> q <TARGET> <LINK_FLAGS> <OBJECTS>")
+endif()
+if(NOT DEFINED CMAKE_CXX_ARCHIVE_FINISH)
+  set(CMAKE_CXX_ARCHIVE_FINISH "<CMAKE_RANLIB> <TARGET>")
+endif()
+
+# compile a C++ file into an object file
+if(NOT CMAKE_CXX_COMPILE_OBJECT)
+  set(CMAKE_CXX_COMPILE_OBJECT
+    "<CMAKE_CXX_COMPILER> <DEFINES> <INCLUDES> <FLAGS> -o <OBJECT> -c <SOURCE>")
+endif()
+
+if(NOT CMAKE_CXX_LINK_EXECUTABLE)
+  set(CMAKE_CXX_LINK_EXECUTABLE
+    "<CMAKE_CXX_COMPILER> <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+endif()
+
+mark_as_advanced(
+CMAKE_VERBOSE_MAKEFILE
+)
+
+set(CMAKE_CXX_INFORMATION_LOADED 1)
diff --git a/CMakeModules/CMakeTestSYCLCompiler.cmake b/CMakeModules/CMakeTestSYCLCompiler.cmake
new file mode 100644
index 0000000000..e640ff9b30
--- /dev/null
+++ b/CMakeModules/CMakeTestSYCLCompiler.cmake
@@ -0,0 +1,89 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+
+if(CMAKE_CXX_COMPILER_FORCED)
+  # The compiler configuration was forced by the user.
+  # Assume the user has configured all compiler information.
+  set(CMAKE_CXX_COMPILER_WORKS TRUE)
+  return()
+endif()
+
+include(CMakeTestCompilerCommon)
+
+# work around enforced code signing and / or missing executable target type
+set(__CMAKE_SAVED_TRY_COMPILE_TARGET_TYPE ${CMAKE_TRY_COMPILE_TARGET_TYPE})
+if(_CMAKE_FEATURE_DETECTION_TARGET_TYPE)
+  set(CMAKE_TRY_COMPILE_TARGET_TYPE ${_CMAKE_FEATURE_DETECTION_TARGET_TYPE})
+endif()
+
+# Remove any cached result from an older CMake version.
+# We now store this in CMakeCXXCompiler.cmake.
+unset(CMAKE_CXX_COMPILER_WORKS CACHE)
+
+# Try to identify the ABI and configure it into CMakeCXXCompiler.cmake
+include(${CMAKE_ROOT}/Modules/CMakeDetermineCompilerABI.cmake)
+CMAKE_DETERMINE_COMPILER_ABI(CXX ${CMAKE_ROOT}/Modules/CMakeCXXCompilerABI.cpp)
+if(CMAKE_CXX_ABI_COMPILED)
+  # The compiler worked so skip dedicated test below.
+  set(CMAKE_CXX_COMPILER_WORKS TRUE)
+  message(STATUS "Check for working CXX compiler: ${CMAKE_CXX_COMPILER} - skipped")
+endif()
+
+# This file is used by EnableLanguage in cmGlobalGenerator to
+# determine that the selected C++ compiler can actually compile
+# and link the most basic of programs.   If not, a fatal error
+# is set and cmake stops processing commands and will not generate
+# any makefiles or projects.
+if(NOT CMAKE_CXX_COMPILER_WORKS)
+  PrintTestCompilerStatus("CXX")
+  __TestCompiler_setTryCompileTargetType()
+  string(CONCAT __TestCompiler_testCXXCompilerSource
+    "#ifndef __cplusplus\n"
+    "# error \"The CMAKE_CXX_COMPILER is set to a C compiler\"\n"
+    "#endif\n"
+    "int main(){return 0;}\n")
+  # Clear result from normal variable.
+  unset(CMAKE_CXX_COMPILER_WORKS)
+  # Puts test result in cache variable.
+  try_compile(CMAKE_CXX_COMPILER_WORKS
+    SOURCE_FROM_VAR testCXXCompiler.cxx __TestCompiler_testCXXCompilerSource
+    OUTPUT_VARIABLE __CMAKE_CXX_COMPILER_OUTPUT)
+  unset(__TestCompiler_testCXXCompilerSource)
+  # Move result from cache to normal variable.
+  set(CMAKE_CXX_COMPILER_WORKS ${CMAKE_CXX_COMPILER_WORKS})
+  unset(CMAKE_CXX_COMPILER_WORKS CACHE)
+  __TestCompiler_restoreTryCompileTargetType()
+  if(NOT CMAKE_CXX_COMPILER_WORKS)
+    PrintTestCompilerResult(CHECK_FAIL "broken")
+    string(REPLACE "\n" "\n  " _output "${__CMAKE_CXX_COMPILER_OUTPUT}")
+    message(FATAL_ERROR "The C++ compiler\n  \"${CMAKE_CXX_COMPILER}\"\n"
+      "is not able to compile a simple test program.\nIt fails "
+      "with the following output:\n  ${_output}\n\n"
+      "CMake will not be able to correctly generate this project.")
+  endif()
+  PrintTestCompilerResult(CHECK_PASS "works")
+endif()
+
+# Try to identify the compiler features
+include(${CMAKE_ROOT}/Modules/CMakeDetermineCompileFeatures.cmake)
+CMAKE_DETERMINE_COMPILE_FEATURES(CXX)
+
+# Re-configure to save learned information.
+configure_file(
+  ${CMAKE_ROOT}/Modules/CMakeCXXCompiler.cmake.in
+  ${CMAKE_PLATFORM_INFO_DIR}/CMakeCXXCompiler.cmake
+  @ONLY
+  )
+include(${CMAKE_PLATFORM_INFO_DIR}/CMakeCXXCompiler.cmake)
+
+if(CMAKE_CXX_SIZEOF_DATA_PTR)
+  foreach(f ${CMAKE_CXX_ABI_FILES})
+    include(${f})
+  endforeach()
+  unset(CMAKE_CXX_ABI_FILES)
+endif()
+
+set(CMAKE_TRY_COMPILE_TARGET_TYPE ${__CMAKE_SAVED_TRY_COMPILE_TARGET_TYPE})
+unset(__CMAKE_SAVED_TRY_COMPILE_TARGET_TYPE)
+unset(__CMAKE_CXX_COMPILER_OUTPUT)

From b7a6074748d570c2a52f1cfd18e1ef90208063ab Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Wed, 2 Aug 2023 21:32:18 -0400
Subject: [PATCH 351/473] Update CMake language files from CXX to SYCL

---
 CMakeLists.txt                                |  15 +-
 CMakeModules/CMakeCompilerABI.h               |  45 +++
 CMakeModules/CMakeDetermineSYCLCompiler.cmake | 178 +++++-----
 CMakeModules/CMakeSYCLCompiler.cmake.in       |   2 +-
 CMakeModules/CMakeSYCLCompilerABI.cpp         |  12 -
 CMakeModules/CMakeSYCLCompilerId.cpp.in       | 105 ++++++
 CMakeModules/CMakeSYCLInformation.cmake       | 307 +++++++++++-------
 CMakeModules/CMakeTestSYCLCompiler.cmake      |  69 ++--
 CMakeModules/InternalUtils.cmake              |  32 ++
 src/backend/common/Logger.hpp                 |   1 +
 src/backend/oneapi/CMakeLists.txt             |  45 ++-
 src/backend/oneapi/device_manager.cpp         |   8 +-
 .../oneapi/kernel/sort_by_key/CMakeLists.txt  |  11 +-
 test/testHelpers.hpp                          |  15 +
 14 files changed, 567 insertions(+), 278 deletions(-)
 create mode 100644 CMakeModules/CMakeCompilerABI.h
 create mode 100644 CMakeModules/CMakeSYCLCompilerId.cpp.in

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a4c3eef645..e4cc17916f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10,9 +10,8 @@ include(CheckLanguage)
 
 include(CMakeModules/AF_vcpkg_options.cmake)
 
-project(ArrayFire VERSION 3.9.0 LANGUAGES C CXX)
-
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
+project(ArrayFire VERSION 3.9.0 LANGUAGES C CXX)
 
 include(AFconfigure_deps_vars)
 include(AFBuildConfigurations)
@@ -44,10 +43,11 @@ option(AF_WITH_EXTERNAL_PACKAGES_ONLY "Build ArrayFire with External packages on
 if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
   set(AF_REQUIRED REQUIRED)
 endif()
-
-get_filename_component(CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
-if(CXX_COMPILER_NAME STREQUAL "dpcpp" OR CXX_COMPILER_NAME STREQUAL "dpcpp.exe"
-   OR CXX_COMPILER_NAME STREQUAL "icpx" OR CXX_COMPILER_NAME STREQUAL "icx.exe")
+if(CMAKE_SYCL_COMPILER)
+  get_filename_component(SYCL_COMPILER_NAME ${CMAKE_SYCL_COMPILER} NAME)
+endif()
+if(SYCL_COMPILER_NAME STREQUAL "dpcpp" OR SYCL_COMPILER_NAME STREQUAL "dpcpp.exe"
+   OR SYCL_COMPILER_NAME STREQUAL "icpx" OR SYCL_COMPILER_NAME STREQUAL "icx.exe")
   set(MKL_THREAD_LAYER "TBB" CACHE STRING "The thread layer to choose for MKL")
   set(MKL_INTERFACE "ilp64")
   set(MKL_INTERFACE_INTEGER_SIZE 8)
@@ -134,6 +134,9 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
   if(DEFINED ENV{MKLROOT} AND NOT DEFINED MKL_ROOT)
     set(MKL_ROOT "$ENV{MKLROOT}")
   endif()
+  set(DPCPP_COMPILER ON)
+  set(MKL_THREADING "tbb_thread")
+  set(MKL_INTERFACE "ilp64")
   find_package(MKL 2023.1)
 endif()
 
diff --git a/CMakeModules/CMakeCompilerABI.h b/CMakeModules/CMakeCompilerABI.h
new file mode 100644
index 0000000000..c5ce4dd9ab
--- /dev/null
+++ b/CMakeModules/CMakeCompilerABI.h
@@ -0,0 +1,45 @@
+
+/* Size of a pointer-to-data in bytes.  */
+#define SIZEOF_DPTR (sizeof(void*))
+const char info_sizeof_dptr[] = {
+  /* clang-format off */
+  'I', 'N', 'F', 'O', ':', 's', 'i', 'z', 'e', 'o', 'f', '_', 'd', 'p', 't',
+  'r', '[', ('0' + ((SIZEOF_DPTR / 10) % 10)), ('0' + (SIZEOF_DPTR % 10)), ']',
+  '\0'
+  /* clang-format on */
+};
+
+/* Byte order.  Only one of these will have bytes in the right order.  */
+static unsigned short const info_byte_order_big_endian[] = {
+  /* INFO:byte_order string for BIG_ENDIAN */
+  0x494E, 0x464F, 0x3A62, 0x7974, 0x655F, 0x6F72, 0x6465, 0x725B,
+  0x4249, 0x475F, 0x454E, 0x4449, 0x414E, 0x5D00, 0x0000
+};
+static unsigned short const info_byte_order_little_endian[] = {
+  /* INFO:byte_order string for LITTLE_ENDIAN */
+  0x4E49, 0x4F46, 0x623A, 0x7479, 0x5F65, 0x726F, 0x6564, 0x5B72,
+  0x494C, 0x5454, 0x454C, 0x455F, 0x444E, 0x4149, 0x5D4E, 0x0000
+};
+
+/* Application Binary Interface.  */
+
+/* Check for (some) ARM ABIs.
+ * See e.g. http://wiki.debian.org/ArmEabiPort for some information on this. */
+#if defined(__GNU__) && defined(__ELF__) && defined(__ARM_EABI__)
+#  define ABI_ID "ELF ARMEABI"
+#elif defined(__GNU__) && defined(__ELF__) && defined(__ARMEB__)
+#  define ABI_ID "ELF ARM"
+#elif defined(__GNU__) && defined(__ELF__) && defined(__ARMEL__)
+#  define ABI_ID "ELF ARM"
+
+#elif defined(__linux__) && defined(__ELF__) && defined(__amd64__) &&         \
+  defined(__ILP32__)
+#  define ABI_ID "ELF X32"
+
+#elif defined(__ELF__)
+#  define ABI_ID "ELF"
+#endif
+
+#if defined(ABI_ID)
+static char const info_abi[] = "INFO:abi[" ABI_ID "]";
+#endif
diff --git a/CMakeModules/CMakeDetermineSYCLCompiler.cmake b/CMakeModules/CMakeDetermineSYCLCompiler.cmake
index c4ddf75e3f..669e8a79e3 100644
--- a/CMakeModules/CMakeDetermineSYCLCompiler.cmake
+++ b/CMakeModules/CMakeDetermineSYCLCompiler.cmake
@@ -3,81 +3,82 @@
 
 
 # determine the compiler to use for C++ programs
-# NOTE, a generator may set CMAKE_CXX_COMPILER before
+# NOTE, a generator may set CMAKE_SYCL_COMPILER before
 # loading this file to force a compiler.
-# use environment variable CXX first if defined by user, next use
-# the cmake variable CMAKE_GENERATOR_CXX which can be defined by a generator
+# use environment variable SYCL first if defined by user, next use
+# the cmake variable CMAKE_GENERATOR_SYCL which can be defined by a generator
 # as a default compiler
 # If the internal cmake variable _CMAKE_TOOLCHAIN_PREFIX is set, this is used
 # as prefix for the tools (e.g. arm-elf-g++, arm-elf-ar etc.)
 #
 # Sets the following variables:
-#   CMAKE_CXX_COMPILER
-#   CMAKE_COMPILER_IS_GNUCXX
+#   CMAKE_SYCL_COMPILER
+#   CMAKE_COMPILER_IS_GNUSYCL
 #   CMAKE_AR
 #   CMAKE_RANLIB
 #
 # If not already set before, it also sets
 #   _CMAKE_TOOLCHAIN_PREFIX
 
-include(${CMAKE_ROOT}/Modules/CMakeDetermineCompiler.cmake)
+#list(APPEND CMAKE_MODULE_PATH ${CMAKE_ROOT})
+include(CMakeDetermineCompiler)
 
 # Load system-specific compiler preferences for this language.
-include(Platform/${CMAKE_SYSTEM_NAME}-Determine-CXX OPTIONAL)
-include(Platform/${CMAKE_SYSTEM_NAME}-CXX OPTIONAL)
-if(NOT CMAKE_CXX_COMPILER_NAMES)
-  set(CMAKE_CXX_COMPILER_NAMES CC)
+#include(Platform/${CMAKE_SYSTEM_NAME}-Determine-SYCL OPTIONAL)
+#include(Platform/${CMAKE_SYSTEM_NAME}-SYCL OPTIONAL)
+if(NOT CMAKE_SYCL_COMPILER_NAMES)
+  set(CMAKE_SYCL_COMPILER_NAMES icpx)
 endif()
 
 if(${CMAKE_GENERATOR} MATCHES "Visual Studio")
 elseif("${CMAKE_GENERATOR}" MATCHES "Green Hills MULTI")
 elseif("${CMAKE_GENERATOR}" MATCHES "Xcode")
-  set(CMAKE_CXX_COMPILER_XCODE_TYPE sourcecode.cpp.cpp)
-  _cmake_find_compiler_path(CXX)
+  set(CMAKE_SYCL_COMPILER_XCODE_TYPE sourcecode.cpp.cpp)
+  _cmake_find_compiler_path(SYCL)
 else()
-  if(NOT CMAKE_CXX_COMPILER)
-    set(CMAKE_CXX_COMPILER_INIT NOTFOUND)
-
-    # prefer the environment variable CXX
-    if(NOT $ENV{CXX} STREQUAL "")
-      get_filename_component(CMAKE_CXX_COMPILER_INIT $ENV{CXX} PROGRAM PROGRAM_ARGS CMAKE_CXX_FLAGS_ENV_INIT)
-      if(CMAKE_CXX_FLAGS_ENV_INIT)
-        set(CMAKE_CXX_COMPILER_ARG1 "${CMAKE_CXX_FLAGS_ENV_INIT}" CACHE STRING "Arguments to CXX compiler")
+  if(NOT CMAKE_SYCL_COMPILER)
+    set(CMAKE_SYCL_COMPILER_INIT NOTFOUND)
+
+    # prefer the environment variable SYCL
+    if(NOT $ENV{SYCL} STREQUAL "")
+      get_filename_component(CMAKE_SYCL_COMPILER_INIT $ENV{SYCL} PROGRAM PROGRAM_ARGS CMAKE_SYCL_FLAGS_ENV_INIT)
+      if(CMAKE_SYCL_FLAGS_ENV_INIT)
+        set(CMAKE_SYCL_COMPILER_ARG1 "${CMAKE_SYCL_FLAGS_ENV_INIT}" CACHE STRING "Arguments to SYCL compiler")
       endif()
-      if(NOT EXISTS ${CMAKE_CXX_COMPILER_INIT})
-        message(FATAL_ERROR "Could not find compiler set in environment variable CXX:\n$ENV{CXX}.\n${CMAKE_CXX_COMPILER_INIT}")
+      if(NOT EXISTS ${CMAKE_SYCL_COMPILER_INIT})
+        message(FATAL_ERROR "Could not find compiler set in environment variable SYCL:\n$ENV{SYCL}.\n${CMAKE_SYCL_COMPILER_INIT}")
       endif()
     endif()
 
     # next prefer the generator specified compiler
-    if(CMAKE_GENERATOR_CXX)
-      if(NOT CMAKE_CXX_COMPILER_INIT)
-        set(CMAKE_CXX_COMPILER_INIT ${CMAKE_GENERATOR_CXX})
+    if(CMAKE_GENERATOR_SYCL)
+      if(NOT CMAKE_SYCL_COMPILER_INIT)
+        set(CMAKE_SYCL_COMPILER_INIT ${CMAKE_GENERATOR_SYCL})
       endif()
     endif()
 
     # finally list compilers to try
-    if(NOT CMAKE_CXX_COMPILER_INIT)
-      set(CMAKE_CXX_COMPILER_LIST CC ${_CMAKE_TOOLCHAIN_PREFIX}c++ ${_CMAKE_TOOLCHAIN_PREFIX}g++ aCC cl bcc xlC)
+    if(NOT CMAKE_SYCL_COMPILER_INIT)
+      set(CMAKE_SYCL_COMPILER_LIST icpx icx)
       if(NOT CMAKE_HOST_WIN32)
         # FIXME(#24314): Add support for the GNU-like icpx compiler driver
         # on Windows, first introduced by Intel oneAPI 2023.0.
-        list(APPEND CMAKE_CXX_COMPILER_LIST icpx)
+        list(APPEND CMAKE_SYCL_COMPILER_LIST icpx)
       endif()
-      list(APPEND CMAKE_CXX_COMPILER_LIST icx clang++)
     endif()
 
-    _cmake_find_compiler(CXX)
+    _cmake_find_compiler(SYCL)
   else()
-    _cmake_find_compiler_path(CXX)
+    _cmake_find_compiler_path(SYCL)
   endif()
-  mark_as_advanced(CMAKE_CXX_COMPILER)
+  mark_as_advanced(CMAKE_SYCL_COMPILER)
 
   # Each entry in this list is a set of extra flags to try
   # adding to the compile line to see if it helps produce
   # a valid identification file.
-  set(CMAKE_CXX_COMPILER_ID_TEST_FLAGS_FIRST)
-  set(CMAKE_CXX_COMPILER_ID_TEST_FLAGS
+  set(CMAKE_SYCL_COMPILER_ID_TEST_FLAGS_FIRST)
+  set(CMAKE_SYCL_COMPILER_ID_TEST_FLAGS
+    "-fsycl"
     # Try compiling to an object file only.
     "-c"
     # IAR does not detect language automatically
@@ -94,64 +95,65 @@ else()
     )
 endif()
 
-if(CMAKE_CXX_COMPILER_TARGET)
-  set(CMAKE_CXX_COMPILER_ID_TEST_FLAGS_FIRST "-c --target=${CMAKE_CXX_COMPILER_TARGET}")
+if(CMAKE_SYCL_COMPILER_TARGET)
+  set(CMAKE_SYCL_COMPILER_ID_TEST_FLAGS_FIRST "-c --target=${CMAKE_SYCL_COMPILER_TARGET}")
 endif()
 
 # Build a small source file to identify the compiler.
-if(NOT CMAKE_CXX_COMPILER_ID_RUN)
-  set(CMAKE_CXX_COMPILER_ID_RUN 1)
+if(NOT CMAKE_SYCL_COMPILER_ID_RUN)
+  set(CMAKE_SYCL_COMPILER_ID_RUN 1)
 
   # Try to identify the compiler.
-  set(CMAKE_CXX_COMPILER_ID)
-  set(CMAKE_CXX_PLATFORM_ID)
+  set(CMAKE_SYCL_COMPILER_ID)
+  set(CMAKE_SYCL_PLATFORM_ID)
   file(READ ${CMAKE_ROOT}/Modules/CMakePlatformId.h.in
-    CMAKE_CXX_COMPILER_ID_PLATFORM_CONTENT)
+    CMAKE_SYCL_COMPILER_ID_PLATFORM_CONTENT)
 
   # The IAR compiler produces weird output.
   # See https://gitlab.kitware.com/cmake/cmake/-/issues/10176#note_153591
-  list(APPEND CMAKE_CXX_COMPILER_ID_VENDORS IAR)
-  set(CMAKE_CXX_COMPILER_ID_VENDOR_FLAGS_IAR )
-  set(CMAKE_CXX_COMPILER_ID_VENDOR_REGEX_IAR "IAR .+ Compiler")
+  list(APPEND CMAKE_SYCL_COMPILER_ID_VENDORS IAR)
+  set(CMAKE_SYCL_COMPILER_ID_VENDOR_FLAGS_IAR )
+  set(CMAKE_SYCL_COMPILER_ID_VENDOR_REGEX_IAR "IAR .+ Compiler")
 
   # Match the link line from xcodebuild output of the form
   #  Ld ...
   #      ...
-  #      /path/to/cc ...CompilerIdCXX/...
+  #      /path/to/cc ...CompilerIdSYCL/...
   # to extract the compiler front-end for the language.
-  set(CMAKE_CXX_COMPILER_ID_TOOL_MATCH_REGEX "\nLd[^\n]*(\n[ \t]+[^\n]*)*\n[ \t]+([^ \t\r\n]+)[^\r\n]*-o[^\r\n]*CompilerIdCXX/(\\./)?(CompilerIdCXX.(framework|xctest|build/[^ \t\r\n]+)/)?CompilerIdCXX[ \t\n\\\"]")
-  set(CMAKE_CXX_COMPILER_ID_TOOL_MATCH_INDEX 2)
+  set(CMAKE_SYCL_COMPILER_ID_TOOL_MATCH_REGEX "\nLd[^\n]*(\n[ \t]+[^\n]*)*\n[ \t]+([^ \t\r\n]+)[^\r\n]*-o[^\r\n]*CompilerIdSYCL/(\\./)?(CompilerIdSYCL.(framework|xctest|build/[^ \t\r\n]+)/)?CompilerIdSYCL[ \t\n\\\"]")
+  set(CMAKE_SYCL_COMPILER_ID_TOOL_MATCH_INDEX 2)
 
   include(${CMAKE_ROOT}/Modules/CMakeDetermineCompilerId.cmake)
-  CMAKE_DETERMINE_COMPILER_ID(CXX CXXFLAGS CMakeCXXCompilerId.cpp)
+  set(SYCLFLAGS "-fsycl -Werror")
+  CMAKE_DETERMINE_COMPILER_ID(SYCL SYCLFLAGS CMakeSYCLCompilerId.cpp)
 
-  _cmake_find_compiler_sysroot(CXX)
+  _cmake_find_compiler_sysroot(SYCL)
 
   # Set old compiler and platform id variables.
-  if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    set(CMAKE_COMPILER_IS_GNUCXX 1)
+  if(CMAKE_SYCL_COMPILER_ID STREQUAL "GNU")
+    set(CMAKE_COMPILER_IS_GNUSYCL 1)
   endif()
 else()
-  if(NOT DEFINED CMAKE_CXX_COMPILER_FRONTEND_VARIANT)
-    # Some toolchain files set our internal CMAKE_CXX_COMPILER_ID_RUN
-    # variable but are not aware of CMAKE_CXX_COMPILER_FRONTEND_VARIANT.
+  if(NOT DEFINED CMAKE_SYCL_COMPILER_FRONTEND_VARIANT)
+    # Some toolchain files set our internal CMAKE_SYCL_COMPILER_ID_RUN
+    # variable but are not aware of CMAKE_SYCL_COMPILER_FRONTEND_VARIANT.
     # They pre-date our support for the GNU-like variant targeting the
     # MSVC ABI so we do not consider that here.
-    if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang"
-      OR "x${CMAKE_CXX_COMPILER_ID}" STREQUAL "xIntelLLVM")
-      if("x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC")
-        set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "MSVC")
+    if(CMAKE_SYCL_COMPILER_ID STREQUAL "Clang"
+      OR "x${CMAKE_SYCL_COMPILER_ID}" STREQUAL "xIntelLLVM")
+      if("x${CMAKE_SYCL_SIMULATE_ID}" STREQUAL "xMSVC")
+        set(CMAKE_SYCL_COMPILER_FRONTEND_VARIANT "MSVC")
       else()
-        set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "GNU")
+        set(CMAKE_SYCL_COMPILER_FRONTEND_VARIANT "GNU")
       endif()
     else()
-      set(CMAKE_CXX_COMPILER_FRONTEND_VARIANT "")
+      set(CMAKE_SYCL_COMPILER_FRONTEND_VARIANT "")
     endif()
   endif()
 endif()
 
 if (NOT _CMAKE_TOOLCHAIN_LOCATION)
-  get_filename_component(_CMAKE_TOOLCHAIN_LOCATION "${CMAKE_CXX_COMPILER}" PATH)
+  get_filename_component(_CMAKE_TOOLCHAIN_LOCATION "${CMAKE_SYCL_COMPILER}" PATH)
 endif ()
 
 # if we have a g++ cross compiler, they have usually some prefix, like
@@ -165,18 +167,18 @@ endif ()
 
 if (NOT _CMAKE_TOOLCHAIN_PREFIX)
 
-  if("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU|Clang|QCC|LCC")
-    get_filename_component(COMPILER_BASENAME "${CMAKE_CXX_COMPILER}" NAME)
+  if("${CMAKE_SYCL_COMPILER_ID}" MATCHES "GNU|Clang|QCC|LCC")
+    get_filename_component(COMPILER_BASENAME "${CMAKE_SYCL_COMPILER}" NAME)
     if (COMPILER_BASENAME MATCHES "^(.+-)?(clang\\+\\+|[gc]\\+\\+|clang-cl)(-[0-9]+(\\.[0-9]+)*)?(-[^.]+)?(\\.exe)?$")
       set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_MATCH_1})
       set(_CMAKE_TOOLCHAIN_SUFFIX ${CMAKE_MATCH_3})
       set(_CMAKE_COMPILER_SUFFIX ${CMAKE_MATCH_5})
-    elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
-      if(CMAKE_CXX_COMPILER_TARGET)
-        set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_CXX_COMPILER_TARGET}-)
+    elseif("${CMAKE_SYCL_COMPILER_ID}" MATCHES "Clang")
+      if(CMAKE_SYCL_COMPILER_TARGET)
+        set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_SYCL_COMPILER_TARGET}-)
       endif()
     elseif(COMPILER_BASENAME MATCHES "QCC(\\.exe)?$")
-      if(CMAKE_CXX_COMPILER_TARGET MATCHES "gcc_nto([a-z0-9]+_[0-9]+|[^_le]+)(le)")
+      if(CMAKE_SYCL_COMPILER_TARGET MATCHES "gcc_nto([a-z0-9]+_[0-9]+|[^_le]+)(le)")
         set(_CMAKE_TOOLCHAIN_PREFIX nto${CMAKE_MATCH_1}-)
       endif()
     endif ()
@@ -186,9 +188,9 @@ if (NOT _CMAKE_TOOLCHAIN_PREFIX)
     if ("${_CMAKE_TOOLCHAIN_PREFIX}" MATCHES "(.+-)?llvm-$")
       set(_CMAKE_TOOLCHAIN_PREFIX ${CMAKE_MATCH_1})
     endif ()
-  elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "TI")
+  elseif("${CMAKE_SYCL_COMPILER_ID}" MATCHES "TI")
     # TI compilers are named e.g. cl6x, cl470 or armcl.exe
-    get_filename_component(COMPILER_BASENAME "${CMAKE_CXX_COMPILER}" NAME)
+    get_filename_component(COMPILER_BASENAME "${CMAKE_SYCL_COMPILER}" NAME)
     if (COMPILER_BASENAME MATCHES "^(.+)?cl([^.]+)?(\\.exe)?$")
       set(_CMAKE_TOOLCHAIN_PREFIX "${CMAKE_MATCH_1}")
       set(_CMAKE_TOOLCHAIN_SUFFIX "${CMAKE_MATCH_2}")
@@ -198,40 +200,40 @@ if (NOT _CMAKE_TOOLCHAIN_PREFIX)
 
 endif ()
 
-set(_CMAKE_PROCESSING_LANGUAGE "CXX")
+set(_CMAKE_PROCESSING_LANGUAGE "SYCL")
 include(CMakeFindBinUtils)
-include(Compiler/${CMAKE_CXX_COMPILER_ID}-FindBinUtils OPTIONAL)
+include(Compiler/${CMAKE_SYCL_COMPILER_ID}-FindBinUtils OPTIONAL)
 unset(_CMAKE_PROCESSING_LANGUAGE)
 
-if(CMAKE_CXX_COMPILER_SYSROOT)
-  string(CONCAT _SET_CMAKE_CXX_COMPILER_SYSROOT
-    "set(CMAKE_CXX_COMPILER_SYSROOT \"${CMAKE_CXX_COMPILER_SYSROOT}\")\n"
-    "set(CMAKE_COMPILER_SYSROOT \"${CMAKE_CXX_COMPILER_SYSROOT}\")")
+if(CMAKE_SYCL_COMPILER_SYSROOT)
+  string(CONCAT _SET_CMAKE_SYCL_COMPILER_SYSROOT
+    "set(CMAKE_SYCL_COMPILER_SYSROOT \"${CMAKE_SYCL_COMPILER_SYSROOT}\")\n"
+    "set(CMAKE_COMPILER_SYSROOT \"${CMAKE_SYCL_COMPILER_SYSROOT}\")")
 else()
-  set(_SET_CMAKE_CXX_COMPILER_SYSROOT "")
+  set(_SET_CMAKE_SYCL_COMPILER_SYSROOT "")
 endif()
 
-if(CMAKE_CXX_COMPILER_ARCHITECTURE_ID)
-  set(_SET_CMAKE_CXX_COMPILER_ARCHITECTURE_ID
-    "set(CMAKE_CXX_COMPILER_ARCHITECTURE_ID ${CMAKE_CXX_COMPILER_ARCHITECTURE_ID})")
+if(CMAKE_SYCL_COMPILER_ARCHITECTURE_ID)
+  set(_SET_CMAKE_SYCL_COMPILER_ARCHITECTURE_ID
+    "set(CMAKE_SYCL_COMPILER_ARCHITECTURE_ID ${CMAKE_SYCL_COMPILER_ARCHITECTURE_ID})")
 else()
-  set(_SET_CMAKE_CXX_COMPILER_ARCHITECTURE_ID "")
+  set(_SET_CMAKE_SYCL_COMPILER_ARCHITECTURE_ID "")
 endif()
 
-if(MSVC_CXX_ARCHITECTURE_ID)
-  set(SET_MSVC_CXX_ARCHITECTURE_ID
-    "set(MSVC_CXX_ARCHITECTURE_ID ${MSVC_CXX_ARCHITECTURE_ID})")
+if(MSVC_SYCL_ARCHITECTURE_ID)
+  set(SET_MSVC_SYCL_ARCHITECTURE_ID
+    "set(MSVC_SYCL_ARCHITECTURE_ID ${MSVC_SYCL_ARCHITECTURE_ID})")
 endif()
 
-if(CMAKE_CXX_XCODE_ARCHS)
+if(CMAKE_SYCL_XCODE_ARCHS)
   set(SET_CMAKE_XCODE_ARCHS
-    "set(CMAKE_XCODE_ARCHS \"${CMAKE_CXX_XCODE_ARCHS}\")")
+    "set(CMAKE_XCODE_ARCHS \"${CMAKE_SYCL_XCODE_ARCHS}\")")
 endif()
 
 # configure all variables set in this file
-configure_file(${CMAKE_ROOT}/Modules/CMakeCXXCompiler.cmake.in
-  ${CMAKE_PLATFORM_INFO_DIR}/CMakeCXXCompiler.cmake
+configure_file(${ArrayFire_SOURCE_DIR}/CMakeModules/CMakeSYCLCompiler.cmake.in
+  ${CMAKE_PLATFORM_INFO_DIR}/CMakeSYCLCompiler.cmake
   @ONLY
   )
 
-set(CMAKE_CXX_COMPILER_ENV_VAR "CXX")
+set(CMAKE_SYCL_COMPILER_ENV_VAR "SYCL")
diff --git a/CMakeModules/CMakeSYCLCompiler.cmake.in b/CMakeModules/CMakeSYCLCompiler.cmake.in
index 50edc9e474..e0193afb13 100644
--- a/CMakeModules/CMakeSYCLCompiler.cmake.in
+++ b/CMakeModules/CMakeSYCLCompiler.cmake.in
@@ -39,7 +39,7 @@ set(CMAKE_SYCL_COMPILER_ID_RUN 1)
 set(CMAKE_SYCL_SOURCE_FILE_EXTENSIONS C;M;c++;cc;cpp;cxx;m;mm;mpp;CPP;ixx;cppm)
 set(CMAKE_SYCL_IGNORE_EXTENSIONS inl;h;hpp;HPP;H;o;O;obj;OBJ;def;DEF;rc;RC)
 
-foreach (lang C OBJC OBJSYCL)
+foreach (lang SYCL)
   if (CMAKE_${lang}_COMPILER_ID_RUN)
     foreach(extension IN LISTS CMAKE_${lang}_SOURCE_FILE_EXTENSIONS)
       list(REMOVE_ITEM CMAKE_SYCL_SOURCE_FILE_EXTENSIONS ${extension})
diff --git a/CMakeModules/CMakeSYCLCompilerABI.cpp b/CMakeModules/CMakeSYCLCompilerABI.cpp
index fe7c926993..cac613b114 100644
--- a/CMakeModules/CMakeSYCLCompilerABI.cpp
+++ b/CMakeModules/CMakeSYCLCompilerABI.cpp
@@ -3,7 +3,6 @@
 #endif
 
 #include "CMakeCompilerABI.h"
-#include <sycl/sycl.hpp>
 
 int main(int argc, char* argv[])
 {
@@ -16,16 +15,5 @@ int main(int argc, char* argv[])
 #endif
   static_cast<void>(argv);
 
-  int count = 0;
-  auto platforms = sycl::platform::get_platforms();
-  for(sycl::platform &platform : platforms) {
-    count += platform.get_devices().size();
-  }
-
-  if(count == 0) {
-    std::fprintf(stderr, "No SYCL devices found.\n");
-    return -1;
-  }
-
   return require;
 }
diff --git a/CMakeModules/CMakeSYCLCompilerId.cpp.in b/CMakeModules/CMakeSYCLCompilerId.cpp.in
new file mode 100644
index 0000000000..913dbc7932
--- /dev/null
+++ b/CMakeModules/CMakeSYCLCompilerId.cpp.in
@@ -0,0 +1,105 @@
+/* This source file must have a .cpp extension so that all C++ compilers
+   recognize the extension without flags.  Borland does not know .cxx for
+   example.  */
+#ifndef __cplusplus
+# error "A C compiler has been selected for C++."
+#endif
+
+#if !defined(__has_include)
+/* If the compiler does not have __has_include, pretend the answer is
+   always no.  */
+#  define __has_include(x) 0
+#endif
+
+@CMAKE_SYCL_COMPILER_ID_CONTENT@
+
+/* Construct the string literal in pieces to prevent the source from
+   getting matched.  Store it in a pointer rather than an array
+   because some compilers will just produce instructions to fill the
+   array rather than assigning a pointer to a static array.  */
+char const* info_compiler = "INFO" ":" "compiler[" COMPILER_ID "]";
+#ifdef SIMULATE_ID
+char const* info_simulate = "INFO" ":" "simulate[" SIMULATE_ID "]";
+#endif
+
+#ifdef __QNXNTO__
+char const* qnxnto = "INFO" ":" "qnxnto[]";
+#endif
+
+#if defined(__CRAYXT_COMPUTE_LINUX_TARGET)
+char const *info_cray = "INFO" ":" "compiler_wrapper[CrayPrgEnv]";
+#endif
+
+@CMAKE_SYCL_COMPILER_ID_PLATFORM_CONTENT@
+@CMAKE_SYCL_COMPILER_ID_ERROR_FOR_TEST@
+
+#if defined(__INTEL_COMPILER) && defined(_MSVC_LANG) && _MSVC_LANG < 201403L
+#  if defined(__INTEL_CXX11_MODE__)
+#    if defined(__cpp_aggregate_nsdmi)
+#      define CXX_STD 201402L
+#    else
+#      define CXX_STD 201103L
+#    endif
+#  else
+#    define CXX_STD 199711L
+#  endif
+#elif defined(_MSC_VER) && defined(_MSVC_LANG)
+#  define CXX_STD _MSVC_LANG
+#else
+#  define CXX_STD __cplusplus
+#endif
+
+const char* info_language_standard_default = "INFO" ":" "standard_default["
+#if CXX_STD > 202002L
+  "23"
+#elif CXX_STD > 201703L
+  "20"
+#elif CXX_STD >= 201703L
+  "17"
+#elif CXX_STD >= 201402L
+  "14"
+#elif CXX_STD >= 201103L
+  "11"
+#else
+  "98"
+#endif
+"]";
+
+const char* info_language_extensions_default = "INFO" ":" "extensions_default["
+#if (defined(__clang__) || defined(__GNUC__) || defined(__xlC__) ||           \
+     defined(__TI_COMPILER_VERSION__)) &&                                     \
+  !defined(__STRICT_ANSI__)
+  "ON"
+#else
+  "OFF"
+#endif
+"]";
+
+/*--------------------------------------------------------------------------*/
+
+int main(int argc, char* argv[])
+{
+  int require = 0;
+  require += info_compiler[argc];
+  require += info_platform[argc];
+  require += info_arch[argc];
+#ifdef COMPILER_VERSION_MAJOR
+  require += info_version[argc];
+#endif
+#ifdef COMPILER_VERSION_INTERNAL
+  require += info_version_internal[argc];
+#endif
+#ifdef SIMULATE_ID
+  require += info_simulate[argc];
+#endif
+#ifdef SIMULATE_VERSION_MAJOR
+  require += info_simulate_version[argc];
+#endif
+#if defined(__CRAYXT_COMPUTE_LINUX_TARGET)
+  require += info_cray[argc];
+#endif
+  require += info_language_standard_default[argc];
+  require += info_language_extensions_default[argc];
+  (void)argv;
+  return require;
+}
diff --git a/CMakeModules/CMakeSYCLInformation.cmake b/CMakeModules/CMakeSYCLInformation.cmake
index 53abf378d5..5e9714327a 100644
--- a/CMakeModules/CMakeSYCLInformation.cmake
+++ b/CMakeModules/CMakeSYCLInformation.cmake
@@ -1,6 +1,11 @@
 # Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
 # file Copyright.txt or https://cmake.org/licensing for details.
 
+# make sure default modules are accesible
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_ROOT}/Modules)
+message(${CMAKE_MODULE_PATH})
+
+set(CMAKE_SYCL_COMPILER_ID IntelLLVM)
 
 # This file sets the basic flags for the C++ language in CMake.
 # It also loads the available platform file for the system-compiler
@@ -13,49 +18,109 @@ include(CMakeLanguageInformation)
 # some compilers use different extensions (e.g. sdcc uses .rel)
 # so set the extension here first so it can be overridden by the compiler specific file
 if(UNIX)
-  set(CMAKE_CXX_OUTPUT_EXTENSION .o)
+  set(CMAKE_SYCL_OUTPUT_EXTENSION .o)
 else()
-  set(CMAKE_CXX_OUTPUT_EXTENSION .obj)
+  set(CMAKE_SYCL_OUTPUT_EXTENSION .obj)
 endif()
 
 set(_INCLUDED_FILE 0)
 
 # Load compiler-specific information.
-if(CMAKE_CXX_COMPILER_ID)
-  include(Compiler/${CMAKE_CXX_COMPILER_ID}-CXX OPTIONAL)
+if(CMAKE_SYCL_COMPILER_ID)
+  #include(Compiler/${CMAKE_SYCL_COMPILER_ID}-CXX OPTIONAL)
 endif()
 
 set(CMAKE_BASE_NAME)
-get_filename_component(CMAKE_BASE_NAME "${CMAKE_CXX_COMPILER}" NAME_WE)
+get_filename_component(CMAKE_BASE_NAME "${CMAKE_SYCL_COMPILER}" NAME_WE)
 # since the gnu compiler has several names force g++
-if(CMAKE_COMPILER_IS_GNUCXX)
+if(CMAKE_COMPILER_IS_GNUSYCL)
   set(CMAKE_BASE_NAME g++)
 endif()
 
+include(Compiler/${CMAKE_SYCL_COMPILER_ID} OPTIONAL)
+__compiler_intel_llvm(SYCL)
 
-# load a hardware specific file, mostly useful for embedded compilers
-if(CMAKE_SYSTEM_PROCESSOR)
-  if(CMAKE_CXX_COMPILER_ID)
-    include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}-CXX-${CMAKE_SYSTEM_PROCESSOR} OPTIONAL RESULT_VARIABLE _INCLUDED_FILE)
+if("x${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "xMSVC")
+  set(CMAKE_SYCL_COMPILE_OPTIONS_EXPLICIT_LANGUAGE -TP)
+  set(CMAKE_SYCL_CLANG_TIDY_DRIVER_MODE "cl")
+  set(CMAKE_SYCL_INCLUDE_WHAT_YOU_USE_DRIVER_MODE "cl")
+  if((NOT DEFINED CMAKE_DEPENDS_USE_COMPILER OR CMAKE_DEPENDS_USE_COMPILER)
+      AND CMAKE_GENERATOR MATCHES "Makefiles|WMake"
+      AND CMAKE_DEPFILE_FLAGS_SYCL)
+    set(CMAKE_SYCL_DEPENDS_USE_COMPILER TRUE)
+  endif()
+else()
+  set(CMAKE_SYCL_COMPILE_OPTIONS_EXPLICIT_LANGUAGE -x c++)
+  if((NOT DEFINED CMAKE_DEPENDS_USE_COMPILER OR CMAKE_DEPENDS_USE_COMPILER)
+      AND CMAKE_GENERATOR MATCHES "Makefiles|WMake"
+      AND CMAKE_DEPFILE_FLAGS_SYCL)
+    # dependencies are computed by the compiler itself
+    set(CMAKE_SYCL_DEPFILE_FORMAT gcc)
+    set(CMAKE_SYCL_DEPENDS_USE_COMPILER TRUE)
   endif()
-  if (NOT _INCLUDED_FILE)
-    include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_BASE_NAME}-${CMAKE_SYSTEM_PROCESSOR} OPTIONAL)
-  endif ()
+
+  set(CMAKE_SYCL_COMPILE_OPTIONS_VISIBILITY_INLINES_HIDDEN "-fvisibility-inlines-hidden")
+
+  string(APPEND CMAKE_SYCL_FLAGS_MINSIZEREL_INIT " -DNDEBUG")
+  string(APPEND CMAKE_SYCL_FLAGS_RELEASE_INIT " -DNDEBUG")
+  string(APPEND CMAKE_SYCL_FLAGS_RELWITHDEBINFO_INIT " -DNDEBUG")
 endif()
 
-# load the system- and compiler specific files
-if(CMAKE_CXX_COMPILER_ID)
-  include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}-CXX OPTIONAL RESULT_VARIABLE _INCLUDED_FILE)
+set(CMAKE_SYCL98_STANDARD__HAS_FULL_SUPPORT ON)
+set(CMAKE_SYCL11_STANDARD__HAS_FULL_SUPPORT ON)
+set(CMAKE_SYCL14_STANDARD__HAS_FULL_SUPPORT ON)
+
+if(NOT "x${CMAKE_SYCL_SIMULATE_ID}" STREQUAL "xMSVC")
+  set(CMAKE_SYCL98_STANDARD_COMPILE_OPTION  "-std=c++98")
+  set(CMAKE_SYCL98_EXTENSION_COMPILE_OPTION "-std=gnu++98")
+
+  set(CMAKE_SYCL11_STANDARD_COMPILE_OPTION  "-std=c++11")
+  set(CMAKE_SYCL11_EXTENSION_COMPILE_OPTION "-std=gnu++11")
+
+  set(CMAKE_SYCL14_STANDARD_COMPILE_OPTION  "-std=c++14")
+  set(CMAKE_SYCL14_EXTENSION_COMPILE_OPTION "-std=gnu++14")
+
+  set(CMAKE_SYCL17_STANDARD_COMPILE_OPTION  "-std=c++17")
+  set(CMAKE_SYCL17_EXTENSION_COMPILE_OPTION "-std=gnu++17")
+
+  set(CMAKE_SYCL20_STANDARD_COMPILE_OPTION  "-std=c++20")
+  set(CMAKE_SYCL20_EXTENSION_COMPILE_OPTION "-std=gnu++20")
+
+  set(CMAKE_SYCL23_STANDARD_COMPILE_OPTION  "-std=c++2b")
+  set(CMAKE_SYCL23_EXTENSION_COMPILE_OPTION "-std=gnu++2b")
+else()
+  set(CMAKE_SYCL98_STANDARD_COMPILE_OPTION  "")
+  set(CMAKE_SYCL98_EXTENSION_COMPILE_OPTION "")
+
+  set(CMAKE_SYCL11_STANDARD_COMPILE_OPTION  "")
+  set(CMAKE_SYCL11_EXTENSION_COMPILE_OPTION "")
+
+  set(CMAKE_SYCL14_STANDARD_COMPILE_OPTION  "-Qstd:c++14")
+  set(CMAKE_SYCL14_EXTENSION_COMPILE_OPTION "-Qstd:c++14")
+
+  set(CMAKE_SYCL17_STANDARD_COMPILE_OPTION  "-Qstd:c++17")
+  set(CMAKE_SYCL17_EXTENSION_COMPILE_OPTION "-Qstd:c++17")
+
+  set(CMAKE_SYCL20_STANDARD_COMPILE_OPTION  "-Qstd:c++20")
+  set(CMAKE_SYCL20_EXTENSION_COMPILE_OPTION "-Qstd:c++20")
+
+  set(CMAKE_SYCL23_STANDARD_COMPILE_OPTION  "-Qstd:c++2b")
+  set(CMAKE_SYCL23_EXTENSION_COMPILE_OPTION "-Qstd:c++2b")
 endif()
-if (NOT _INCLUDED_FILE)
-  include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_BASE_NAME} OPTIONAL
-          RESULT_VARIABLE _INCLUDED_FILE)
-endif ()
 
-# load any compiler-wrapper specific information
-if (CMAKE_CXX_COMPILER_WRAPPER)
-  __cmake_include_compiler_wrapper(CXX)
-endif ()
+include(Platform/${CMAKE_EFFECTIVE_SYSTEM_NAME}-${CMAKE_SYCL_COMPILER_ID} OPTIONAL RESULT_VARIABLE _INCLUDED_FILE)
+
+if(WIN32)
+  set(_COMPILE_CXX " /TP")
+  __windows_compiler_intel(SYCL)
+elseif(UNIX AND NOT APPLE)
+  __linux_compiler_intel_llvm(SYCL)
+  # This should be -isystem but icpx throws an error on Ubuntu
+  # when you include /usr/include as a system header
+  set(CMAKE_INCLUDE_SYSTEM_FLAG_SYCL "-I ")
+else()
+  __apple_compiler_intel_llvm(SYCL)
+endif()
 
 # We specify the compiler information in the system file for some
 # platforms, but this language may not have been enabled when the file
@@ -65,11 +130,11 @@ if (NOT _INCLUDED_FILE)
   include(Platform/${CMAKE_SYSTEM_NAME} OPTIONAL)
 endif ()
 
-if(CMAKE_CXX_SIZEOF_DATA_PTR)
-  foreach(f ${CMAKE_CXX_ABI_FILES})
+if(CMAKE_SYCL_SIZEOF_DATA_PTR)
+  foreach(f ${CMAKE_SYCL_ABI_FILES})
     include(${f})
   endforeach()
-  unset(CMAKE_CXX_ABI_FILES)
+  unset(CMAKE_SYCL_ABI_FILES)
 endif()
 
 # This should be included before the _INIT variables are
@@ -84,118 +149,118 @@ if(CMAKE_USER_MAKE_RULES_OVERRIDE)
   set(CMAKE_USER_MAKE_RULES_OVERRIDE "${_override}")
 endif()
 
-if(CMAKE_USER_MAKE_RULES_OVERRIDE_CXX)
+if(CMAKE_USER_MAKE_RULES_OVERRIDE_SYCL)
   # Save the full path of the file so try_compile can use it.
-  include(${CMAKE_USER_MAKE_RULES_OVERRIDE_CXX} RESULT_VARIABLE _override)
-  set(CMAKE_USER_MAKE_RULES_OVERRIDE_CXX "${_override}")
+  include(${CMAKE_USER_MAKE_RULES_OVERRIDE_SYCL} RESULT_VARIABLE _override)
+  set(CMAKE_USER_MAKE_RULES_OVERRIDE_SYCL "${_override}")
 endif()
 
 
 # Create a set of shared library variable specific to C++
 # For 90% of the systems, these are the same flags as the C versions
 # so if these are not set just copy the flags from the c version
-if(NOT CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS)
-  set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS})
+if(NOT CMAKE_SHARED_LIBRARY_CREATE_SYCL_FLAGS)
+  set(CMAKE_SHARED_LIBRARY_CREATE_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
 endif()
 
-if(NOT CMAKE_CXX_COMPILE_OPTIONS_PIC)
-  set(CMAKE_CXX_COMPILE_OPTIONS_PIC ${CMAKE_C_COMPILE_OPTIONS_PIC})
+if(NOT CMAKE_SYCL_COMPILE_OPTIONS_PIC)
+  set(CMAKE_SYCL_COMPILE_OPTIONS_PIC ${CMAKE_CXX_COMPILE_OPTIONS_PIC})
 endif()
 
-if(NOT CMAKE_CXX_COMPILE_OPTIONS_PIE)
-  set(CMAKE_CXX_COMPILE_OPTIONS_PIE ${CMAKE_C_COMPILE_OPTIONS_PIE})
+if(NOT CMAKE_SYCL_COMPILE_OPTIONS_PIE)
+  set(CMAKE_SYCL_COMPILE_OPTIONS_PIE ${CMAKE_CXX_COMPILE_OPTIONS_PIE})
 endif()
-if(NOT CMAKE_CXX_LINK_OPTIONS_PIE)
-  set(CMAKE_CXX_LINK_OPTIONS_PIE ${CMAKE_C_LINK_OPTIONS_PIE})
+if(NOT CMAKE_SYCL_LINK_OPTIONS_PIE)
+  set(CMAKE_SYCL_LINK_OPTIONS_PIE ${CMAKE_CXX_LINK_OPTIONS_PIE})
 endif()
-if(NOT CMAKE_CXX_LINK_OPTIONS_NO_PIE)
-  set(CMAKE_CXX_LINK_OPTIONS_NO_PIE ${CMAKE_C_LINK_OPTIONS_NO_PIE})
+if(NOT CMAKE_SYCL_LINK_OPTIONS_NO_PIE)
+  set(CMAKE_SYCL_LINK_OPTIONS_NO_PIE ${CMAKE_CXX_LINK_OPTIONS_NO_PIE})
 endif()
 
-if(NOT CMAKE_CXX_COMPILE_OPTIONS_DLL)
-  set(CMAKE_CXX_COMPILE_OPTIONS_DLL ${CMAKE_C_COMPILE_OPTIONS_DLL})
+if(NOT CMAKE_SYCL_COMPILE_OPTIONS_DLL)
+  set(CMAKE_SYCL_COMPILE_OPTIONS_DLL ${CMAKE_CXX_COMPILE_OPTIONS_DLL})
 endif()
 
-if(NOT CMAKE_SHARED_LIBRARY_CXX_FLAGS)
-  set(CMAKE_SHARED_LIBRARY_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_C_FLAGS})
+if(NOT CMAKE_SHARED_LIBRARY_SYCL_FLAGS)
+  set(CMAKE_SHARED_LIBRARY_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
 endif()
 
-if(NOT DEFINED CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS)
-  set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_C_FLAGS})
+if(NOT DEFINED CMAKE_SHARED_LIBRARY_LINK_SYCL_FLAGS)
+  set(CMAKE_SHARED_LIBRARY_LINK_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS})
 endif()
 
-if(NOT CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG)
-  set(CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG})
+if(NOT CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG)
+  set(CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
 endif()
 
-if(NOT CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP)
-  set(CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG_SEP})
+if(NOT CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG_SEP)
+  set(CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
 endif()
 
-if(NOT CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG)
-  set(CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG ${CMAKE_SHARED_LIBRARY_RPATH_LINK_C_FLAG})
+if(NOT CMAKE_SHARED_LIBRARY_RPATH_LINK_SYCL_FLAG)
+  set(CMAKE_SHARED_LIBRARY_RPATH_LINK_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG})
 endif()
 
-if(NOT DEFINED CMAKE_EXE_EXPORTS_CXX_FLAG)
-  set(CMAKE_EXE_EXPORTS_CXX_FLAG ${CMAKE_EXE_EXPORTS_C_FLAG})
+if(NOT DEFINED CMAKE_EXE_EXPORTS_SYCL_FLAG)
+  set(CMAKE_EXE_EXPORTS_SYCL_FLAG ${CMAKE_EXE_EXPORTS_CXX_FLAG})
 endif()
 
-if(NOT DEFINED CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG)
-  set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_C_FLAG})
+if(NOT DEFINED CMAKE_SHARED_LIBRARY_SONAME_SYCL_FLAG)
+  set(CMAKE_SHARED_LIBRARY_SONAME_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG})
 endif()
 
-if(NOT CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG)
-  set(CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG})
+if(NOT CMAKE_EXECUTABLE_RUNTIME_SYCL_FLAG)
+  set(CMAKE_EXECUTABLE_RUNTIME_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG})
 endif()
 
-if(NOT CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG_SEP)
-  set(CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP})
+if(NOT CMAKE_EXECUTABLE_RUNTIME_SYCL_FLAG_SEP)
+  set(CMAKE_EXECUTABLE_RUNTIME_SYCL_FLAG_SEP ${CMAKE_SHARED_LIBRARY_RUNTIME_SYCL_FLAG_SEP})
 endif()
 
-if(NOT CMAKE_EXECUTABLE_RPATH_LINK_CXX_FLAG)
-  set(CMAKE_EXECUTABLE_RPATH_LINK_CXX_FLAG ${CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG})
+if(NOT CMAKE_EXECUTABLE_RPATH_LINK_SYCL_FLAG)
+  set(CMAKE_EXECUTABLE_RPATH_LINK_SYCL_FLAG ${CMAKE_SHARED_LIBRARY_RPATH_LINK_SYCL_FLAG})
 endif()
 
-if(NOT DEFINED CMAKE_SHARED_LIBRARY_LINK_CXX_WITH_RUNTIME_PATH)
-  set(CMAKE_SHARED_LIBRARY_LINK_CXX_WITH_RUNTIME_PATH ${CMAKE_SHARED_LIBRARY_LINK_C_WITH_RUNTIME_PATH})
+if(NOT DEFINED CMAKE_SHARED_LIBRARY_LINK_SYCL_WITH_RUNTIME_PATH)
+  set(CMAKE_SHARED_LIBRARY_LINK_SYCL_WITH_RUNTIME_PATH ${CMAKE_SHARED_LIBRARY_LINK_CXX_WITH_RUNTIME_PATH})
 endif()
 
-if(NOT CMAKE_INCLUDE_FLAG_CXX)
-  set(CMAKE_INCLUDE_FLAG_CXX ${CMAKE_INCLUDE_FLAG_C})
+if(NOT CMAKE_INCLUDE_FLAG_SYCL)
+  set(CMAKE_INCLUDE_FLAG_SYCL ${CMAKE_INCLUDE_FLAG_C})
 endif()
 
 # for most systems a module is the same as a shared library
 # so unless the variable CMAKE_MODULE_EXISTS is set just
 # copy the values from the LIBRARY variables
 if(NOT CMAKE_MODULE_EXISTS)
-  set(CMAKE_SHARED_MODULE_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_CXX_FLAGS})
-  set(CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS})
+  set(CMAKE_SHARED_MODULE_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_SYCL_FLAGS})
+  set(CMAKE_SHARED_MODULE_CREATE_SYCL_FLAGS ${CMAKE_SHARED_LIBRARY_CREATE_SYCL_FLAGS})
 endif()
 
 # repeat for modules
-if(NOT CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS)
-  set(CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS ${CMAKE_SHARED_MODULE_CREATE_C_FLAGS})
+if(NOT CMAKE_SHARED_MODULE_CREATE_SYCL_FLAGS)
+  set(CMAKE_SHARED_MODULE_CREATE_SYCL_FLAGS ${CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS})
 endif()
 
-if(NOT CMAKE_SHARED_MODULE_CXX_FLAGS)
-  set(CMAKE_SHARED_MODULE_CXX_FLAGS ${CMAKE_SHARED_MODULE_C_FLAGS})
+if(NOT CMAKE_SHARED_MODULE_SYCL_FLAGS)
+  set(CMAKE_SHARED_MODULE_SYCL_FLAGS ${CMAKE_SHARED_MODULE_CXX_FLAGS})
 endif()
 
-# Initialize CXX link type selection flags from C versions.
+# Initialize SYCL link type selection flags from C versions.
 foreach(type SHARED_LIBRARY SHARED_MODULE EXE)
-  if(NOT CMAKE_${type}_LINK_STATIC_CXX_FLAGS)
-    set(CMAKE_${type}_LINK_STATIC_CXX_FLAGS
-      ${CMAKE_${type}_LINK_STATIC_C_FLAGS})
+  if(NOT CMAKE_${type}_LINK_STATIC_SYCL_FLAGS)
+    set(CMAKE_${type}_LINK_STATIC_SYCL_FLAGS
+      ${CMAKE_${type}_LINK_STATIC_CXX_FLAGS})
   endif()
-  if(NOT CMAKE_${type}_LINK_DYNAMIC_CXX_FLAGS)
-    set(CMAKE_${type}_LINK_DYNAMIC_CXX_FLAGS
-      ${CMAKE_${type}_LINK_DYNAMIC_C_FLAGS})
+  if(NOT CMAKE_${type}_LINK_DYNAMIC_SYCL_FLAGS)
+    set(CMAKE_${type}_LINK_DYNAMIC_SYCL_FLAGS
+      ${CMAKE_${type}_LINK_DYNAMIC_CXX_FLAGS})
   endif()
 endforeach()
 
 if(CMAKE_EXECUTABLE_FORMAT STREQUAL "ELF")
-  if(NOT DEFINED CMAKE_CXX_LINK_WHAT_YOU_USE_FLAG)
-    set(CMAKE_CXX_LINK_WHAT_YOU_USE_FLAG "LINKER:--no-as-needed")
+  if(NOT DEFINED CMAKE_SYCL_LINK_WHAT_YOU_USE_FLAG)
+    set(CMAKE_SYCL_LINK_WHAT_YOU_USE_FLAG "LINKER:--no-as-needed")
   endif()
   if(NOT DEFINED CMAKE_LINK_WHAT_YOU_USE_CHECK)
     set(CMAKE_LINK_WHAT_YOU_USE_CHECK ldd -u -r)
@@ -206,33 +271,33 @@ endif()
 # on the initial values computed in the platform/*.cmake files
 # use _INIT variables so that this only happens the first time
 # and you can set these flags in the cmake cache
-set(CMAKE_CXX_FLAGS_INIT "$ENV{CXXFLAGS} ${CMAKE_CXX_FLAGS_INIT}")
+set(CMAKE_SYCL_FLAGS_INIT "-fsycl $ENV{SYCLFLAGS} ${CMAKE_SYCL_FLAGS_INIT}")
 
-cmake_initialize_per_config_variable(CMAKE_CXX_FLAGS "Flags used by the CXX compiler")
+cmake_initialize_per_config_variable(CMAKE_SYCL_FLAGS "Flags used by the SYCL compiler")
 
-if(CMAKE_CXX_STANDARD_LIBRARIES_INIT)
-  set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES_INIT}"
+if(CMAKE_SYCL_STANDARD_LIBRARIES_INIT)
+  set(CMAKE_SYCL_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES_INIT}"
     CACHE STRING "Libraries linked by default with all C++ applications.")
-  mark_as_advanced(CMAKE_CXX_STANDARD_LIBRARIES)
+  mark_as_advanced(CMAKE_SYCL_STANDARD_LIBRARIES)
 endif()
 
-if(NOT CMAKE_CXX_COMPILER_LAUNCHER AND DEFINED ENV{CMAKE_CXX_COMPILER_LAUNCHER})
-  set(CMAKE_CXX_COMPILER_LAUNCHER "$ENV{CMAKE_CXX_COMPILER_LAUNCHER}"
-    CACHE STRING "Compiler launcher for CXX.")
+if(NOT CMAKE_SYCL_COMPILER_LAUNCHER AND DEFINED ENV{CMAKE_SYCL_COMPILER_LAUNCHER})
+  set(CMAKE_SYCL_COMPILER_LAUNCHER "$ENV{CMAKE_SYCL_COMPILER_LAUNCHER}"
+    CACHE STRING "Compiler launcher for SYCL.")
 endif()
 
-if(NOT CMAKE_CXX_LINKER_LAUNCHER AND DEFINED ENV{CMAKE_CXX_LINKER_LAUNCHER})
-  set(CMAKE_CXX_LINKER_LAUNCHER "$ENV{CMAKE_CXX_LINKER_LAUNCHER}"
-    CACHE STRING "Linker launcher for CXX.")
+if(NOT CMAKE_SYCL_LINKER_LAUNCHER AND DEFINED ENV{CMAKE_SYCL_LINKER_LAUNCHER})
+  set(CMAKE_SYCL_LINKER_LAUNCHER "$ENV{CMAKE_SYCL_LINKER_LAUNCHER}"
+    CACHE STRING "Linker launcher for SYCL.")
 endif()
 
 include(CMakeCommonLanguageInclude)
 
 # now define the following rules:
-# CMAKE_CXX_CREATE_SHARED_LIBRARY
-# CMAKE_CXX_CREATE_SHARED_MODULE
-# CMAKE_CXX_COMPILE_OBJECT
-# CMAKE_CXX_LINK_EXECUTABLE
+# CMAKE_SYCL_CREATE_SHARED_LIBRARY
+# CMAKE_SYCL_CREATE_SHARED_MODULE
+# CMAKE_SYCL_COMPILE_OBJECT
+# CMAKE_SYCL_LINK_EXECUTABLE
 
 # variables supplied by the generator at use time
 # <TARGET>
@@ -243,54 +308,54 @@ include(CMakeCommonLanguageInclude)
 # <FLAGS>
 # <LINK_FLAGS>
 
-# CXX compiler information
-# <CMAKE_CXX_COMPILER>
-# <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS>
-# <CMAKE_CXX_SHARED_MODULE_CREATE_FLAGS>
-# <CMAKE_CXX_LINK_FLAGS>
+# SYCL compiler information
+# <CMAKE_SYCL_COMPILER>
+# <CMAKE_SHARED_LIBRARY_CREATE_SYCL_FLAGS>
+# <CMAKE_SYCL_SHARED_MODULE_CREATE_FLAGS>
+# <CMAKE_SYCL_LINK_FLAGS>
 
 # Static library tools
 # <CMAKE_AR>
 # <CMAKE_RANLIB>
 
-
 # create a shared C++ library
-if(NOT CMAKE_CXX_CREATE_SHARED_LIBRARY)
-  set(CMAKE_CXX_CREATE_SHARED_LIBRARY
-      "<CMAKE_CXX_COMPILER> <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
+if(NOT CMAKE_SYCL_CREATE_SHARED_LIBRARY)
+  set(CMAKE_SYCL_CREATE_SHARED_LIBRARY
+      "<CMAKE_SYCL_COMPILER> <CMAKE_SHARED_LIBRARY_SYCL_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_SYCL_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> <OBJECTS> <LINK_LIBRARIES>")
 endif()
 
 # create a c++ shared module copy the shared library rule by default
-if(NOT CMAKE_CXX_CREATE_SHARED_MODULE)
-  set(CMAKE_CXX_CREATE_SHARED_MODULE ${CMAKE_CXX_CREATE_SHARED_LIBRARY})
+if(NOT CMAKE_SYCL_CREATE_SHARED_MODULE)
+  set(CMAKE_SYCL_CREATE_SHARED_MODULE ${CMAKE_SYCL_CREATE_SHARED_LIBRARY})
 endif()
 
 
 # Create a static archive incrementally for large object file counts.
-# If CMAKE_CXX_CREATE_STATIC_LIBRARY is set it will override these.
-if(NOT DEFINED CMAKE_CXX_ARCHIVE_CREATE)
-  set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> qc <TARGET> <LINK_FLAGS> <OBJECTS>")
+# If CMAKE_SYCL_CREATE_STATIC_LIBRARY is set it will override these.
+if(NOT DEFINED CMAKE_SYCL_ARCHIVE_CREATE)
+  set(CMAKE_SYCL_ARCHIVE_CREATE "<CMAKE_AR> qc <TARGET> <LINK_FLAGS> <OBJECTS>")
 endif()
-if(NOT DEFINED CMAKE_CXX_ARCHIVE_APPEND)
-  set(CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> q <TARGET> <LINK_FLAGS> <OBJECTS>")
+if(NOT DEFINED CMAKE_SYCL_ARCHIVE_APPEND)
+  set(CMAKE_SYCL_ARCHIVE_APPEND "<CMAKE_AR> q <TARGET> <LINK_FLAGS> <OBJECTS>")
 endif()
-if(NOT DEFINED CMAKE_CXX_ARCHIVE_FINISH)
-  set(CMAKE_CXX_ARCHIVE_FINISH "<CMAKE_RANLIB> <TARGET>")
+if(NOT DEFINED CMAKE_SYCL_ARCHIVE_FINISH)
+  set(CMAKE_SYCL_ARCHIVE_FINISH "<CMAKE_RANLIB> <TARGET>")
 endif()
 
 # compile a C++ file into an object file
-if(NOT CMAKE_CXX_COMPILE_OBJECT)
-  set(CMAKE_CXX_COMPILE_OBJECT
-    "<CMAKE_CXX_COMPILER> <DEFINES> <INCLUDES> <FLAGS> -o <OBJECT> -c <SOURCE>")
+if(NOT CMAKE_SYCL_COMPILE_OBJECT)
+  set(CMAKE_SYCL_COMPILE_OBJECT
+    "<CMAKE_SYCL_COMPILER> <DEFINES> <INCLUDES> <FLAGS> -o <OBJECT> -c <SOURCE>")
 endif()
 
-if(NOT CMAKE_CXX_LINK_EXECUTABLE)
-  set(CMAKE_CXX_LINK_EXECUTABLE
-    "<CMAKE_CXX_COMPILER> <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+if(NOT CMAKE_SYCL_LINK_EXECUTABLE)
+  set(CMAKE_SYCL_LINK_EXECUTABLE
+    "<CMAKE_SYCL_COMPILER> <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
 endif()
 
+
 mark_as_advanced(
 CMAKE_VERBOSE_MAKEFILE
 )
 
-set(CMAKE_CXX_INFORMATION_LOADED 1)
+set(CMAKE_SYCL_INFORMATION_LOADED 1)
diff --git a/CMakeModules/CMakeTestSYCLCompiler.cmake b/CMakeModules/CMakeTestSYCLCompiler.cmake
index e640ff9b30..e2f37a2da0 100644
--- a/CMakeModules/CMakeTestSYCLCompiler.cmake
+++ b/CMakeModules/CMakeTestSYCLCompiler.cmake
@@ -2,10 +2,10 @@
 # file Copyright.txt or https://cmake.org/licensing for details.
 
 
-if(CMAKE_CXX_COMPILER_FORCED)
+if(CMAKE_SYCL_COMPILER_FORCED)
   # The compiler configuration was forced by the user.
   # Assume the user has configured all compiler information.
-  set(CMAKE_CXX_COMPILER_WORKS TRUE)
+  set(CMAKE_SYCL_COMPILER_WORKS TRUE)
   return()
 endif()
 
@@ -18,16 +18,16 @@ if(_CMAKE_FEATURE_DETECTION_TARGET_TYPE)
 endif()
 
 # Remove any cached result from an older CMake version.
-# We now store this in CMakeCXXCompiler.cmake.
-unset(CMAKE_CXX_COMPILER_WORKS CACHE)
+# We now store this in CMakeSYCLCompiler.cmake.
+unset(CMAKE_SYCL_COMPILER_WORKS CACHE)
 
-# Try to identify the ABI and configure it into CMakeCXXCompiler.cmake
-include(${CMAKE_ROOT}/Modules/CMakeDetermineCompilerABI.cmake)
-CMAKE_DETERMINE_COMPILER_ABI(CXX ${CMAKE_ROOT}/Modules/CMakeCXXCompilerABI.cpp)
-if(CMAKE_CXX_ABI_COMPILED)
+# Try to identify the ABI and configure it into CMakeSYCLCompiler.cmake
+include(CMakeDetermineCompilerABI)
+CMAKE_DETERMINE_COMPILER_ABI(SYCL ${ArrayFire_SOURCE_DIR}/CMakeModules/CMakeSYCLCompilerABI.cpp)
+if(CMAKE_SYCL_ABI_COMPILED)
   # The compiler worked so skip dedicated test below.
-  set(CMAKE_CXX_COMPILER_WORKS TRUE)
-  message(STATUS "Check for working CXX compiler: ${CMAKE_CXX_COMPILER} - skipped")
+  set(CMAKE_SYCL_COMPILER_WORKS TRUE)
+  message(STATUS "Check for working SYCL compiler: ${CMAKE_SYCL_COMPILER} - skipped")
 endif()
 
 # This file is used by EnableLanguage in cmGlobalGenerator to
@@ -35,29 +35,29 @@ endif()
 # and link the most basic of programs.   If not, a fatal error
 # is set and cmake stops processing commands and will not generate
 # any makefiles or projects.
-if(NOT CMAKE_CXX_COMPILER_WORKS)
-  PrintTestCompilerStatus("CXX")
+if(NOT CMAKE_SYCL_COMPILER_WORKS)
+  PrintTestCompilerStatus("SYCL")
   __TestCompiler_setTryCompileTargetType()
-  string(CONCAT __TestCompiler_testCXXCompilerSource
+  file(WRITE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testSYCLCompiler.cxx
     "#ifndef __cplusplus\n"
-    "# error \"The CMAKE_CXX_COMPILER is set to a C compiler\"\n"
+    "# error \"The CMAKE_SYCL_COMPILER is set to a C compiler\"\n"
     "#endif\n"
     "int main(){return 0;}\n")
   # Clear result from normal variable.
-  unset(CMAKE_CXX_COMPILER_WORKS)
+  unset(CMAKE_SYCL_COMPILER_WORKS)
   # Puts test result in cache variable.
-  try_compile(CMAKE_CXX_COMPILER_WORKS
-    SOURCE_FROM_VAR testCXXCompiler.cxx __TestCompiler_testCXXCompilerSource
-    OUTPUT_VARIABLE __CMAKE_CXX_COMPILER_OUTPUT)
-  unset(__TestCompiler_testCXXCompilerSource)
+  try_compile(CMAKE_SYCL_COMPILER_WORKS ${CMAKE_BINARY_DIR}
+    ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testSYCLCompiler.cxx
+    OUTPUT_VARIABLE __CMAKE_SYCL_COMPILER_OUTPUT)
+  unset(__TestCompiler_testSYCLCompilerSource)
   # Move result from cache to normal variable.
-  set(CMAKE_CXX_COMPILER_WORKS ${CMAKE_CXX_COMPILER_WORKS})
-  unset(CMAKE_CXX_COMPILER_WORKS CACHE)
+  set(CMAKE_SYCL_COMPILER_WORKS ${CMAKE_SYCL_COMPILER_WORKS})
+  unset(CMAKE_SYCL_COMPILER_WORKS CACHE)
   __TestCompiler_restoreTryCompileTargetType()
-  if(NOT CMAKE_CXX_COMPILER_WORKS)
+  if(NOT CMAKE_SYCL_COMPILER_WORKS)
     PrintTestCompilerResult(CHECK_FAIL "broken")
-    string(REPLACE "\n" "\n  " _output "${__CMAKE_CXX_COMPILER_OUTPUT}")
-    message(FATAL_ERROR "The C++ compiler\n  \"${CMAKE_CXX_COMPILER}\"\n"
+    string(REPLACE "\n" "\n  " _output "${__CMAKE_SYCL_COMPILER_OUTPUT}")
+    message(FATAL_ERROR "The C++ compiler\n  \"${CMAKE_SYCL_COMPILER}\"\n"
       "is not able to compile a simple test program.\nIt fails "
       "with the following output:\n  ${_output}\n\n"
       "CMake will not be able to correctly generate this project.")
@@ -66,24 +66,25 @@ if(NOT CMAKE_CXX_COMPILER_WORKS)
 endif()
 
 # Try to identify the compiler features
-include(${CMAKE_ROOT}/Modules/CMakeDetermineCompileFeatures.cmake)
-CMAKE_DETERMINE_COMPILE_FEATURES(CXX)
+include(CMakeDetermineCompileFeatures)
+CMAKE_DETERMINE_COMPILE_FEATURES(SYCL)
 
+set(CMAKE_TRY_COMPILE_CONFIGURATION "")
 # Re-configure to save learned information.
 configure_file(
-  ${CMAKE_ROOT}/Modules/CMakeCXXCompiler.cmake.in
-  ${CMAKE_PLATFORM_INFO_DIR}/CMakeCXXCompiler.cmake
+  ${ArrayFire_SOURCE_DIR}/CMakeModules/CMakeSYCLCompiler.cmake.in
+  ${CMAKE_PLATFORM_INFO_DIR}/CMakeSYCLCompiler.cmake
   @ONLY
-  )
-include(${CMAKE_PLATFORM_INFO_DIR}/CMakeCXXCompiler.cmake)
+)
+include(${CMAKE_PLATFORM_INFO_DIR}/CMakeSYCLCompiler.cmake)
 
-if(CMAKE_CXX_SIZEOF_DATA_PTR)
-  foreach(f ${CMAKE_CXX_ABI_FILES})
+if(CMAKE_SYCL_SIZEOF_DATA_PTR)
+  foreach(f ${CMAKE_SYCL_ABI_FILES})
     include(${f})
   endforeach()
-  unset(CMAKE_CXX_ABI_FILES)
+  unset(CMAKE_SYCL_ABI_FILES)
 endif()
 
 set(CMAKE_TRY_COMPILE_TARGET_TYPE ${__CMAKE_SAVED_TRY_COMPILE_TARGET_TYPE})
 unset(__CMAKE_SAVED_TRY_COMPILE_TARGET_TYPE)
-unset(__CMAKE_CXX_COMPILER_OUTPUT)
+unset(__CMAKE_SYCL_COMPILER_OUTPUT)
diff --git a/CMakeModules/InternalUtils.cmake b/CMakeModules/InternalUtils.cmake
index 863cbaed22..8d29718365 100644
--- a/CMakeModules/InternalUtils.cmake
+++ b/CMakeModules/InternalUtils.cmake
@@ -39,6 +39,37 @@ check_cxx_compiler_flag(-Rno-debug-disables-optimization has_cxx_debug-disables-
 function(arrayfire_set_default_cxx_flags target)
   target_compile_options(${target}
     PRIVATE
+
+      $<$<BOOL:${CMAKE_SYCL_COMPILER}>:
+        $<$<COMPILE_LANGUAGE:SYCL>:
+                # OpenCL targets need this flag to avoid
+                # ignored attribute warnings in the OpenCL
+                # headers
+                -Wno-ignored-attributes
+                -Wall
+                -Wno-unqualified-std-cast-call
+                -Werror=reorder-ctor
+                #-fp-model precise
+                $<$<BOOL:${AF_WITH_FAST_MATH}>: -ffast-math -fno-errno-math -fno-trapping-math -fno-signed-zeros -mno-ieee-fp>
+                $<$<NOT:$<BOOL:${AF_WITH_FAST_MATH}>>: $<IF:$<PLATFORM_ID:Windows>,/fp=precise,-fp-model=precise>>
+                $<$<CONFIG:Debug>:-Rno-debug-disables-optimization>
+
+                $<$<PLATFORM_ID:Windows>: /wd4251
+                                          /wd4068
+                                          /wd4275
+                                          /wd4668
+                                          /wd4710
+                                          /wd4505
+                                          /we5038
+                                          /bigobj
+                                          /EHsc
+                                          /nologo
+                                          # MSVC incorrectly sets the cplusplus to 199711L even if the compiler supports
+                                          # c++11 features. This flag sets it to the correct standard supported by the
+                                          # compiler
+                                          $<$<BOOL:${cplusplus_define}>:/Zc:__cplusplus>
+                                          $<$<BOOL:${cxx_compliance}>:/permissive-> >
+            >>
       $<$<COMPILE_LANGUAGE:CXX>:
               # C4068: Warnings about unknown pragmas
               # C4668: Warnings about unknown defintions
@@ -53,6 +84,7 @@ function(arrayfire_set_default_cxx_flags target)
                                           /we5038
                                           /bigobj
                                           /EHsc
+                                          /nologo
                                           # MSVC incorrectly sets the cplusplus to 199711L even if the compiler supports
                                           # c++11 features. This flag sets it to the correct standard supported by the
                                           # compiler
diff --git a/src/backend/common/Logger.hpp b/src/backend/common/Logger.hpp
index a004e773fb..a9a8feaa0b 100644
--- a/src/backend/common/Logger.hpp
+++ b/src/backend/common/Logger.hpp
@@ -22,6 +22,7 @@
 /* Intel ICC/ICPC */
 // Fix the warning code here, if any
 #elif defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC diagnostic push
 /* GNU GCC/G++ */
 #elif defined(_MSC_VER)
 /* Microsoft Visual Studio */
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index d4c7245311..4ecb470ef9 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -5,6 +5,10 @@
 #The complete license agreement can be obtained at:
 #http:  // arrayfire.com/licenses/BSD-3-Clause
 
+if(AF_BUILD_ONEAPI)
+    enable_language(SYCL)
+endif()
+
 include(InternalUtils)
 include(build_cl2hpp)
 include(FileToString)
@@ -260,6 +264,26 @@ target_sources(afoneapi
     kernel/wrap_dilated.hpp
 )
 
+function(set_sycl_language)
+  foreach(target ${ARGV})
+    set_target_properties(${target}
+      PROPERTIES
+        LINKER_LANGUAGE SYCL)
+
+    get_target_property(TGT_SOURCES ${target} SOURCES)
+    if(NOT TGT_SOURCES)
+      get_target_property(TGT_SOURCES ${target} INTERFACE_SOURCES)
+    endif()
+
+    foreach(FILE ${TGT_SOURCES})
+      get_filename_component(FILE_EXTENSION ${FILE} EXT)
+      if(FILE_EXTENSION STREQUAL ".cpp")
+        set_source_files_properties(${FILE} PROPERTIES LANGUAGE SYCL)
+      endif()
+    endforeach()
+  endforeach()
+endfunction()
+
 set(kernel_src
   ${CMAKE_CURRENT_SOURCE_DIR}/../opencl/kernel/KParam.hpp
   ${CMAKE_CURRENT_SOURCE_DIR}/../opencl/kernel/jit.cl
@@ -301,10 +325,11 @@ target_include_directories(afoneapi
 
 target_compile_options(afoneapi
   PRIVATE
-    -fsycl
+  $<$<COMPILE_LANGUAGE:SYCL>:
     -fno-sycl-id-queries-fit-in-int
     -sycl-std=2020
-    -fno-sycl-rdc
+    $<$<PLATFORM_ID:Linux>: -fno-sycl-rdc>
+    >
 )
 
 target_compile_definitions(afoneapi
@@ -322,8 +347,6 @@ cmake_host_system_information(RESULT NumberOfThreads
 
 target_link_libraries(afoneapi
   PRIVATE
-    -fsycl
-    -fvisibility-inlines-hidden
     c_api_interface
     cpp_api_interface
     oneapi_sort_by_key
@@ -331,14 +354,20 @@ target_link_libraries(afoneapi
     OpenCL::OpenCL
     OpenCL::cl2hpp
     -fno-sycl-id-queries-fit-in-int
-    -fno-sycl-rdc
-    -fsycl-device-code-split=per_kernel
-    -fsycl-link-huge-device-code
+    $<$<PLATFORM_ID:Linux>:-fsycl-link-huge-device-code>
+    $<$<PLATFORM_ID:Linux>:-fvisibility-inlines-hidden>
+    $<$<PLATFORM_ID:Linux>:-fno-sycl-rdc>
     -fsycl-max-parallel-link-jobs=${NumberOfThreads}
     MKL::MKL_DPCPP
   )
+  set_sycl_language(afcommon_interface
+    oneapi_sort_by_key
+    c_api_interface
+    cpp_api_interface
+    afoneapi)
+
 
-af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR})
+#af_split_debug_info(afoneapi ${AF_INSTALL_LIB_DIR})
 
 install(TARGETS afoneapi
   EXPORT ArrayFireoneAPITargets
diff --git a/src/backend/oneapi/device_manager.cpp b/src/backend/oneapi/device_manager.cpp
index ac06d5768c..56125382a0 100644
--- a/src/backend/oneapi/device_manager.cpp
+++ b/src/backend/oneapi/device_manager.cpp
@@ -104,13 +104,7 @@ DeviceManager::DeviceManager()
     // Iterate through platforms, get all available devices and store them
     for (auto& platform : platforms) {
         vector<sycl::device> current_devices;
-        try {
-            current_devices = platform.get_devices();
-        } catch (sycl::exception& err) {
-            printf("DeviceManager::DeviceManager() exception: %s\n",
-                   err.what());
-            throw;
-        }
+        current_devices = platform.get_devices();
         AF_TRACE("Found {} devices on platform {}", current_devices.size(),
                  platform.get_info<sycl::info::platform::name>());
 
diff --git a/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt b/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
index ce184639eb..394d593d6e 100644
--- a/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
+++ b/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
@@ -20,6 +20,10 @@ foreach(SBK_TYPE ${SBK_TYPES})
         "${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/sort_by_key_impl.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key_impl.hpp"
     )
+
+  set_source_files_properties("${CMAKE_CURRENT_SOURCE_DIR}/kernel/sort_by_key/sort_by_key_impl.cpp"
+    PROPERTIES
+      LANGUAGE SYCL)
   set_target_properties(oneapi_sort_by_key_${SBK_TYPE}
     PROPERTIES
       COMPILE_DEFINITIONS "TYPE=${SBK_TYPE};AFDLL;$<TARGET_PROPERTY:Boost::boost,INTERFACE_COMPILE_DEFINITIONS>"
@@ -41,12 +45,17 @@ foreach(SBK_TYPE ${SBK_TYPES})
       ..
       )
 
+  target_compile_options(oneapi_sort_by_key_${SBK_TYPE}
+    PRIVATE
+      $<$<COMPILE_LANGUAGE:SYCL>: -fno-sycl-id-queries-fit-in-int
+                                  -sycl-std=2020
+                                  $<$<PLATFORM_ID:Linux>: -fno-sycl-rdc>>)
+
   target_include_directories(oneapi_sort_by_key_${SBK_TYPE}
     SYSTEM PRIVATE
       ${span-lite_SOURCE_DIR}/include
       $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>)
 
-  target_compile_options(oneapi_sort_by_key_${SBK_TYPE} PUBLIC -fsycl)
   set_target_properties(oneapi_sort_by_key_${SBK_TYPE} PROPERTIES POSITION_INDEPENDENT_CODE ON)
   target_sources(oneapi_sort_by_key
     INTERFACE $<TARGET_OBJECTS:oneapi_sort_by_key_${SBK_TYPE}>)
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 3f1beb55bb..84ac83839f 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -7,13 +7,17 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 #pragma once
+#ifdef __GNUC__
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-function"
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wparentheses"
+#endif
 #include <half.hpp>
+#ifdef __GNUC__
 #pragma GCC diagnostic pop
+#endif
 #include <af/array.h>
 #include <af/defines.h>
 #include <af/dim4.hpp>
@@ -49,11 +53,20 @@ std::ostream &operator<<(std::ostream &os, const af_half &val);
     do { (void)(expr); } while (0)
 
 namespace aft {
+#ifdef __GNUC__
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#elif defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable : 4996)
+#endif
 typedef intl intl;
 typedef uintl uintl;
+#ifdef __GNUC__
 #pragma GCC diagnostic pop
+#elif defined(_MSC_VER)
+#pragma warning(pop)
+#endif
 }  // namespace aft
 
 using aft::intl;
@@ -630,4 +643,6 @@ ::testing::AssertionResult assertArrayEq(std::string aName, std::string bName,
                                          const af_array a, const af_array b,
                                          TestOutputArrayInfo *metadata);
 
+#ifdef __GNUC__
 #pragma GCC diagnostic pop
+#endif

From 26486330f581be8550a13420724e64f481c977d3 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 14 Aug 2023 16:48:01 -0400
Subject: [PATCH 352/473] Set cmake_minimum_version for oneAPI. Fix compiler id

---
 .github/workflows/unix_cpu_build.yml | 4 ++--
 CMakeLists.txt                       | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 3146358772..460aaa9d34 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -63,7 +63,7 @@ jobs:
         needs: [clang-format, documentation]
         env:
           NINJA_VER: 1.10.2
-          CMAKE_VER: 3.10.2
+          CMAKE_VER: 3.16.3
         strategy:
             fail-fast: false
             matrix:
@@ -93,7 +93,7 @@ jobs:
                   chmod +x ninja
                   ${GITHUB_WORKSPACE}/ninja --version
 
-            - name: Download CMake 3.10.2 for Linux
+            - name: Download CMake 3.16.3 for Linux
               if: matrix.os != 'macos-latest'
               env:
                   OS_NAME: ${{ matrix.os }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e4cc17916f..deafa7a759 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -5,7 +5,11 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.10.2)
+if(AF_BUILD_ONEAPI)
+  cmake_minimum_required(VERSION 3.20)
+else()
+  cmake_minimum_required(VERSION 3.16.3)
+endif()
 include(CheckLanguage)
 
 include(CMakeModules/AF_vcpkg_options.cmake)

From 0514a5da43736d22a39bce4b56e61e7e0fc464d3 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 15 Aug 2023 12:06:30 -0400
Subject: [PATCH 353/473] Source tbb because of the new default threading
 backend for CPU

---
 .github/workflows/unix_cpu_build.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/unix_cpu_build.yml b/.github/workflows/unix_cpu_build.yml
index 460aaa9d34..07ffba36f7 100644
--- a/.github/workflows/unix_cpu_build.yml
+++ b/.github/workflows/unix_cpu_build.yml
@@ -151,7 +151,7 @@ jobs:
                   sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
                   sudo sh -c 'echo deb https://apt.repos.intel.com/oneapi all main > /etc/apt/sources.list.d/oneAPI.list'
                   sudo apt-get -qq update
-                  sudo apt-get install -y intel-oneapi-mkl-devel
+                  sudo apt-get install -y intel-oneapi-mkl-devel intel-oneapi-tbb-devel
                   if [ "$CC" == 'icx' ]; then sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp; fi
                   echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> ${GITHUB_ENV}
 
@@ -171,10 +171,10 @@ jobs:
                   branch=$(git rev-parse --abbrev-ref HEAD)
                   buildname=$(if [ -z "$prnum" ]; then echo "$branch"; else echo "PR-$prnum"; fi)
                   dashboard=$(if [ -z "$prnum" ]; then echo "Continuous"; else echo "Experimental"; fi)
-                  backend=$(if [ "$USE_MKL" == 1 ]; then echo "Intel-MKL"; else echo "FFTW/LAPACK/BLAS"; fi)
+                  backend=$(if [ "$USE_MKL" == true ]; then echo "Intel-MKL"; else echo "FFTW/LAPACK/BLAS"; fi)
                   buildname="$buildname-cpu-$BLAS_BACKEND"
                   cmake_rpath=$(if [ $OS_NAME == 'macos-latest' ]; then echo "-DCMAKE_INSTALL_RPATH=/opt/arrayfire/lib"; fi)
-                  if [ "$CC" == 'icx' ]; then source /opt/intel/oneapi/setvars.sh intel64; fi
+                  if [ "$CC" == 'icx' ] || [ "$USE_MKL" == true ]; then source /opt/intel/oneapi/setvars.sh; fi
                   mkdir build && cd build && unset VCPKG_ROOT
                   ${CMAKE_PROGRAM} -G Ninja \
                       -DCMAKE_MAKE_PROGRAM:FILEPATH=${GITHUB_WORKSPACE}/ninja \
@@ -189,7 +189,8 @@ jobs:
             - name: Build and Test
               env:
                   CC: ${{ matrix.compiler }}
+                  USE_MKL: ${{ matrix.blas_backend == 'MKL' }}
               run: |
                   cd ${GITHUB_WORKSPACE}/build
-                  if [ "$CC" == 'icx' ]; then source /opt/intel/oneapi/setvars.sh intel64; fi
+                  if [ "$CC" == 'icx' ] || [ "$USE_MKL" == true ]; then source /opt/intel/oneapi/setvars.sh; fi
                   ctest -D Experimental --track ${CTEST_DASHBOARD} -T Test -T Submit -R cpu -j2

From bda893a1d280dcbf3284f536408edee3c95e4c0d Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 7 Aug 2023 14:07:59 -0700
Subject: [PATCH 354/473] fix wrong number of elements in createStrided for
 oneapi

---
 src/backend/oneapi/Array.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index f227f8def3..8ff64d78ec 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -196,8 +196,7 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
         data = memAlloc<T>(info.elements());
         getQueue()
             .submit([&](sycl::handler &h) {
-                h.copy(in_data,
-                       data->get_access(h, sycl::range(info.elements())));
+                h.copy(in_data, data->get_access(h, sycl::range(info.total())));
             })
             .wait();
     }

From 8aabf16f74396494459e3db1abef62be37366d79 Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Mon, 7 Aug 2023 17:12:24 -0700
Subject: [PATCH 355/473] implements events in oneapi backend

---
 src/backend/oneapi/Event.cpp | 64 ++++++++++++++++--------------------
 src/backend/oneapi/Event.hpp | 13 ++++----
 2 files changed, 35 insertions(+), 42 deletions(-)

diff --git a/src/backend/oneapi/Event.cpp b/src/backend/oneapi/Event.cpp
index 056c6cf950..60bc8bcb77 100644
--- a/src/backend/oneapi/Event.cpp
+++ b/src/backend/oneapi/Event.cpp
@@ -24,56 +24,50 @@ namespace arrayfire {
 namespace oneapi {
 /// \brief Creates a new event and marks it in the queue
 Event makeEvent(sycl::queue& queue) {
-    ONEAPI_NOT_SUPPORTED("makeEvent");
-    return Event();
+    Event e;
+    if (e.create() == 0) { e.mark(queue); }
+    return e;
 }
 
 af_event createEvent() {
-    ONEAPI_NOT_SUPPORTED("");
-    return 0;
-    // auto e = make_unique<Event>();
-    // // Ensure the default CL command queue is initialized
-    // getQueue();
-    // if (e->create() != CL_SUCCESS) {
-    //     AF_ERROR("Could not create event", AF_ERR_RUNTIME);
-    // }
-    // Event& ref = *e.release();
-    // return getHandle(ref);
+    auto e = make_unique<Event>();
+    // Ensure the default CL command queue is initialized
+    getQueue();
+    if (e->create() != 0) {
+        AF_ERROR("Could not create event", AF_ERR_RUNTIME);
+    }
+    Event& ref = *e.release();
+    return getHandle(ref);
 }
 
 void markEventOnActiveQueue(af_event eventHandle) {
-    ONEAPI_NOT_SUPPORTED("");
-    // Event& event = getEvent(eventHandle);
-    //// Use the currently-active stream
-    // if (event.mark(getQueue()()) != CL_SUCCESS) {
-    //    AF_ERROR("Could not mark event on active queue", AF_ERR_RUNTIME);
-    //}
+    Event& event = getEvent(eventHandle);
+    // Use the currently-active stream
+    if (event.mark(getQueue()) != 0) {
+        AF_ERROR("Could not mark event on active queue", AF_ERR_RUNTIME);
+    }
 }
 
 void enqueueWaitOnActiveQueue(af_event eventHandle) {
-    ONEAPI_NOT_SUPPORTED("");
-    // Event& event = getEvent(eventHandle);
-    //// Use the currently-active stream
-    // if (event.enqueueWait(getQueue()()) != CL_SUCCESS) {
-    //    AF_ERROR("Could not enqueue wait on active queue for event",
-    //             AF_ERR_RUNTIME);
-    //}
+    Event& event = getEvent(eventHandle);
+    // Use the currently-active stream
+    if (event.enqueueWait(getQueue()) != 0) {
+        AF_ERROR("Could not enqueue wait on active queue for event",
+                 AF_ERR_RUNTIME);
+    }
 }
 
 void block(af_event eventHandle) {
-    ONEAPI_NOT_SUPPORTED("");
-    // Event& event = getEvent(eventHandle);
-    // if (event.block() != CL_SUCCESS) {
-    //    AF_ERROR("Could not block on active queue for event", AF_ERR_RUNTIME);
-    //}
+    Event& event = getEvent(eventHandle);
+    if (event.block() != 0) {
+        AF_ERROR("Could not block on active queue for event", AF_ERR_RUNTIME);
+    }
 }
 
 af_event createAndMarkEvent() {
-    ONEAPI_NOT_SUPPORTED("");
-    return 0;
-    // af_event handle = createEvent();
-    // markEventOnActiveQueue(handle);
-    // return handle;
+    af_event handle = createEvent();
+    markEventOnActiveQueue(handle);
+    return handle;
 }
 
 }  // namespace oneapi
diff --git a/src/backend/oneapi/Event.hpp b/src/backend/oneapi/Event.hpp
index ae7fdd8c29..44af139cda 100644
--- a/src/backend/oneapi/Event.hpp
+++ b/src/backend/oneapi/Event.hpp
@@ -17,33 +17,32 @@ namespace arrayfire {
 namespace oneapi {
 class OneAPIEventPolicy {
    public:
-    using EventType = sycl::event;
+    using EventType = sycl::event *;
     using QueueType = sycl::queue;
-    // using ErrorType = sycl::exception; //does this make sense
     using ErrorType = int;
 
     static ErrorType createAndMarkEvent(EventType *e) noexcept {
-        // Events are created when you mark them
+        *e = new sycl::event;
         return 0;
     }
 
     static ErrorType markEvent(EventType *e, QueueType stream) noexcept {
-        // return clEnqueueMarkerWithWaitList(stream, 0, nullptr, e);
+        **e = stream.ext_oneapi_submit_barrier();
         return 0;
     }
 
     static ErrorType waitForEvent(EventType *e, QueueType stream) noexcept {
-        // return clEnqueueMarkerWithWaitList(stream, 1, e, nullptr);
+        stream.ext_oneapi_submit_barrier({**e});
         return 0;
     }
 
     static ErrorType syncForEvent(EventType *e) noexcept {
-        // return clWaitForEvents(1, e);
+        (*e)->wait();
         return 0;
     }
 
     static ErrorType destroyEvent(EventType *e) noexcept {
-        // return clReleaseEvent(*e);
+        delete *e;
         return 0;
     }
 };

From b1f2f86361924f7cb1160120a1e8cb62510c649e Mon Sep 17 00:00:00 2001
From: syurkevi <stefan@arrayfire.com>
Date: Fri, 4 Aug 2023 11:12:23 -0700
Subject: [PATCH 356/473] corrects double checks for reduce tests

---
 test/reduce.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/reduce.cpp b/test/reduce.cpp
index 0726a11791..0b8317a960 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -411,6 +411,9 @@ class ReduceByKeyP : public ::testing::TestWithParam<reduce_by_key_params *> {
         if (noHalfTests(params->vType_)) {
             GTEST_SKIP() << "Half not supported on this device";
         }
+        if (noDoubleTests(GetParam()->vType_)) {
+            GTEST_SKIP() << "Double not supported on this device";
+        }
 
         keys = ptrToArray(params->iSize, params->iKeys_, params->kType_);
         vals = ptrToArray(params->iSize, params->iVals_, params->vType_);
@@ -1967,6 +1970,9 @@ class RaggedReduceMaxRangeP : public ::testing::TestWithParam<ragged_params *> {
         if (noHalfTests(params->vType_)) {
             GTEST_SKIP() << "Half not supported on this device";
         }
+        if (noDoubleTests(GetParam()->vType_)) {
+            GTEST_SKIP() << "Double not supported on this device";
+        }
 
         const size_t rdim_size = params->reduceDimLen_;
         const int dim          = params->reduceDim_;
@@ -2324,6 +2330,7 @@ TEST(Reduce, Test_Sum_Global_Array_nanval) {
 
 TEST(Reduce, nanval_issue_3255) {
     SKIP_IF_FAST_MATH_ENABLED();
+    SUPPORTED_TYPE_CHECK(double);
     char *info_str;
     af_array ikeys, ivals, okeys, ovals;
     dim_t dims[1] = {8};

From b803eb802fffc72bab3a628d5ea122a19b1e090d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 13 Jun 2023 18:11:26 -0400
Subject: [PATCH 357/473] Move device checks out of getInfo into getArray

---
 src/api/c/array.cpp                | 55 +++++++++++++++---------------
 src/api/c/binary.cpp               | 16 ++++-----
 src/api/c/blas.cpp                 |  8 ++---
 src/api/c/cast.cpp                 |  4 +--
 src/api/c/device.cpp               |  4 +--
 src/api/c/handle.cpp               |  2 +-
 src/api/c/handle.hpp               | 16 +++++++--
 src/api/c/sparse.cpp               |  2 +-
 src/api/c/sparse_handle.hpp        |  4 ++-
 src/backend/common/ArrayInfo.cpp   |  8 +----
 src/backend/common/SparseArray.cpp | 10 +++++-
 src/backend/common/SparseArray.hpp |  7 ++++
 src/backend/cpu/Array.cpp          |  8 ++++-
 src/backend/cpu/Array.hpp          |  7 ++++
 src/backend/cuda/Array.cpp         | 10 +++++-
 src/backend/cuda/Array.hpp         |  7 ++++
 src/backend/oneapi/Array.cpp       | 10 +++++-
 src/backend/oneapi/Array.hpp       |  7 ++++
 src/backend/opencl/Array.cpp       | 10 +++++-
 src/backend/opencl/Array.hpp       |  7 ++++
 test/array.cpp                     |  2 +-
 21 files changed, 143 insertions(+), 61 deletions(-)

diff --git a/src/api/c/array.cpp b/src/api/c/array.cpp
index 173c52171c..4e1877e364 100644
--- a/src/api/c/array.cpp
+++ b/src/api/c/array.cpp
@@ -20,6 +20,7 @@ using af::dim4;
 using arrayfire::copyData;
 using arrayfire::copySparseArray;
 using arrayfire::getSparseArrayBase;
+using arrayfire::getUseCount;
 using arrayfire::releaseHandle;
 using arrayfire::releaseSparseHandle;
 using arrayfire::retainSparseHandle;
@@ -192,24 +193,24 @@ af_err af_copy_array(af_array *out, const af_array in) {
 // Strong Exception Guarantee
 af_err af_get_data_ref_count(int *use_count, const af_array in) {
     try {
-        const ArrayInfo &info = getInfo(in, false, false);
+        const ArrayInfo &info = getInfo(in, false);
         const af_dtype type   = info.getType();
 
         int res;
         switch (type) {
-            case f32: res = getArray<float>(in).useCount(); break;
-            case c32: res = getArray<cfloat>(in).useCount(); break;
-            case f64: res = getArray<double>(in).useCount(); break;
-            case c64: res = getArray<cdouble>(in).useCount(); break;
-            case b8: res = getArray<char>(in).useCount(); break;
-            case s32: res = getArray<int>(in).useCount(); break;
-            case u32: res = getArray<uint>(in).useCount(); break;
-            case u8: res = getArray<uchar>(in).useCount(); break;
-            case s64: res = getArray<intl>(in).useCount(); break;
-            case u64: res = getArray<uintl>(in).useCount(); break;
-            case s16: res = getArray<short>(in).useCount(); break;
-            case u16: res = getArray<ushort>(in).useCount(); break;
-            case f16: res = getArray<half>(in).useCount(); break;
+            case f32: res = getUseCount<float>(in); break;
+            case c32: res = getUseCount<cfloat>(in); break;
+            case f64: res = getUseCount<double>(in); break;
+            case c64: res = getUseCount<cdouble>(in); break;
+            case b8: res = getUseCount<char>(in); break;
+            case s32: res = getUseCount<int>(in); break;
+            case u32: res = getUseCount<uint>(in); break;
+            case u8: res = getUseCount<uchar>(in); break;
+            case s64: res = getUseCount<intl>(in); break;
+            case u64: res = getUseCount<uintl>(in); break;
+            case s16: res = getUseCount<short>(in); break;
+            case u16: res = getUseCount<ushort>(in); break;
+            case f16: res = getUseCount<half>(in); break;
             default: TYPE_ERROR(1, type);
         }
         std::swap(*use_count, res);
@@ -221,7 +222,7 @@ af_err af_get_data_ref_count(int *use_count, const af_array in) {
 af_err af_release_array(af_array arr) {
     try {
         if (arr == 0) { return AF_SUCCESS; }
-        const ArrayInfo &info = getInfo(arr, false, false);
+        const ArrayInfo &info = getInfo(arr, false);
         af_dtype type         = info.getType();
 
         if (info.isSparse()) {
@@ -335,7 +336,7 @@ af_err af_write_array(af_array arr, const void *data, const size_t bytes,
 af_err af_get_elements(dim_t *elems, const af_array arr) {
     try {
         // Do not check for device mismatch
-        *elems = getInfo(arr, false, false).elements();
+        *elems = getInfo(arr, false).elements();
     }
     CATCHALL
     return AF_SUCCESS;
@@ -344,7 +345,7 @@ af_err af_get_elements(dim_t *elems, const af_array arr) {
 af_err af_get_type(af_dtype *type, const af_array arr) {
     try {
         // Do not check for device mismatch
-        *type = getInfo(arr, false, false).getType();
+        *type = getInfo(arr, false).getType();
     }
     CATCHALL
     return AF_SUCCESS;
@@ -354,7 +355,7 @@ af_err af_get_dims(dim_t *d0, dim_t *d1, dim_t *d2, dim_t *d3,
                    const af_array in) {
     try {
         // Do not check for device mismatch
-        const ArrayInfo &info = getInfo(in, false, false);
+        const ArrayInfo &info = getInfo(in, false);
         *d0                   = info.dims()[0];
         *d1                   = info.dims()[1];
         *d2                   = info.dims()[2];
@@ -367,7 +368,7 @@ af_err af_get_dims(dim_t *d0, dim_t *d1, dim_t *d2, dim_t *d3,
 af_err af_get_numdims(unsigned *nd, const af_array in) {
     try {
         // Do not check for device mismatch
-        const ArrayInfo &info = getInfo(in, false, false);
+        const ArrayInfo &info = getInfo(in, false);
         *nd                   = info.ndims();
     }
     CATCHALL
@@ -375,14 +376,14 @@ af_err af_get_numdims(unsigned *nd, const af_array in) {
 }
 
 #undef INSTANTIATE
-#define INSTANTIATE(fn1, fn2)                                  \
-    af_err fn1(bool *result, const af_array in) {              \
-        try {                                                  \
-            const ArrayInfo &info = getInfo(in, false, false); \
-            *result               = info.fn2();                \
-        }                                                      \
-        CATCHALL                                               \
-        return AF_SUCCESS;                                     \
+#define INSTANTIATE(fn1, fn2)                           \
+    af_err fn1(bool *result, const af_array in) {       \
+        try {                                           \
+            const ArrayInfo &info = getInfo(in, false); \
+            *result               = info.fn2();         \
+        }                                               \
+        CATCHALL                                        \
+        return AF_SUCCESS;                              \
     }
 
 INSTANTIATE(af_is_empty, isEmpty)
diff --git a/src/api/c/binary.cpp b/src/api/c/binary.cpp
index dc5eddf4bc..50590568f8 100644
--- a/src/api/c/binary.cpp
+++ b/src/api/c/binary.cpp
@@ -279,8 +279,8 @@ af_err af_add(af_array *out, const af_array lhs, const af_array rhs,
               const bool batchMode) {
     try {
         // Check if inputs are sparse
-        const ArrayInfo &linfo = getInfo(lhs, false, true);
-        const ArrayInfo &rinfo = getInfo(rhs, false, true);
+        const ArrayInfo &linfo = getInfo(lhs, false);
+        const ArrayInfo &rinfo = getInfo(rhs, false);
 
         if (linfo.isSparse() && rinfo.isSparse()) {
             return af_arith_sparse<af_add_t>(out, lhs, rhs);
@@ -301,8 +301,8 @@ af_err af_mul(af_array *out, const af_array lhs, const af_array rhs,
               const bool batchMode) {
     try {
         // Check if inputs are sparse
-        const ArrayInfo &linfo = getInfo(lhs, false, true);
-        const ArrayInfo &rinfo = getInfo(rhs, false, true);
+        const ArrayInfo &linfo = getInfo(lhs, false);
+        const ArrayInfo &rinfo = getInfo(rhs, false);
 
         if (linfo.isSparse() && rinfo.isSparse()) {
             // return af_arith_sparse<af_mul_t>(out, lhs, rhs);
@@ -327,8 +327,8 @@ af_err af_sub(af_array *out, const af_array lhs, const af_array rhs,
               const bool batchMode) {
     try {
         // Check if inputs are sparse
-        const ArrayInfo &linfo = getInfo(lhs, false, true);
-        const ArrayInfo &rinfo = getInfo(rhs, false, true);
+        const ArrayInfo &linfo = getInfo(lhs, false);
+        const ArrayInfo &rinfo = getInfo(rhs, false);
 
         if (linfo.isSparse() && rinfo.isSparse()) {
             return af_arith_sparse<af_sub_t>(out, lhs, rhs);
@@ -350,8 +350,8 @@ af_err af_div(af_array *out, const af_array lhs, const af_array rhs,
               const bool batchMode) {
     try {
         // Check if inputs are sparse
-        const ArrayInfo &linfo = getInfo(lhs, false, true);
-        const ArrayInfo &rinfo = getInfo(rhs, false, true);
+        const ArrayInfo &linfo = getInfo(lhs, false);
+        const ArrayInfo &rinfo = getInfo(rhs, false);
 
         if (linfo.isSparse() && rinfo.isSparse()) {
             // return af_arith_sparse<af_div_t>(out, lhs, rhs);
diff --git a/src/api/c/blas.cpp b/src/api/c/blas.cpp
index 0946d42083..0cd8fddd8d 100644
--- a/src/api/c/blas.cpp
+++ b/src/api/c/blas.cpp
@@ -134,8 +134,8 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs,
                const af_mat_prop optRhs, const void *alpha, const af_array lhs,
                const af_array rhs, const void *beta) {
     try {
-        const ArrayInfo &lhsInfo = getInfo(lhs, false, true);
-        const ArrayInfo &rhsInfo = getInfo(rhs, true, true);
+        const ArrayInfo &lhsInfo = getInfo(lhs, false);
+        const ArrayInfo &rhsInfo = getInfo(rhs, true);
 
         af_dtype lhs_type = lhsInfo.getType();
         af_dtype rhs_type = rhsInfo.getType();
@@ -227,8 +227,8 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs,
 af_err af_matmul(af_array *out, const af_array lhs, const af_array rhs,
                  const af_mat_prop optLhs, const af_mat_prop optRhs) {
     try {
-        const ArrayInfo &lhsInfo = getInfo(lhs, false, true);
-        const ArrayInfo &rhsInfo = getInfo(rhs, true, true);
+        const ArrayInfo &lhsInfo = getInfo(lhs, false);
+        const ArrayInfo &rhsInfo = getInfo(rhs, true);
 
         if (lhsInfo.isSparse()) {
             return af_sparse_matmul(out, lhs, rhs, optLhs, optRhs);
diff --git a/src/api/c/cast.cpp b/src/api/c/cast.cpp
index 20e47a1a2d..328c81ca65 100644
--- a/src/api/c/cast.cpp
+++ b/src/api/c/cast.cpp
@@ -34,7 +34,7 @@ using detail::uintl;
 using detail::ushort;
 
 static af_array cast(const af_array in, const af_dtype type) {
-    const ArrayInfo& info = getInfo(in, false, true);
+    const ArrayInfo& info = getInfo(in, false);
 
     if (info.getType() == type) { return retain(in); }
 
@@ -68,7 +68,7 @@ static af_array cast(const af_array in, const af_dtype type) {
 
 af_err af_cast(af_array* out, const af_array in, const af_dtype type) {
     try {
-        const ArrayInfo& info = getInfo(in, false, true);
+        const ArrayInfo& info = getInfo(in, false);
 
         af_dtype inType = info.getType();
         if ((inType == c32 || inType == c64) &&
diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp
index 1b6ef9fb93..ef37888523 100644
--- a/src/api/c/device.cpp
+++ b/src/api/c/device.cpp
@@ -80,7 +80,7 @@ af_err af_get_available_backends(int* result) {
 af_err af_get_backend_id(af_backend* result, const af_array in) {
     try {
         if (in) {
-            const ArrayInfo& info = getInfo(in, false, false);
+            const ArrayInfo& info = getInfo(in, false);
             *result               = info.getBackendId();
         } else {
             return AF_ERR_ARG;
@@ -93,7 +93,7 @@ af_err af_get_backend_id(af_backend* result, const af_array in) {
 af_err af_get_device_id(int* device, const af_array in) {
     try {
         if (in) {
-            const ArrayInfo& info = getInfo(in, false, false);
+            const ArrayInfo& info = getInfo(in, false);
             *device               = static_cast<int>(info.getDevId());
         } else {
             return AF_ERR_ARG;
diff --git a/src/api/c/handle.cpp b/src/api/c/handle.cpp
index 7a93847826..243bfdba63 100644
--- a/src/api/c/handle.cpp
+++ b/src/api/c/handle.cpp
@@ -29,7 +29,7 @@ using detail::ushort;
 namespace arrayfire {
 
 af_array retain(const af_array in) {
-    const ArrayInfo &info = getInfo(in, false, false);
+    const ArrayInfo &info = getInfo(in, false);
     af_dtype ty           = info.getType();
 
     if (info.isSparse()) {
diff --git a/src/api/c/handle.hpp b/src/api/c/handle.hpp
index 97243ac353..add7a7c612 100644
--- a/src/api/c/handle.hpp
+++ b/src/api/c/handle.hpp
@@ -40,8 +40,7 @@ af_array createHandleFromDeviceData(const af::dim4 &d, af_dtype dtype,
                                     void *data);
 
 namespace common {
-const ArrayInfo &getInfo(const af_array arr, bool sparse_check = true,
-                         bool device_check = true);
+const ArrayInfo &getInfo(const af_array arr, bool sparse_check = true);
 
 template<typename To>
 detail::Array<To> castArray(const af_array &in);
@@ -53,6 +52,7 @@ const detail::Array<T> &getArray(const af_array &arr) {
     const detail::Array<T> *A = static_cast<const detail::Array<T> *>(arr);
     if ((af_dtype)af::dtype_traits<T>::af_type != A->getType())
         AF_ERROR("Invalid type for input array.", AF_ERR_INTERNAL);
+    checkAndMigrate(*A);
     return *A;
 }
 
@@ -61,9 +61,21 @@ detail::Array<T> &getArray(af_array &arr) {
     detail::Array<T> *A = static_cast<detail::Array<T> *>(arr);
     if ((af_dtype)af::dtype_traits<T>::af_type != A->getType())
         AF_ERROR("Invalid type for input array.", AF_ERR_INTERNAL);
+    checkAndMigrate(*A);
     return *A;
 }
 
+/// Returns the use count
+///
+/// \note This function is called separately because we cannot call getArray in
+/// case the data was built on a different context. so we are avoiding the check
+/// and migrate function
+template<typename T>
+int getUseCount(const af_array &arr) {
+    detail::Array<T> *A = static_cast<detail::Array<T> *>(arr);
+    return A->useCount();
+}
+
 template<typename T>
 af_array getHandle(const detail::Array<T> &A) {
     detail::Array<T> *ret = new detail::Array<T>(A);
diff --git a/src/api/c/sparse.cpp b/src/api/c/sparse.cpp
index 917864dcaf..db57b0077b 100644
--- a/src/api/c/sparse.cpp
+++ b/src/api/c/sparse.cpp
@@ -347,7 +347,7 @@ af_err af_sparse_convert_to(af_array *out, const af_array in,
                             const af_storage destStorage) {
     try {
         // Handle dense case
-        const ArrayInfo &info = getInfo(in, false, true);
+        const ArrayInfo &info = getInfo(in, false);
         if (!info.isSparse()) {  // If input is dense
             return af_create_sparse_array_from_dense(out, in, destStorage);
         }
diff --git a/src/api/c/sparse_handle.hpp b/src/api/c/sparse_handle.hpp
index e99bbb36e5..62c5289ebc 100644
--- a/src/api/c/sparse_handle.hpp
+++ b/src/api/c/sparse_handle.hpp
@@ -30,6 +30,7 @@ const common::SparseArray<T> &getSparseArray(const af_array &arr) {
     const common::SparseArray<T> *A =
         static_cast<const common::SparseArray<T> *>(arr);
     ARG_ASSERT(0, A->isSparse() == true);
+    checkAndMigrate(*A);
     return *A;
 }
 
@@ -37,6 +38,7 @@ template<typename T>
 common::SparseArray<T> &getSparseArray(af_array &arr) {
     common::SparseArray<T> *A = static_cast<common::SparseArray<T> *>(arr);
     ARG_ASSERT(0, A->isSparse() == true);
+    checkAndMigrate(*A);
     return *A;
 }
 
@@ -62,7 +64,7 @@ af_array retainSparseHandle(const af_array in) {
 // based on castArray in handle.hpp
 template<typename To>
 common::SparseArray<To> castSparse(const af_array &in) {
-    const ArrayInfo &info = getInfo(in, false, true);
+    const ArrayInfo &info = getInfo(in, false);
     using namespace common;
 
 #define CAST_SPARSE(Ti)                                                          \
diff --git a/src/backend/common/ArrayInfo.cpp b/src/backend/common/ArrayInfo.cpp
index d919c942f8..60c55c3e52 100644
--- a/src/backend/common/ArrayInfo.cpp
+++ b/src/backend/common/ArrayInfo.cpp
@@ -221,8 +221,7 @@ dim4 toStride(const vector<af_seq> &seqs, const af::dim4 &parentDims) {
 namespace arrayfire {
 namespace common {
 
-const ArrayInfo &getInfo(const af_array arr, bool sparse_check,
-                         bool device_check) {
+const ArrayInfo &getInfo(const af_array arr, bool sparse_check) {
     const ArrayInfo *info = nullptr;
     memcpy(&info, &arr, sizeof(af_array));
 
@@ -230,11 +229,6 @@ const ArrayInfo &getInfo(const af_array arr, bool sparse_check,
     // are accepted Otherwise only regular Array<T> is accepted
     if (sparse_check) { ARG_ASSERT(0, info->isSparse() == false); }
 
-    if (device_check && info->getDevId() != static_cast<unsigned>(
-                                                detail::getActiveDeviceId())) {
-        AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE);
-    }
-
     return *info;
 }
 
diff --git a/src/backend/common/SparseArray.cpp b/src/backend/common/SparseArray.cpp
index ac91a29f31..ed9680c6a5 100644
--- a/src/backend/common/SparseArray.cpp
+++ b/src/backend/common/SparseArray.cpp
@@ -171,6 +171,13 @@ void destroySparseArray(SparseArray<T> *sparse) {
     delete sparse;
 }
 
+template<typename T>
+void checkAndMigrate(const SparseArray<T> &arr) {
+    checkAndMigrate(arr.getColIdx());
+    checkAndMigrate(arr.getRowIdx());
+    checkAndMigrate(arr.getValues());
+}
+
 ////////////////////////////////////////////////////////////////////////////
 // Sparse Array Class Implementations
 ////////////////////////////////////////////////////////////////////////////
@@ -250,7 +257,8 @@ SparseArray<T>::SparseArray(const SparseArray<T> &other, bool copy)
     template SparseArray<T>::SparseArray(                                    \
         const af::dim4 &_dims, const Array<T> &_values,                      \
         const Array<int> &_rowIdx, const Array<int> &_colIdx,                \
-        const af::storage _storage, bool _copy)
+        const af::storage _storage, bool _copy);                             \
+    template void checkAndMigrate(const SparseArray<T> &arr)
 
 // Instantiate only floating types
 INSTANTIATE(float);
diff --git a/src/backend/common/SparseArray.hpp b/src/backend/common/SparseArray.hpp
index 860f7814ac..046a92fbe7 100644
--- a/src/backend/common/SparseArray.hpp
+++ b/src/backend/common/SparseArray.hpp
@@ -248,5 +248,12 @@ class SparseArray {
     friend void destroySparseArray<T>(SparseArray<T> *sparse);
 };
 
+/// Checks if the Array object can be migrated to the current device and if not,
+/// an error is thrown
+///
+/// \param[in] arr The Array that will be checked.
+template<typename T>
+void checkAndMigrate(const SparseArray<T> &arr);
+
 }  // namespace common
 }  // namespace arrayfire
diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index 88f4bcabee..dc0b5d5dad 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -134,6 +134,11 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
     }
 }
 
+template<typename T>
+void checkAndMigrate(const Array<T> &arr) {
+    return;
+}
+
 template<typename T>
 void Array<T>::eval() {
     evalMultiple<T>({this});
@@ -353,7 +358,8 @@ void Array<T>::setDataDims(const dim4 &new_dims) {
         Array<T> & arr, const void *const data, const size_t bytes);          \
     template void evalMultiple<T>(vector<Array<T> *> arrays);                 \
     template kJITHeuristics passesJitHeuristics<T>(span<Node *> n);           \
-    template void Array<T>::setDataDims(const dim4 &new_dims);
+    template void Array<T>::setDataDims(const dim4 &new_dims);                \
+    template void checkAndMigrate<T>(const Array<T> &arr);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/cpu/Array.hpp b/src/backend/cpu/Array.hpp
index 3c7b54c5ec..7afed3501e 100644
--- a/src/backend/cpu/Array.hpp
+++ b/src/backend/cpu/Array.hpp
@@ -127,6 +127,13 @@ void *getRawPtr(const Array<T> &arr) {
     return (void *)(arr.get(false));
 }
 
+/// Checks if the Array object can be migrated to the current device and if not,
+/// an error is thrown
+///
+/// \param[in] arr The Array that will be checked.
+template<typename T>
+void checkAndMigrate(const Array<T> &arr);
+
 // Array Array Implementation
 template<typename T>
 class Array {
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index eb71a9f7a2..9af853cb22 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -57,6 +57,13 @@ std::shared_ptr<BufferNode<T>> bufferNodePtr() {
         static_cast<af::dtype>(dtype_traits<T>::af_type));
 }
 
+template<typename T>
+void checkAndMigrate(const Array<T> &arr) {
+    if (arr.getDevId() != detail::getActiveDeviceId()) {
+        AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE);
+    }
+}
+
 template<typename T>
 Array<T>::Array(const af::dim4 &dims)
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
@@ -468,7 +475,8 @@ void Array<T>::setDataDims(const dim4 &new_dims) {
         Array<T> & arr, const void *const data, const size_t bytes);          \
     template void evalMultiple<T>(std::vector<Array<T> *> arrays);            \
     template kJITHeuristics passesJitHeuristics<T>(span<Node *> n);           \
-    template void Array<T>::setDataDims(const dim4 &new_dims);
+    template void Array<T>::setDataDims(const dim4 &new_dims);                \
+    template void checkAndMigrate<T>(const Array<T> &arr);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp
index 7e1324d016..caf1a90357 100644
--- a/src/backend/cuda/Array.hpp
+++ b/src/backend/cuda/Array.hpp
@@ -34,6 +34,13 @@ using af::dim4;
 template<typename T>
 class Array;
 
+/// Checks if the Array object can be migrated to the current device and if not,
+/// an error is thrown
+///
+/// \param[in] arr The Array that will be checked.
+template<typename T>
+void checkAndMigrate(const Array<T> &arr);
+
 template<typename T>
 void evalNodes(Param<T> out, common::Node *node);
 
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 8ff64d78ec..f2ef09c044 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -88,6 +88,13 @@ void verifyTypeSupport<arrayfire::common::half>() {
 }
 }  // namespace
 
+template<typename T>
+void checkAndMigrate(const Array<T> &arr) {
+    if (arr.getDevId() != detail::getActiveDeviceId()) {
+        AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE);
+    }
+}
+
 template<typename T>
 Array<T>::Array(const dim4 &dims)
     : info(getActiveDeviceId(), dims, 0, calcStrides(dims),
@@ -564,7 +571,8 @@ size_t Array<T>::getAllocatedBytes() const {
     template kJITHeuristics passesJitHeuristics<T>(span<Node *> node);       \
     template void *getDevicePtr<T>(const Array<T> &arr);                     \
     template void Array<T>::setDataDims(const dim4 &new_dims);               \
-    template size_t Array<T>::getAllocatedBytes() const;
+    template size_t Array<T>::getAllocatedBytes() const;                     \
+    template void checkAndMigrate<T>(const Array<T> &arr);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/oneapi/Array.hpp b/src/backend/oneapi/Array.hpp
index e0b0962222..5e7ec490f1 100644
--- a/src/backend/oneapi/Array.hpp
+++ b/src/backend/oneapi/Array.hpp
@@ -51,6 +51,13 @@ using af::dim4;
 template<typename T>
 class Array;
 
+/// Checks if the Array object can be migrated to the current device and if not,
+/// an error is thrown
+///
+/// \param[in] arr The Array that will be checked.
+template<typename T>
+void checkAndMigrate(const Array<T> &arr);
+
 template<typename T>
 void evalMultiple(std::vector<Array<T> *> arrays);
 
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 2d3bc40e0b..c54476d38d 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -192,6 +192,13 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
     }
 }
 
+template<typename T>
+void checkAndMigrate(const Array<T> &arr) {
+    if (arr.getDevId() != detail::getActiveDeviceId()) {
+        AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE);
+    }
+}
+
 template<typename T>
 void Array<T>::eval() {
     if (isReady()) { return; }
@@ -552,7 +559,8 @@ size_t Array<T>::getAllocatedBytes() const {
     template kJITHeuristics passesJitHeuristics<T>(span<Node *> node);        \
     template void *getDevicePtr<T>(const Array<T> &arr);                      \
     template void Array<T>::setDataDims(const dim4 &new_dims);                \
-    template size_t Array<T>::getAllocatedBytes() const;
+    template size_t Array<T>::getAllocatedBytes() const;                      \
+    template void checkAndMigrate<T>(const Array<T> & arr);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/opencl/Array.hpp b/src/backend/opencl/Array.hpp
index 3a672d00f6..5bd6d422c4 100644
--- a/src/backend/opencl/Array.hpp
+++ b/src/backend/opencl/Array.hpp
@@ -41,6 +41,13 @@ using af::dim4;
 template<typename T>
 class Array;
 
+/// Checks if the Array object can be migrated to the current device and if not,
+/// an error is thrown
+///
+/// \param[in] arr The Array that will be checked.
+template<typename T>
+void checkAndMigrate(const Array<T> &arr);
+
 template<typename T>
 void evalMultiple(std::vector<Array<T> *> arrays);
 
diff --git a/test/array.cpp b/test/array.cpp
index 5962797083..4ba6452b2c 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -473,7 +473,7 @@ TEST(DeviceId, Same) {
 
 TEST(DeviceId, Different) {
     int ndevices = getDeviceCount();
-    if (ndevices < 2) return;
+    if (ndevices < 2) GTEST_SKIP() << "Skipping mult-GPU test";
     int id0 = getDevice();
     int id1 = (id0 + 1) % ndevices;
 

From bf7d1c80d55f79980cc9c9b5f6e003e17e5e8159 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 15 Jun 2023 22:00:15 -0400
Subject: [PATCH 358/473] Allow access to buffers on other devices with the
 same context

---
 src/backend/cuda/Array.cpp            |  4 ++-
 src/backend/cuda/device_manager.cpp   | 23 +++++++++++++++++
 src/backend/cuda/device_manager.hpp   |  9 +++++++
 src/backend/cuda/platform.cpp         |  6 +++++
 src/backend/cuda/platform.hpp         |  7 ++++++
 src/backend/opencl/Array.cpp          | 11 +++++---
 src/backend/opencl/device_manager.cpp | 28 +++++++++++++++------
 src/backend/opencl/device_manager.hpp |  2 ++
 src/backend/opencl/platform.cpp       | 36 ++++++++++++++++++---------
 src/backend/opencl/platform.hpp       |  7 ++++++
 test/array.cpp                        | 21 +++++++++++++++-
 11 files changed, 129 insertions(+), 25 deletions(-)

diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index 9af853cb22..12a66f1293 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -59,7 +59,9 @@ std::shared_ptr<BufferNode<T>> bufferNodePtr() {
 
 template<typename T>
 void checkAndMigrate(const Array<T> &arr) {
-    if (arr.getDevId() != detail::getActiveDeviceId()) {
+    int arr_id = arr.getDevId();
+    int cur_id = detail::getActiveDeviceId();
+    if (!isDeviceBufferAccessible(arr_id, cur_id)) {
         AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE);
     }
 }
diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index c60bf35437..9e7cc2d68b 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -613,6 +613,29 @@ DeviceManager::DeviceManager()
 
     sortDevices();
 
+    // Set all default peer access to false
+    for (auto &dev_map : device_peer_access_map)
+        for (auto &dev_access : dev_map) { dev_access = false; }
+
+    // Enable peer 2 peer access to device memory if available
+    for (int i = 0; i < nDevices; i++) {
+        for (int j = 0; j < nDevices; j++) {
+            if (i != j) {
+                int can_access_peer;
+                CUDA_CHECK(cudaDeviceCanAccessPeer(&can_access_peer, i, j));
+                if (can_access_peer) {
+                    CUDA_CHECK(cudaSetDevice(i));
+                    AF_TRACE("Peer access enabled for {}({}) and {}({})", i,
+                             cuDevices[i].prop.name, j, cuDevices[j].prop.name);
+                    CUDA_CHECK(cudaDeviceEnablePeerAccess(j, 0));
+                    device_peer_access_map[i][j] = true;
+                }
+            } else {
+                device_peer_access_map[i][j] = true;
+            }
+        }
+    }
+
     // Initialize all streams to 0.
     // Streams will be created in setActiveDevice()
     for (int i = 0; i < MAX_DEVICES; i++) {
diff --git a/src/backend/cuda/device_manager.hpp b/src/backend/cuda/device_manager.hpp
index 9275386011..ca43efaf1f 100644
--- a/src/backend/cuda/device_manager.hpp
+++ b/src/backend/cuda/device_manager.hpp
@@ -11,6 +11,7 @@
 
 #include <platform.hpp>
 
+#include <array>
 #include <memory>
 #include <mutex>
 #include <string>
@@ -95,6 +96,8 @@ class DeviceManager {
 
     friend std::pair<int, int> getComputeCapability(const int device);
 
+    friend bool isDeviceBufferAccessible(int buf_device_id, int execution_id);
+
    private:
     DeviceManager();
 
@@ -117,6 +120,12 @@ class DeviceManager {
 
     std::shared_ptr<spdlog::logger> logger;
 
+    /// A matrix of booleans where true indicates that the corresponding
+    /// corrdinate devices can access each other buffers. False indicates
+    /// buffers need to be copied over to the other device
+    std::array<std::array<bool, MAX_DEVICES>, MAX_DEVICES>
+        device_peer_access_map;
+
     std::vector<cudaDevice_t> cuDevices;
     std::vector<std::pair<int, int>> devJitComputes;
 
diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 3fab99bb7f..52a22cdbaf 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -208,6 +208,12 @@ DeviceManager::~DeviceManager() {
     }
 }
 
+bool isDeviceBufferAccessible(int buf_device_id, int execution_id) {
+    DeviceManager &mngr = DeviceManager::getInstance();
+    return buf_device_id == execution_id ||
+           mngr.device_peer_access_map[buf_device_id][execution_id];
+}
+
 int getBackend() { return AF_BACKEND_CUDA; }
 
 string getDeviceInfo(int device) noexcept {
diff --git a/src/backend/cuda/platform.hpp b/src/backend/cuda/platform.hpp
index cac1281b59..be9f0b9996 100644
--- a/src/backend/cuda/platform.hpp
+++ b/src/backend/cuda/platform.hpp
@@ -88,6 +88,13 @@ cudaStream_t getStream(int device);
 
 cudaStream_t getActiveStream();
 
+/// Returns true if the buffer on device buf_device_id can be accessed by
+/// kernels on device execution_id
+///
+/// \param[in] buf_device_id The device id of the buffer
+/// \param[in] execution_id The device where the buffer will be accessed.
+bool isDeviceBufferAccessible(int buf_device_id, int execution_id);
+
 /// Return a handle to the stream for the device.
 ///
 /// \param[in] device The device of the returned stream
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index c54476d38d..b4e66373a5 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -194,8 +194,13 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
 
 template<typename T>
 void checkAndMigrate(const Array<T> &arr) {
-    if (arr.getDevId() != detail::getActiveDeviceId()) {
-        AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE);
+    int arr_id = arr.getDevId();
+    int cur_id = detail::getActiveDeviceId();
+    if (!isDeviceBufferAccessible(arr_id, cur_id)) {
+        AF_ERROR(
+            "The array's device context does not match the current device's "
+            "context",
+            AF_ERR_DEVICE);
     }
 }
 
@@ -560,7 +565,7 @@ size_t Array<T>::getAllocatedBytes() const {
     template void *getDevicePtr<T>(const Array<T> &arr);                      \
     template void Array<T>::setDataDims(const dim4 &new_dims);                \
     template size_t Array<T>::getAllocatedBytes() const;                      \
-    template void checkAndMigrate<T>(const Array<T> & arr);
+    template void checkAndMigrate<T>(const Array<T> &arr);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index a8ca6e96c9..1e628af521 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -171,6 +171,14 @@ static inline bool compare_default(const unique_ptr<Device>& ldev,
     return l_mem > r_mem;
 }
 
+/// Class to compare two devices for sorting in a map
+class deviceLess {
+   public:
+    bool operator()(const cl::Device& lhs, const cl::Device& rhs) const {
+        return lhs() < rhs();
+    }
+};
+
 DeviceManager::DeviceManager()
     : logger(common::loggerFactory("platform"))
     , mUserDeviceOffset(0)
@@ -216,6 +224,7 @@ DeviceManager::DeviceManager()
 
     AF_TRACE("Found {} OpenCL platforms", platforms.size());
 
+    std::map<cl::Device, cl::Context, deviceLess> mDeviceContextMap;
     // Iterate through platforms, get all available devices and store them
     for (auto& platform : platforms) {
         vector<Device> current_devices;
@@ -227,11 +236,15 @@ DeviceManager::DeviceManager()
         }
         AF_TRACE("Found {} devices on platform {}", current_devices.size(),
                  platform.getInfo<CL_PLATFORM_NAME>());
-        for (auto& dev : current_devices) {
-            mDevices.emplace_back(make_unique<Device>(dev));
-            AF_TRACE("Found device {} on platform {}",
-                     dev.getInfo<CL_DEVICE_NAME>(),
-                     platform.getInfo<CL_PLATFORM_NAME>());
+        if (!current_devices.empty()) {
+            cl::Context ctx(current_devices);
+            for (auto& dev : current_devices) {
+                mDeviceContextMap[dev] = ctx;
+                mDevices.emplace_back(make_unique<Device>(dev));
+                AF_TRACE("Found device {} on platform {}",
+                         dev.getInfo<CL_DEVICE_NAME>(),
+                         platform.getInfo<CL_PLATFORM_NAME>());
+            }
         }
     }
 
@@ -250,10 +263,9 @@ DeviceManager::DeviceManager()
     for (int i = 0; i < nDevices; i++) {
         cl_platform_id device_platform =
             devices[i]->getInfo<CL_DEVICE_PLATFORM>();
-        cl_context_properties cps[3] = {
-            CL_CONTEXT_PLATFORM, (cl_context_properties)(device_platform), 0};
         try {
-            mContexts.push_back(make_unique<Context>(*devices[i], cps));
+            mContexts.emplace_back(
+                make_unique<cl::Context>(mDeviceContextMap[*devices[i]]));
             mQueues.push_back(make_unique<CommandQueue>(
                 *mContexts.back(), *devices[i], cl::QueueProperties::None));
             mIsGLSharingOn.push_back(false);
diff --git a/src/backend/opencl/device_manager.hpp b/src/backend/opencl/device_manager.hpp
index 432758bd87..69ddd80d2d 100644
--- a/src/backend/opencl/device_manager.hpp
+++ b/src/backend/opencl/device_manager.hpp
@@ -139,6 +139,8 @@ class DeviceManager {
 
     friend afcl::platform getActivePlatformVendor();
 
+    friend bool isDeviceBufferAccessible(int buf_device_id, int execution_id);
+
    public:
     static const int MAX_DEVICES = 32;
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index 165eded95f..ac07c3b818 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -278,6 +278,16 @@ afcl::platform getActivePlatformVendor() {
     return devMngr.mPlatforms[get<1>(devId)].second;
 }
 
+bool isDeviceBufferAccessible(int buf_device_id, int execution_id) {
+    DeviceManager& devMngr = DeviceManager::getInstance();
+
+    common::lock_guard_t lock(devMngr.deviceMutex);
+
+    return buf_device_id == execution_id ||
+           *devMngr.mContexts[buf_device_id] ==
+               *devMngr.mContexts[execution_id];
+}
+
 const Context& getContext() {
     device_id_t& devId = tlocalActiveDeviceId();
 
@@ -330,9 +340,9 @@ vector<Version> getOpenCLCDeviceVersion(const Device& device) {
     auto platform_version = device_platform.getInfo<CL_PLATFORM_VERSION>();
     vector<Version> out;
 
-    /// The ifdef allows us to support BUILDING ArrayFire with older versions of
-    /// OpenCL where as the if condition in the ifdef allows us to support older
-    /// versions of OpenCL at runtime
+    /// The ifdef allows us to support BUILDING ArrayFire with older
+    /// versions of OpenCL where as the if condition in the ifdef allows us
+    /// to support older versions of OpenCL at runtime
 #ifdef CL_DEVICE_OPENCL_C_ALL_VERSIONS
     if (platform_version.substr(7).c_str()[0] >= '3') {
         vector<cl_name_version> device_versions =
@@ -519,24 +529,25 @@ void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que) {
     {
         common::lock_guard_t lock(devMngr.deviceMutex);
 
-        auto tDevice  = make_unique<cl::Device>(dev, true);
-        auto tContext = make_unique<cl::Context>(ctx, true);
+        cl::Device tDevice(dev, true);
+        cl::Context tContext(ctx, true);
         auto tQueue =
-            (que == NULL ? make_unique<cl::CommandQueue>(*tContext, *tDevice)
+            (que == NULL ? make_unique<cl::CommandQueue>(tContext, tDevice)
                          : make_unique<cl::CommandQueue>(que, true));
         // FIXME: add OpenGL Interop for user provided contexts later
         devMngr.mIsGLSharingOn.push_back(false);
         devMngr.mDeviceTypes.push_back(
-            static_cast<int>(tDevice->getInfo<CL_DEVICE_TYPE>()));
+            static_cast<int>(tDevice.getInfo<CL_DEVICE_TYPE>()));
 
-        auto device_platform = tDevice->getInfo<CL_DEVICE_PLATFORM>();
+        auto device_platform = tDevice.getInfo<CL_DEVICE_PLATFORM>();
         devMngr.mPlatforms.push_back(
             std::make_pair<std::unique_ptr<cl::Platform>, afcl_platform>(
                 make_unique<cl::Platform>(device_platform, true),
-                getPlatformEnum(*tDevice)));
+                getPlatformEnum(tDevice)));
 
-        devMngr.mDevices.push_back(move(tDevice));
-        devMngr.mContexts.push_back(move(tContext));
+        devMngr.mDevices.emplace_back(make_unique<cl::Device>(move(tDevice)));
+        devMngr.mContexts.emplace_back(
+            make_unique<cl::Context>(move(tContext)));
         devMngr.mQueues.push_back(move(tQueue));
         nDevices = static_cast<int>(devMngr.mDevices.size()) - 1;
 
@@ -594,7 +605,8 @@ void removeDeviceContext(cl_device_id dev, cl_context ctx) {
         common::lock_guard_t lock(devMngr.deviceMutex);
 
         const int dCount = static_cast<int>(devMngr.mDevices.size());
-        for (int i = 0; i < dCount; ++i) {
+        for (int i = static_cast<int>(devMngr.mUserDeviceOffset); i < dCount;
+             ++i) {
             if (devMngr.mDevices[i]->operator()() == dev &&
                 devMngr.mContexts[i]->operator()() == ctx) {
                 deleteIdx = i;
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index 94ab6dff52..c14c25f399 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -187,5 +187,12 @@ afcl::platform getPlatformEnum(cl::Device dev);
 
 void setActiveContext(int device);
 
+/// Returns true if the buffer on device buf_device_id can be accessed by
+/// kernels on device execution_id
+///
+/// \param[in] buf_device_id The device id of the buffer
+/// \param[in] execution_id The device where the buffer will be accessed.
+bool isDeviceBufferAccessible(int buf_device_id, int execution_id);
+
 }  // namespace opencl
 }  // namespace arrayfire
diff --git a/test/array.cpp b/test/array.cpp
index 4ba6452b2c..bcf6fa997e 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -491,7 +491,8 @@ TEST(DeviceId, Different) {
 
         af_array c;
         af_err err = af_matmul(&c, a.get(), b.get(), AF_MAT_NONE, AF_MAT_NONE);
-        ASSERT_EQ(err, AF_ERR_DEVICE);
+        af::sync();
+        ASSERT_EQ(err, AF_SUCCESS);
     }
 
     setDevice(id1);
@@ -657,3 +658,21 @@ TEST(Array, InitializerListFixDim4) {
     af::array b{dim4(3, 3), data.data()};
     ASSERT_ARRAYS_EQ(constant(3.14, 3, 3), b);
 }
+
+TEST(Array, OtherDevice) {
+    if (af::getDeviceCount() == 1) GTEST_SKIP() << "Single device. Skipping";
+    af::setDevice(0);
+    af::info();
+    af::array a = constant(3, 5, 5);
+    a.eval();
+    af::setDevice(1);
+    af::info();
+    af::array b = constant(2, 5, 5);
+    b.eval();
+
+    af::array c = a + b;
+    af::eval(c);
+    af::sync();
+    af::setDevice(0);
+    ASSERT_ARRAYS_EQ(constant(5, 5, 5), c);
+}

From 21b5a169cc76307904da2426d452f2c45c37b8cc Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 17 Jun 2023 16:51:16 -0400
Subject: [PATCH 359/473] Update OpenCL getQueue to accept the device id

---
 src/backend/opencl/device_manager.hpp | 2 +-
 src/backend/opencl/platform.cpp       | 6 +++---
 src/backend/opencl/platform.hpp       | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/backend/opencl/device_manager.hpp b/src/backend/opencl/device_manager.hpp
index 69ddd80d2d..4b27a8f885 100644
--- a/src/backend/opencl/device_manager.hpp
+++ b/src/backend/opencl/device_manager.hpp
@@ -105,7 +105,7 @@ class DeviceManager {
 
     friend const cl::Context& getContext();
 
-    friend cl::CommandQueue& getQueue();
+    friend cl::CommandQueue& getQueue(int device_id);
 
     friend cl_command_queue getQueueHandle(int device_id);
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index ac07c3b818..eb9bc320e4 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -306,9 +306,9 @@ cl_command_queue getQueueHandle(int device_id) {
     return (*(devMngr.mQueues[device_id]))();
 }
 
-CommandQueue& getQueue() {
-    device_id_t& devId = tlocalActiveDeviceId();
-
+CommandQueue& getQueue(int device_id) {
+    device_id_t devId     = (device_id = -1) ? tlocalActiveDeviceId()
+      : make_pair<unsigned, unsigned>(device_id, device_id);
     DeviceManager& devMngr = DeviceManager::getInstance();
 
     common::lock_guard_t lock(devMngr.deviceMutex);
diff --git a/src/backend/opencl/platform.hpp b/src/backend/opencl/platform.hpp
index c14c25f399..30124d9aa2 100644
--- a/src/backend/opencl/platform.hpp
+++ b/src/backend/opencl/platform.hpp
@@ -65,7 +65,7 @@ int& getMaxJitSize();
 
 const cl::Context& getContext();
 
-cl::CommandQueue& getQueue();
+cl::CommandQueue& getQueue(int device_id = -1);
 
 /// Return a cl_command_queue handle to the queue for the device.
 ///

From 5c32bb11dcdc2325a8420764694dd01639b1825f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 17 Jun 2023 17:10:24 -0400
Subject: [PATCH 360/473] Use getInfo instead of getArray in releaseHandle to
 get device id

---
 src/api/c/handle.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/api/c/handle.cpp b/src/api/c/handle.cpp
index 243bfdba63..9c980af9f0 100644
--- a/src/api/c/handle.cpp
+++ b/src/api/c/handle.cpp
@@ -139,9 +139,9 @@ dim4 verifyDims(const unsigned ndims, const dim_t *const dims) {
 
 template<typename T>
 void releaseHandle(const af_array arr) {
-    auto &Arr      = getArray<T>(arr);
+    auto &info     = getInfo(arr);
     int old_device = detail::getActiveDeviceId();
-    int array_id   = Arr.getDevId();
+    int array_id   = info.getDevId();
     if (array_id != old_device) {
         detail::setDevice(array_id);
         detail::destroyArray(static_cast<detail::Array<T> *>(arr));

From 97ccdc08136157ef055fb4595f86d425dfdcdaca Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Sat, 17 Jun 2023 18:16:54 -0400
Subject: [PATCH 361/473] Allow access to arrays on devices from other contexts
 and non-peer devices

---
 src/api/c/handle.hpp               |  2 +-
 src/backend/common/SparseArray.cpp |  6 +++---
 src/backend/cuda/Array.cpp         | 13 ++++++++++---
 src/backend/cuda/Array.hpp         |  3 ++-
 src/backend/opencl/Array.cpp       | 22 ++++++++++++++++------
 src/backend/opencl/Array.hpp       |  3 ++-
 src/backend/opencl/platform.cpp    |  5 +++--
 7 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/src/api/c/handle.hpp b/src/api/c/handle.hpp
index add7a7c612..b2e3df97cc 100644
--- a/src/api/c/handle.hpp
+++ b/src/api/c/handle.hpp
@@ -52,7 +52,7 @@ const detail::Array<T> &getArray(const af_array &arr) {
     const detail::Array<T> *A = static_cast<const detail::Array<T> *>(arr);
     if ((af_dtype)af::dtype_traits<T>::af_type != A->getType())
         AF_ERROR("Invalid type for input array.", AF_ERR_INTERNAL);
-    checkAndMigrate(*A);
+    checkAndMigrate(*const_cast<detail::Array<T> *>(A));
     return *A;
 }
 
diff --git a/src/backend/common/SparseArray.cpp b/src/backend/common/SparseArray.cpp
index ed9680c6a5..052dc97e86 100644
--- a/src/backend/common/SparseArray.cpp
+++ b/src/backend/common/SparseArray.cpp
@@ -173,9 +173,9 @@ void destroySparseArray(SparseArray<T> *sparse) {
 
 template<typename T>
 void checkAndMigrate(const SparseArray<T> &arr) {
-    checkAndMigrate(arr.getColIdx());
-    checkAndMigrate(arr.getRowIdx());
-    checkAndMigrate(arr.getValues());
+    checkAndMigrate(const_cast<Array<int> &>(arr.getColIdx()));
+    checkAndMigrate(const_cast<Array<int> &>(arr.getRowIdx()));
+    checkAndMigrate(const_cast<Array<T> &>(arr.getValues()));
 }
 
 ////////////////////////////////////////////////////////////////////////////
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index 12a66f1293..9193f329de 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -58,11 +58,18 @@ std::shared_ptr<BufferNode<T>> bufferNodePtr() {
 }
 
 template<typename T>
-void checkAndMigrate(const Array<T> &arr) {
+void checkAndMigrate(Array<T> &arr) {
     int arr_id = arr.getDevId();
     int cur_id = detail::getActiveDeviceId();
     if (!isDeviceBufferAccessible(arr_id, cur_id)) {
-        AF_ERROR("Input Array not created on current device", AF_ERR_DEVICE);
+        static auto getLogger = [&] { return spdlog::get("platform"); };
+        AF_TRACE("Migrating array from {} to {}.", arr_id, cur_id);
+        auto migrated_data = memAlloc<T>(arr.elements());
+        CUDA_CHECK(
+            cudaMemcpyPeerAsync(migrated_data.get(), getDeviceNativeId(cur_id),
+                                arr.get(), getDeviceNativeId(arr_id),
+                                arr.elements() * sizeof(T), getActiveStream()));
+        arr.data.reset(migrated_data.release(), memFree);
     }
 }
 
@@ -478,7 +485,7 @@ void Array<T>::setDataDims(const dim4 &new_dims) {
     template void evalMultiple<T>(std::vector<Array<T> *> arrays);            \
     template kJITHeuristics passesJitHeuristics<T>(span<Node *> n);           \
     template void Array<T>::setDataDims(const dim4 &new_dims);                \
-    template void checkAndMigrate<T>(const Array<T> &arr);
+    template void checkAndMigrate<T>(Array<T> & arr);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/cuda/Array.hpp b/src/backend/cuda/Array.hpp
index caf1a90357..82e8bb9583 100644
--- a/src/backend/cuda/Array.hpp
+++ b/src/backend/cuda/Array.hpp
@@ -39,7 +39,7 @@ class Array;
 ///
 /// \param[in] arr The Array that will be checked.
 template<typename T>
-void checkAndMigrate(const Array<T> &arr);
+void checkAndMigrate(Array<T> &arr);
 
 template<typename T>
 void evalNodes(Param<T> out, common::Node *node);
@@ -305,6 +305,7 @@ class Array {
     friend void destroyArray<T>(Array<T> *arr);
     friend void *getDevicePtr<T>(const Array<T> &arr);
     friend void *getRawPtr<T>(const Array<T> &arr);
+    friend void checkAndMigrate<T>(Array<T> &arr);
 };
 
 }  // namespace cuda
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index b4e66373a5..21dec5166c 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -193,14 +193,24 @@ Array<T>::Array(const dim4 &dims, const dim4 &strides, dim_t offset_,
 }
 
 template<typename T>
-void checkAndMigrate(const Array<T> &arr) {
+void checkAndMigrate(Array<T> &arr) {
     int arr_id = arr.getDevId();
     int cur_id = detail::getActiveDeviceId();
     if (!isDeviceBufferAccessible(arr_id, cur_id)) {
-        AF_ERROR(
-            "The array's device context does not match the current device's "
-            "context",
-            AF_ERR_DEVICE);
+        auto getLogger = [&] { return spdlog::get("platform"); };
+        AF_TRACE("Migrating array from {} to {}.", arr_id, cur_id);
+        auto migrated_data           = memAlloc<T>(arr.elements());
+        void *mapped_migrated_buffer = getQueue().enqueueMapBuffer(
+            *migrated_data, CL_TRUE, CL_MAP_READ, 0, arr.elements());
+        setDevice(arr_id);
+        Buffer &buf = *arr.get();
+        getQueue().enqueueReadBuffer(buf, CL_TRUE, 0, arr.elements(),
+                                     mapped_migrated_buffer);
+        setDevice(cur_id);
+        getQueue().enqueueUnmapMemObject(*migrated_data,
+                                         mapped_migrated_buffer);
+        arr.data.reset(migrated_data.release(), bufferFree);
+        arr.setId(cur_id);
     }
 }
 
@@ -565,7 +575,7 @@ size_t Array<T>::getAllocatedBytes() const {
     template void *getDevicePtr<T>(const Array<T> &arr);                      \
     template void Array<T>::setDataDims(const dim4 &new_dims);                \
     template size_t Array<T>::getAllocatedBytes() const;                      \
-    template void checkAndMigrate<T>(const Array<T> &arr);
+    template void checkAndMigrate<T>(Array<T> & arr);
 
 INSTANTIATE(float)
 INSTANTIATE(double)
diff --git a/src/backend/opencl/Array.hpp b/src/backend/opencl/Array.hpp
index 5bd6d422c4..05b0468333 100644
--- a/src/backend/opencl/Array.hpp
+++ b/src/backend/opencl/Array.hpp
@@ -46,7 +46,7 @@ class Array;
 ///
 /// \param[in] arr The Array that will be checked.
 template<typename T>
-void checkAndMigrate(const Array<T> &arr);
+void checkAndMigrate(Array<T> &arr);
 
 template<typename T>
 void evalMultiple(std::vector<Array<T> *> arrays);
@@ -330,6 +330,7 @@ class Array {
     friend void destroyArray<T>(Array<T> *arr);
     friend void *getDevicePtr<T>(const Array<T> &arr);
     friend void *getRawPtr<T>(const Array<T> &arr);
+    friend void checkAndMigrate<T>(Array<T> &arr);
 };
 
 }  // namespace opencl
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index eb9bc320e4..d6406a32e1 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -307,8 +307,9 @@ cl_command_queue getQueueHandle(int device_id) {
 }
 
 CommandQueue& getQueue(int device_id) {
-    device_id_t devId     = (device_id = -1) ? tlocalActiveDeviceId()
-      : make_pair<unsigned, unsigned>(device_id, device_id);
+    device_id_t devId =
+        (device_id = -1) ? tlocalActiveDeviceId()
+                         : make_pair<unsigned, unsigned>(device_id, device_id);
     DeviceManager& devMngr = DeviceManager::getInstance();
 
     common::lock_guard_t lock(devMngr.deviceMutex);

From e43ecf8bf68c8907be718d362e9d8d4d3423435c Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 17 Aug 2023 13:34:41 -0400
Subject: [PATCH 362/473] Update minimum cmake version for examples to 3.5 to
 avoid warnings

---
 examples/CMakeLists.txt                  | 2 +-
 examples/benchmarks/CMakeLists.txt       | 2 +-
 examples/computer_vision/CMakeLists.txt  | 2 +-
 examples/financial/CMakeLists.txt        | 2 +-
 examples/getting_started/CMakeLists.txt  | 2 +-
 examples/graphics/CMakeLists.txt         | 2 +-
 examples/helloworld/CMakeLists.txt       | 2 +-
 examples/image_processing/CMakeLists.txt | 2 +-
 examples/lin_algebra/CMakeLists.txt      | 2 +-
 examples/machine_learning/CMakeLists.txt | 2 +-
 examples/pde/CMakeLists.txt              | 2 +-
 examples/unified/CMakeLists.txt          | 2 +-
 12 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index f69eff6e1f..91280e485e 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 cmake_policy(VERSION 3.5)
 project(ArrayFire-Examples
   VERSION 3.7.0
diff --git a/examples/benchmarks/CMakeLists.txt b/examples/benchmarks/CMakeLists.txt
index d5ece4b562..9cf8197317 100644
--- a/examples/benchmarks/CMakeLists.txt
+++ b/examples/benchmarks/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Benchmarks
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/computer_vision/CMakeLists.txt b/examples/computer_vision/CMakeLists.txt
index 7314d29148..7113816566 100644
--- a/examples/computer_vision/CMakeLists.txt
+++ b/examples/computer_vision/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Computer-Vision
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/financial/CMakeLists.txt b/examples/financial/CMakeLists.txt
index 9cc2435b25..f2b82d4de8 100644
--- a/examples/financial/CMakeLists.txt
+++ b/examples/financial/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Financial
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/getting_started/CMakeLists.txt b/examples/getting_started/CMakeLists.txt
index f0ee51249a..790afd3d1f 100644
--- a/examples/getting_started/CMakeLists.txt
+++ b/examples/getting_started/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Getting-Started
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/graphics/CMakeLists.txt b/examples/graphics/CMakeLists.txt
index d59a506278..dd2918b641 100644
--- a/examples/graphics/CMakeLists.txt
+++ b/examples/graphics/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Graphics
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/helloworld/CMakeLists.txt b/examples/helloworld/CMakeLists.txt
index 3567873958..0aa58ca2c9 100644
--- a/examples/helloworld/CMakeLists.txt
+++ b/examples/helloworld/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-HelloWorld
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/image_processing/CMakeLists.txt b/examples/image_processing/CMakeLists.txt
index 12307b679f..cfcd109922 100644
--- a/examples/image_processing/CMakeLists.txt
+++ b/examples/image_processing/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Image-Processing
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/lin_algebra/CMakeLists.txt b/examples/lin_algebra/CMakeLists.txt
index baba1a4181..b08aceeeee 100644
--- a/examples/lin_algebra/CMakeLists.txt
+++ b/examples/lin_algebra/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Linear-Algebra
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/machine_learning/CMakeLists.txt b/examples/machine_learning/CMakeLists.txt
index 9c2c3ade6c..d1cbcc9541 100644
--- a/examples/machine_learning/CMakeLists.txt
+++ b/examples/machine_learning/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Linear-Algebra
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/pde/CMakeLists.txt b/examples/pde/CMakeLists.txt
index 0b74e6165f..23a89ace31 100644
--- a/examples/pde/CMakeLists.txt
+++ b/examples/pde/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-PDE
   VERSION 3.5.0
   LANGUAGES CXX)
diff --git a/examples/unified/CMakeLists.txt b/examples/unified/CMakeLists.txt
index 330a9c4af7..42ab6432f0 100644
--- a/examples/unified/CMakeLists.txt
+++ b/examples/unified/CMakeLists.txt
@@ -5,7 +5,7 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-cmake_minimum_required(VERSION 3.0)
+cmake_minimum_required(VERSION 3.5)
 project(ArrayFire-Example-Unified
   VERSION 3.5.0
   LANGUAGES CXX)

From 0ea179f9ee7a03fc550a57680d5252c82d5272b9 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 17 Aug 2023 13:35:27 -0400
Subject: [PATCH 363/473] Update CMakeSYCLInformation linker flags for
 executables

---
 CMakeModules/CMakeSYCLInformation.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeModules/CMakeSYCLInformation.cmake b/CMakeModules/CMakeSYCLInformation.cmake
index 5e9714327a..df850959f1 100644
--- a/CMakeModules/CMakeSYCLInformation.cmake
+++ b/CMakeModules/CMakeSYCLInformation.cmake
@@ -350,7 +350,7 @@ endif()
 
 if(NOT CMAKE_SYCL_LINK_EXECUTABLE)
   set(CMAKE_SYCL_LINK_EXECUTABLE
-    "<CMAKE_SYCL_COMPILER> <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
+    "<CMAKE_SYCL_COMPILER> <FLAGS> <CMAKE_SYCL_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
 endif()
 
 

From 09fab2fdbae3c23226816e47c405bb0a3e1fae43 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 17 Aug 2023 13:36:04 -0400
Subject: [PATCH 364/473] Separate system includes in cuda_unified and
 cuda_cuda tests

---
 test/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 0cb3cbfe51..5f606e14f8 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -379,8 +379,9 @@ if(CUDA_FOUND)
       add_executable(${target} cuda.cu)
       target_include_directories(${target}
         PRIVATE
-          ${CMAKE_SOURCE_DIR}
-          ${CMAKE_CURRENT_SOURCE_DIR}
+        ${CMAKE_SOURCE_DIR}
+        ${CMAKE_CURRENT_SOURCE_DIR})
+      target_include_directories(${target}
         SYSTEM PRIVATE
           ${ArrayFire_SOURCE_DIR}/extern/half/include)
       if(${backend} STREQUAL "unified")

From 541687a0276a2019dd0be4cfdb339be514f28758 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Thu, 17 Aug 2023 13:36:28 -0400
Subject: [PATCH 365/473] Add guard around GTest::gtest alias

---
 test/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 5f606e14f8..cf7e66255f 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -40,7 +40,9 @@ elseif(NOT TARGET GTest::gtest)
   target_compile_options(gtest
     PRIVATE
       $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>)
-  add_library(GTest::gtest ALIAS gtest)
+  if(NOT TARGET GTest::gtest)
+    add_library(GTest::gtest ALIAS gtest)
+  endif()
   # Hide gtest project variables
   mark_as_advanced(
     BUILD_SHARED_LIBS

From 30d9f0ba86e690f74e7126df2ed714d1493947f5 Mon Sep 17 00:00:00 2001
From: John Melonakos <john.melonakos@intel.com>
Date: Thu, 17 Aug 2023 20:14:55 -0700
Subject: [PATCH 366/473] updates arith, data, lapack, blas documentation
 (#3485)

* updates algorithm, arith, data, lapack, blas documentation

---------

Co-authored-by: syurkevi <stefan@arrayfire.com>
---
 docs/details/algorithm.dox |  367 +++++-----
 docs/details/arith.dox     |   33 +-
 docs/details/blas.dox      |   40 +-
 docs/details/data.dox      |  100 +--
 docs/details/lapack.dox    |  229 +++---
 docs/details/random.dox    |    8 +-
 include/af/algorithm.h     | 1352 +++++++++++++++++++-----------------
 include/af/arith.h         |  828 ++++++++++++----------
 include/af/blas.h          |  276 ++++----
 include/af/data.h          | 1253 +++++++++++++++++----------------
 include/af/lapack.h        |  491 +++++++------
 include/af/random.h        |  345 ++++-----
 12 files changed, 2878 insertions(+), 2444 deletions(-)

diff --git a/docs/details/algorithm.dox b/docs/details/algorithm.dox
index 38b3c26d5a..055750098c 100644
--- a/docs/details/algorithm.dox
+++ b/docs/details/algorithm.dox
@@ -1,74 +1,76 @@
 /*!
 \page batch_detail_algo algorithm
-
-This function performs the operation across all batches present in the input simultaneously.
-
+This function runs across all batches in the input simultaneously.
 */
 
 
+
 /**
 \addtogroup arrayfire_func
 @{
-\defgroup reduce_func_sum sum
 
+
+
+\defgroup reduce_func_sum sum
 \ingroup reduce_mat
 
-Find the sum of values in the input
+Sum array elements over a given dimension.
 
-This table defines the return value types for the corresponding input types
+This table defines output types for corresponding input types:
 
 Input Type          | Output Type
 --------------------|---------------------
 f32, f64, c32, c64  | same as input
-s32, u32, s64, u64  | same as input
+s32, s64, u32, u64  | same as input
 s16                 | s32
 u16, u8, b8         | u32
 
 \copydoc batch_detail_algo
 
-\defgroup reduce_func_sum_by_key sumByKey
 
+
+\defgroup reduce_func_sum_by_key sumByKey
 \ingroup reduce_mat
 
-Finds the sum of an input array according to an array of keys.
+Sum array elements over a given dimension, according to an array of keys.
+
 The values corresponding to each group of consecutive equal keys will be summed
-together.  Keys can repeat, however only consecutive key values will be
+together. Keys can repeat; however, only consecutive key values will be
 considered for each reduction. If a key value is repeated somewhere else in the
-keys array it will be considered the start of a new reduction.  There are two
+keys array it will be considered the start of a new reduction. There are two
 outputs: the reduced set of consecutive keys and the corresponding final
-reduced values.  An example demonstrating the reduction behavior can be seen in
-the following snippet.
+set of reduced values.
+
+An example demonstrating the reduction behavior can be seen in the following
+snippet.
 
 \snippet test/reduce.cpp ex_reduce_sum_by_key
 
-The keys input type must be an integer type(s32 or u32).
-This table defines the return types for the corresponding values type
+The keys' input type must be integer (s32 or u32).
+
+This table defines output types for corresponding input types:
 
 Input Type          | Output Type
 --------------------|---------------------
 f32, f64, c32, c64  | same as input
-s32, u32, s64, u64  | same as input
+s32, s64, u32, u64  | same as input
 s16                 | s32
 u16, u8, b8         | u32
 f16                 | f32
 
-The input keys must be a 1-D vector matching the size of the reduced dimension.
-In the case of multiple dimensions in the input values array, the dim parameter
-specifies which dimension to reduce along.  An example of multi-dimensional
-reduce by key can be seen below:
+The keys array must be 1-dimensional matching the size of the reduced
+dimension. An example of multi-dimensional reduce-by-key can be seen below:
 
 \snippet test/reduce.cpp ex_reduce_sum_by_key_dim
 
 
 
-
 \defgroup reduce_func_product product
-
 \ingroup reduce_mat
 
-Find the product of values in the input
+Multiply array elements over a given dimension.
 
-This table defines the return value types for the corresponding input types
+This table defines output types for corresponding input types:
 
 Input Type          | Output Type
 --------------------|---------------------
@@ -79,23 +81,28 @@ u16, u8, b8         | u32
 
 \copydoc batch_detail_algo
 
-\defgroup reduce_func_product_by_key productByKey
 
+
+\defgroup reduce_func_product_by_key productByKey
 \ingroup reduce_mat
 
-Finds the product of an input array according to an array of keys.
+Multiply array elements over a given dimension, according to an array of keys.
+
 The values corresponding to each group of consecutive equal keys will be
-multiplied together.  Keys can repeat, however only consecutive key values will
+multiplied together. Keys can repeat; however, only consecutive key values will
 be considered for each reduction. If a key value is repeated somewhere else in
-the keys array it will be considered the start of a new reduction.  There are
+the keys array it will be considered the start of a new reduction. There are
 two outputs: the reduced set of consecutive keys and the corresponding final
-reduced values.  An example demonstrating the reduction behavior can be seen in
-the following snippet.
+set of reduced values.
+
+An example demonstrating the reduction behavior can be seen in the following
+snippet.
 
 \snippet test/reduce.cpp ex_reduce_product_by_key
 
-The keys input type must be an integer type(s32 or u32).
-This table defines the return types for the corresponding values type
+The keys' input type must be integer (s32 or u32).
+
+This table defines output types for corresponding input types:
 
 Input Type          | Output Type
 --------------------|---------------------
@@ -105,208 +112,210 @@ s16                 | s32
 u16, u8, b8         | u32
 f16                 | f32
 
-The input keys must be a 1-D vector matching the size of the reduced dimension.
-In the case of multiple dimensions in the input values array, the dim parameter
-specifies which dimension to reduce along.  An example of multi-dimensional
-reduce by key can be seen below:
+The keys array must be 1-dimenstional matching the size of the reduced
+dimension. An example of multi-dimensional reduce-by-key can be seen below:
 
 \snippet test/reduce.cpp ex_reduce_product_by_key_dim
 
 
 
-
 \defgroup reduce_func_min min
-
 \ingroup reduce_mat
 
-Find the minimum values and their locations
+Return the minimum along a given dimension.
 
 \copydoc batch_detail_algo
 
-\defgroup reduce_func_min_by_key minByKey
 
+
+\defgroup reduce_func_min_by_key minByKey
 \ingroup reduce_mat
 
-Finds the min of an input array according to an array of keys.  The minimum
-will be found of all values corresponding to each group of consecutive equal
-keys.  Keys can repeat, however only consecutive key values will be considered
-for each reduction. If a key value is repeated somewhere else in the keys array
-it will be considered the start of a new reduction.  There are two outputs:
-the reduced set of consecutive keys and the corresponding final reduced
-values.  An example demonstrating the reduction behavior can be seen in the
-following snippet.
+Return the minimum along a given dimension, according to an array of keys.
+
+The minimum is returned from the values corresponding to each group of 
+consecutive equal keys. Keys can repeat; however, only consecutive key values
+will be considered for each reduction. If a key value is repeated somewhere
+else in the keys array it will be considered the start of a new reduction.
+There are two outputs: the reduced set of consecutive keys and the
+corresponding final set of reduced values.
+
+An example demonstrating the reduction behavior can be seen in the following
+snippet.
 
 \snippet test/reduce.cpp ex_reduce_min_by_key
 
-The keys input type must be an integer type(s32 or u32).
-The values return type will be the same as the values input type.
+The keys' input type must be integer (s32 or u32).
 
-The input keys must be a 1-D vector matching the size of the reduced dimension.
-In the case of multiple dimensions in the input values array, the dim parameter
-specifies which dimension to reduce along.  An example of multi-dimensional
-reduce by key can be seen below:
+The output type is the same as input type.
+
+The keys array must be 1-dimenstional matching the size of the reduced
+dimension. An example of multi-dimensional reduce-by-key can be seen below:
 
 \snippet test/reduce.cpp ex_reduce_min_by_key_dim
 
 
-\defgroup reduce_func_max max
 
+\defgroup reduce_func_max max
 \ingroup reduce_mat
 
-Find the maximum values and their locations
+Return the maximum along a given dimension.
 
 \copydoc batch_detail_algo
 
 
-\defgroup reduce_func_max_by_key maxByKey
 
+\defgroup reduce_func_max_by_key maxByKey
 \ingroup reduce_mat
 
-Finds the max of an input array according to an array of keys.  The maximum
-will be found of all values corresponding to each group of consecutive equal
-keys.  Keys can repeat, however only consecutive key values will be considered
-for each reduction. If a key value is repeated somewhere else in the keys array
-it will be considered the start of a new reduction.  There are two outputs:
-the reduced set of consecutive keys and the corresponding final reduced
-values.  An example demonstrating the reduction behavior can be seen in the
-following snippet.
+Return the maximum along a given dimension, according to an array of keys.
+
+The maximum is returned from the values corresponding to each group of 
+consecutive equal keys. Keys can repeat; however, only consecutive key values
+will be considered for each reduction. If a key value is repeated somewhere
+else in the keys array it will be considered the start of a new reduction.
+There are two outputs: the reduced set of consecutive keys and the
+corresponding final set of reduced values.
+
+An example demonstrating the reduction behavior can be seen in the following
+snippet.
 
 \snippet test/reduce.cpp ex_reduce_max_by_key
 
-The keys input type must be an integer type(s32 or u32).
-The values return type will be the same as the values input type.
+The keys' input type must be integer (s32 or u32).
+
+The output type is the same as input type.
 
-The input keys must be a 1-D vector matching the size of the reduced dimension.
-In the case of multiple dimensions in the input values array, the dim parameter
-specifies which dimension to reduce along.  An example of multi-dimensional
-reduce by key can be seen below:
+The keys array must be 1-dimenstional matching the size of the reduced
+dimension. An example of multi-dimensional reduce-by-key can be seen below:
 
 \snippet test/reduce.cpp ex_reduce_max_by_key_dim
 
 
 
 \defgroup reduce_func_all_true allTrue
-\brief Test if all values in an array are true
-
 \ingroup reduce_mat
 
-Find if of all of the values in input are true
+Check if all values along a given dimension are true.
 
-Return type is b8 for all input types
+Return type is `b8` for all input types.
 
 \copydoc batch_detail_algo
 
-\defgroup reduce_func_all_true_by_key allTrueByKey
-\brief Calculate if all values that share the same consecutive keys are true
 
+
+\defgroup reduce_func_all_true_by_key allTrueByKey
 \ingroup reduce_mat
 
-Finds if all of the values of an input array are true according to an array of
-keys.  All values corresponding to each group of consecutive equal keys will be
-tested to make sure all are true.  Keys can repeat, however only consecutive
-key values will be considered for each reduction. If a key value is repeated
+Check if all values along a given dimension are true, according to an array of
+keys.
+
+All values corresponding to each group of consecutive equal keys will be tested
+to make sure all are true. Keys can repeat; however, only consecutive key
+values will be considered for each reduction. If a key value is repeated
 somewhere else in the keys array it will be considered the start of a new
-reduction.  There are two outputs: the reduced set of consecutive keys and the
-corresponding final reduced values.  An example demonstrating the reduction
-behavior can be seen in the following snippet.
+reduction. There are two outputs: the reduced set of consecutive keys and the
+corresponding final set of reduced values.
+
+An example demonstrating the reduction behavior can be seen in the following
+snippet.
 
 \snippet test/reduce.cpp ex_reduce_alltrue_by_key
 
-The keys input type must be an integer type(s32 or u32).
-The values return type will be of type b8.
+The keys' input type must be integer (s32 or u32).
 
-The input keys must be a 1-D vector matching the size of the reduced dimension.
-In the case of multiple dimensions in the input values array, the dim parameter
-specifies which dimension to reduce along.  An example of multi-dimensional
-reduce by key can be seen below:
+The output type is `b8`.
 
-\snippet test/reduce.cpp ex_reduce_alltrue_by_key_dim
+The keys array must be 1-dimenstional matching the size of the reduced
+dimension. An example of multi-dimensional reduce-by-key can be seen below:
 
+\snippet test/reduce.cpp ex_reduce_alltrue_by_key_dim
 
 
 
 \defgroup reduce_func_any_true anytrue
-\brief Calculate if any values in an array are true
-
 \ingroup reduce_mat
 
-Find if of any of the values in input are true
+Check if any values along a given dimension are true.
 
-Return type is b8 for all input types
+The output type is `b8`.
 
 \copydoc batch_detail_algo
 
-\defgroup reduce_func_anytrue_by_key anyTrueByKey
-\brief Calculate if any values that share the same consecutive keys are true
 
+
+\defgroup reduce_func_anytrue_by_key anyTrueByKey
 \ingroup reduce_mat
 
-Finds if any of the values of an input array are true according to an array of
-keys.  All values corresponding to each group of consecutive equal keys will be
-tested to make sure any are true.  Keys can repeat, however only consecutive
-key values will be considered for each reduction. If a key value is repeated
+Check if any values along a given dimension are true, according to an array of
+keys.
+
+Values corresponding to each group of consecutive equal keys will be tested to
+check if any are true. Keys can repeat; however, only consecutive key
+values will be considered for each reduction. If a key value is repeated
 somewhere else in the keys array it will be considered the start of a new
-reduction.  There are two outputs: the reduced set of consecutive keys and the
-corresponding final reduced values.  An example demonstrating the reduction
-behavior can be seen in the following snippet.
+reduction. There are two outputs: the reduced set of consecutive keys and the
+corresponding final set of reduced values.
+
+An example demonstrating the reduction behavior can be seen in the following
+snippet.
 
 \snippet test/reduce.cpp ex_reduce_anytrue_by_key
 
-The keys input type must be an integer type(s32 or u32).
-The values return type will be of type u8.
+The keys' input type must be integer (s32 or u32).
 
-The input keys must be a 1-D vector matching the size of the reduced dimension.
-In the case of multiple dimensions in the input values array, the dim parameter
-specifies which dimension to reduce along.  An example of multi-dimensional
-reduce by key can be seen below:
+The output type is `b8`.
+
+The keys array must be 1-dimenstional matching the size of the reduced
+dimension. An example of multi-dimensional reduce-by-key can be seen below:
 
 \snippet test/reduce.cpp ex_reduce_anytrue_by_key_dim
 
 
-\defgroup reduce_func_count count
 
+\defgroup reduce_func_count count
 \ingroup reduce_mat
 
-Count the number of non-zero elements in the input
+Count non-zero values in an array along a given dimension.
 
-Return type is u32 for all input types
+The output type is `u32`.
 
 \copydoc batch_detail_algo
 
-\defgroup reduce_func_count_by_key countByKey
 
+
+\defgroup reduce_func_count_by_key countByKey
 \ingroup reduce_mat
 
-Counts the non-zero values of an input array according to an array of keys.
+Count non-zero values in an array, according to an array of keys.
+
 All non-zero values corresponding to each group of consecutive equal keys will
-be counted.  Keys can repeat, however only consecutive key values will be
+be counted. Keys can repeat; however, only consecutive key values will be
 considered for each reduction. If a key value is repeated somewhere else in the
-keys array it will be considered the start of a new reduction.  There are two
-outputs: the reduced set of consecutive keys and the corresponding final
-reduced values.  An example demonstrating the reduction behavior can be seen in
-the following snippet.
+keys array it will be considered the start of a new reduction. There are two
+outputs: the reduced set of consecutive keys and the corresponding final set of
+reduced values.
+
+An example demonstrating the reduction behavior can be seen in the following
+snippet.
 
 \snippet test/reduce.cpp ex_reduce_count_by_key
 
-The keys input type must be an integer type(s32 or u32).
-The values return type will be of type u32.
+The keys' input type must be integer (s32 or u32).
 
-The input keys must be a 1-D vector matching the size of the reduced dimension.
-In the case of multiple dimensions in the input values array, the dim parameter
-specifies which dimension to reduce along.  An example of multi-dimensional
-reduce by key can be seen below:
+The output type is `u32`.
 
-\snippet test/reduce.cpp ex_reduce_count_by_key_dim
+The keys array must be 1-dimenstional matching the size of the reduced
+dimension. An example of multi-dimensional reduce-by-key can be seen below:
 
+\snippet test/reduce.cpp ex_reduce_count_by_key_dim
 
 
 
 \defgroup scan_func_accum accum
-\brief Cumulative sum (inclusive). Also known as a scan
-
 \ingroup scan_mat
 
-Calculate the cumulative sum (inclusive) along the specified dimension
+Evaluate the cumulative sum (inclusive) along a given dimension.
 
 For a 1D array \f$X\f$, the inclusive cumulative sum calculates \f$x_i =
 \sum_{p=0}^{i}x_p\f$ for every \f$x \in X\f$. Here is a simple example for the
@@ -314,7 +323,7 @@ For a 1D array \f$X\f$, the inclusive cumulative sum calculates \f$x_i =
 
 \snippet test/scan.cpp ex_accum_1D
 
-For 2D arrays (and higher dimensions), you can specify the dimension along which
+For 2D arrays and higher dimensions, you can specify the dimension along which
 the cumulative sum will be calculated. Thus, the formula above will be
 calculated for all array slices along the specified dimension (in the 2D case
 for example, this looks like \f$x_{i,j} = \sum_{p=0}^{j}x_{i,p}\f$ if the second
@@ -325,12 +334,12 @@ required to be specified in the C API):
 \snippet test/scan.cpp ex_accum_2D
 
 The output array type may be different from the input array type. The following
-table defines the corresponding output types for each input type:
+table defines corresponding output types for each input type:
 
 Input Type          | Output Type
 --------------------|---------------------
 f32, f64, c32, c64  | same as input
-s32, u32, s64, u64  | same as input
+s32, s64, u32, u64  | same as input
 s16                 | s32
 u16, u8, b8         | u32
 
@@ -338,151 +347,147 @@ u16, u8, b8         | u32
 
 
 
-\defgroup scan_func_where where
-
+\defgroup scan_func_scan scan
 \ingroup scan_mat
 
-Locate the indices of non-zero elements
-
-Return type is u32 for all input types
+Scan an array (generalized) over a given dimension.
 
-The locations are provided by flattening the input into a linear array.
+Perform inclusive or exclusive scan using a given binary operation along a
+given dimension.
 
+Binary operations can be [add](\ref AF_BINARY_ADD), [mul](\ref AF_BINARY_MUL),
+[min](\ref AF_BINARY_MIN), [max](\ref AF_BINARY_MAX) as defined by \ref
+af_binary_op.
 
 
-\defgroup scan_func_scan scan
 
+\defgroup scan_func_scanbykey scanByKey
 \ingroup scan_mat
 
-Inclusive or exclusive scan of an array
+Scan an array (generalized) over a given dimension, according to an array of
+keys.
 
 Perform inclusive or exclusive scan using a given binary operation along a
-given dimension.
+given dimension using a key.
 
 Binary operations can be [add](\ref AF_BINARY_ADD), [mul](\ref AF_BINARY_MUL),
-[min](\ref AF_BINARY_MIN), [max](\ref AF_BINARY_MAX) as defined by \ref af_binary_op.
-
+[min](\ref AF_BINARY_MIN), [max](\ref AF_BINARY_MAX) as defined by \ref
+af_binary_op.
 
 
-\defgroup scan_func_scanbykey scanByKey
 
+\defgroup scan_func_where where
 \ingroup scan_mat
 
-Inclusive or exclusive scan of an array by key
+Locate the indices of the non-zero values in an array.
 
-Perform inclusive or exclusive scan using a given binary operation along a
-given dimension using a key.
+Output type is `u32`.
 
-Binary operations can be [add](\ref AF_BINARY_ADD), [mul](\ref AF_BINARY_MUL),
-[min](\ref AF_BINARY_MIN), [max](\ref AF_BINARY_MAX) as defined by \ref af_binary_op.
+The locations are provided by flattening the input into a linear array.
 
 
 
 \defgroup calc_func_diff1 diff1
-
 \ingroup calc_mat
 
-First order numerical difference along specified dimension
+Calculate the first order difference in an array over a given dimension.
 
 \copydoc batch_detail_algo
 
 
 
 \defgroup calc_func_diff2 diff2
-
 \ingroup calc_mat
 
-Second order numerical difference along specified dimension
+Calculate the second order difference in an array over a given dimension.
 
 \copydoc batch_detail_algo
 
 
 
 \defgroup sort_func_sort sort
-
 \ingroup sort_mat
 
-Sort input arrays
-
-Sort an multi dimensional array
+Sort an array over a given dimension.
 
 
 
 \defgroup sort_func_sort_index sortIndex
-
 \ingroup sort_mat
 
-Sort input arrays get the sorted indices
+Sort an array over a given dimension and return the original indices.
 
-Sort a multi dimensional array and return sorted indices. Index array is of
-type u32.
+Output type is `u32`.
 
 
 
 \defgroup sort_func_sort_keys sortByKey
-
 \ingroup sort_mat
 
-Sort input arrays based on keys
-
-Sort a multi dimensional array based on keys
+Sort an array over a given dimension, according to an array of keys.
 
 
 
 \defgroup set_func_unique setunique
-
 \ingroup set_mat
 
-Finds unique values from an input set. The input must be a one-dimensional array. Batching is not currently supported.
+Return the unique values in an array.
+
+The input must be a one-dimensional array. Batching is not currently supported.
 
-A simple example of finding the unique values of a set using setUnique() can be seen below:
+An example, unsorted:
 
 \snippet test/set.cpp ex_set_unique_simple
 
 The function can be sped up if it is known that the inputs are sorted.
 
+An example, sorted (ascending):
+
 \snippet test/set.cpp ex_set_unique_sorted
 
 The inputs can be sorted in ascending or descending order.
 
-\snippet test/set.cpp ex_set_unique_desc
-
-
+An example, sorted (descending):
 
+\snippet test/set.cpp ex_set_unique_desc
 
 
 
 \defgroup set_func_union setunion
-
 \ingroup set_mat
 
-Find the union of two sets. The inputs must be one-dimensional arrays. Batching is not currently supported.
+Evaluate the union of two arrays.
+
+The inputs must be one-dimensional arrays. Batching is not currently supported.
 
-A simple example of finding the union of two sets using setUnion() can be seen below:
+An example:
 
 \snippet test/set.cpp ex_set_union_simple
 
-The function can be sped up if it is known that each input is sorted in increasing order and its values are unique.
+The function can be sped up if the input is sorted in increasing order and its
+values are unique.
 
 \snippet test/set.cpp ex_set_union
 
 
 
-
 \defgroup set_func_intersect setintersect
-
 \ingroup set_mat
 
-Find the intersection of two sets. The inputs must be one-dimensional arrays. Batching is not currently supported.
+Evaluate the intersection of two arrays.
+
+The inputs must be one-dimensional arrays. Batching is not currently supported.
 
-A simple example of finding the intersection of two sets using setIntersect() can be seen below:
+An example:
 
 \snippet test/set.cpp ex_set_intersect_simple
 
-The function can be sped up if it is known that each input is sorted in increasing order and its values are unique.
+The function can be sped up if the input is sorted in increasing order and its
+values are unique.
 
 \snippet test/set.cpp ex_set_intersect
 
 
+
 @}
 */
diff --git a/docs/details/arith.dox b/docs/details/arith.dox
index 2e123f7ba8..3a118bc890 100644
--- a/docs/details/arith.dox
+++ b/docs/details/arith.dox
@@ -1,6 +1,7 @@
 /*!
 \page arith_real_only arith_real
-\note This function supports real inputs only. Complex inputs are not yet supported.
+\note This function only supports real inputs; complex inputs are not yet
+supported.
 */
 
 /*!
@@ -19,28 +20,28 @@
 \defgroup arith_func_add add
 \ingroup arith_mat
 
-Elementwise addition
+Elementwise addition.
 
 
 
 \defgroup arith_func_sub sub
 \ingroup arith_mat
 
-Elementwise subtraction
+Elementwise subtraction.
 
 
 
 \defgroup arith_func_mul mul
 \ingroup arith_mat
 
-Elementwise multiply
+Elementwise multiply.
 
 
 
 \defgroup arith_func_div div
 \ingroup arith_mat
 
-Elementwise division
+Elementwise division.
 
 
 
@@ -67,7 +68,8 @@ Check if the elements of one array are greater than those of another array.
 
 Less than or equal to, an elementwise comparison of two arrays.
 
-Check if the elements of one array are less than or equal to those of another array.
+Check if the elements of one array are less than or equal to those of another
+array.
 
 
 \defgroup arith_func_ge ge
@@ -75,14 +77,15 @@ Check if the elements of one array are less than or equal to those of another ar
 
 Greater than or equal to, an elementwise comparison of two arrays.
 
-Check if the elements of one array are greater than or equal to those of another array.
+Check if the elements of one array are greater than or equal to those of
+another array.
 
 
 
 \defgroup arith_func_eq eq
 \ingroup logic_mat
 
-\brief Equal to, an elementwise comparison of two arrays.
+Equal to, an elementwise comparison of two arrays.
 
 Check if the elements of one array are equal to those of another array.
 
@@ -91,7 +94,7 @@ Check if the elements of one array are equal to those of another array.
 \defgroup arith_func_neq neq
 \ingroup logic_mat
 
-\brief Not equal to, an elementwise comparison of two arrays.
+Not equal to, an elementwise comparison of two arrays.
 
 Check if the elements of one array are not equal to those of another array.
 
@@ -384,10 +387,14 @@ Create complex arrays.
 
 Complex arrays are created from any of the following four inputs:
 
-1. a single real array, returning zeros for the imaginary component. See `array b` in the example.
-2. two real arrays, one for the real component and one for the imaginary component. See `array c` in the example.
-3. a single real array for the real component and a single scalar for each imaginary component. See `array d` in the example.
-4. a single scalar for each real component and a single real array for the imaginary component. See `array e` in the example.
+1. a single real array, returning zeros for the imaginary component. See
+   `array b` in the example.
+2. two real arrays, one for the real component and one for the imaginary
+   component. See `array c` in the example.
+3. a single real array for the real component and a single scalar for each
+   imaginary component. See `array d` in the example.
+4. a single scalar for each real component and a single real array for the
+   imaginary component. See `array e` in the example.
 
 __Examples:__
 
diff --git a/docs/details/blas.dox b/docs/details/blas.dox
index b8757d81fb..943e77a502 100644
--- a/docs/details/blas.dox
+++ b/docs/details/blas.dox
@@ -1,29 +1,18 @@
 /**
 \addtogroup arrayfire_func
 @{
-\defgroup blas_func_dot dot
-
-\ingroup blas_mat
-
-\brief Calculate the dot product of a vector
-
-Scalar dot product between two vectors.  Also referred to as the inner
-product.
-
-=======================================================================
 
 \defgroup blas_func_matmul matmul
-\ingroup blas_mat
 
-\brief Matrix multiplication using array
+Matrix multiplication.
 
 Performs a matrix multiplication on the two input arrays after performing the
 operations specified in the options. The operations are done while reading the
 data from memory. This results in no additional memory being used for temporary
 buffers.
 
-Batched matrix multiplications are supported. Given below are the supported
-types of batch operations for any given set of two matrices A and B.
+Batched matrix multiplications are supported. The supported types of batch
+operations for any given set of two matrices A and B are given below,
 
 | Size of Input Matrix A     | Size of Input Matrix B     | Output Matrix Size          |
 |:--------------------------:|:--------------------------:|:---------------------------:|
@@ -32,8 +21,8 @@ types of batch operations for any given set of two matrices A and B.
 | \f$ \{ M, K,  1,  1 \} \f$ | \f$ \{ K, N, b2, b3 \} \f$ |  \f$ \{ M, N, b2, b3 \} \f$ |
 | \f$ \{ M, K, b2, b3 \} \f$ | \f$ \{ K, N,  1,  1 \} \f$ |  \f$ \{ M, N, b2, b3 \} \f$ |
 
-where M, K, N are dimensions of the matrix and b2, b3 indicate batch size along the
-respective dimension.
+where `M`, `K`, `N` are dimensions of the matrix and `b2`, `b3` indicate batch
+size along the respective dimension.
 
 For the last two entries in the above table, the 2D matrix is broadcasted to
 match the dimensions of 3D/4D array. This broadcast doesn't involve any additional
@@ -43,14 +32,24 @@ memory allocations either on host or device.
 for Sparse-Dense matrix multiplication. See the notes of the function for usage
 and restrictions.
 
+\ingroup blas_mat
 
 =======================================================================
 
-\defgroup blas_func_transpose transpose
+\defgroup blas_func_dot dot
+
+Compute the dot product.
+
+Scalar dot product between two vectors, also referred to as the inner
+product.
+
 \ingroup blas_mat
-\ingroup manip_mat
 
-\brief Transpose a matrix.
+=======================================================================
+
+\defgroup blas_func_transpose transpose
+
+Transpose a matrix.
 
 Reverse or permute the dimensions of an array; returns the modified array.
 For an array a with two dimensions, `transpose(a)` gives the matrix transpose.
@@ -70,6 +69,9 @@ __Examples:__
 
 \snippet test/transpose.cpp ex_blas_func_transpose
 
+\ingroup blas_mat
+\ingroup manip_mat
+
 =======================================================================
 
 @}
diff --git a/docs/details/data.dox b/docs/details/data.dox
index 99a94f1202..bb96a4c61f 100644
--- a/docs/details/data.dox
+++ b/docs/details/data.dox
@@ -4,20 +4,9 @@
 
 \defgroup data_func_constant constant
 
-\brief Create a array from a scalar input value
+Create an array from a scalar input value.
 
-The array created has the same value at all locations
-
-\ingroup data_mat
-\ingroup arrayfire_func
-
-=======================================================================
-
-\defgroup data_func_pad pad
-
-\brief Pad an array
-
-Pad the input array using a constant or values from input along border
+Generate an array with elements set to a specified value.
 
 \ingroup data_mat
 \ingroup arrayfire_func
@@ -26,7 +15,7 @@ Pad the input array using a constant or values from input along border
 
 \defgroup data_func_identity identity
 
-\brief Create an identity array with diagonal values 1
+Generate an identity matrix.
 
 \code
 array a = identity(5, 3);
@@ -45,7 +34,8 @@ array a = identity(5, 3);
 
 \defgroup data_func_range range
 
-\brief Create an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions.
+Generate an array with `[0, n-1]` values along the a specified dimension and
+tiled across other dimensions.
 
 __Examples:__
 
@@ -58,7 +48,8 @@ __Examples:__
 
 \defgroup data_func_iota iota
 
-\brief Create an sequence [0, dims.elements() - 1] and modify to specified dimensions dims and then tile it according to tile_dims
+Generate an array with `[0, n-1]` values modified to specified dimensions and
+tiling.
 
 \code
 // Generate [0, 5x3 - 1] in dimensions 5, 3
@@ -87,7 +78,12 @@ array b = iota(dim4(5, 3), dim4(1, 2))
 =======================================================================
 
 \defgroup data_func_diag diag
-\brief Extract diagonal from a matrix when \p extract is set to true. Create a diagonal matrix from input array when \p extract is set to false
+
+Extract the diagonal from an array.
+
+If `extract` is true, an array is extracted containing diagonal of the matrix,
+while a false condition returns a diagonal matrix.
+
 
 \code
 // Extraction
@@ -140,9 +136,10 @@ array b = diag(a, -1, false);
 
 \defgroup manip_func_join join
 
-\brief Join up to 4 arrays along specified dimension.
+Join up to 4 arrays along specified dimension.
 
-Requires that all dimensions except the join dimension must be the same for all arrays.
+Requires that all dimensions except the join dimension must be the same for all
+arrays.
 
 \ingroup manip_mat
 \ingroup arrayfire_func
@@ -151,13 +148,14 @@ Requires that all dimensions except the join dimension must be the same for all
 
 \defgroup manip_func_tile tile
 
-\brief Repeat the contents of the input array along the specified dimensions
+Generate a tiled array by repeating an array's contents along a specified
+dimension.
 
 Creates copies of the input array and concatenates them with each other, such
 that the output array will have as many copies of the input array as the user
-specifies, along each dimension. In this sense, the output array is essentially
-a set of "tiles", where each copy of the input array (including the original) is
-a "tile" (hence the name of this function).
+specifies along each dimension. In this sense, the output array is a set of
+"tiles" where each copy of the input array, including the original, is
+a "tile".
 
 Given below are some examples. The input array looks like this:
 
@@ -184,7 +182,7 @@ dimension:
 
 \defgroup manip_func_reorder reorder
 
-\brief Reorder an array according to the specified dimensions.
+Reorder an array.
 
 Exchanges data of an array such that the requested change in dimension
 is satisfied. The linear ordering of data within the array is preserved.
@@ -201,7 +199,7 @@ a [2 2 3 1]
     2.0000     4.0000
 
 
-reorder(a, 1, 0, 2) [2 2 3 1]  //equivalent to a transpose
+reorder(a, 1, 0, 2) [2 2 3 1]  // equivalent to a transpose
     1.0000     2.0000
     3.0000     4.0000
 
@@ -229,9 +227,9 @@ reorder(a, 2, 0, 1) [3 2 2 1]
 
 \defgroup manip_func_shift shift
 
-\brief Circular shift slong specified dimensions
+Shift an array.
 
-Shifts the values in a circular fashion along the specified dimesion.
+Circular shift array values along a specified dimesion.
 
 \ingroup manip_mat
 \ingroup arrayfire_func
@@ -240,9 +238,10 @@ Shifts the values in a circular fashion along the specified dimesion.
 
 \defgroup manip_func_moddims moddims
 
-\brief Modify the dimensions of an array without changing the order of its elements.
+Modify the dimensions of an array without changing the order of its elements.
 
-This function only modifies array metadata and requires no computation. It is a NOOP.
+This function only modifies array metadata and requires no computation. It is a
+NOOP.
 
 __Examples:__
 
@@ -255,9 +254,9 @@ __Examples:__
 
 \defgroup manip_func_flat flat
 
-\brief Flatten the input to a single dimension
+Flatten an array.
 
-Simply returns the array as a vector. This is a noop.
+Simply returns the array as a vector. This is a NOOP.
 
 \ingroup manip_mat
 \ingroup arrayfire_func
@@ -266,9 +265,9 @@ Simply returns the array as a vector. This is a noop.
 
 \defgroup manip_func_flip flip
 
-\brief Flip the input along specified dimension
+Flip the input along a specified dimension.
 
-Mirrors the array along the specified dimensions.
+Mirrors the array along the specified dimension.
 
 \ingroup manip_mat
 \ingroup arrayfire_func
@@ -277,7 +276,7 @@ Mirrors the array along the specified dimensions.
 
 \defgroup data_func_lower lower
 
-\brief Create a lower triangular matrix from input array
+Return the lower triangular matrix from an input array.
 
 \ingroup data_mat
 \ingroup arrayfire_func
@@ -286,7 +285,7 @@ Mirrors the array along the specified dimensions.
 
 \defgroup data_func_upper upper
 
-\brief Create a upper triangular matrix from input array
+Return the upper triangular matrix from an input array.
 
 \ingroup data_mat
 \ingroup arrayfire_func
@@ -295,13 +294,12 @@ Mirrors the array along the specified dimensions.
 
 \defgroup data_func_select select
 
-\brief Selects elements from two arrays based on the values of a binary
-       conditional array.
+Select elements based on a conditional array.
 
-Creates a new array that is composed of values either from array \p a or array
-\p b, based on a third conditional array. For all non-zero elements in the
-conditional array, the output array will contain values from \p a. Otherwise the
-output will contain values from \p b.
+Creates a new array that is composed of values either from array `a` or array
+`b`, based on a third conditional array. For all non-zero elements in the
+conditional array, the output array will contain values from `a`. Otherwise the
+output will contain values from `b`.
 
 \snippet test/select.cpp ex_data_select
 
@@ -309,7 +307,7 @@ is equivalent to:
 
 \snippet test/select.cpp ex_data_select_c
 
-The conditional array must be a b8 typed array.
+The conditional array must be a \ref b8 typed array.
 
 The select function can perform batched operations based on the size of each of
 the inputs. The following table describes the input and output sizes for
@@ -330,15 +328,27 @@ supported batched configurations.
 
 \defgroup data_func_replace replace
 
-\brief Replace elements of an array based on a conditional array
+Replace elements of an array with elements of another array.
 
-- Input values are retained when corresponding elements from condition array are true.
-- Input values are replaced when corresponding elements from condition array are false.
+Input values are retained when corresponding elements from the conditional
+array are true. Input values are replaced when corresponding elements from the
+conditional array are false.
 
 \ingroup manip_mat
 \ingroup arrayfire_func
 
 =======================================================================
 
+\defgroup data_func_pad pad
+
+Pad an array.
+
+Pad the input array using a constant or values from input along the border.
+
+\ingroup data_mat
+\ingroup arrayfire_func
+
+=======================================================================
+
 @}
 */
diff --git a/docs/details/lapack.dox b/docs/details/lapack.dox
index bf977b0c0c..995d47129b 100644
--- a/docs/details/lapack.dox
+++ b/docs/details/lapack.dox
@@ -1,25 +1,47 @@
 /**
 \addtogroup arrayfire_func
 @{
-\defgroup lapack_factor_func_lu lu
+
+\defgroup lapack_factor_func_svd svd
+
+Perform singular value decomposition.
+
+This function factorizes a matrix \f$A\f$ into two unitary matrices, \f$U\f$
+and \f$V^T\f$, and a diagonal matrix \f$S\f$, such that \f$A = USV^T\f$. If
+\f$A\f$ has \f$M\f$ rows and \f$N\f$ columns (\f$M \times N\f$), then \f$U\f$
+will be \f$M \times M\f$, \f$V\f$ will be \f$N \times N\f$, and \f$S\f$ will be
+\f$M \times N\f$. However, for \f$S\f$, this function only returns the non-zero
+diagonal elements as a sorted (in descending order) 1D array.
+
+To reconstruct the original matrix \f$A\f$ from the individual factors, the
+following code snippet can be used:
+
+\snippet test/svd_dense.cpp ex_svd_reg
+
+When memory is a concern, and \f$A\f$ is dispensable, \ref af::svdInPlace() can
+be used. However, this in-place version is currently limited to input arrays
+where \f$M \geq N\f$.
 
 \ingroup lapack_factor_mat
 
-\brief Perform LU decomposition
+===============================================================================
 
-This function decomposes input matrix **A** into a lower triangle **L**, an upper triangle **U** such that
+\defgroup lapack_factor_func_lu lu
 
-    \f$A = L * U\f$
+Perform LU decomposition.
 
-For stability, a permutation array **P** is also used to modify the formula in the following manner.
+This function decomposes input matrix \f$A\f$ into a lower triangle \f$L\f$, an
+upper triangle \f$U\f$ such that \f$A = L * U\f$.
 
-    \f$A(P, span) = L * U\f$
+For stability, a permutation array \f$P\f$ is also used to modify the formula
+in the following manner, \f$A(P, span) = L * U\f$.
 
-This operation can be performed in ArrayFire using the following code snippet.
+This operation can be performed in ArrayFire, using the following code snippet.
 
 \snippet test/lu_dense.cpp ex_lu_unpacked
 
-The permuted version of the original matrix can be reconstructed using the following snippet.
+The permuted version of the original matrix can be reconstructed, using the
+following snippet.
 
 \snippet test/lu_dense.cpp ex_lu_recon
 
@@ -57,115 +79,98 @@ a_perm [3 3 1 1]
     1.0000     4.0000     7.0000
 \endcode
 
-When memory is a concern, users can perform the LU decomposition in place as shown below.
+When memory is a concern, users can perform the LU decomposition in place as
+shown below.
 
 \snippet test/lu_dense.cpp ex_lu_packed
 
-The lower and upper triangle matrices can be obtained if necessary in the following manner.
+The lower and upper triangle matrices can be obtained if necessary in the
+following manner.
 
 \snippet test/lu_dense.cpp ex_lu_extract
 
-LU decompositions has many applications including <a href="http://en.wikipedia.org/wiki/LU_decomposition#Solving_linear_equations">solving a system of linear equations</a>. Check \ref af::solveLU fore more information.
-
-=======================================================================
-
-\defgroup lapack_factor_func_qr qr
+LU decompositions have many applications including
+<a href="http://en.wikipedia.org/wiki/LU_decomposition#Solving_linear_equations">
+solving a system of linear equations</a>. Check \ref af::solveLU for more
+information.
 
 \ingroup lapack_factor_mat
 
-\brief Perform QR decomposition
-
-This function decomposes input matrix **A** into an orthogonal matrix **Q** and an upper triangular matrix **R** such that
+===============================================================================
 
-     \f$A = Q * R\f$
+\defgroup lapack_factor_func_qr qr
 
-     \f$Q * Q^T = I\f$
+Perform QR decomposition.
 
-Where **I** is an identity matrix. The matrix **Q** is a square matrix of size **max(M, N)** where **M** and **N** are rows and columns of **A** respectively. The matrix **R** is the same size as **A*.
+This function decomposes input matrix \f$A\f$ into an orthogonal matrix \f$Q\f$
+and an upper triangular matrix \f$R\f$ such that, \f$A = Q * R\f$ and
+\f$Q * Q^T = I\f$, where \f$I\f$ is an identity matrix. The matrix \f$Q\f$ is a
+square matrix of size \f$max(M, N)\f$ where \f$M\f$ and \f$N\f$ are rows and
+columns of \f$A\f$ respectively. The matrix \f$R\f$ is the same size as
+\f$A\f$.
 
 This operation can be performed in ArrayFire using the following code snippet.
 
 \snippet test/qr_dense.cpp ex_qr_unpacked
 
-The additional parameter **Tau** can be used to speed up solving over and under determined system of equations.
+The additional parameter `tau` can be used to speed up solving over- and
+under-determined systems of equations.
 
 The original matrix can be reconstructed using the following code snippet.
 
 \snippet test/qr_dense.cpp ex_qr_recon
 
-When memory is a concern, users can perform QR decomposition in place as shown below.
+When memory is a concern, users can perform QR decomposition in place as shown
+below.
 
 \snippet test/qr_dense.cpp ex_qr_packed
 
-=======================================================================
-
-\defgroup lapack_factor_func_cholesky cholesky
-
 \ingroup lapack_factor_mat
 
-\brief Perform Cholesky decomposition
+===============================================================================
 
-This function decomposes a <a href="http://en.wikipedia.org/wiki/Positive-definite_matrix">positive definite</a> matrix **A** into two triangular matrices such that
+\defgroup lapack_factor_func_cholesky cholesky
 
-     \f$A = L * U\f$
+Perform Cholesky decomposition.
 
-     \f$L = U^T\f$
+This function decomposes a
+<a href="http://en.wikipedia.org/wiki/Positive-definite_matrix">positive
+definite</a> matrix \f$A\f$ into two triangular matrices such that,
+\f$A = L * U\f$ and \f$L = U^T\f$.
 
-Only one of **L** and **U** is stored to conserve space when solving linear equations.
+Only one of \f$L\f$ and \f$U\f$ is stored to conserve space when solving linear
+equations.
 
 This operation can be performed in ArrayFire using the following code snippet.
 
 \snippet test/cholesky_dense.cpp ex_chol_reg
 
-When memory is a concern, users can perform Cholesky decomposition in place as shown below.
+When memory is a concern, users can perform Cholesky decomposition in place as
+shown below.
 
 \snippet test/cholesky_dense.cpp ex_chol_inplace
 
-=======================================================================
-
-\defgroup lapack_factor_func_svd svd
-
 \ingroup lapack_factor_mat
 
-\brief Computes the singular value decomposition of a matrix
-
-This function factorizes a matrix \f$A\f$ into two unitary matrices, \f$U\f$ and
-\f$V^T\f$, and a diagonal matrix \f$S\f$, such that \f$A = USV^T\f$. If \f$A\f$
-has \f$M\f$ rows and \f$N\f$ columns (\f$M \times N\f$), then \f$U\f$ will be
-\f$M \times M\f$, \f$V\f$ will be \f$N \times N\f$, and \f$S\f$ will be
-\f$M \times N\f$. However, for \f$S\f$, this function only returns the non-zero
-diagonal elements as a sorted (in descending order) 1D array.
-
-To reconstruct the original matrix \f$A\f$ from the individual factors, the
-following code snippet can be used:
-
-\snippet test/svd_dense.cpp ex_svd_reg
-
-When memory is a concern, and \f$A\f$ is dispensable, \ref af::svdInPlace() can be
-used. However, this in-place version is currently limited to input arrays where
-\f$M \geq N\f$.
-
-=======================================================================
+===============================================================================
 
 \defgroup lapack_solve_func_gen solve
 
-\ingroup lapack_solve_mat
-
-\brief Solve a system of equations
+Solve a system of equations.
 
-This function takes a co-efficient matrix **A** and an output matrix **B**  as inputs to solve the following equation for **X**
-
-     \f$A * X = B\f$
+This function takes a co-efficient matrix \f$A\f$ and an output matrix \f$B\f$
+as inputs to solve the following equation for \f$X\f$, \f$A * X = B\f$.
 
 This operation can be done in ArrayFire using the following code snippet.
 
 \snippet test/solve_common.hpp ex_solve
 
-The results can be verified by reconstructing the output matrix using \ref af::matmul in the following manner.
+The results can be verified by reconstructing the output matrix using \ref
+af::matmul in the following manner,
 
 \snippet test/solve_common.hpp ex_solve_recon
 
-The sample output can be seen below
+The sample output can be seen below.
 
 \code
 A [3 3 1 1]
@@ -189,52 +194,57 @@ B1 [3 1 1 1]
    39.0000
 \endcode
 
-If the coefficient matrix is known to be a triangular matrix, \ref AF_MAT_LOWER or \ref AF_MAT_UPPER can be passed to make solve faster.
+If the coefficient matrix is known to be a triangular matrix, \ref AF_MAT_LOWER
+or \ref AF_MAT_UPPER can be passed to make solve faster.
 
-The sample code snippets for solving a lower triangular matrix can be seen below.
+The sample code snippets for solving a lower triangular matrix can be seen
+below.
 
 \snippet test/solve_common.hpp ex_solve_lower
 
-Similarily, the code snippet for solving an upper triangular matrix can be seen below.
+Similarily, the code snippet for solving an upper triangular matrix can be seen
+below.
 
 \snippet test/solve_common.hpp ex_solve_upper
 
 See also: \ref af::solveLU
 
-=======================================================================
-
-\defgroup lapack_solve_lu_func_gen solveLU
-
 \ingroup lapack_solve_mat
 
-\brief Solve a system of equations
+===============================================================================
+
+\defgroup lapack_solve_lu_func_gen solveLU
 
-This function takes a co-efficient matrix **A** and an output matrix **B**  as inputs to solve the following equation for **X**
+Solve a system of equations.
 
-     \f$A * X = B\f$
+This function takes a co-efficient matrix \f$A\f$ and an output matrix \f$B\f$
+as inputs to solve the following equation for \f$X\f$, \f$A * X = B\f$.
 
 This operation can be done in ArrayFire using the following code snippet.
 
 \snippet test/solve_common.hpp ex_solve_lu
 
-This function along with \ref af::lu split up the task af::solve performs for square matrices.
+This function, along with \ref af::lu, split up the task af::solve performs for
+square matrices.
 
-\note This function is beneficial over \ref af::solve only in long running application where the coefficient matrix **A** stays the same, but the observed variables keep changing.
+This function is beneficial over \ref af::solve only in long running
+application where the coefficient matrix \f$A\f$ stays the same, but the
+observed variables keep changing.
 
+\ingroup lapack_solve_mat
 
-=======================================================================
+===============================================================================
 
 \defgroup lapack_ops_func_inv inverse
 
-\ingroup lapack_ops_mat
-
-\brief Invert a matrix
+Invert a matrix.
 
-This function inverts a square matrix **A**. The code snippet to demonstrate this can be seen below.
+This function inverts a square matrix \f$A\f$. The code snippet to demonstrate
+this can be seen below.
 
 \snippet test/inverse_dense.cpp ex_inverse
 
-The sample output can be seen below
+The sample output can be seen below.
 
 \code
 A [3 3 1 1]
@@ -254,71 +264,74 @@ I [3 3 1 1]
 
 \endcode
 
-=======================================================================
+\ingroup lapack_ops_mat
 
-\defgroup lapack_ops_func_pinv pinverse
+===============================================================================
 
-\ingroup lapack_ops_mat
+\defgroup lapack_ops_func_pinv pinverse
 
-\brief Pseudo-invert a matrix
+Pseudo-invert (Moore-Penrose) a matrix.
 
 This function calculates the Moore-Penrose pseudoinverse of a matrix \f$A\f$,
-using \ref af::svd at its core. If \f$A\f$ is of size \f$M \times N\f$, then its
-pseudoinverse \f$A^+\f$ will be of size \f$N \times M\f$.
+using \ref af::svd at its core. If \f$A\f$ is of size \f$M \times N\f$, then
+its pseudoinverse \f$A^+\f$ will be of size \f$N \times M\f$.
 
 This calculation can be batched if the input array is three or four-dimensional
 \f$(M \times N \times P \times Q\f$, with \f$Q=1\f$ for only three dimensions
-\f$)\f$. Each \f$M \times N\f$ slice along the third dimension will have its own
-pseudoinverse, for a total of \f$P \times Q\f$ pseudoinverses in the output array
-\f$(N \times M \times P \times Q)\f$.
+\f$)\f$. Each \f$M \times N\f$ slice along the third dimension will have its
+own pseudoinverse, for a total of \f$P \times Q\f$ pseudoinverses in the output
+array \f$(N \times M \times P \times Q)\f$.
 
-Here's an example snippet of its usage. In this example, we have a matrix \f$A\f$
-and we compute its pseudoinverse \f$A^+\f$. This condition must hold:
+Below is an example snippet of its usage. In this example, we have a matrix
+\f$A\f$ and compute its pseudoinverse \f$A^+\f$. This condition must hold:
 \f$AA^+A=A\f$, given that the two matrices are pseudoinverses of each other (in
 fact, this is one of the Moore-Penrose conditions):
 
 \snippet test/pinverse.cpp ex_pinverse
 
-==================================================================================
+\ingroup lapack_ops_mat
+
+===============================================================================
 
 \defgroup lapack_ops_func_rank rank
 
-\ingroup lapack_ops_mat
+Find the rank of a matrix.
 
-\brief Find the rank of the input matrix.
+This function uses \ref af::qr to find the rank of the input matrix within the
+given tolerance.
 
-This function uses \ref af::qr to find the rank of the input matrix within the given tolerance.
+\ingroup lapack_ops_mat
 
-=====================================================================================
+===============================================================================
 
 \defgroup lapack_ops_func_det det
 
-\ingroup lapack_ops_mat
+Find the determinant of a matrix.
 
-\brief Find the determinant of the input matrix.
+This function requires scratch space equal to the input array.
 
-
-\note This function requires scratch space equal to the input array
+\ingroup lapack_ops_mat
 
 ===============================================================================
 
 \defgroup lapack_ops_func_norm norm
 
-\ingroup lapack_ops_mat
+Find the norm of a matrix
 
-\brief Find the norm of the input matrix
+This function can return the norm using various metrics based on the `type`
+parameter.
 
-This function can return the norm using various metrics based on the type paramter.
+\ref AF_NORM_MATRIX_2 is currently not supported.
 
-\note \ref AF_NORM_MATRIX_2 is currently not supported.
+\ingroup lapack_ops_mat
 
 ===============================================================================
 
 \defgroup lapack_helper_func_available isLAPACKAvailable
 
-\ingroup lapack_helper
+\brief Returns true if ArrayFire is compiled with LAPACK support
 
-\brief Returns true is ArrayFire is compiled with LAPACK support
+\ingroup lapack_helper
 
 ===============================================================================
 
diff --git a/docs/details/random.dox b/docs/details/random.dox
index 63ca846106..d2400fcbbe 100644
--- a/docs/details/random.dox
+++ b/docs/details/random.dox
@@ -5,7 +5,7 @@
 
 \brief Random Number Generation Functions
 
-Functions to generate and manage random numbers and random number engines
+Functions to generate and manage random numbers and random number engines.
 
 \ingroup data_mat
 
@@ -16,7 +16,7 @@ Functions to generate and manage random numbers and random number engines
 
 \defgroup random_func_random_engine randomEngine
 
-\brief Functions to create, modify, use, and destroy randomEngine objects
+\brief Functions to create, modify, use, and destroy randomEngine objects.
 
 A \ref af::randomEngine object can be used to generate psuedo random numbers
 using various types of random number generation algorithms defined by \ref
@@ -76,7 +76,7 @@ returned by \ref af_get_default_random_engine.
 
 \defgroup random_func_set_seed setSeed
 
-\brief Set the seed for random number generation
+\brief Set the seed for random number generation.
 
 Sets the seed for the current default random engine.
 
@@ -86,7 +86,7 @@ Sets the seed for the current default random engine.
 
 \defgroup random_func_get_seed getSeed
 
-\brief Returns the seed for random number generation
+\brief Returns the seed for random number generation.
 
 Returns the seed for the current default random engine.
 
diff --git a/include/af/algorithm.h b/include/af/algorithm.h
index 801792a32a..4949d0894d 100644
--- a/include/af/algorithm.h
+++ b/include/af/algorithm.h
@@ -16,62 +16,60 @@ namespace af
     class array;
 
     /**
-       C++ Interface for sum of elements in an array
+       C++ Interface to sum array elements over a given dimension.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the add operation occurs
-       \return    result of sum all values along dimension \p dim
+       \param[in] in  input array
+       \param[in] dim dimension along which the summation occurs, -1 denotes
+                      the first non-singleton dimension
+       \return        sum
 
        \ingroup reduce_func_sum
-
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
     */
     AFAPI array sum(const array &in, const int dim = -1);
 
 #if AF_API_VERSION >= 31
     /**
-       C++ Interface for sum of elements in an array while replacing nan values
+       C++ Interface to sum array elements over a given dimension, replacing
+       any NaNs with a specified value.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the add operation occurs
-       \param[in]  nanval   The value that will replace the NaNs in \p in
-       \return    result of sum all values along dimension \p dim
+       \param[in] in     input array
+       \param[in] dim    dimension along which the summation occurs
+       \param[in] nanval value that replaces NaNs
+       \return           sum
 
        \ingroup reduce_func_sum
-
     */
     AFAPI array sum(const array &in, const int dim, const double nanval);
 #endif
 
 #if AF_API_VERSION >= 37
     /**
-       C++ Interface for sum of elements along given dimension by key
+       C++ Interface to sum array elements over a given dimension, according to
+       an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the sum of all values in \p vals along
-                            \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the add operation occurs
+       \param[out] keys_out reduced keys
+       \param[out] vals_out sum
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the summation occurs, -1
+                            denotes the first non-singleton dimension
 
        \ingroup reduce_func_sum_by_key
-
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
     */
     AFAPI void sumByKey(array &keys_out, array &vals_out,
                         const array &keys, const array &vals,
-                        const int dim=-1);
+                        const int dim = -1);
 
     /**
-       C++ Interface for sum of elements along given dimension by key while replacing nan values
+       C++ Interface to sum array elements over a given dimension, replacing
+       any NaNs with a specified value, according to an array of keys.
 
-       \param[out] keys_out Will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out Will contain the sum of all values in \p vals along
-                            \p dim according to \p keys
-       \param[in]  keys     Is the key array
-       \param[in]  vals     Is the array containing the values to be reduced
-       \param[in]  dim      The dimension along which the add operation occurs
-       \param[in]  nanval   The value that will replace the NaNs in \p vals
+       \param[out] keys_out reduced keys
+       \param[out] vals_out sum
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the summation occurs
+       \param[in]  nanval   value that replaces NaNs
 
        \ingroup reduce_func_sum_by_key
     */
@@ -81,27 +79,26 @@ namespace af
 #endif
 
     /**
-       C++ Interface for product of elements in an array
+       C++ Interface to multiply array elements over a given dimension.
 
-       \param[in] in     The input array
-       \param[in] dim    The dimension along which the multiply operation occurs
-       \return    result of product all values along dimension \p dim
+       \param[in] in  input array
+       \param[in] dim dimension along which the product occurs, -1 denotes the
+                      first non-singleton dimension
+       \return        product
 
        \ingroup reduce_func_product
-
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
     */
     AFAPI array product(const array &in, const int dim = -1);
 
 #if AF_API_VERSION >= 31
     /**
-       C++ Interface for product of elements in an array while replacing nan
-       values
+       C++ Interface to multiply array elements over a given dimension,
+       replacing any NaNs with a specified value.
 
-       \param[in] in      The input array
-       \param[in] dim     The dimension along which the multiply operation occurs
-       \param[in] nanval  The value that will replace the NaNs in \p in
-       \return    result of product all values along dimension \p dim
+       \param[in] in     input array
+       \param[in] dim    dimension along which the product occurs
+       \param[in] nanval value that replaces NaNs
+       \return           product
 
        \ingroup reduce_func_product
     */
@@ -110,35 +107,33 @@ namespace af
 
 #if AF_API_VERSION >= 37
     /**
-       C++ Interface for product of elements in an array according to a key
+       C++ Interface to multiply array elements over a given dimension,
+       according to an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the product of all values in \p vals
-                            along \p dim according to \p keys
-       \param[in]  keys     The key array
-       \param[in]  vals     The array containing the values to be reduced
-       \param[in]  dim      The dimension along which the product operation occurs
+       \param[out] keys_out reduced keys
+       \param[out] vals_out product
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the product occurs, -1
+                            denotes the first non-singleton dimension
 
        \ingroup reduce_func_product_by_key
-
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
     */
     AFAPI void productByKey(array &keys_out, array &vals_out,
                             const array &keys, const array &vals,
                             const int dim = -1);
 
     /**
-       C++ Interface for product of elements in an array according to a key
-       while replacing nan values
+       C++ Interface to multiply array elements over a given dimension,
+       replacing any NaNs with a specified value, according to an array of
+       keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p
-                            dim
-       \param[out] vals_out will contain the product of all values in \p
-                            vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the product operation occurs
-       \param[in] nanval  The value that will replace the NaNs in \p vals
+       \param[out] keys_out reduced keys
+       \param[out] vals_out product
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the product occurs
+       \param[in]  nanval   value that replaces NaNs
 
        \ingroup reduce_func_product_by_key
 
@@ -149,33 +144,34 @@ namespace af
 #endif
 
     /**
-       C++ Interface for minimum values in an array
+       C++ Interface to return the minimum along a given dimension.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the minimum value needs to be extracted
-       \return    result of minimum all values along dimension \p dim
+       NaN values are ignored.
 
-       \ingroup reduce_func_min
+       \param[in] in  input array
+       \param[in] dim dimension along which the minimum is found, -1 denotes
+                      the first non-singleton dimension
+       \return        minimum
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are ignored
+       \ingroup reduce_func_min
     */
     AFAPI array min(const array &in, const int dim = -1);
 
 #if AF_API_VERSION >= 37
     /**
-       C++ Interface for minimum values in an array according to a key
+       C++ Interface to return the minimum along a given dimension, according
+       to an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the minimum of all values in \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the min operation occurs
+       NaN values are ignored.
 
-       \ingroup reduce_func_min_by_key
+       \param[out] keys_out reduced keys
+       \param[out] vals_out minimum
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the minimum is found, -1
+                            denotes the first non-singleton dimension
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are ignored
+       \ingroup reduce_func_min_by_key
     */
     AFAPI void minByKey(array &keys_out, array &vals_out,
                         const array &keys, const array &vals,
@@ -183,33 +179,34 @@ namespace af
 #endif
 
     /**
-       C++ Interface for maximum values in an array
+       C++ Interface to return the maximum along a given dimension.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the maximum value needs to be extracted
-       \return    result of maximum all values along dimension \p dim
+       NaN values are ignored.
 
-       \ingroup reduce_func_max
+       \param[in] in  input array
+       \param[in] dim dimension along which the maximum is found, -1 denotes
+                      the first non-singleton dimension
+       \return        maximum
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are ignored
+       \ingroup reduce_func_max
     */
     AFAPI array max(const array &in, const int dim = -1);
 
 #if AF_API_VERSION >= 37
     /**
-       C++ Interface for maximum values in an array according to a key
+       C++ Interface to return the maximum along a given dimension, according
+       to an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the maximum of all values in \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the max operation occurs
+       NaN values are ignored.
 
-       \ingroup reduce_func_max_by_key
+       \param[out] keys_out reduced keys
+       \param[out] vals_out maximum
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the maximum is found, -1
+                            denotes the first non-singleton dimension
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are ignored
+       \ingroup reduce_func_max_by_key
     */
     AFAPI void maxByKey(array &keys_out, array &vals_out,
                         const array &keys, const array &vals,
@@ -218,50 +215,51 @@ namespace af
 
 #if AF_API_VERSION >= 38
     /**
-       C++ Interface for ragged max values in an array
-       Uses an additional input array to determine the number of elements to use along the reduction axis.
+       C++ Interface to return the ragged maximum along a given dimension.
 
-       \param[out] val will contain the maximum ragged values in \p in along \p dim according to \p ragged_len
-       \param[out] idx will contain the locations of the maximum ragged values in \p in along \p dim according to \p ragged_len
-       \param[in] in contains the input values to be reduced
-       \param[in] ragged_len array containing number of elements to use when reducing along \p dim
-       \param[in] dim The dimension along which the max operation occurs
+       Input parameter `ragged_len` sets the number of elements to consider.
 
-       \ingroup reduce_func_max
+       NaN values are ignored.
+
+       \param[out] val        ragged maximum
+       \param[out] idx        locations of the maximum ragged values
+       \param[in]  in         input array
+       \param[in]  ragged_len array containing the number of elements to use
+       \param[in]  dim        dimension along which the maximum is found
 
-       \note NaN values are ignored
+       \ingroup reduce_func_max
     */
     AFAPI void max(array &val, array &idx, const array &in, const array &ragged_len, const int dim);
 #endif
 
     /**
-       C++ Interface for checking all true values in an array
+       C++ Interface to check if all values along a given dimension are true.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the values are checked to be all true
-       \return    result of checking if values along dimension \p dim are all true
+       NaN values are ignored.
 
-       \ingroup reduce_func_all_true
+       \param[in] in  input array
+       \param[in] dim dimension along which the check occurs, -1 denotes the
+                      first non-singleton dimension
+       \return        array containing 1's if all true; 0's otherwise
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are ignored
+       \ingroup reduce_func_all_true
     */
     AFAPI array allTrue(const array &in, const int dim = -1);
 
 #if AF_API_VERSION >= 37
     /**
-       C++ Interface for checking all true values in an array according to a key
+       C++ Interface to check if all values along a given dimension are true,
+       according to an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the reduced and of all values in \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the all true operation occurs
+       NaN values are ignored.
 
-       \ingroup reduce_func_alltrue_by_key
+       \param[out] keys_out reduced keys
+       \param[out] vals_out array containing 1's if all true; 0's otherwise
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the check occurs
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are ignored
+       \ingroup reduce_func_alltrue_by_key
     */
     AFAPI void allTrueByKey(array &keys_out, array &vals_out,
                             const array &keys, const array &vals,
@@ -269,33 +267,33 @@ namespace af
 #endif
 
     /**
-       C++ Interface for checking any true values in an array
+       C++ Interface to check if any values along a given dimension are true.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the values are checked to be any true
-       \return    result of checking if values along dimension \p dim are any true
+       NaN values are ignored.
 
-       \ingroup reduce_func_any_true
+       \param[in] in  input array
+       \param[in] dim dimension along which the check occurs, -1 denotes the
+                      first non-singleton dimension
+       \return        array containing 1's if any true; 0's otherwise
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are ignored
+       \ingroup reduce_func_any_true
     */
     AFAPI array anyTrue(const array &in, const int dim = -1);
 
 #if AF_API_VERSION >= 37
     /**
-       C++ Interface for checking any true values in an array according to a key
+       C++ Interface to check if any values along a given dimension are true,
+       according to an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the reduced or of all values in \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the any true operation occurs
+       NaN values are ignored.
 
-       \ingroup reduce_func_anytrue_by_key
+       \param[out] keys_out reduced keys
+       \param[out] vals_out array containing 1's if any true; 0's otherwise
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the check occurs
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are ignored
+       \ingroup reduce_func_anytrue_by_key
     */
     AFAPI void anyTrueByKey(array &keys_out, array &vals_out,
                             const array &keys, const array &vals,
@@ -303,33 +301,35 @@ namespace af
 #endif
 
     /**
-       C++ Interface for counting non-zero values in an array
+       C++ Interface to count non-zero values in an array along a given
+       dimension.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the the number of non-zero values are counted
-       \return    the number of non-zero values along dimension \p dim
+       NaN values are treated as non-zero.
 
-       \ingroup reduce_func_count
+       \param[in] in  input array
+       \param[in] dim dimension along which the count occurs, -1 denotes the
+                      first non-singleton dimension
+       \return        count
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are treated as non zero.
+       \ingroup reduce_func_count
     */
     AFAPI array count(const array &in, const int dim = -1);
 
 #if AF_API_VERSION >= 37
     /**
-       C++ Interface for counting non-zero values in an array according to a key
+       C++ Interface to count non-zero values in an array, according to an
+       array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the count of all values in \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the count operation occurs
+       NaN values are treated as non-zero.
 
-       \ingroup reduce_func_count_by_key
+       \param[out] keys_out reduced keys
+       \param[out] vals_out count
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the count occurs, -1 denotes
+                            the first non-singleton dimension
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
-       \note NaN values are treated as non zero.
+       \ingroup reduce_func_count_by_key
     */
     AFAPI void countByKey(array &keys_out, array &vals_out,
                           const array &keys, const array &vals,
@@ -337,10 +337,13 @@ namespace af
 #endif
 
     /**
-       C++ Interface for sum of all elements in an array
+       C++ Interface to sum array elements over all dimensions.
 
-       \param[in] in is the input array
-       \return    the sum of all values of \p in
+       Results in a single value as an output, which may be a single element
+       `af::array`.
+
+       \param[in] in  input array
+       \return        sum
 
        \ingroup reduce_func_sum
     */
@@ -348,12 +351,15 @@ namespace af
 
 #if AF_API_VERSION >= 31
     /**
-       C++ Interface for sum of all elements in an array while replacing nan
-       values
+       C++ Interface to sum array elements over all dimensions, replacing any
+       NaNs with a specified value.
+
+       Results in a single value as an output, which may be a single element
+       `af::array`.
 
-       \param[in] in is the input array
-       \param[in] nanval  The value that will replace the NaNs in \p in
-       \return    the sum of all values of \p in
+       \param[in] in     input array
+       \param[in] nanval value that replaces NaNs
+       \return           sum
 
        \ingroup reduce_func_sum
     */
@@ -361,10 +367,11 @@ namespace af
 #endif
 
     /**
-       C++ Interface for product of all elements in an array
+       C++ Interface to multiply array elements over the first non-singleton
+       dimension.
 
-       \param[in] in is the input array
-       \return    the product of all values of \p in
+       \param[in] in input array
+       \return       product
 
        \ingroup reduce_func_product
     */
@@ -372,143 +379,155 @@ namespace af
 
 #if AF_API_VERSION >= 31
     /**
-       C++ Interface for product of all elements in an array while replacing nan
-       values
+       C++ Interface to multiply array elements over the first non-singleton
+       dimension, replacing any NaNs with a specified value.
 
-       \param[in] in is the input array
-       \param[in] nanval  The value that will replace the NaNs in \p in
-       \return    the product of all values of \p in
+       \param[in] in     input array
+       \param[in] nanval value that replaces NaNs
+       \return           product
 
        \ingroup reduce_func_product
     */
     template<typename T> T product(const array &in, double nanval);
 #endif
 
-
     /**
-       C++ Interface for getting minimum value of an array
+       C++ Interface to return the minimum along the first non-singleton
+       dimension.
 
-       \param[in] in is the input array
-       \return    the minimum of all values of \p in
+       NaN values are ignored.
 
-       \ingroup reduce_func_min
+       \param[in] in input array
+       \return       minimum
 
-       \note NaN values are ignored
+       \ingroup reduce_func_min
     */
     template<typename T> T min(const array &in);
 
     /**
-       C++ Interface for getting maximum value of an array
+       C++ Interface to return the maximum along the first non-singleton
+       dimension.
 
-       \param[in] in is the input array
-       \return    the maximum of all values of \p in
+       NaN values are ignored.
 
-       \ingroup reduce_func_max
+       \param[in] in input array
+       \return       maximum
 
-       \note NaN values are ignored
+       \ingroup reduce_func_max
     */
     template<typename T> T max(const array &in);
 
     /**
-       C++ Interface for checking if all values in an array are true
+       C++ Interface to check if all values along the first non-singleton
+       dimension are true.
 
-       \param[in] in is the input array
-       \return    true if all values of \p in are true, false otherwise
+       NaN values are ignored.
 
-       \ingroup reduce_func_all_true
+       \param[in] in input array
+       \return       array containing 1's if all true; 0's otherwise
 
-       \note NaN values are ignored
+       \ingroup reduce_func_all_true
     */
     template<typename T> T allTrue(const array &in);
 
     /**
-       C++ Interface for checking if any values in an array are true
+       C++ Interface to check if any values along the first non-singleton
+       dimension are true.
 
-       \param[in] in is the input array
-       \return    true if any values of \p in are true, false otherwise
+       NaN values are ignored.
 
-       \ingroup reduce_func_any_true
+       \param[in] in input array
+       \return       array containing 1's if any true; 0's otherwise
 
-       \note NaN values are ignored
+       \ingroup reduce_func_any_true
     */
     template<typename T> T anyTrue(const array &in);
 
     /**
-       C++ Interface for counting total number of non-zero values in an array
+       C++ Interface to count non-zero values along the first non-singleton
+       dimension.
 
-       \param[in] in is the input array
-       \return    the number of non-zero values in \p in
+       NaN values are treated as non-zero.
 
-       \ingroup reduce_func_count
+       \param[in] in input array
+       \return       count
 
-       \note NaN values are treated as non zero
+       \ingroup reduce_func_count
     */
     template<typename T> T count(const array &in);
 
     /**
-       C++ Interface for getting minimum values and their locations in an array
-
-       \param[out] val will contain the minimum values along dimension \p dim
-       \param[out] idx will contain the locations of minimum all values along dimension \p dim
-       \param[in]  in is the input array
-       \param[in]  dim The dimension along which the minimum value needs to be extracted
+       C++ Interface to return the minimum and its location along a given
+       dimension.
 
-       \ingroup reduce_func_min
+       NaN values are ignored.
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
+       \param[out] val minimum
+       \param[out] idx location
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the minimum is found, -1 denotes
+                       the first non-singleton dimension
 
-       \note NaN values are ignored
+       \ingroup reduce_func_min
     */
     AFAPI void min(array &val, array &idx, const array &in, const int dim = -1);
 
     /**
-       C++ Interface for getting maximum values and their locations in an array
-
-       \param[out] val will contain the maximum values along dimension \p dim
-       \param[out] idx will contain the locations of maximum all values along dimension \p dim
-       \param[in]  in is the input array
-       \param[in]  dim The dimension along which the maximum value needs to be extracted
+       C++ Interface to return the maximum and its location along a given
+       dimension.
 
-       \ingroup reduce_func_max
+       NaN values are ignored.
 
-       \note \p dim is -1 by default. -1 denotes the first non-singleton dimension.
+       \param[out] val maximum
+       \param[out] idx location
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the maximum is found, -1 denotes
+                       the first non-singleton dimension
 
-       \note NaN values are ignored
+       \ingroup reduce_func_max
     */
     AFAPI void max(array &val, array &idx, const array &in, const int dim = -1);
 
     /**
-       C++ Interface for getting minimum value and its location from the entire array
+       C++ Interface to return the minimum and its location over all
+       dimensions.
 
-       \param[out] val will contain the minimum values in the input
-       \param[out] idx will contain the locations of minimum all values in the input
-       \param[in]  in is the input array
+       NaN values are ignored.
 
-       \ingroup reduce_func_min
+       Often used to return values directly to the host.
+
+       \param[out] val minimum
+       \param[out] idx location
+       \param[in]  in  input array
 
-       \note NaN values are ignored
+       \ingroup reduce_func_min
     */
     template<typename T> void min(T *val, unsigned *idx, const array &in);
 
     /**
-       C++ Interface for getting maximum value and its location from the entire array
+       C++ Interface to return the maximum and its location over all
+       dimensions.
 
-       \param[out] val contains the maximum values in the input
-       \param[out] idx contains the locations of maximum all values in the input
-       \param[in]  in is the input array
+       NaN values are ignored.
 
-       \ingroup reduce_func_max
+       Often used to return values directly to the host.
 
-       \note NaN values are ignored
+       \param[out] val maximum
+       \param[out] idx location
+       \param[in]  in  input array
+
+       \ingroup reduce_func_max
     */
     template<typename T> void max(T *val, unsigned *idx, const array &in);
 
     /**
-       C++ Interface for computing the cumulative sum (inclusive) of an array
+       C++ Interface to evaluate the cumulative sum (inclusive) along a given
+       dimension.
 
-       \param[in] in is the input array
-       \param[in] dim is the dimension along which the inclusive sum is calculated
-       \return the output containing inclusive sums of the input
+       \param[in] in  input array
+       \param[in] dim dimension along which the sum is accumulated, 0 denotes
+                      the first non-singleton dimension
+       \return        cumulative sum
 
        \ingroup scan_func_accum
     */
@@ -516,13 +535,14 @@ namespace af
 
 #if AF_API_VERSION >=34
     /**
-       C++ Interface generalized scan of an array
+       C++ Interface to scan an array (generalized) over a given dimension.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which scan is performed
-       \param[in] op is the type of binary operation used
-       \param[in] inclusive_scan is flag specifying whether scan is inclusive
-       \return the output containing scan of the input
+       \param[in] in             input array
+       \param[in] dim            dimension along which the scan occurs, 0
+                                 denotes the first non-singleton dimension
+       \param[in] op             type of binary operation used
+       \param[in] inclusive_scan flag specifying whether the scan is inclusive
+       \return                   scan
 
        \ingroup scan_func_scan
     */
@@ -530,14 +550,16 @@ namespace af
                      binaryOp op = AF_BINARY_ADD, bool inclusive_scan = true);
 
     /**
-       C++ Interface generalized scan by key of an array
+       C++ Interface to scan an array (generalized) over a given dimension,
+       according to an array of keys.
 
-       \param[in] key is the key array
-       \param[in] in is the input array
-       \param[in] dim The dimension along which scan is performed
-       \param[in] op is the type of binary operations used
-       \param[in] inclusive_scan is flag specifying whether scan is inclusive
-       \return the output containing scan of the input
+       \param[in] key            keys array
+       \param[in] in             input array
+       \param[in] dim            dimension along which the scan occurs, 0
+                                 denotes the first non-singleton dimension
+       \param[in] op             type of binary operation used
+       \param[in] inclusive_scan flag specifying whether the scan is inclusive
+       \return                   scan
 
        \ingroup scan_func_scanbykey
     */
@@ -546,44 +568,49 @@ namespace af
 #endif
 
     /**
-       C++ Interface for finding the locations of non-zero values in an array
+       C++ Interface to locate the indices of the non-zero values in an array.
 
-       \param[in] in is the input array.
-       \return linear indices where \p in is non-zero
+       \param[in] in input array
+       \return       linear indices where `in` is non-zero
 
        \ingroup scan_func_where
     */
     AFAPI array where(const array &in);
 
     /**
-       C++ Interface for calculating first order differences in an array
+       C++ Interface to calculate the first order difference in an array over a
+       given dimension.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which numerical difference is performed
-       \return array of first order numerical difference
+       \param[in] in  input array
+       \param[in] dim dimension along which the difference occurs, 0
+                      denotes the first non-singleton dimension
+       \return        first order numerical difference
 
        \ingroup calc_func_diff1
     */
     AFAPI array diff1(const array &in, const int dim = 0);
 
     /**
-       C++ Interface for calculating second order differences in an array
+       C++ Interface to calculate the second order difference in an array over
+       a given dimension.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which numerical difference is performed
-       \return array of second order numerical difference
+       \param[in] in  input array
+       \param[in] dim dimension along which the difference occurs, 0
+                      denotes the first non-singleton dimension
+       \return        second order numerical difference
 
        \ingroup calc_func_diff2
     */
     AFAPI array diff2(const array &in, const int dim = 0);
 
     /**
-       C++ Interface for sorting an array
+       C++ Interface to sort an array over a given dimension.
 
-       \param[in] in is the input array
-       \param[in] dim The dimension along which numerical difference is performed
+       \param[in] in          input array
+       \param[in] dim         dimension along which the sort occurs, 0 denotes
+                              the first non-singleton dimension
        \param[in] isAscending specifies the sorting order
-       \return the sorted output
+       \return                sorted output
 
        \ingroup sort_func_sort
     */
@@ -591,27 +618,32 @@ namespace af
                      const bool isAscending = true);
 
     /**
-       C++ Interface for sorting an array and getting original indices
+       C++ Interface to sort an array over a given dimension and to return the
+       original indices.
 
-       \param[out] out will contain the sorted output
-       \param[out] indices will contain the indices in the original input
-       \param[in] in is the input array
-       \param[in] dim The dimension along which numerical difference is performed
-       \param[in] isAscending specifies the sorting order
+       \param[out] out         sorted output
+       \param[out] indices     indices from the input
+       \param[in]  in          input array
+       \param[in]  dim         dimension along which the sort occurs, 0 denotes
+                               the first non-singleton dimension
+       \param[in]  isAscending specifies the sorting order
 
        \ingroup sort_func_sort_index
     */
     AFAPI void  sort(array &out, array &indices, const array &in, const unsigned dim = 0,
                      const bool isAscending = true);
+
     /**
-       C++ Interface for sorting an array based on keys
+       C++ Interface to sort an array over a given dimension, according to an
+       array of keys.
 
-       \param[out] out_keys will contain the keys based on sorted values
-       \param[out] out_values will contain the sorted values
-       \param[in] keys is the input array
-       \param[in] values The dimension along which numerical difference is performed
-       \param[in] dim The dimension along which numerical difference is performed
-       \param[in] isAscending specifies the sorting order
+       \param[out] out_keys    sorted keys
+       \param[out] out_values  sorted output
+       \param[in]  keys        keys array
+       \param[in]  values      input array
+       \param[in]  dim         dimension along which the sort occurs, 0 denotes
+                               the first non-singleton dimension
+       \param[in]  isAscending specifies the sorting order
 
        \ingroup sort_func_sort_keys
     */
@@ -620,23 +652,23 @@ namespace af
                      const bool isAscending = true);
 
     /**
-       C++ Interface for getting unique values
+       C++ Interface to return the unique values in an array.
 
-       \param[in] in is the input array
-       \param[in] is_sorted if true, skips the sorting steps internally
-       \return the unique values from \p in
+       \param[in] in        input array
+       \param[in] is_sorted if true, skip the sorting steps internally
+       \return              unique values
 
        \ingroup set_func_unique
     */
     AFAPI array setUnique(const array &in, const bool is_sorted=false);
 
     /**
-       C++ Interface for finding the union of two arrays
+       C++ Interface to evaluate the union of two arrays.
 
-       \param[in] first is the first input array
-       \param[in] second is the second input array
-       \param[in] is_unique if true, skips calling unique internally
-       \return all unique values present in \p first and \p second (union) in increasing order
+       \param[in] first     input array
+       \param[in] second    input array
+       \param[in] is_unique if true, skip calling setUnique internally
+       \return              union, values in increasing order
 
        \ingroup set_func_union
     */
@@ -644,12 +676,12 @@ namespace af
                          const bool is_unique=false);
 
     /**
-       C++ Interface for finding the intersection of two arrays
+       C++ Interface to evaluate the intersection of two arrays.
 
-       \param[in] first is the first input array
-       \param[in] second is the second input array
-       \param[in] is_unique if true, skips calling unique internally
-       \return unique values that are present in both \p first and \p second(intersection) in increasing order
+       \param[in] first     input array
+       \param[in] second    input array
+       \param[in] is_unique if true, skip calling setUnique internally
+       \return              intersection, values in increasing order
 
        \ingroup set_func_intersect
     */
@@ -663,12 +695,13 @@ extern "C" {
 #endif
 
     /**
-       C Interface for sum of elements in an array
+       C Interface to sum array elements over a given dimension.
 
-       \param[out] out will contain the sum of all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the add operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out sum
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the summation occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_sum
     */
@@ -676,11 +709,14 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for sum of all elements in an array, resulting in an array
+       C Interface to sum array elements over all dimensions.
 
-       \param[out] out will contain the sum of all values in \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       Results in a single element `af::array`.
+
+       \param[out] out sum
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_sum
     */
@@ -689,13 +725,15 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       C Interface for sum of elements in an array while replacing nans
+       C Interface to sum array elements over a given dimension, replacing any
+       NaNs with a specified value.
 
-       \param[out] out will contain the sum of all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the add operation occurs
-       \param[in] nanval  The value that will replace the NaNs in \p in
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out    sum
+       \param[in]  in     input array
+       \param[in]  dim    dimension along which the summation occurs
+       \param[in]  nanval value that replaces NaNs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_sum
     */
@@ -705,13 +743,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for sum of all elements in an array, resulting in an array with
-       nan substitution
+       C Interface to sum array elements over all dimensions, replacing any
+       NaNs with a specified value.
+
+       Results in a single element `af::array`.
 
-       \param[out] out will contain the sum of all values in \p in
-       \param[in] in is the input array
-       \param[in] nanval  The value that will replace the NaNs in \p in
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out    sum
+       \param[in]  in     input array
+       \param[in]  nanval value that replaces NaNs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_sum
     */
@@ -720,14 +761,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-       C Interface for sum of elements in an array according to key
+       C Interface to sum array elements over a given dimension, according to
+       an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the sum of all values in \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the add operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] keys_out reduced keys
+       \param[out] vals_out sum
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the summation occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_sum_by_key
     */
@@ -735,20 +778,17 @@ extern "C" {
                                const af_array keys, const af_array vals, const int dim);
 
     /**
-       C Interface for sum of elements in an array according to key while
-       replacing nans
-
-       \param[out] keys_out will contain the reduced keys in \p vals along \p
-                            dim
-       \param[out] vals_out will contain the sum of all values in \p vals
-                            along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the add operation occurs
-       \param[in] nanval  The value that will replace the NaNs in \p vals
+       C Interface to sum array elements over a given dimension, replacing any
+       NaNs with a specified value, according to an array of keys.
 
-
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] keys_out reduced keys
+       \param[out] vals_out sum
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the summation occurs
+       \param[in]  nanval   value that replaces NaNs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_sum_by_key
     */
@@ -758,12 +798,13 @@ extern "C" {
 #endif
 
     /**
-       C Interface for product of elements in an array
+       C Interface to multiply array elements over a given dimension.
 
-       \param[out] out will contain the product of all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the multiply operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out product
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the product occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_product
     */
@@ -771,11 +812,14 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for product of elements in an array, resulting in an array
+       C Interface to multiply array elements over all dimensions.
+
+       Results in a single element `af::array`.
 
-       \param[out] out will contain the product of all values in \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out product
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_product
     */
@@ -784,14 +828,15 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       C Interface for product of elements in an array while replacing nans
+       C Interface to multiply array elements over a given dimension, replacing
+       any NaNs with a specified value.
 
-       \param[out] out will contain the product of all values in \p in along \p
-                       dim
-       \param[in] in   is the input array
-       \param[in] dim  The dimension along which the product operation occurs
-       \param[in] nanval  The value that will replace the NaNs in \p in
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out    product
+       \param[in]  in     input array
+       \param[in]  dim    dimension along with the product occurs
+       \param[in]  nanval value that replaces NaNs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_product
     */
@@ -800,13 +845,14 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for product of elements in an array, resulting in an array
-       while replacing nans
+       C Interface to multiply array elements over all dimensions, replacing
+       any NaNs with a specified value.
 
-       \param[out] out will contain the product of all values in \p in
-       \param[in] in   is the input array
-       \param[in] nanval  The value that will replace the NaNs in \p in
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out    product
+       \param[in]  in     input array
+       \param[in]  nanval value that replaces NaNs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_product
     */
@@ -815,14 +861,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-       C Interface for product of elements in an array according to key
+       C Interface to multiply array elements over a given dimension, according
+       to an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the product of all values in \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the product operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] keys_out reduced keys
+       \param[out] vals_out product
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the product occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_product_by_key
     */
@@ -830,18 +878,17 @@ extern "C" {
                                    const af_array keys, const af_array vals, const int dim);
 
     /**
-       C Interface for product of elements in an array according to key while
-       replacing nans
+       C Interface to multiply array elements over a given dimension, replacing
+       any NaNs with a specified value, according to an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p
-                            dim
-       \param[out] vals_out will contain the product of all values in \p
-                            vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the product operation occurs
-       \param[in] nanval  The value that will replace the NaNs in \p vals
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] keys_out reduced keys
+       \param[out] vals_out product
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the product occurs
+       \param[in]  nanval   value that replaces NaNs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_product_by_key
     */
@@ -851,12 +898,13 @@ extern "C" {
 #endif
 
     /**
-       C Interface for minimum values in an array
+       C Interface to return the minimum along a given dimension.
 
-       \param[out] out will contain the minimum of all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the minimum value is extracted
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out minimum
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the minimum is found
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_min
     */
@@ -864,14 +912,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-       C Interface for minimum values in an array according to key
+       C Interface to return the minimum along a given dimension, according to
+       an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the minimum of all values in \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the minimum value is extracted
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] keys_out reduced keys
+       \param[out] vals_out minimum
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the minimum is found
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_min_by_key
     */
@@ -881,12 +931,13 @@ extern "C" {
 #endif
 
     /**
-       C Interface for maximum values in an array
+       C Interface to return the maximum along a given dimension.
 
-       \param[out] out will contain the maximum of all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the maximum value is extracted
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out  maximum
+       \param[in]  in   input array
+       \param[in]  dim dimension along which the maximum is found
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_max
     */
@@ -894,16 +945,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-       C Interface for maximum values in an array according to key
+       C Interface to return the maximum along a given dimension, according to
+       an array of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p
-                            dim
-       \param[out] vals_out will contain the maximum of all values in \p
-                            vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the maximum value is extracted
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] keys_out reduced keys
+       \param[out] vals_out maximum
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the maximum is found
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_max_by_key
     */
@@ -914,30 +965,35 @@ extern "C" {
 
 #if AF_API_VERSION >= 38
     /**
-       C Interface for finding ragged max values in an array
-       Uses an additional input array to determine the number of elements to use along the reduction axis.
+       C Interface to return the ragged maximum over a given dimension.
 
-       \param[out] val will contain the maximum ragged values in \p in along \p dim according to \p ragged_len
-       \param[out] idx will contain the locations of the maximum ragged values in \p in along \p dim according to \p ragged_len
-       \param[in] in contains the input values to be reduced
-       \param[in] ragged_len array containing number of elements to use when reducing along \p dim
-       \param[in] dim The dimension along which the max operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       Input parameter `ragged_len` sets the number of elements to consider.
 
-       \ingroup reduce_func_max
+       NaN values are ignored.
+
+       \param[out] val        ragged maximum
+       \param[out] idx        locations of the maximum ragged values
+       \param[in]  in         input array
+       \param[in]  ragged_len array containing the number of elements to use
+       \param[in]  dim        dimension along which the maximum is found
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-       \note NaN values are ignored
+       \ingroup reduce_func_max
     */
     AFAPI af_err af_max_ragged(af_array *val, af_array *idx, const af_array in, const af_array ragged_len, const int dim);
 #endif
 
     /**
-       C Interface for checking all true values in an array
+       C Interface  to check if all values along a given dimension are true.
 
-       \param[out] out will contain the result of "and" operation all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the "and" operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       NaN values are ignored.
+
+       \param[out] out array containing 1's if all true; 0's otherwise
+       \param[in]  in  input array
+       \param[in]  dim dimention along which the check occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_all_true
     */
@@ -945,15 +1001,18 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-       C Interface for checking all true values in an array according to key
+       C Interface to check if all values along a given dimension are true,
+       according to an array of keys.
+
+       NaN values are ignored.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the the reduced and of all values in
-                            \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the "and" operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] keys_out reduced keys
+       \param[out] vals_out array containing 1's if all true; 0's otherwise
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the check occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_alltrue_by_key
     */
@@ -963,12 +1022,15 @@ extern "C" {
 #endif
 
     /**
-       C Interface for checking any true values in an array
+       C Interface to check if any values along a given dimension are true.
 
-       \param[out] out will contain the result of "or" operation all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the "or" operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       NaN values are ignored.
+
+       \param[out] out array containing 1's if any true; 0's otherwise
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the check occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_any_true
     */
@@ -976,15 +1038,17 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-       C Interface for checking any true values in an array according to key
+       C Interface to check if any values along a given dimension are true.
+
+       NaN values are ignored.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the reduced or of all values in
-                            \p vals along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the "or" operation occurs
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] keys_out reduced keys
+       \param[out] vals_out array containing 1's if any true; 0's otherwise
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimensions along which the check occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_anytrue_by_key
     */
@@ -994,12 +1058,16 @@ extern "C" {
 #endif
 
     /**
-       C Interface for counting non-zero values in an array
+       C Interface to count non-zero values in an array along a given
+       dimension.
+
+       NaN values are treated as non-zero.
 
-       \param[out] out will contain the number of non-zero values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the non-zero values are counted
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out count
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the count occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_count
     */
@@ -1007,15 +1075,18 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-       C Interface for counting non-zero values in an array according to key
+       C Interface to count non-zero values in an array, according to an array
+       of keys.
 
-       \param[out] keys_out will contain the reduced keys in \p vals along \p dim
-       \param[out] vals_out will contain the count of all values in \p vals
-                            along \p dim according to \p keys
-       \param[in] keys is the key array
-       \param[in] vals is the array containing the values to be reduced
-       \param[in] dim The dimension along which the non-zero values are counted
-       \return \ref AF_SUCCESS if the execution completes properly
+       NaN values are treated as non-zero.
+
+       \param[out] keys_out reduced keys
+       \param[out] vals_out count
+       \param[in]  keys     keys array
+       \param[in]  vals     input array
+       \param[in]  dim      dimension along which the count occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_count_by_key
     */
@@ -1025,16 +1096,15 @@ extern "C" {
 #endif
 
     /**
-       C Interface for sum of all elements in an array
+       C Interface to sum array elements over all dimensions.
 
-       \param[out] real will contain the real part of adding all elements in
-                        input \p in
-       \param[out] imag will contain the imaginary part of adding all elements
-                        in input \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       If `in` is real, `imag` will be set to zeros.
 
-       \note \p imag is always set to 0 when \p in is real
+       \param[out] real sum of all real components
+       \param[out] imag sum of all imaginary components
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_sum
     */
@@ -1042,17 +1112,17 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       C Interface for sum of all elements in an array while replacing nans
+       C Interface to sum array elements over all dimensions, replacing any
+       NaNs with a specified value.
 
-       \param[out] real will contain the real part of adding all elements in
-                        input \p in
-       \param[out] imag will contain the imaginary part of adding all elements
-                        in input \p in
-       \param[in] in is the input array
-       \param[in] nanval is the value which replaces nan
-       \return \ref AF_SUCCESS if the execution completes properly
+       If `in` is real, `imag` will be set to zeros.
 
-       \note \p imag is always set to 0 when \p in is real
+       \param[out] real   sum of all real components
+       \param[out] imag   sum of all imaginary components
+       \param[in]  in     input array
+       \param[in]  nanval value that replaces NaNs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_sum
     */
@@ -1061,14 +1131,15 @@ extern "C" {
 #endif
 
     /**
-       C Interface for product of all elements in an array
+       C Interface to multiply array elements over all dimensions.
 
-       \param[out] real will contain the real part of multiplying all elements in input \p in
-       \param[out] imag will contain the imaginary part of multiplying all elements in input \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       If `in` is real, `imag` will be set to zeros.
 
-       \note \p imag is always set to 0 when \p in is real
+       \param[out] real product of all real components
+       \param[out] imag product of all imaginary components
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_product
     */
@@ -1076,17 +1147,17 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       C Interface for product of all elements in an array while replacing nans
+       C Interface to multiply array elements over all dimensions, replacing
+       any NaNs with a specified value.
 
-       \param[out] real   will contain the real part of multiplication of all
-                          elements in input \p in
-       \param[out] imag   will contain the imaginary part of multiplication of
-                          all elements in input \p in
-       \param[in]  in     is the input array
-       \param[in]  nanval is the value which replaces nan
-       \return \ref AF_SUCCESS if the execution completes properly
+       If `in` is real, `imag` will be set to zeros.
 
-       \note \p imag is always set to 0 when \p in is real
+       \param[out] real   product of all real components
+       \param[out] imag   product of all imaginary components
+       \param[in]  in     input array
+       \param[in]  nanval value that replaces NaNs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_product
     */
@@ -1095,14 +1166,15 @@ extern "C" {
 #endif
 
     /**
-       C Interface for getting minimum value of an array
+       C Interface to return the minimum over all dimensions.
 
-       \param[out] real will contain the real part of minimum value of all elements in input \p in
-       \param[out] imag will contain the imaginary part of minimum value of all elements in input \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       If `in` is real, `imag` will be set to zeros.
 
-       \note \p imag is always set to 0 when \p in is real.
+       \param[out] real real component of the minimum
+       \param[out] imag imaginary component of the minimum
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_min
     */
@@ -1110,11 +1182,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for minimum values in an array, returning an array
+       C Interface to return the minimum over all dimensions.
 
-       \param[out] out will contain the minimum of all values in \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out minimum
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_min
     */
@@ -1122,14 +1195,15 @@ extern "C" {
 #endif
 
     /**
-       C Interface for getting maximum value of an array
+       C Interface to return the maximum over all dimensions.
 
-       \param[out] real will contain the real part of maximum value of all elements in input \p in
-       \param[out] imag will contain the imaginary part of maximum value of all elements in input \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       If `in` is real, `imag` will be set to zeros.
 
-       \note \p imag is always set to 0 when \p in is real.
+       \param[out] real real component of the maximum
+       \param[out] imag imaginary component of the maximum
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_max
     */
@@ -1137,13 +1211,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for getting maximum value of an array, returning an array
-
-       \param[out] out will contain the maximum of all values in \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       C Interface to return the maximum over all dimensions.
 
-       \note \p imag is always set to 0 when \p in is real.
+       \param[out] out maximum
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_max
     */
@@ -1151,14 +1224,13 @@ extern "C" {
 #endif
 
     /**
-       C Interface for checking if all values in an array are true
-
-       \param[out] real is 1 if all values of input \p in are true, 0 otherwise.
-       \param[out] imag is always set to 0.
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
-
-       \note \p imag is always set to 0.
+       C Interface to check if all values over all dimensions are true.
+ 
+       \param[out] real 1 if all true; 0 otherwise
+       \param[out] imag 0
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_all_true
     */
@@ -1166,14 +1238,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for checking if all values in an array are true,
-       while returning an af_array
-
-       \param[out] out will contain 1 if all values of input \p in are true, 0 otherwise
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
-
-       \note \p imag is always set to 0.
+       C Interface to check if all values over all dimensions are true.
+ 
+       \param[out] out 1 if all true; 0 otherwise
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_all_true
     */
@@ -1181,14 +1251,13 @@ extern "C" {
 #endif
 
     /**
-       C Interface for checking if any values in an array are true
-
-       \param[out] real is 1 if any value of input \p in is true, 0 otherwise.
-       \param[out] imag is always set to 0.
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       C Interface to check if any values over all dimensions are true.
 
-       \note \p imag is always set to 0.
+       \param[out] real 1 if any true; 0 otherwise
+       \param[out] imag 0
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_any_true
     */
@@ -1196,14 +1265,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for checking if any values in an array are true,
-       while returning an af_array
+       C Interface to check if any values over all dimensions are true.
 
-       \param[out] out will contain 1 if any value of input \p in is true, 0 otherwise
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
-
-       \note \p imag is always set to 0.
+       \param[out] out 1 if any true; 0 otherwise
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_any_true
     */
@@ -1211,14 +1278,13 @@ extern "C" {
 #endif
 
     /**
-       C Interface for counting total number of non-zero values in an array
-
-       \param[out] real will contain the number of non-zero values in \p in.
-       \param[out] imag is always set to 0.
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       C Interface to count non-zero values over all dimensions.
 
-       \note \p imag is always set to 0.
+       \param[out] real count
+       \param[out] imag 0
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_count
     */
@@ -1226,12 +1292,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       C Interface for counting total number of non-zero values in an array,
-       while returning an af_array
+       C Interface to count non-zero values over all dimensions.
 
-       \param[out] out contain the number of non-zero values in \p in.
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out count
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_count
     */
@@ -1239,13 +1305,15 @@ extern "C" {
 #endif
 
     /**
-       C Interface for getting minimum values and their locations in an array
+       C Interface to return the minimum and its location along a given
+       dimension.
 
-       \param[out] out will contain the minimum of all values in \p in along \p dim
-       \param[out] idx will contain the location of minimum of all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the minimum value is extracted
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out minimum
+       \param[out] idx location
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the minimum is found
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_min
     */
@@ -1253,13 +1321,15 @@ extern "C" {
                          const int dim);
 
     /**
-       C Interface for getting maximum values and their locations in an array
+       C Interface to return the maximum and its location along a given
+       dimension.
 
-       \param[out] out will contain the maximum of all values in \p in along \p dim
-       \param[out] idx will contain the location of maximum of all values in \p in along \p dim
-       \param[in] in is the input array
-       \param[in] dim The dimension along which the maximum value is extracted
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out maximum
+       \param[out] idx location
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the maximum is found
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_max
     */
@@ -1267,15 +1337,16 @@ extern "C" {
                          const int dim);
 
     /**
-       C Interface for getting minimum value and its location from the entire array
+       C Interface to return the minimum and its location over all dimensions.
 
-       \param[out] real will contain the real part of minimum value of all elements in input \p in
-       \param[out] imag will contain the imaginary part of minimum value of all elements in input \p in
-       \param[out] idx will contain the location of minimum of all values in \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       NaN values are ignored.
 
-       \note \p imag is always set to 0 when \p in is real.
+       \param[out] real real component of the minimum
+       \param[out] imag imaginary component of the minimum; 0 if `idx` is real
+       \param[out] idx  location
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_min
     */
@@ -1283,27 +1354,30 @@ extern "C" {
                              const af_array in);
 
     /**
-       C Interface for getting maximum value and it's location from the entire array
+       C Interface to return the maximum and its location over all dimensions.
 
-       \param[out] real will contain the real part of maximum value of all elements in input \p in
-       \param[out] imag will contain the imaginary part of maximum value of all elements in input \p in
-       \param[out] idx will contain the location of maximum of all values in \p in
-       \param[in] in is the input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       NaN values are ignored.
 
-       \note \p imag is always set to 0 when \p in is real.
+       \param[out] real real component of the maximum
+       \param[out] imag imaginary component of the maximum; 0 if `idx` is real
+       \param[out] idx  location
+       \param[in]  in   input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup reduce_func_max
     */
     AFAPI af_err af_imax_all(double *real, double *imag, unsigned *idx, const af_array in);
 
     /**
-       C Interface for computing the cumulative sum (inclusive) of an array
+       C Interface to evaluate the cumulative sum (inclusive) along a given
+       dimension.
 
-       \param[out] out will contain inclusive sums of the input
-       \param[in] in is the input array
-       \param[in] dim is the dimension along which the inclusive sum is calculated
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out cumulative sum
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the sum is accumulated
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup scan_func_accum
     */
@@ -1311,14 +1385,15 @@ extern "C" {
 
 #if AF_API_VERSION >=34
     /**
-       C Interface generalized scan of an array
+       C Interface to scan an array (generalized) over a given dimension.
 
-       \param[out] out will contain scan of the input
-       \param[in] in is the input array
-       \param[in] dim The dimension along which scan is performed
-       \param[in] op is the type of binary operations used
-       \param[in] inclusive_scan is flag specifying whether scan is inclusive
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out            scan
+       \param[in]  in             input array
+       \param[in]  dim            dimension along which the scan occurs
+       \param[in]  op             type of binary operation used
+       \param[in]  inclusive_scan flag specifying whether the scan is inclusive
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup scan_func_scan
     */
@@ -1326,15 +1401,17 @@ extern "C" {
                          af_binary_op op, bool inclusive_scan);
 
     /**
-       C Interface generalized scan by key of an array
+       C Interface to scan an array (generalized) over a given dimension,
+       according to an array of keys.
 
-       \param[out] out will contain scan of the input
-       \param[in] key is the key array
-       \param[in] in is the input array
-       \param[in] dim The dimension along which scan is performed
-       \param[in] op is the type of binary operations used
-       \param[in] inclusive_scan is flag specifying whether scan is inclusive
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out            scan
+       \param[in]  key            keys array
+       \param[in]  in             input array
+       \param[in]  dim            dimension along which the scan occurs
+       \param[in]  op             type of binary operation used
+       \param[in]  inclusive_scan flag specifying whether the scan is inclusive
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup scan_func_scanbykey
     */
@@ -1345,48 +1422,54 @@ extern "C" {
 #endif
 
     /**
-       C Interface for finding the locations of non-zero values in an array
+       C Interface to locate the indices of the non-zero values in an array.
 
-       \param[out] idx will contain indices where \p in is non-zero
-       \param[in] in is the input array.
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] idx linear indices where `in` is non-zero
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup scan_func_where
     */
     AFAPI af_err af_where(af_array *idx, const af_array in);
 
     /**
-       C Interface for calculating first order differences in an array
+       C Interface to calculate the first order difference in an array over a
+       given dimension.
 
-       \param[out] out will contain the first order numerical differences of \p in
-       \param[in] in is the input array
-       \param[in] dim The dimension along which numerical difference is performed
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out first order numerical difference
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the difference occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup calc_func_diff1
     */
     AFAPI af_err af_diff1(af_array *out, const af_array in, const int dim);
 
     /**
-       C Interface for calculating second order differences in an array
+       C Interface to calculate the second order difference in an array over a
+       given dimension.
 
-       \param[out] out will contain the second order numerical differences of \p in
-       \param[in] in is the input array
-       \param[in] dim The dimension along which numerical difference is performed
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out second order numerical difference
+       \param[in]  in  input array
+       \param[in]  dim dimension along which the difference occurs
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup calc_func_diff2
     */
     AFAPI af_err af_diff2(af_array *out, const af_array in, const int dim);
 
     /**
-       C Interface for sorting an array
+       C Interface to sort an array over a given dimension.
 
-       \param[out] out will contain the sorted output
-       \param[in] in is the input array
-       \param[in] dim The dimension along which numerical difference is performed
-       \param[in] isAscending specifies the sorting order
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out         sorted output
+       \param[in]  in          input array
+       \param[in]  dim         dimension along which the sort occurs
+       \param[in]  isAscending specifies the sorting order
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup sort_func_sort
     */
@@ -1394,29 +1477,33 @@ extern "C" {
                          const bool isAscending);
 
     /**
-       C Interface for sorting an array and getting original indices
+       C Interface to sort an array over a given dimension and to return the
+       original indices.
 
-       \param[out] out will contain the sorted output
-       \param[out] indices will contain the indices in the original input
-       \param[in] in is the input array
-       \param[in] dim The dimension along which numerical difference is performed
-       \param[in] isAscending specifies the sorting order
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out         sorted output
+       \param[out] indices     indices from the input
+       \param[in]  in          input array
+       \param[in]  dim         dimension along which the sort occurs
+       \param[in]  isAscending specifies the sorting order
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup sort_func_sort_index
     */
     AFAPI af_err af_sort_index(af_array *out, af_array *indices, const af_array in,
                                const unsigned dim, const bool isAscending);
     /**
-       C Interface for sorting an array based on keys
+       C Interface to sort an array over a given dimension, according to an
+       array of keys.
 
-       \param[out] out_keys will contain the keys based on sorted values
-       \param[out] out_values will contain the sorted values
-       \param[in] keys is the input array
-       \param[in] values The dimension along which numerical difference is performed
-       \param[in] dim The dimension along which numerical difference is performed
-       \param[in] isAscending specifies the sorting order
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out_keys    sorted keys
+       \param[out] out_values  sorted output
+       \param[in]  keys        keys array
+       \param[in]  values      input array
+       \param[in]  dim         dimension along which the sort occurs
+       \param[in]  isAscending specifies the sorting order
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup sort_func_sort_keys
     */
@@ -1425,25 +1512,27 @@ extern "C" {
                                 const unsigned dim, const bool isAscending);
 
     /**
-       C Interface for getting unique values
+       C Interface to return the unique values in an array.
 
-       \param[out] out will contain the unique values from \p in
-       \param[in] in is the input array
-       \param[in] is_sorted if true, skips the sorting steps internally
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out       unique values
+       \param[in]  in        input array
+       \param[in]  is_sorted if true, skip the sorting steps internally
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup set_func_unique
     */
     AFAPI af_err af_set_unique(af_array *out, const af_array in, const bool is_sorted);
 
     /**
-       C Interface for finding the union of two arrays
+       C Interface to evaluate the union of two arrays.
 
-       \param[out] out will contain the union of \p first and \p second
-       \param[in] first is the first input array
-       \param[in] second is the second input array
-       \param[in] is_unique if true, skips calling unique internally
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out       union, values in increasing order
+       \param[in]  first     input array
+       \param[in]  second    input array
+       \param[in]  is_unique if true, skip calling unique internally
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup set_func_union
     */
@@ -1451,13 +1540,14 @@ extern "C" {
                               const af_array second, const bool is_unique);
 
     /**
-       C Interface for finding the intersection of two arrays
+       C Interface to evaluate the intersection of two arrays.
 
-       \param[out] out will contain the intersection of \p first and \p second
-       \param[in] first is the first input array
-       \param[in] second is the second input array
-       \param[in] is_unique if true, skips calling unique internally
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out       intersection, values in increasing order
+       \param[in]  first     input array
+       \param[in]  second    input array
+       \param[in]  is_unique if true, skip calling unique internally
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup set_func_intersect
     */
diff --git a/include/af/arith.h b/include/af/arith.h
index ea9be6c328..9b02e668b6 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -18,25 +18,27 @@ namespace af
     ///
     /// \param[in] lhs input array
     /// \param[in] rhs input array
-    /// \return minimum of \p lhs and \p rhs
+    /// \return        minimum
     ///
     /// \ingroup arith_func_min
     AFAPI array min    (const array &lhs, const array &rhs);
 
-    /// C++ Interface to find the elementwise minimum between an array and a scalar value.
+    /// C++ Interface to find the elementwise minimum between an array and a
+    /// scalar value.
     ///
     /// \param[in] lhs input array
     /// \param[in] rhs scalar value
-    /// \return minimum of \p lhs and \p rhs
+    /// \return        minimum
     ///
     /// \ingroup arith_func_min
     AFAPI array min    (const array &lhs, const double rhs);
 
-    /// C++ Interface to find the elementwise minimum between an array and a scalar value.
+    /// C++ Interface to find the elementwise minimum between an array and a
+    /// scalar value.
     ///
     /// \param[in] lhs scalar value
     /// \param[in] rhs input array
-    /// \return minimum of \p lhs and \p rhs
+    /// \return        minimum
     ///
     /// \ingroup arith_func_min
     AFAPI array min    (const double lhs, const array &rhs);
@@ -45,25 +47,27 @@ namespace af
     ///
     /// \param[in] lhs input array
     /// \param[in] rhs input array
-    /// \return maximum of \p lhs and \p rhs
+    /// \return        maximum
     ///
     /// \ingroup arith_func_max
     AFAPI array max    (const array &lhs, const array &rhs);
 
-    /// C++ Interface to find the elementwise maximum between an array and a scalar value.
+    /// C++ Interface to find the elementwise maximum between an array and a
+    /// scalar value.
     ///
     /// \param[in] lhs input array
     /// \param[in] rhs scalar value
-    /// \return maximum of \p lhs and \p rhs
+    /// \return        maximum
     ///
     /// \ingroup arith_func_max
     AFAPI array max    (const array &lhs, const double rhs);
 
-    /// C++ Interface to find the elementwise maximum between an array and a scalar value.
+    /// C++ Interface to find the elementwise maximum between an array and a
+    /// scalar value.
     ///
     /// \param[in] lhs input array
     /// \param[in] rhs scalar value
-    /// \return maximum of \p lhs and \p rhs
+    /// \return        maximum
     ///
     /// \ingroup arith_func_max
     AFAPI array max    (const double lhs, const array &rhs);
@@ -75,7 +79,7 @@ namespace af
     /// \param[in] in input array
     /// \param[in] lo lower limit; can be an array or a scalar
     /// \param[in] hi upper limit; can be an array or a scalar
-    /// \return array containing values from \p in clamped between \p lo and \p hi
+    /// \return       clamped array
     /// 
     /// \ingroup arith_func_clamp
     AFAPI array clamp(const array &in, const array &lo, const array &hi);
@@ -102,7 +106,7 @@ namespace af
     ///
     /// \param[in] lhs numerator; can be an array or a scalar
     /// \param[in] rhs denominator; can be an array or a scalar
-    /// \return remainder of \p lhs divided by \p rhs
+    /// \return        remainder
     /// 
     /// \ingroup arith_func_rem
     AFAPI array rem    (const array &lhs, const array &rhs);
@@ -119,7 +123,7 @@ namespace af
     ///
     /// \param[in] lhs dividend; can be an array or a scalar
     /// \param[in] rhs divisor; can be an array or a scalar
-    /// \return \p lhs modulo \p rhs
+    /// \return        modulus
     /// 
     /// \ingroup arith_func_mod
     AFAPI array mod    (const array &lhs, const array &rhs);
@@ -134,15 +138,16 @@ namespace af
     /// C++ Interface to calculate the absolute value.
     ///
     /// \param[in] in input array
-    /// \return absolute value
+    /// \return       absolute value
     ///
     /// \ingroup arith_func_abs
     AFAPI array abs    (const array &in);
 
-    /// C++ Interface to calculate the phase angle (in radians) of a complex array.
+    /// C++ Interface to calculate the phase angle (in radians) of a complex
+    /// array.
     ///
     /// \param[in] in input array, typically complex
-    /// \return phase angle (in radians)
+    /// \return       phase angle (in radians)
     /// 
     /// \ingroup arith_func_arg
     AFAPI array arg    (const array &in);
@@ -150,7 +155,7 @@ namespace af
     /// C++ Interface to return the sign of elements in an array.
     ///
     /// \param[in] in input array
-    /// \return array containing 1's for negative values; 0's otherwise
+    /// \return       array containing 1's for negative values; 0's otherwise
     /// 
     /// \ingroup arith_func_sign
     AFAPI array sign  (const array &in);
@@ -158,7 +163,7 @@ namespace af
     /// C++ Interface to round numbers.
     ///
     /// \param[in] in input array
-    /// \return numbers rounded to nearest integer
+    /// \return       nearest integer
     ///
     /// \ingroup arith_func_round
     AFAPI array round  (const array &in);
@@ -166,7 +171,7 @@ namespace af
     /// C++ Interface to truncate numbers.
     ///
     /// \param[in] in input array
-    /// \return nearest integer not greater in magnitude than \p in
+    /// \return       nearest integer not greater in magnitude than `in`
     /// 
     /// \ingroup arith_func_trunc
     AFAPI array trunc  (const array &in);
@@ -174,7 +179,7 @@ namespace af
     /// C++ Interface to floor numbers.
     ///
     /// \param[in] in input array
-    /// \return values rounded to nearest integer less than or equal to current value
+    /// \return       nearest integer less than or equal to `in`
     ///
     /// \ingroup arith_func_floor
     AFAPI array floor  (const array &in);
@@ -182,7 +187,7 @@ namespace af
     /// C++ Interface to ceil numbers.
     ///
     /// \param[in] in input array
-    /// \return values rounded to nearest integer greater than or equal to current value
+    /// \return       nearest integer greater than or equal to `in`
     ///
     /// \ingroup arith_func_ceil
     AFAPI array ceil   (const array &in);
@@ -192,11 +197,11 @@ namespace af
     /// C++ Interface to calculate the length of the hypotenuse of two inputs.
     ///
     /// Calculates the hypotenuse of two inputs. The inputs can be both arrays
-    /// or an array and a scalar.
+    /// or can be an array and a scalar.
     ///
     /// \param[in] lhs length of first side
     /// \param[in] rhs length of second side
-    /// \return length of the hypotenuse
+    /// \return        length of the hypotenuse
     AFAPI array hypot  (const array &lhs, const array &rhs);
 
     /// \copydoc hypot(const array&, const array&)
@@ -209,7 +214,7 @@ namespace af
     /// C++ Interface to evaluate the sine function.
     ///
     /// \param[in] in input array
-    /// \return sine
+    /// \return       sine
     ///
     /// \ingroup arith_func_sin
     AFAPI array sin    (const array &in);
@@ -217,7 +222,7 @@ namespace af
     /// C++ Interface to evaluate the cosine function.
     ///
     /// \param[in] in input array
-    /// \return cosine
+    /// \return       cosine
     ///
     /// \ingroup arith_func_cos
     AFAPI array cos    (const array &in);
@@ -225,7 +230,7 @@ namespace af
     /// C++ Interface to evaluate the tangent function.
     ///
     /// \param[in] in input array
-    /// \return tangent
+    /// \return       tangent
     ///
     /// \ingroup arith_func_tan
     AFAPI array tan    (const array &in);
@@ -233,7 +238,7 @@ namespace af
     /// C++ Interface to evaluate the inverse sine function.
     ///
     /// \param[in] in input array
-    /// \return inverse sine
+    /// \return       inverse sine
     ///
     /// \ingroup arith_func_asin
     AFAPI array asin   (const array &in);
@@ -241,7 +246,7 @@ namespace af
     /// C++ Interface to evaluate the inverse cosine function.
     ///
     /// \param[in] in input array
-    /// \return inverse cosine
+    /// \return       inverse cosine
     ///
     /// \ingroup arith_func_acos
     AFAPI array acos   (const array &in);
@@ -249,7 +254,7 @@ namespace af
     /// C++ Interface to evaluate the inverse tangent function.
     ///
     /// \param[in] in input array
-    /// \return inverse tangent
+    /// \return       inverse tangent
     ///
     /// \ingroup arith_func_atan
     AFAPI array atan   (const array &in);
@@ -260,7 +265,7 @@ namespace af
     ///
     /// \param[in] lhs value of numerator
     /// \param[in] rhs value of denominator
-    /// \return inverse tangent of the inputs
+    /// \return        inverse tangent of the inputs
     AFAPI array atan2  (const array &lhs, const array &rhs);
 
     /// \copydoc atan2(const array&, const array&)
@@ -273,7 +278,7 @@ namespace af
     /// C++ Interface to evaluate the hyperbolic sine function.
     ///
     /// \param[in] in input array
-    /// \return hyperbolic sine
+    /// \return       hyperbolic sine
     ///
     /// \ingroup arith_func_sinh
     AFAPI array sinh(const array& in);
@@ -281,7 +286,7 @@ namespace af
     /// C++ Interface to evaluate the hyperbolic cosine function.
     ///
     /// \param[in] in input array
-    /// \return hyperbolic cosine
+    /// \return       hyperbolic cosine
     ///
     /// \ingroup arith_func_cosh
     AFAPI array cosh(const array& in);
@@ -289,7 +294,7 @@ namespace af
     /// C++ Interface to evaluate the hyperbolic tangent function.
     ///
     /// \param[in] in input array
-    /// \return hyperbolic tangent
+    /// \return       hyperbolic tangent
     ///
     /// \ingroup arith_func_tanh
     AFAPI array tanh(const array& in);
@@ -297,7 +302,7 @@ namespace af
     /// C++ Interface to evaluate the inverse hyperbolic sine function.
     ///
     /// \param[in] in input array
-    /// \return inverse hyperbolic sine
+    /// \return       inverse hyperbolic sine
     ///
     /// \ingroup arith_func_asinh
     AFAPI array asinh(const array& in);
@@ -305,7 +310,7 @@ namespace af
     /// C++ Interface to evaluate the inverse hyperbolic cosine function.
     ///
     /// \param[in] in input array
-    /// \return inverse hyperbolic cosine
+    /// \return       inverse hyperbolic cosine
     ///
     /// \ingroup arith_func_acosh
     AFAPI array acosh(const array& in);
@@ -313,7 +318,7 @@ namespace af
     /// C++ Interface to evaluate the inverse hyperbolic tangent function.
     ///
     /// \param[in] in input array
-    /// \return inverse hyperbolic tangent
+    /// \return       inverse hyperbolic tangent
     ///
     /// \ingroup arith_func_atanh
     AFAPI array atanh(const array& in);
@@ -322,36 +327,44 @@ namespace af
     /// @{
     /// C++ Interface to create a complex array from a single real array.
     ///
-    /// \param[in] in a real array
-    /// \return the returned complex array
+    /// \param[in] in input array
+    /// \return       complex array
     AFAPI array complex(const array& in);
  
     /// C++ Interface to create a complex array from two real arrays.
     ///
-    /// \param[in] real_ a real array to be assigned as the real component of the returned complex array
-    /// \param[in] imag_ a real array to be assigned as the imaginary component of the returned complex array
-    /// \return the returned complex array
+    /// \param[in] real_ input array to be assigned as the real component of
+    ///                  the returned complex array
+    /// \param[in] imag_ input array to be assigned as the imaginary component
+    ///                  of the returned complex array
+    /// \return          complex array
     AFAPI array complex(const array &real_, const array &imag_);
 
-    /// C++ Interface to create a complex array from a single real array for the real component and a single scalar for each imaginary component.
+    /// C++ Interface to create a complex array from a single real array for
+    /// the real component and a single scalar for each imaginary component.
     ///
-    /// \param[in] real_ a real array to be assigned as the real component of the returned complex array
-    /// \param[in] imag_ a single scalar to be assigned as the imaginary component of each value of the returned complex array
-    /// \return the returned complex array
+    /// \param[in] real_ input array to be assigned as the real component of
+    ///                  the returned complex array
+    /// \param[in] imag_ single scalar to be assigned as the imaginary
+    ///                  component of each value of the returned complex array
+    /// \return          complex array
     AFAPI array complex(const array &real_, const double imag_);
 
-    /// C++ Interface to create a complex array from a single scalar for each real component and a single real array for the imaginary component.
+    /// C++ Interface to create a complex array from a single scalar for each
+    /// real component and a single real array for the imaginary component.
     ///
-    /// \param[in] real_ a single scalar to be assigned as the real component of each value of the returned complex array
-    /// \param[in] imag_ a real array to be assigned as the imaginary component of the returned complex array
-    /// \return the returned complex array
+    /// \param[in] real_ single scalar to be assigned as the real component of
+    ///                  each value of the returned complex array
+    /// \param[in] imag_ input array to be assigned as the imaginary component
+    ///                  of the returned complex array
+    /// \return          complex array
     AFAPI array complex(const double real_, const array &imag_);
     /// @}
 
     /// C++ Interface to return the real part of a complex array.
     ///
     /// \param[in] in input complex array
-    /// \return real part
+    /// \return       real part
     ///
     /// \ingroup arith_func_real
     AFAPI array real   (const array &in);
@@ -359,7 +372,7 @@ namespace af
     /// C++ Interface to return the imaginary part of a complex array.
     ///
     /// \param[in] in input complex array
-    /// \return imaginary part
+    /// \return       imaginary part
     ///
     /// \ingroup arith_func_imag
     AFAPI array imag   (const array &in);
@@ -367,7 +380,7 @@ namespace af
     /// C++ Interface to calculate the complex conjugate of an input array.
     ///
     /// \param[in] in input complex array
-    /// \return complex conjugate
+    /// \return       complex conjugate
     ///
     /// \ingroup arith_func_conjg
     AFAPI array conjg  (const array &in);
@@ -375,8 +388,8 @@ namespace af
     /// C++ Interface to evaluate the nth root.
     ///
     /// \param[in] nth_root nth root
-    /// \param[in] value value
-    /// \return \p nth_root th root of \p value
+    /// \param[in] value    value
+    /// \return             `nth_root` th root of `value`
     ///
     /// \ingroup arith_func_root
     AFAPI array root    (const array &nth_root, const array &value);
@@ -384,8 +397,8 @@ namespace af
     /// C++ Interface to evaluate the nth root.
     ///
     /// \param[in] nth_root nth root
-    /// \param[in] value value
-    /// \return \p nth_root th root of \p value
+    /// \param[in] value    value
+    /// \return             `nth_root` th root of `value`
     ///
     /// \ingroup arith_func_root
     AFAPI array root    (const array &nth_root, const double value);
@@ -393,8 +406,8 @@ namespace af
     /// C++ Interface to evaluate the nth root.
     ///
     /// \param[in] nth_root nth root
-    /// \param[in] value value
-    /// \return \p nth_root th root of \p value
+    /// \param[in] value    value
+    /// \return             `nth_root` th root of `value`
     ///
     /// \ingroup arith_func_root
     AFAPI array root    (const double nth_root, const array &value);
@@ -404,11 +417,12 @@ namespace af
     /// @{
     /// C++ Interface to raise a base to a power (or exponent).
     ///
-    /// Computes the value of \p base raised to the power of \p exponent. The inputs can be two arrays or an array and a scalar.
+    /// Computes the value of `base` raised to the power of `exponent`. The
+    /// inputs can be two arrays or an array and a scalar.
     ///
-    /// \param[in] base base
+    /// \param[in] base     base
     /// \param[in] exponent exponent
-    /// \return \p base raised to the power of \p exponent
+    /// \return             `base` raised to the power of `exponent`
     AFAPI array pow    (const array &base, const array &exponent);
 
     /// \copydoc pow(const array&, const array&)
@@ -419,8 +433,8 @@ namespace af
 
     /// C++ Interface to raise 2 to a power (or exponent).
     ///
-    /// \param[in] in exponent
-    /// \return 2 raised to the power
+    /// \param[in] in power
+    /// \return       2 raised to the power
     ///
     AFAPI array pow2    (const array &in);
     /// @}
@@ -428,10 +442,10 @@ namespace af
 #if AF_API_VERSION >= 31
     /// C++ Interface to evaluate the logistical sigmoid function.
     ///
-    /// \param[in] in input
-    /// \return sigmoid
+    /// Computes \f$\frac{1}{1+e^{-x}}\f$.
     /// 
-    /// \note Computes `1/(1+e^-x)`.
+    /// \param[in] in input
+    /// \return       sigmoid
     ///
     /// \ingroup arith_func_sigmoid
     AFAPI array sigmoid (const array &in);
@@ -440,57 +454,61 @@ namespace af
     /// C++ Interface to evaluate the exponential.
     ///
     /// \param[in] in exponent
-    /// \return exponential
+    /// \return       exponential
     ///
     /// \ingroup arith_func_exp
     AFAPI array exp    (const array &in);
 
-    /// C++ Interface to evaluate the exponential of an array minus 1, `exp(in) - 1`.
+    /// C++ Interface to evaluate the exponential of an array minus 1,
+    /// `exp(in) - 1`.
     ///
+    /// This function is useful when `in` is small.
+    /// 
     /// \param[in] in exponent
-    /// \return the exponential minus 1
+    /// \return       exponential minus 1
     ///
-    /// \note This function is useful when \p in is small
     /// \ingroup arith_func_expm1
     AFAPI array expm1  (const array &in);
 
     /// C++ Interface to evaluate the error function.
     ///
-    /// \param[in] in input
-    /// \return error function
+    /// \param[in] in input array
+    /// \return       error function
     ///
     /// \ingroup arith_func_erf
     AFAPI array erf    (const array &in);
 
     /// C++ Interface to evaluate the complementary error function.
     ///
-    /// \param[in] in input
-    /// \return complementary error function
+    /// \param[in] in input array
+    /// \return       complementary error function
     ///
     /// \ingroup arith_func_erfc
     AFAPI array erfc   (const array &in);
 
     /// C++ Interface to evaluate the natural logarithm.
     ///
-    /// \param[in] in input
-    /// \return natural logarithm
+    /// \param[in] in input array
+    /// \return       natural logarithm
     ///
     /// \ingroup arith_func_log
     AFAPI array log    (const array &in);
 
-    /// C++ Interface to evaluate the natural logarithm of 1 + input, `ln(1+in)`.
-    ///
+    /// C++ Interface to evaluate the natural logarithm of 1 + input,
+    /// `ln(1+in)`.
+    /// 
+    /// This function is useful when `in` is small.
+    /// 
     /// \param[in] in input
     /// \return natural logarithm of `1 + input`
     ///
-    /// \note This function is useful when \p in is small
     /// \ingroup arith_func_log1p
     AFAPI array log1p  (const array &in);
 
     /// C++ Interface to evaluate the base 10 logarithm.
     ///
     /// \param[in] in input
-    /// \return base 10 logarithm
+    /// \return       base 10 logarithm
     ///
     /// \ingroup arith_func_log10
     AFAPI array log10  (const array &in);
@@ -498,7 +516,7 @@ namespace af
     /// C++ Interface to evaluate the base 2 logarithm.
     ///
     /// \param[in] in input
-    /// \return base 2 logarithm
+    /// \return       base 2 logarithm
     ///
     /// \ingroup explog_func_log2
     AFAPI array log2   (const array &in);
@@ -506,7 +524,7 @@ namespace af
     /// C++ Interface to evaluate the square root.
     ///
     /// \param[in] in input
-    /// \return square root
+    /// \return       square root
     ///
     /// \ingroup arith_func_sqrt
     AFAPI array sqrt   (const array &in);
@@ -515,7 +533,7 @@ namespace af
     /// C++ Interface to evaluate the reciprocal square root.
     ///
     /// \param[in] in input
-    /// \return reciprocal square root
+    /// \return       reciprocal square root
     ///
     /// \ingroup arith_func_rsqrt
     AFAPI array rsqrt   (const array &in);
@@ -524,7 +542,7 @@ namespace af
     /// C++ Interface to evaluate the cube root.
     ///
     /// \param[in] in input
-    /// \return cube root
+    /// \return       cube root
     ///
     /// \ingroup arith_func_cbrt
     AFAPI array cbrt   (const array &in);
@@ -532,7 +550,7 @@ namespace af
     /// C++ Interface to calculate the factorial.
     ///
     /// \param[in] in input
-    /// \return the factorial function
+    /// \return       factorial
     ///
     /// \ingroup arith_func_factorial
     AFAPI array factorial (const array &in);
@@ -540,15 +558,16 @@ namespace af
     /// C++ Interface to evaluate the gamma function.
     ///
     /// \param[in] in input
-    /// \return gamma function
+    /// \return       gamma function
     ///
     /// \ingroup arith_func_tgamma
     AFAPI array tgamma (const array &in);
 
-    /// C++ Interface to evaluate the logarithm of the absolute value of the gamma function.
+    /// C++ Interface to evaluate the logarithm of the absolute value of the
+    /// gamma function.
     ///
     /// \param[in] in input
-    /// \return logarithm of the absolute value of the gamma function
+    /// \return       logarithm of the absolute value of the gamma function
     ///
     /// \ingroup arith_func_lgamma
     AFAPI array lgamma (const array &in);
@@ -556,7 +575,7 @@ namespace af
     /// C++ Interface to check which values are zero.
     ///
     /// \param[in] in input
-    /// \return array containing 1's where input is 0; 0's otherwise
+    /// \return       array containing 1's where input is 0; 0's otherwise
     ///
     /// \ingroup arith_func_iszero
     AFAPI array iszero (const array &in);
@@ -564,7 +583,8 @@ namespace af
     /// C++ Interface to check if values are infinite.
     ///
     /// \param[in] in input
-    /// \return array containing 1's where input is Inf or -Inf; 0's otherwise
+    /// \return       array containing 1's where input is Inf or -Inf; 0's
+    ///               otherwise
     ///
     /// \ingroup arith_func_isinf
     AFAPI array isInf  (const array &in);
@@ -572,7 +592,7 @@ namespace af
     /// C++ Interface to check if values are NaN.
     ///
     /// \param[in] in input
-    /// \return array containing 1's where input is NaN; 0's otherwise
+    /// \return       array containing 1's where input is NaN; 0's otherwise
     ///
     /// \ingroup arith_func_isnan
     AFAPI array isNaN  (const array &in);
@@ -586,11 +606,12 @@ extern "C" {
     /**
        C Interface to add two arrays.
 
-       \param[out] out sum of \p lhs and \p rhs
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   +
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_add
     */
@@ -599,11 +620,12 @@ extern "C" {
     /**
        C Interface to subtract one array from another array.
 
-       \param[out] out subtraction of \p lhs - \p rhs
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   -
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_sub
     */
@@ -612,11 +634,12 @@ extern "C" {
     /**
        C Interface to multiply two arrays.
 
-       \param[out] out product of \p lhs and \p rhs
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   *
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_mul
     */
@@ -625,89 +648,113 @@ extern "C" {
     /**
        C Interface to divide one array by another array.
 
-       \param[out] out result of \p lhs / \p rhs.
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   \
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_div
     */
     AFAPI af_err af_div   (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to perform a less-than comparison between corresponding elements of two arrays.
+       C Interface to perform a less-than comparison between corresponding
+       elements of two arrays.
+
+       Output type is b8.
 
-       \param[out] out result of \p lhs < \p rhs; type is b8
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   1's where `lhs < rhs`, else 0's
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup logic_func_lt
     */
     AFAPI af_err af_lt    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to perform a greater-than comparison between corresponding elements of two arrays.
+       C Interface to perform a greater-than comparison between corresponding
+       elements of two arrays.
 
-       \param[out] out result of \p lhs > \p rhs; type is b8
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       Output type is b8.
+
+       \param[out] out   1's where `lhs > rhs`, else 0's
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_gt
     */
     AFAPI af_err af_gt    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to perform a less-than-or-equal comparison between corresponding elements of two arrays.
+       C Interface to perform a less-than-or-equal comparison between
+       corresponding elements of two arrays.
+
+       Output type is b8.
 
-       \param[out] out result of \p lhs <= \p rhs; type is b8
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   1's where `lhs <= rhs`, else 0's
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_le
     */
     AFAPI af_err af_le    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to perform a greater-than-or-equal comparison between corresponding elements of two arrays.
+       C Interface to perform a greater-than-or-equal comparison between
+       corresponding elements of two arrays.
 
-       \param[out] out result of \p lhs >= \p rhs; type is b8
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       Output type is b8.
+
+       \param[out] out   1's where `lhs >= rhs`, else 0's
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_ge
     */
     AFAPI af_err af_ge    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to check if corresponding elements of two arrays are equal
+       C Interface to check if corresponding elements of two arrays are equal.
+
+       Output type is b8.
 
-       \param[out] out result of `lhs == rhs`; type is b8
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   1's where `lhs == rhs`, else 0's
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_eq
     */
     AFAPI af_err af_eq    (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to check if corresponding elements of two arrays are not equal
+       C Interface to check if corresponding elements of two arrays are not
+       equal.
+
+       Output type is b8.
 
-       \param[out] out result of `lhs != rhs`; type is b8
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   1's where `lhs != rhs`, else 0's
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_neq
     */
@@ -716,11 +763,14 @@ extern "C" {
     /**
        C Interface to evaluate the logical AND of two arrays.
 
-       \param[out] out result of \p lhs && \p rhs; type is b8
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       Output type is b8.
+
+       \param[out] out   1's where `lhs && rhs`, else 0's
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_and
     */
@@ -729,11 +779,14 @@ extern "C" {
     /**
        C Interface the evaluate the logical OR of two arrays.
 
-       \param[out] out result of \p lhs || \p rhs; type is b8
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       Output type is b8.
+
+       \param[out] out   1's where `lhs || rhs`, else 0's
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_or
     */
@@ -742,9 +795,12 @@ extern "C" {
     /**
        C Interface to evaluate the logical NOT of an array.
 
-       \param[out] out result of logical NOT; type is b8
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       Output type is b8.
+
+       \param[out] out !, logical NOT
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_not
     */
@@ -754,9 +810,10 @@ extern "C" {
     /**
        C Interface to evaluate the bitwise NOT of an array.
 
-       \param[out] out result of bitwise NOT
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out ~, bitwise NOT
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_bitnot
     */
@@ -766,11 +823,12 @@ extern "C" {
     /**
        C Interface to evaluate the bitwise AND of two arrays.
 
-       \param[out] out result of \p lhs & \p rhs
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   &, bitwise AND
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_bitand
     */
@@ -779,11 +837,12 @@ extern "C" {
     /**
        C Interface to evaluate the bitwise OR of two arrays.
 
-       \param[out] out result of \p lhs | \p rhs
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   |, bitwise OR
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_bitor
     */
@@ -792,11 +851,12 @@ extern "C" {
     /**
        C Interface to evaluate the bitwise XOR of two arrays.
 
-       \param[out] out result of \p lhs ^ \p rhs
-       \param[in] lhs first input
-       \param[in] rhs second input
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   ^, bitwise XOR
+       \param[in]  lhs   first input
+       \param[in]  rhs   second input
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_bitxor
     */
@@ -805,11 +865,12 @@ extern "C" {
     /**
        C Interface to shift the bits of integer arrays left.
 
-       \param[out] out result of the left shift
-       \param[in] lhs values to shift
-       \param[in] rhs n bits to shift
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   left shift
+       \param[in]  lhs   values to shift
+       \param[in]  rhs   n bits to shift
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_shiftl
     */
@@ -818,11 +879,12 @@ extern "C" {
     /**
        C Interface to shift the bits of integer arrays right.
 
-       \param[out] out result of the right shift
-       \param[in] lhs values to shift
-       \param[in] rhs n bits to shift
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   right shift
+       \param[in]  lhs   values to shift
+       \param[in]  rhs   n bits to shift
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_shiftr
     */
@@ -832,15 +894,16 @@ extern "C" {
        C Interface to cast an array from one type to another.
 
        This function casts an af_array object from one type to another. If the
-       type of the original array is the same as \p type then the same array is
+       type of the original array is the same as `type` then the same array is
        returned.
 
-       \note Consecitive casting operations may be may be optimized out if the
+       Consecutive casting operations may be may be optimized out if the
        original type of the af_array is the same as the final type. For example
-       if the original type is f64 which is then cast to f32 and then back to
-       f64, then the cast to f32 will be skipped and that operation will *NOT*
+       if the original type is f64, which is cast to f32 and then back to
+       f64, then the cast to f32 is skipped and that operation will *NOT*
        be performed by ArrayFire. The following table shows which casts will
        be optimized out. outer -> inner -> outer
+
        | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | u8 | b8 | s64 | u64 | s16 | u16 | f16 |
        |---------|-----|-----|-----|-----|-----|-----|----|----|-----|-----|-----|-----|-----|
        | f32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
@@ -856,14 +919,16 @@ extern "C" {
        | s16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
        | u16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
        | f16     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-       If you want to avoid this behavior use af_eval after the first cast
+
+       If you want to avoid this behavior use, af_eval after the first cast
        operation. This will ensure that the cast operation is performed on the
        af_array.
 
-       \param[out] out values in the specified type
-       \param[in] in input
-       \param[in] type target data type \ref af_dtype
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out  values in the specified type
+       \param[in]  in   input
+       \param[in]  type target data type \ref af_dtype
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_cast
     */
@@ -872,24 +937,27 @@ extern "C" {
     /**
        C Interface to find the elementwise minimum between two arrays.
 
-       \param[out] out minimum of \p lhs and \p rhs
-       \param[in] lhs input array
-       \param[in] rhs input array
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   minimum
+       \param[in]  lhs   input array
+       \param[in]  rhs   input array
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_min
     */
     AFAPI af_err af_minof (af_array *out, const af_array lhs, const af_array rhs, const bool batch);
 
     /**
-       C Interface to find the elementwise minimum between an array and a scalar value.
+       C Interface to find the elementwise minimum between an array and a
+       scalar value.
 
-       \param[out] out maximum of \p lhs and \p rhs
-       \param[in] lhs input array
-       \param[in] rhs input array
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   maximum
+       \param[in]  lhs   input array
+       \param[in]  rhs   input array
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_max
     */
@@ -899,12 +967,13 @@ extern "C" {
     /**
        C Interface to clamp an array between an upper and a lower limit.
 
-       \param[out] out array containing values from \p in clamped between \p lo and \p hi
-       \param[in] in input array
-       \param[in] lo lower limit array
-       \param[in] hi upper limit array
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   clamped array
+       \param[in]  in    input array
+       \param[in]  lo    lower limit array
+       \param[in]  hi    upper limit array
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_clamp
     */
@@ -915,11 +984,12 @@ extern "C" {
     /**
        C Interface to calculate the remainder.
 
-       \param[out] out remainder of \p lhs divided by \p rhs
-       \param[in] lhs numerator
-       \param[in] rhs denominator
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   remainder
+       \param[in]  lhs   numerator
+       \param[in]  rhs   denominator
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_rem
     */
@@ -928,11 +998,12 @@ extern "C" {
     /**
        C Interface to calculate the modulus.
 
-       \param[out] out \p lhs modulo \p rhs
-       \param[in] lhs dividend
-       \param[in] rhs divisor
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   modulus
+       \param[in]  lhs   dividend
+       \param[in]  rhs   divisor
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_mod
     */
@@ -942,19 +1013,22 @@ extern "C" {
        C Interface to calculate the absolute value.
 
        \param[out] out absolute value
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_abs
     */
     AFAPI af_err af_abs     (af_array *out, const af_array in);
 
     /**
-       C Interface to calculate the phase angle (in radians) of a complex array.
+       C Interface to calculate the phase angle (in radians) of a complex
+       array.
 
        \param[out] out phase angle (in radians)
-       \param[in] in input array, typically complex
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array, typically complex
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_arg
     */
@@ -964,8 +1038,9 @@ extern "C" {
        C Interface to calculate the sign of elements in an array.
 
        \param[out] out array containing 1's for negative values; 0's otherwise
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_sign
     */
@@ -974,9 +1049,10 @@ extern "C" {
     /**
        C Interface to round numbers.
 
-       \param[out] out values rounded to nearest integer
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out nearest integer
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_round
     */
@@ -985,9 +1061,10 @@ extern "C" {
     /**
        C Interface to truncate numbers.
 
-       \param[out] out nearest integer not greater in magnitude than \p in
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out nearest integer not greater in magnitude than `in`
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_trunc
     */
@@ -996,9 +1073,10 @@ extern "C" {
     /**
        C Interface to floor numbers.
 
-       \param[out] out values rounded to nearest integer less than or equal to \p in
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out nearest integer less than or equal to `in`
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_floor
     */
@@ -1007,9 +1085,10 @@ extern "C" {
     /**
        C Interface to ceil numbers.
 
-       \param[out] out values rounded to nearest integer greater than or equal to \p in
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out nearest integer greater than or equal to `in`
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_ceil
     */
@@ -1018,11 +1097,12 @@ extern "C" {
     /**
        C Interface to calculate the length of the hypotenuse of two inputs.
 
-       \param[out] out length of the hypotenuse
-       \param[in] lhs length of first side
-       \param[in] rhs length of second side
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   length of the hypotenuse
+       \param[in]  lhs   length of first side
+       \param[in]  rhs   length of second side
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_floor
     */
@@ -1032,8 +1112,9 @@ extern "C" {
        C Interface to evaluate the sine function.
 
        \param[out] out sine
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_sin
     */
@@ -1043,8 +1124,9 @@ extern "C" {
        C Interface to evaluate the cosine function.
 
        \param[out] out cosine
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_cos
     */
@@ -1054,8 +1136,9 @@ extern "C" {
        C Interface to evaluate the tangent function.
 
        \param[out] out tangent
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_tan
     */
@@ -1065,8 +1148,9 @@ extern "C" {
        C Interface to evaluate the inverse sine function.
 
        \param[out] out inverse sine
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_asin
     */
@@ -1076,8 +1160,9 @@ extern "C" {
        C Interface to evaluate the inverse cosine function.
 
        \param[out] out inverse cos
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_acos
     */
@@ -1087,8 +1172,9 @@ extern "C" {
        C Interface to evaluate the inverse tangent function.
 
        \param[out] out inverse tangent
-       \param[in] in input array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_atan
     */
@@ -1097,11 +1183,12 @@ extern "C" {
     /**
        C Interface to evaluate the inverse tangent of two arrays.
 
-       \param[out] out inverse tangent of two arrays
-       \param[in] lhs numerator
-       \param[in] rhs denominator
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   inverse tangent of two arrays
+       \param[in]  lhs   numerator
+       \param[in]  rhs   denominator
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_atan
     */
@@ -1111,8 +1198,9 @@ extern "C" {
        C Interface to evaluate the hyperbolic sine function.
 
        \param[out] out hyperbolic sine
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_sinh
     */
@@ -1122,8 +1210,9 @@ extern "C" {
        C Interface to evaluate the hyperbolic cosine function.
 
        \param[out] out hyperbolic cosine
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_cosh
     */
@@ -1133,8 +1222,9 @@ extern "C" {
        C Interface to evaluate the hyperbolic tangent function.
 
        \param[out] out hyperbolic tangent
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_tanh
     */
@@ -1144,8 +1234,9 @@ extern "C" {
        C Interface to evaluate the inverse hyperbolic sine function.
 
        \param[out] out inverse hyperbolic sine
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_asinh
     */
@@ -1155,8 +1246,9 @@ extern "C" {
        C Interface to evaluate the inverse hyperbolic cosine function.
 
        \param[out] out inverse hyperbolic cosine
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_acosh
     */
@@ -1166,8 +1258,9 @@ extern "C" {
        C Interface to evaluate the inverse hyperbolic tangent function.
 
        \param[out] out inverse hyperbolic tangent
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_atanh
     */
@@ -1177,8 +1270,9 @@ extern "C" {
        C Interface to create a complex array from a single real array.
 
        \param[out] out complex array
-       \param[in] in real array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  real array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_cplx
     */
@@ -1187,11 +1281,14 @@ extern "C" {
     /**
        C Interface to create a complex array from two real arrays.
 
-       \param[out] out complex array
-       \param[in] real real array to be assigned as the real component of the returned complex array
-       \param[in] imag real array to be assigned as the imaginary component of the returned complex array
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   complex array
+       \param[in]  real  real array to be assigned as the real component of the
+                         returned complex array
+       \param[in]  imag  real array to be assigned as the imaginary component
+                         of the returned complex array
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_cplx
     */
@@ -1201,8 +1298,9 @@ extern "C" {
        C Interface to return the real part of a complex array.
 
        \param[out] out real part
-       \param[in] in complex array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  complex array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_real
     */
@@ -1212,8 +1310,9 @@ extern "C" {
        C Interface to return the imaginary part of a complex array.
 
        \param[out] out imaginary part
-       \param[in] in complex array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  complex array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_imag
     */
@@ -1223,8 +1322,9 @@ extern "C" {
        C Interface to evaluate the complex conjugate of an input array.
 
        \param[out] out complex conjugate
-       \param[in] in complex array
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  complex array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_conjg
     */
@@ -1233,11 +1333,12 @@ extern "C" {
     /**
        C Interface to evaluate the nth root.
 
-       \param[out] out \p lhs th root of \p rhs
-       \param[in] lhs nth root
-       \param[in] rhs value
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   `lhs` th root of `rhs`
+       \param[in]  lhs   nth root
+       \param[in]  rhs   value
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_root
     */
@@ -1247,11 +1348,12 @@ extern "C" {
     /**
        C Interface to raise a base to a power (or exponent).
 
-       \param[out] out \p lhs raised to the power of \p rhs
-       \param[in] lhs base
-       \param[in] rhs exponent
-       \param[in] batch specifies if operations need to be performed in batch mode
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out   `lhs` raised to the power of `rhs`
+       \param[in]  lhs   base
+       \param[in]  rhs   exponent
+       \param[in]  batch batch mode
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_pow
     */
@@ -1260,9 +1362,10 @@ extern "C" {
     /**
        C Interface to raise 2 to a power (or exponent).
 
-       \param[out] out 2 raised to the power of \p in
-       \param[in] in exponent
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out 2 raised to the power of `in`
+       \param[in]  in  exponent
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_pow2
     */
@@ -1272,11 +1375,12 @@ extern "C" {
     /**
        C Interface to evaluate the logistical sigmoid function.
 
-       Computes `1/(1+e^-x)`.
+       Computes \f$\frac{1}{1+e^{-x}}\f$.
 
        \param[out] out output of the logistic sigmoid function
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_sigmoid
     */
@@ -1286,20 +1390,23 @@ extern "C" {
     /**
        C Interface to evaluate the exponential.
 
-       \param[out] out e raised to the power of \p in
-       \param[in] in exponent
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out e raised to the power of `in`
+       \param[in]  in  exponent
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_exp
     */
     AFAPI af_err af_exp     (af_array *out, const af_array in);
 
     /**
-       C Interface to evaluate the exponential of an array minus 1, `exp(in) - 1`.
+       C Interface to evaluate the exponential of an array minus 1,
+       `exp(in) - 1`.
 
        \param[out] out exponential of `in - 1`
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_expm1
     */
@@ -1309,8 +1416,9 @@ extern "C" {
        C Interface to evaluate the error function.
 
        \param[out] out error function value
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_erf
     */
@@ -1320,8 +1428,9 @@ extern "C" {
        C Interface to evaluate the complementary error function.
 
        \param[out] out complementary error function
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_erfc
     */
@@ -1331,8 +1440,9 @@ extern "C" {
        C Interface to evaluate the natural logarithm.
 
        \param[out] out natural logarithm
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_log
     */
@@ -1342,8 +1452,9 @@ extern "C" {
        C Interface to evaluate the natural logarithm of 1 + input, `ln(1+in)`.
 
        \param[out] out logarithm of `in + 1`
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_log1p
     */
@@ -1353,8 +1464,9 @@ extern "C" {
        C Interface to evaluate the base 10 logarithm.
 
        \param[out] out base 10 logarithm
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_log10
     */
@@ -1364,8 +1476,9 @@ extern "C" {
        C Interface to evaluate the base 2 logarithm.
 
        \param[out] out base 2 logarithm
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup explog_func_log2
     */
@@ -1375,8 +1488,9 @@ extern "C" {
        C Interface to evaluate the square root.
 
        \param[out] out square root
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_sqrt
     */
@@ -1387,8 +1501,9 @@ extern "C" {
       C Interface to evaluate the reciprocal square root.
 
       \param[out] out reciprocal square root
-      \param[in] in input
-      \return \ref AF_SUCCESS if the execution completes properly
+      \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
       \ingroup arith_func_rsqrt
     */
@@ -1398,8 +1513,9 @@ extern "C" {
        C Interface to evaluate the cube root.
 
        \param[out] out cube root
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_cbrt
     */
@@ -1409,8 +1525,9 @@ extern "C" {
        C Interface to calculate the factorial.
 
        \param[out] out factorial
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_factorial
     */
@@ -1420,54 +1537,61 @@ extern "C" {
        C Interface to evaluate the gamma function.
 
        \param[out] out gamma function
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_tgamma
     */
     AFAPI af_err af_tgamma   (af_array *out, const af_array in);
 
     /**
-       C Interface to evaluate the logarithm of the absolute value of the gamma function.
+       C Interface to evaluate the logarithm of the absolute value of the
+       gamma function.
 
        \param[out] out logarithm of the absolute value of the gamma function
-       \param[in] in input
-       \return \ref AF_SUCCESS if the execution completes properly
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup arith_func_lgamma
     */
     AFAPI af_err af_lgamma   (af_array *out, const af_array in);
 
     /**
-        C Interface to check if values are zero.
+       C Interface to check if values are zero.
 
-        \param[out] out array containing 1's where input is 0; 0's otherwise
-        \param[in] in input
-        \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out array containing 1's where input is 0; 0's otherwise
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup arith_func_iszero
+       \ingroup arith_func_iszero
     */
     AFAPI af_err af_iszero  (af_array *out, const af_array in);
 
     /**
-        C Interface to check if values are infinite.
+       C Interface to check if values are infinite.
 
-        \param[out] out array containing 1's where input is Inf or -Inf; 0's otherwise
-        \param[in] in input
-        \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out array containing 1's where input is Inf or -Inf; 0's
+                       otherwise
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup arith_func_isinf
+       \ingroup arith_func_isinf
     */
     AFAPI af_err af_isinf   (af_array *out, const af_array in);
 
     /**
-        C Interface to check if values are NaN.
+       C Interface to check if values are NaN.
 
-        \param[out] out array containing 1's where input is NaN; 0's otherwise
-        \param[in] in input
-        \return \ref AF_SUCCESS if the execution completes properly
+       \param[out] out array containing 1's where input is NaN; 0's otherwise
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup arith_func_isnan
+       \ingroup arith_func_isnan
     */
     AFAPI af_err af_isnan   (af_array *out, const af_array in);
 
diff --git a/include/af/blas.h b/include/af/blas.h
index d20986b215..4580ea2112 100644
--- a/include/af/blas.h
+++ b/include/af/blas.h
@@ -1,4 +1,4 @@
-/*******************************************************
+/********************************************************
  * Copyright (c) 2014, ArrayFire
  * All rights reserved.
  *
@@ -7,15 +7,7 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
-/** \file blas.h
- *
- * Contains BLAS related functions
- *
- * Contains functions for basic BLAS functionallity
- */
-
 #pragma once
-
 #include <af/defines.h>
 
 #ifdef __cplusplus
@@ -23,93 +15,95 @@ namespace af
 {
     class array;
     /**
-        \brief Matrix multiply of two arrays
+       C++ Interface to multiply two matrices.
 
-        \copydetails blas_func_matmul
+       \copydetails blas_func_matmul
 
-        \param[in] lhs The array object on the left hand side
-        \param[in] rhs The array object on the right hand side
-        \param[in] optLhs Transpose left hand side before the function is performed
-        \param[in] optRhs Transpose right hand side before the function is performed
-        \return The result of the matrix multiplication of lhs, rhs
+       `optLhs` and `optRhs` can only be one of \ref AF_MAT_NONE,
+       \ref AF_MAT_TRANS, \ref AF_MAT_CTRANS.
 
-        \note optLhs and optRhs can only be one of \ref AF_MAT_NONE, \ref
-              AF_MAT_TRANS, \ref AF_MAT_CTRANS \note This function is not supported
-              in GFOR
+       This function is not supported in GFOR.
 
-        \note <b> The following applies for Sparse-Dense matrix multiplication.</b>
-        \note This function can be used with one sparse input. The sparse input
-              must always be the \p lhs and the dense matrix must be \p rhs.
-        \note The sparse array can only be of \ref AF_STORAGE_CSR format.
-        \note The returned array is always dense.
-        \note \p optLhs an only be one of \ref AF_MAT_NONE, \ref AF_MAT_TRANS,
-              \ref AF_MAT_CTRANS.
-        \note \p optRhs can only be \ref AF_MAT_NONE.
+       \note <b>The following applies for Sparse-Dense matrix multiplication.</b>
+       \note This function can be used with one sparse input. The sparse input
+             must always be the \p lhs and the dense matrix must be \p rhs.
+       \note The sparse array can only be of \ref AF_STORAGE_CSR format.
+       \note The returned array is always dense.
+       \note \p optLhs an only be one of \ref AF_MAT_NONE, \ref AF_MAT_TRANS,
+             \ref AF_MAT_CTRANS.
+       \note \p optRhs can only be \ref AF_MAT_NONE.
 
-        \ingroup blas_func_matmul
+       \param[in] lhs    input array on the left-hand side
+       \param[in] rhs    input array on the right-hand side
+       \param[in] optLhs transpose the left-hand side prior to multiplication
+       \param[in] optRhs transpose the right-hand side prior to multiplication
+       \return    `lhs` * `rhs`
 
-     */
+       \ingroup blas_func_matmul
+    */
     AFAPI array matmul(const array &lhs, const array &rhs,
                        const matProp optLhs = AF_MAT_NONE,
                        const matProp optRhs = AF_MAT_NONE);
 
     /**
-       \brief Matrix multiply of two arrays
+       C++ Interface to multiply two matrices.
+       The second matrix will be transposed.
 
        \copydetails blas_func_matmul
 
-       \param[in] lhs The array object on the left hand side
-       \param[in] rhs The array object on the right hand side
-       \return The result of the matrix multiplication of \p lhs, transpose(\p rhs)
+       This function is not supported in GFOR.
 
-       \note This function is not supported in GFOR
+       \param[in] lhs input array on the left-hand side
+       \param[in] rhs input array on the right-hand side
+       \return    `lhs` * transpose(`rhs`)
 
        \ingroup blas_func_matmul
     */
     AFAPI array matmulNT(const array &lhs, const array &rhs);
 
     /**
-       \brief Matrix multiply of two arrays
+       C++ Interface to multiply two matrices.
+       The first matrix will be transposed.
 
        \copydetails blas_func_matmul
 
-       \param[in] lhs The array object on the left hand side
-       \param[in] rhs The array object on the right hand side
-       \return The result of the matrix multiplication of transpose(\p lhs), \p rhs
+       This function is not supported in GFOR.
 
-       \note This function is not supported in GFOR
+       \param[in] lhs input array on the left-hand side
+       \param[in] rhs input array on the right-hand side
+       \return    transpose(`lhs`) * `rhs`
 
        \ingroup blas_func_matmul
     */
     AFAPI array matmulTN(const array &lhs, const array &rhs);
 
     /**
-       \brief Matrix multiply of two arrays
+       C++ Interface to multiply two matrices.
+       Both matrices will be transposed.
 
        \copydetails blas_func_matmul
 
-       \param[in] lhs The array object on the left hand side
-       \param[in] rhs The array object on the right hand side
-       \return The result of the matrix multiplication of transpose(\p lhs), transpose(\p rhs)
+       This function is not supported in GFOR.
 
-       \note This function is not supported in GFOR
+       \param[in] lhs input array on the left-hand side
+       \param[in] rhs input array on the right-hand side
+       \return    transpose(`lhs`) * transpose(`rhs`)
 
        \ingroup blas_func_matmul
     */
     AFAPI array matmulTT(const array &lhs, const array &rhs);
 
     /**
-       \brief Chain 2 matrix multiplications
+       C++ Interface to chain multiply three matrices.
 
-       The matrix multiplications are done in a way to reduce temporary memory
+       The matrix multiplications are done in a way to reduce temporary memory.
+
+       This function is not supported in GFOR.
 
        \param[in] a The first array
        \param[in] b The second array
        \param[in] c The third array
-
-       \returns out = a x b x c
-
-       \note This function is not supported in GFOR
+       \return    a x b x c
 
        \ingroup blas_func_matmul
     */
@@ -117,18 +111,17 @@ namespace af
 
 
     /**
-       \brief Chain 3 matrix multiplications
+       C++ Interface to chain multiply three matrices.
 
-       The matrix multiplications are done in a way to reduce temporary memory
+       The matrix multiplications are done in a way to reduce temporary memory.
+
+       This function is not supported in GFOR.
 
        \param[in] a The first array
        \param[in] b The second array
        \param[in] c The third array
        \param[in] d The fourth array
-
-       \returns out = a x b x c x d
-
-       \note This function is not supported in GFOR
+       \returns   a x b x c x d
 
        \ingroup blas_func_matmul
     */
@@ -136,36 +129,34 @@ namespace af
 
 #if AF_API_VERSION >= 35
     /**
-        \brief Dot Product
+        C++ Interface to compute the dot product.
 
-        Scalar dot product between two vectors. Also referred to as the inner
+        Scalar dot product between two vectors, also referred to as the inner
         product.
 
         \code
           // compute scalar dot product
-          array x = randu(100),
-          y = randu(100);
+          array x = randu(100), y = randu(100);
 
           af_print(dot(x, y));
           // OR
           printf("%f\n", dot<float>(x, y));
-
         \endcode
 
-        \tparam T The type of the output
-        \param[in] lhs The array object on the left hand side
-        \param[in] rhs The array object on the right hand side
-        \param[in] optLhs Options for lhs. Currently only \ref AF_MAT_NONE and
-                  AF_MAT_CONJ are supported.
-        \param[in] optRhs Options for rhs. Currently only \ref AF_MAT_NONE and
-        AF_MAT_CONJ are supported \return The result of the dot product of lhs,
-        rhs
-
-        \note optLhs and optRhs can only be one of \ref AF_MAT_NONE or \ref
-              AF_MAT_CONJ
-        \note optLhs = AF_MAT_CONJ and optRhs = AF_MAT_NONE will run
-              conjugate dot operation.
-        \note This function is not supported in GFOR
+       Parameters `optLhs` and `optRhs` can only be one of \ref AF_MAT_NONE or
+       \ref AF_MAT_CONJ. The conjugate dot product can be computed by setting
+       `optLhs = AF_MAT_CONJ` and `optRhs = AF_MAT_NONE`.
+
+       This function is not supported in GFOR.
+
+        \tparam    T      type of the output
+        \param[in] lhs    input array on the left-hand side
+        \param[in] rhs    input array on the right-hand side
+        \param[in] optLhs `lhs` options, only \ref AF_MAT_NONE and \ref
+                          AF_MAT_CONJ are supported
+        \param[in] optRhs `rhs` options, only \ref AF_MAT_NONE and \ref
+                          AF_MAT_CONJ are supported
+        \return    dot product of `lhs` and `rhs`
 
         \ingroup blas_func_dot
     */
@@ -181,20 +172,21 @@ namespace af
                     const matProp optRhs = AF_MAT_NONE);
 
     /**
-        \brief C++ Interface for transposing a matrix
+        C++ Interface to transpose a matrix.
+
+        \param[in] in        input array
+        \param[in] conjugate if true, conjugate transposition is performed
+        \return    transpose
 
-        \param[in] in an input matrix
-        \param[in] conjugate if true, a conjugate transposition is performed
-        \return the transposed matrix
         \ingroup blas_func_transpose
     */
     AFAPI array transpose(const array &in, const bool conjugate = false);
 
     /**
-        \brief C++ Interface for transposing a matrix in-place
+        C++ Interface to transpose a matrix in-place.
 
-        \param[in,out] in the matrix to be transposed in-place
-        \param[in] conjugate if true, a conjugate transposition is performed
+        \param[in,out] in        input array to be transposed in-place
+        \param[in]     conjugate if true, conjugate transposition is performed
 
         \ingroup blas_func_transpose
     */
@@ -208,11 +200,10 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-        \brief BLAS general matrix multiply (GEMM) of two \ref af_array objects
+        C Interface to multiply two matrices.
 
-        \details
-        This provides a general interface to the BLAS level 3 general matrix
-        multiply (GEMM), which is generally defined as:
+        This provides an interface to the BLAS level 3 general matrix multiply
+        (GEMM) of two \ref af_array objects, which is generally defined as:
 
         \f[
         C = \alpha * opA(A)opB(B) + \beta * C
@@ -251,23 +242,15 @@ extern "C" {
 
         \snippet test/blas.cpp ex_af_gemm_overwrite
 
-        \param[in,out] C     Pointer to the output \ref af_array
-
-        \param[in]     opA   Operation to perform on A before the multiplication
-
-        \param[in]     opB   Operation to perform on B before the multiplication
-
-        \param[in]     alpha The alpha value; must be the same type as \p lhs
-                            and \p rhs
-
-        \param[in]     A     Left-hand side operand
-
-        \param[in]     B     Right-hand side operand
-
-        \param[in]     beta  The beta value; must be the same type as \p lhs
-                            and \p rhs
-
-        \return AF_SUCCESS if the operation is successful.
+        \param[in,out] C     `A` * `B` = `C`
+        \param[in]     opA   operation to perform on A before the multiplication
+        \param[in]     opB   operation to perform on B before the multiplication
+        \param[in]     alpha alpha value; must be the same type as `A` and `B`
+        \param[in]     A     input array on the left-hand side
+        \param[in]     B     input array on the right-hand side
+        \param[in]     beta  beta value; must be the same type as `A` and `B`
+        \return        \ref AF_SUCCESS, if function returns successfully, else
+                       an \ref af_err code is given
 
         \ingroup blas_func_matmul
     */
@@ -277,17 +260,9 @@ extern "C" {
 #endif
 
     /**
-        \brief Matrix multiply of two \ref af_array
-
-        \details Performs a matrix multiplication on two arrays (lhs, rhs).
+        C Interface to multiply two matrices.
 
-        \param[out] out Pointer to the output \ref af_array
-        \param[in] lhs A 2D matrix \ref af_array object
-        \param[in] rhs A 2D matrix \ref af_array object
-        \param[in] optLhs Transpose left hand side before the function is performed
-        \param[in] optRhs Transpose right hand side before the function is performed
-
-        \return AF_SUCCESS if the process is successful.
+        Performs matrix multiplication on two arrays.
 
         \note <b> The following applies for Sparse-Dense matrix multiplication.</b>
         \note This function can be used with one sparse input. The sparse input
@@ -298,30 +273,41 @@ extern "C" {
               \ref AF_MAT_CTRANS.
         \note \p optRhs can only be \ref AF_MAT_NONE.
 
+        \param[out] out    `lhs` * `rhs` = `out`
+        \param[in]  lhs    input array on the left-hand side
+        \param[in]  rhs    input array on the right-hand side
+        \param[in]  optLhs transpose `lhs` before the function is performed
+        \param[in]  optRhs transpose `rhs` before the function is performed
+        \return     \ref AF_SUCCESS, if function returns successfully, else
+                    an \ref af_err code is given
+
         \ingroup blas_func_matmul
      */
     AFAPI af_err af_matmul( af_array *out ,
                             const af_array lhs, const af_array rhs,
                             const af_mat_prop optLhs, const af_mat_prop optRhs);
 
-
     /**
-        Scalar dot product between two vectors.  Also referred to as the inner
+        C Interface to compute the dot product.
+
+        Scalar dot product between two vectors, also referred to as the inner
         product.
 
         \code
-        // compute scalar dot product
-        array x = randu(100), y = randu(100);
-        print(dot<float>(x,y));
+          // compute scalar dot product
+          array x = randu(100), y = randu(100);
+          print(dot<float>(x,y));
         \endcode
 
-        \param[out] out The array object with the result of the dot operation
-        \param[in] lhs The array object on the left hand side
-        \param[in] rhs The array object on the right hand side
-        \param[in] optLhs Options for lhs. Currently only \ref AF_MAT_NONE and
-                   AF_MAT_CONJ are supported.
-        \param[in] optRhs Options for rhs. Currently only \ref AF_MAT_NONE and AF_MAT_CONJ are supported
-        \return AF_SUCCESS if the process is successful.
+        \param[out] out    dot product of `lhs` and `rhs`
+        \param[in]  lhs    input array on the left-hand side
+        \param[in]  rhs    input array on the right-hand side
+        \param[in]  optLhs `lhs` options, only \ref AF_MAT_NONE and \ref
+                           AF_MAT_CONJ are supported
+        \param[in]  optRhs `rhs` options, only \ref AF_MAT_NONE and \ref
+                           AF_MAT_CONJ are supported
+        \return     \ref AF_SUCCESS, if function returns successfully, else
+                    an \ref af_err code is given
 
         \ingroup blas_func_dot
     */
@@ -331,18 +317,21 @@ extern "C" {
 
 #if AF_API_VERSION >= 35
     /**
+        C Interface to compute the dot product, scalar result returned on host.
+
         Scalar dot product between two vectors. Also referred to as the inner
         product. Returns the result as a host scalar.
 
-        \param[out] real is the real component of the result of dot operation
-        \param[out] imag is the imaginary component of the result of dot operation
-        \param[in] lhs The array object on the left hand side
-        \param[in] rhs The array object on the right hand side
-        \param[in] optLhs Options for lhs. Currently only \ref AF_MAT_NONE and
-                   AF_MAT_CONJ are supported.
-        \param[in] optRhs Options for rhs. Currently only \ref AF_MAT_NONE and AF_MAT_CONJ are supported
-
-        \return AF_SUCCESS if the process is successful.
+        \param[out] real   real component of the dot product
+        \param[out] imag   imaginary component of the dot product
+        \param[in]  lhs    input array on the left-hand side
+        \param[in]  rhs    input array on the right-hand side
+        \param[in]  optLhs `lhs` options, only \ref AF_MAT_NONE and \ref
+                           AF_MAT_CONJ are supported
+        \param[in]  optRhs `rhs` options, only \ref AF_MAT_NONE and \ref
+                           AF_MAT_CONJ are supported
+        \return     \ref AF_SUCCESS, if function returns successfully, else
+                    an \ref af_err code is given
 
         \ingroup blas_func_dot
     */
@@ -352,22 +341,25 @@ extern "C" {
 #endif
 
     /**
-        \brief C Interface for transposing a matrix
+        C Interface to transpose a matrix.
 
-        \param[out] out the transposed matrix
-        \param[in] in an input matrix
-        \param[in] conjugate if true, a conjugate transposition is performed
+        \param[out] out       transpose
+        \param[in]  in        input array
+        \param[in]  conjugate if true, conjugate transposition is performed
+        \return     \ref AF_SUCCESS, if function returns successfully, else
+                    an \ref af_err code is given
 
-        \return AF_SUCCESS if the process is successful.
         \ingroup blas_func_transpose
     */
     AFAPI af_err af_transpose(af_array *out, af_array in, const bool conjugate);
 
     /**
-        \brief C Interface for transposing a matrix in-place
+        C Interface to transpose a matrix in-place.
 
-        \param[in,out] in is the matrix to be transposed in place
-        \param[in] conjugate if true, a conjugate transposition is performed
+        \param[in,out] in        input array to be transposed in-place
+        \param[in]     conjugate if true, conjugate transposition is performed
+        \return        \ref AF_SUCCESS, if function returns successfully, else
+                       an \ref af_err code is given
 
         \ingroup blas_func_transpose
     */
diff --git a/include/af/data.h b/include/af/data.h
index 1559ea204f..22e1874439 100644
--- a/include/af/data.h
+++ b/include/af/data.h
@@ -17,509 +17,479 @@ namespace af
 {
     class array;
 
-    /**
-        \param[in] val is the value of each element of the array be genrated
-        \param[in] dims is the dimensions of the array to be generated
-        \param[in] ty is the type of the array
-
-        \return array of size \p dims
-
-        \ingroup data_func_constant
-    */
-
+    /// C++ Interface to generate an array with elements set to a specified
+    /// value.
+    ///
+    /// \param[in] val  constant value
+    /// \param[in] dims dimensions of the array to be generated
+    /// \param[in] ty   type
+    /// \return         constant array
+    ///
+    /// \ingroup data_func_constant
     template<typename T>
     array constant(T val, const dim4 &dims, const dtype ty=(af_dtype)dtype_traits<T>::ctype);
 
-    /**
-        \param[in] val is the value of each element of the array to be generated
-        \param[in] d0 is the size of the array to be generated
-        \param[in] ty is the type of the array
-
-        \return array of size \p d0
-
-        \ingroup data_func_constant
-    */
-
+    /// C++ Interface to generate a 1-D array with elements set to a specified
+    /// value.
+    ///
+    /// \param[in] val constant value
+    /// \param[in] d0  size of the first dimension
+    /// \param[in] ty  type
+    /// \return        constant 1-D array
+    ///
+    /// \ingroup data_func_constant
     template<typename T>
     array constant(T val, const dim_t d0, const af_dtype ty=(af_dtype)dtype_traits<T>::ctype);
 
-    /**
-        \param[in] val is the value of each element of the array to be generated
-        \param[in] d0 is the number of rows of the array to be generated
-        \param[in] d1 is the number of columns of the array to be generated
-        \param[in] ty is the type of the array
-
-        \return array of size \p d0 x d1
-
-        \ingroup data_func_constant
-    */
+    /// C++ Interface to generate a 2-D array with elements set to a specified
+    /// value.
+    ///
+    /// \param[in] val constant value
+    /// \param[in] d0  size of the first dimension
+    /// \param[in] d1  size of the second dimension
+    /// \param[in] ty  type
+    /// \return        constant 2-D array
+    ///
+    /// \ingroup data_func_constant
     template<typename T>
     array constant(T val, const dim_t d0, const dim_t d1, const af_dtype ty=(af_dtype)dtype_traits<T>::ctype);
 
-    /**
-        \param[in] val is the value of each element of the array to be generated
-        \param[in] d0 is the size of the 1st dimension of the array to be generated
-        \param[in] d1 is the size of the 2nd dimension of the array to be generated
-        \param[in] d2 is the size of the 3rd dimension of the array to be generated
-        \param[in] ty is the type of the array
-
-        \return array of size \p d0 x d1 x d2
-
-        \ingroup data_func_constant
-    */
+    /// C++ Interface to generate a 3-D array with elements set to a specified
+    /// value.
+    ///
+    /// \param[in] val constant value
+    /// \param[in] d0  size of the first dimension
+    /// \param[in] d1  size of the second dimension
+    /// \param[in] d2  size of the third dimension
+    /// \param[in] ty  type
+    /// \return        constant 3-D array
+    ///
+    /// \ingroup data_func_constant
     template<typename T>
     array constant(T val, const dim_t d0, const dim_t d1, const dim_t d2, const af_dtype ty=(af_dtype)dtype_traits<T>::ctype);
 
-    /**
-        \param[in] val is the value of each element of the array to be generated
-        \param[in] d0 is the size of the 1st dimension of the array to be generated
-        \param[in] d1 is the size of the 2nd dimension of the array to be generated
-        \param[in] d2 is the size of the 3rd dimension of the array to be generated
-        \param[in] d3 is the size of the 4rd dimension of the array to be generated
-        \param[in] ty is the type of the array
-
-        \return array of size \p d0 x d1 x d2 x d3
-
-        \ingroup data_func_constant
-    */
+    /// C++ Interface to generate a 4-D array with elements set to a specified
+    /// value.
+    ///
+    /// \param[in] val constant value
+    /// \param[in] d0  size of the first dimension
+    /// \param[in] d1  size of the second dimension
+    /// \param[in] d2  size of the third dimension
+    /// \param[in] d3  size of the fourth dimension
+    /// \param[in] ty  type
+    /// \return        constant 4-D array
+    ///
+    /// \ingroup data_func_constant
     template<typename T>
     array constant(T val, const dim_t d0, const dim_t d1, const dim_t d2, const dim_t d3, const af_dtype ty=(af_dtype)dtype_traits<T>::ctype);
 
-    /**
-        \param[in] dims is dim4 for size of all dimensions
-        \param[in] ty is the type of array to generate
-
-        \returns an identity array of specified dimension and type
-
-        \ingroup data_func_identity
-    */
+    /// C++ Interface to generate an identity array.
+    ///
+    /// \param[in] dims size
+    /// \param[in] ty   type
+    /// \return         identity array
+    ///
+    /// \ingroup data_func_identity
     AFAPI array identity(const dim4 &dims, const dtype ty=f32);
 
-    /**
-        \param[in] d0 is size of first dimension
-        \param[in] ty is the type of array to generate
-
-        \returns an identity array of specified dimension and type
-
-        \ingroup data_func_identity
-    */
+    /// C++ Interface to generate a 1-D identity array.
+    ///
+    /// \param[in] d0 size of the first dimension
+    /// \param[in] ty type
+    /// \return       identity array
+    ///
+    /// \ingroup data_func_identity
     AFAPI array identity(const dim_t d0, const dtype ty=f32);
 
-    /**
-        \param[in] d0 is size of first dimension
-        \param[in] d1 is size of second dimension
-        \param[in] ty is the type of array to generate
-
-        \returns an identity array of specified dimension and type
-
-        \ingroup data_func_identity
-    */
+    /// C++ Interface to generate a 2-D identity array.
+    ///
+    /// \param[in] d0 size of the first dimension
+    /// \param[in] d1 size of the second dimension
+    /// \param[in] ty type
+    /// \return       identity array
+    ///
+    /// \ingroup data_func_identity
     AFAPI array identity(const dim_t d0, const dim_t d1, const dtype ty=f32);
 
-    /**
-        \param[in] d0 is size of first dimension
-        \param[in] d1 is size of second dimension
-        \param[in] d2 is size of third dimension
-        \param[in] ty is the type of array to generate
-
-        \returns an identity array of specified dimension and type
-
-        \ingroup data_func_identity
-    */
+    /// C++ Interface to generate a 3-D identity array.
+    ///
+    /// \param[in] d0 size of the first dimension
+    /// \param[in] d1 size of the second dimension
+    /// \param[in] d2 size of the third dimension
+    /// \param[in] ty type
+    /// \return       identity array
+    ///
+    /// \ingroup data_func_identity
     AFAPI array identity(const dim_t d0, const dim_t d1,
                          const dim_t d2, const dtype ty=f32);
 
-    /**
-        \param[in] d0 is size of first dimension
-        \param[in] d1 is size of second dimension
-        \param[in] d2 is size of third dimension
-        \param[in] d3 is size of fourth dimension
-        \param[in] ty is the type of array to generate
-
-        \returns an identity array of specified dimension and type
-
-        \ingroup data_func_identity
-    */
+    /// C++ Interface to generate a 4-D identity array.
+    ///
+    /// \param[in] d0 size of the first dimension
+    /// \param[in] d1 size of the second dimension
+    /// \param[in] d2 size of the third dimension
+    /// \param[in] d3 size of the fourth dimension
+    /// \param[in] ty type
+    /// \return       identity array
+    ///
+    /// \ingroup data_func_identity
     AFAPI array identity(const dim_t d0, const dim_t d1,
                          const dim_t d2, const dim_t d3, const dtype ty=f32);
 
-    /**
-    *  C++ Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions of shape `dim4`.
-    *
-        \param[in] dims the `dim4` object describing the shape of the generated array
-        \param[in] seq_dim the dimesion along which `[0, dim[seq_dim] - 1]` is created
-        \param[in] ty the type of the generated array
-
-        \returns the generated array
-
-        \ingroup data_func_range
-    */
+    /// C++ Interface to generate an array with `[0, n-1]` values along the
+    /// `seq_dim` dimension and tiled across other dimensions of shape `dim4`.
+    ///
+    /// \param[in] dims    size
+    /// \param[in] seq_dim dimesion along which the range is created
+    /// \param[in] ty      type
+    /// \return            range array
+    ///
+    /// \ingroup data_func_range
     AFAPI array range(const dim4 &dims, const int seq_dim = -1, const dtype ty=f32);
 
-    /**
-    *  C++ Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions described by dimension parameters.
-    *
-        \param[in] d0 the size of first dimension
-        \param[in] d1 the size of second dimension
-        \param[in] d2 the size of third dimension
-        \param[in] d3 the size of fourth dimension
-        \param[in] seq_dim the dimesion along which `[0, dim[seq_dim] - 1]` is created
-        \param[in] ty the type of the generated array
-
-        \returns the generated array
-
-        \ingroup data_func_range
-    */
+    /// C++ Interface to generate an array with `[0, n-1]` values along the
+    /// `seq_dim` dimension and tiled across other dimensions described by
+    /// dimension parameters.
+    ///
+    /// \param[in] d0      size of the first dimension
+    /// \param[in] d1      size of the second dimension
+    /// \param[in] d2      size of the third dimension
+    /// \param[in] d3      size of the fourth dimension
+    /// \param[in] seq_dim dimesion along which the range is created
+    /// \param[in] ty      type
+    /// \return            range array
+    ///
+    /// \ingroup data_func_range
     AFAPI array range(const dim_t d0, const dim_t d1 = 1, const dim_t d2 = 1,
                       const dim_t d3 = 1, const int seq_dim = -1, const dtype ty=f32);
 
-    /**
-        \param[in] dims is dim4 for unit dimensions of the sequence to be generated
-        \param[in] tile_dims is dim4 for the number of repetitions of the unit dimensions
-        \param[in] ty is the type of array to generate
-
-        \returns an array of integral range specified dimension and type
-
-        \ingroup data_func_iota
-    */
+    /// C++ Interface to generate an array with `[0, n-1]` values modified to
+    /// specified dimensions and tiling.
+    ///
+    /// \param[in] dims      size
+    /// \param[in] tile_dims number of tiled repetitions in each dimension
+    /// \param[in] ty        type
+    /// \return              iota array
+    ///
+    /// \ingroup data_func_iota
     AFAPI array iota(const dim4 &dims, const dim4 &tile_dims = dim4(1), const dtype ty=f32);
 
-    /**
-        \param[in] in is the input array
-        \param[in] num is the diagonal index
-        \param[in] extract when true returns an array containing diagonal of tha matrix
-        and when false returns a matrix with \p in as diagonal
-
-        \returns an array with either the diagonal or the matrix based on \p extract
-
-        \ingroup data_func_diag
-    */
+    /// C++ Interface to extract the diagonal from an array.
+    ///
+    /// \param[in] in      input array
+    /// \param[in] num     diagonal index
+    /// \param[in] extract if true, returns an array containing diagonal of the
+    ///                    matrix; if false, returns a diagonal matrix
+    /// \return            diagonal array (or matrix)
+    ///
+    /// \ingroup data_func_diag
     AFAPI array diag(const array &in, const int num = 0, const bool extract = true);
 
-    /**
-        \brief Join 2 arrays along \p dim
-
-        \param[in] dim is the dimension along which join occurs
-        \param[in] first is the first input array
-        \param[in] second is the second input array
-        \return the array that joins input arrays along the given dimension
-
-        \note empty arrays will be ignored
-
-        \ingroup manip_func_join
-    */
+    /// C++ Interface to join 2 arrays along a dimension.
+    ///
+    /// Empty arrays are ignored.
+    ///
+    /// \param[in] dim    dimension along which the join occurs
+    /// \param[in] first  input array
+    /// \param[in] second input array
+    /// \return           joined array
+    ///
+    /// \ingroup manip_func_join
     AFAPI array join(const int dim, const array &first, const array &second);
 
-    /**
-        \brief Join 3 arrays along \p dim
-
-        \param[in] dim is the dimension along which join occurs
-        \param[in] first is the first input array
-        \param[in] second is the second input array
-        \param[in] third is the third input array
-        \return the array that joins input arrays along the given dimension
-
-        \note empty arrays will be ignored
-
-        \ingroup manip_func_join
-    */
+    /// C++ Interface to join 3 arrays along a dimension.
+    ///
+    /// Empty arrays are ignored.
+    ///
+    /// \param[in] dim    dimension along which the join occurs
+    /// \param[in] first  input array
+    /// \param[in] second input array
+    /// \param[in] third  input array
+    /// \return           joined array
+    ///
+    /// \ingroup manip_func_join
     AFAPI array join(const int dim, const array &first, const array &second, const array &third);
 
-    /**
-        \brief Join 4 arrays along \p dim
-
-        \param[in] dim is the dimension along which join occurs
-        \param[in] first is the first input array
-        \param[in] second is the second input array
-        \param[in] third is the third input array
-        \param[in] fourth is the fourth input array
-        \return the array that joins input arrays along the given dimension
-
-        \note empty arrays will be ignored
-
-        \ingroup manip_func_join
-    */
+    /// C++ Interface to join 4 arrays along a dimension.
+    ///
+    /// Empty arrays are ignored.
+    ///
+    /// \param[in] dim    dimension along which the join occurs
+    /// \param[in] first  input array
+    /// \param[in] second input array
+    /// \param[in] third  input array
+    /// \param[in] fourth input array
+    /// \return           joined array
+    ///
+    /// \ingroup manip_func_join
     AFAPI array join(const int dim, const array &first, const array &second,
                      const array &third, const array &fourth);
 
-    /**
-        \param[in] in is the input array
-        \param[in] x is the number of times \p in is copied along the first dimension
-        \param[in] y is the number of times \p in is copied along the the second dimension
-        \param[in] z is the number of times \p in is copied along the third dimension
-        \param[in] w is the number of times \p in is copied along the fourth dimension
-        \return The tiled version of the input array
-
-        \note \p x, \p y, \p z, and \p w includes the original in the count as
-              well. Thus, if no duplicates are needed in a certain dimension,
-              leave it as 1 (the default value for just one copy)
-
-        \ingroup manip_func_tile
-    */
+    /// C++ Interface to generate a tiled array.
+    ///
+    /// Note, `x`, `y`, `z`, and `w` include the original in the count.
+    ///
+    /// \param[in] in input array
+    /// \param[in] x  number tiles along the first dimension
+    /// \param[in] y  number tiles along the second dimension
+    /// \param[in] z  number tiles along the third dimension
+    /// \param[in] w  number tiles along the fourth dimension
+    /// \return       tiled array
+    ///
+    /// \ingroup manip_func_tile
     AFAPI array tile(const array &in, const unsigned x, const unsigned y=1,
                      const unsigned z=1, const unsigned w=1);
 
-    /**
-        \param[in] in is the input array
-        \param[in] dims specifies the number of times \p in is copied along each dimension
-        \return The tiled version of the input array
-
-        \note Each component of \p dims includes the original in the count as
-              well. Thus, if no duplicates are needed in a certain dimension,
-              leave it as 1 (the default value for just one copy)
-
-        \ingroup manip_func_tile
-    */
+    /// C++ Interface to generate a tiled array.
+    ///
+    /// Each component of `dims` includes the original in the count. Thus, if
+    /// no duplicates are needed in a certain dimension, it is left as 1, the
+    /// default value for just one copy.
+    ///
+    /// \param[in] in   input array
+    /// \param[in] dims number of times `in` is copied along each dimension
+    /// \return         tiled array
+    ///
+    /// \ingroup manip_func_tile
     AFAPI array tile(const array &in, const dim4 &dims);
 
-    /**
-        \param[in] in is the input array
-        \param[in] x specifies which dimension should be first
-        \param[in] y specifies which dimension should be second
-        \param[in] z specifies which dimension should be third
-        \param[in] w specifies which dimension should be fourth
-        \return the reordered output
-
-        \ingroup manip_func_reorder
-    */
+    /// C++ Interface to reorder an array. 
+    ///
+    /// \param[in] in input array
+    /// \param[in] x  specifies which dimension should be first
+    /// \param[in] y  specifies which dimension should be second
+    /// \param[in] z  specifies which dimension should be third
+    /// \param[in] w  specifies which dimension should be fourth
+    /// \return       reordered array
+    ///
+    /// \ingroup manip_func_reorder
     AFAPI array reorder(const array& in, const unsigned x,
                         const unsigned y=1, const unsigned z=2, const unsigned w=3);
 
-    /**
-        \param[in] in is the input array
-        \param[in] x specifies the shift along first dimension
-        \param[in] y specifies the shift along second dimension
-        \param[in] z specifies the shift along third dimension
-        \param[in] w specifies the shift along fourth dimension
-
-        \return the shifted output
-
-        \ingroup manip_func_shift
-    */
+    /// C++ Interface to shift an array.
+    ///
+    /// \param[in] in input array
+    /// \param[in] x  specifies the shift along the first dimension
+    /// \param[in] y  specifies the shift along the second dimension
+    /// \param[in] z  specifies the shift along the third dimension
+    /// \param[in] w  specifies the shift along the fourth dimension
+    /// \return       shifted array
+    ///
+    /// \ingroup manip_func_shift
     AFAPI array shift(const array& in, const int x, const int y=0, const int z=0, const int w=0);
 
-    /**
-    * C++ Interface for modifying the dimensions of an input array to the shape specified by a `dim4` object
-    *
-        \param[in] in the input array
-        \param[in] dims the array of new dimension sizes
-        \return the modded output
-
-        \ingroup manip_func_moddims
-    */
+    /// C++ Interface to modify the dimensions of an input array to a specified
+    /// shape.
+    ///
+    /// \param[in] in   input array
+    /// \param[in] dims new dimension sizes
+    /// \return         modded output
+    ///
+    /// \ingroup manip_func_moddims
     AFAPI array moddims(const array& in, const dim4& dims);
 
-    /**
-    * C++ Interface for modifying the dimensions of an input array to the shape specified by dimension length parameters
-    *
-        \param[in] in the input array
-        \param[in] d0 the new size of the first dimension
-        \param[in] d1 the new size of the second dimension (optional)
-        \param[in] d2 the new size of the third dimension (optional)
-        \param[in] d3 the new size of the fourth dimension (optional)
-        \return the modded output
-
-        \ingroup manip_func_moddims
-    */
+    /// C++ Interface to modify the dimensions of an input array to a specified
+    /// shape.
+    ///
+    /// \param[in] in input array
+    /// \param[in] d0 new size of the first dimension
+    /// \param[in] d1 new size of the second dimension (optional)
+    /// \param[in] d2 new size of the third dimension (optional)
+    /// \param[in] d3 new size of the fourth dimension (optional)
+    /// \return       modded output
+    ///
+    /// \ingroup manip_func_moddims
     AFAPI array moddims(const array& in, const dim_t d0, const dim_t d1=1, const dim_t d2=1, const dim_t d3=1);
 
-    /**
-    * C++ Interface for modifying the dimensions of an input array to the shape specified by an array of `ndims` dimensions
-    *
-        \param[in] in the input array
-        \param[in] ndims the number of dimensions
-        \param[in] dims the array of new dimension sizes
-        \return the modded output
-
-        \ingroup manip_func_moddims
-    */
+    /// C++ Interface to modify the dimensions of an input array to a specified
+    /// shape.
+    ///
+    /// \param[in] in    input array
+    /// \param[in] ndims number of dimensions
+    /// \param[in] dims  new dimension sizes
+    /// \return          modded output
+    ///
+    /// \ingroup manip_func_moddims
     AFAPI array moddims(const array& in, const unsigned ndims, const dim_t* const dims);
 
-    /**
-        \param[in] in is the input array
-        \return the flat array
-
-        \ingroup manip_func_flat
-    */
+    /// C++ Interface to flatten an array.
+    ///
+    /// \param[in] in input array
+    /// \return       flat array
+    ///
+    /// \ingroup manip_func_flat
     AFAPI array flat(const array &in);
 
-    /**
-        \param[in] in is the input array
-        \param[in] dim is the dimensions to flip the array
-        \return the flipped array
-
-        \ingroup manip_func_flip
-    */
+    /// C++ Interface to flip an array.
+    ///
+    /// \param[in] in  input array
+    /// \param[in] dim dimension to flip
+    /// \return        flipped array
+    ///
+    /// \ingroup manip_func_flip
     AFAPI array flip(const array &in, const unsigned dim);
 
-    /**
-        \param[in] in is the input matrix
-        \param[in] is_unit_diag is a boolean parameter specifying if the diagonal elements should be 1
-        \return the lower triangle array
-
-        \ingroup data_func_lower
-    */
+    /// C++ Interface to return the lower triangle array.
+    ///
+    /// \param[in] in           input array
+    /// \param[in] is_unit_diag boolean specifying if diagonal elements are 1's
+    /// \return                 lower triangle array
+    ///
+    /// \ingroup data_func_lower
     AFAPI array lower(const array &in, bool is_unit_diag=false);
 
-    /**
-        \param[in] in is the input matrix
-        \param[in] is_unit_diag is a boolean parameter specifying if the diagonal elements should be 1
-        \return the upper triangle matrix
-
-        \ingroup data_func_upper
-    */
+    /// C++ Interface to return the upper triangle array.
+    ///
+    /// \param[in] in           input array
+    /// \param[in] is_unit_diag boolean specifying if diagonal elements are 1's
+    /// \return                 upper triangle matrix
+    ///
+    /// \ingroup data_func_upper
     AFAPI array upper(const array &in, bool is_unit_diag=false);
 
 #if AF_API_VERSION >= 31
-    /**
-       \param[in]  cond is the conditional array
-       \param[in]  a is the array containing elements from the true part of the condition
-       \param[in]  b is the array containing elements from the false part of the condition
-       \return  the output containing elements of \p a when \p cond is true else elements from \p b
-
-       \ingroup data_func_select
-    */
+    /// C++ Interface to select elements based on a conditional array.
+    ///
+    /// \param[in] cond conditional array
+    /// \param[in] a    when true, select array element
+    /// \param[in] b    when false, select array element
+    /// \return         `a` when `cond` is true, else `b`
+    ///
+    /// \ingroup data_func_select
     AFAPI array select(const array &cond, const array  &a, const array  &b);
 #endif
 
 #if AF_API_VERSION >= 31
-    /**
-       \param[in]  cond is the conditional array
-       \param[in]  a is the array containing elements from the true part of the condition
-       \param[in]  b is a scalar assigned to \p out when \p cond is false
-       \return  the output containing elements of \p a when \p cond is true else the value \p b
-
-       \ingroup data_func_select
-    */
+    /// C++ Interface to select elements based on a conditional array.
+    ///
+    /// \param[in] cond conditional array
+    /// \param[in] a    when true, select array element
+    /// \param[in] b    when false, select scalar value
+    /// \return         `a` when `cond` is true, else `b`
+    ///
+    /// \ingroup data_func_select
     AFAPI array select(const array &cond, const array  &a, const double &b);
 #endif
 
 #if AF_API_VERSION >= 31
-    /**
-       \param[in]  cond is the conditional array
-       \param[in]  a is a scalar assigned to \p out when \p cond is true
-       \param[in]  b is the array containing elements from the false part of the condition
-       \return  the output containing the value \p a when \p cond is true else elements from \p b
-
-       \ingroup data_func_select
-    */
+    /// C++ Interface to select elements based on a conditional array.
+    ///
+    /// \param[in] cond conditional array
+    /// \param[in] a    when true, select scalar value
+    /// \param[in] b    when false, select array element
+    /// \return         `a` when `cond` is true, else `b`
+    ///
+    /// \ingroup data_func_select
     AFAPI array select(const array &cond, const double &a, const array  &b);
 #endif
 
 #if AF_API_VERSION >= 31
-    /**
-       \param[inout]  a is the input array
-       \param[in]  cond is the conditional array.
-       \param[in]  b is the replacement array.
-
-       \note Values of \p a are replaced with corresponding values of \p b, when \p cond is false.
-
-       \ingroup data_func_replace
-    */
+    /// C++ Interface to replace elements of an array with elements of another
+    /// array.
+    ///
+    /// Elements of `a` are replaced with corresponding elements of `b` when
+    /// `cond` is false.
+    ///
+    /// \param[inout] a    input array
+    /// \param[in]    cond conditional array
+    /// \param[in]    b    replacement array
+    ///
+    /// \ingroup data_func_replace
     AFAPI void replace(array &a, const array  &cond, const array  &b);
 #endif
 
 #if AF_API_VERSION >= 31
-    /**
-       \param[inout]  a is the input array
-       \param[in]  cond is the conditional array.
-       \param[in]  b is the replacement value.
-
-       \note Values of \p a are replaced with value \p b, when \p cond is false.
-
-       \ingroup data_func_replace
-    */
+    /// C++ Interface to replace elements of an array with a scalar value.
+    ///
+    /// Elements of `a` are replaced with a scalar value when `cond` is false.
+    ///
+    /// \param[inout] a    input array
+    /// \param[in]    cond conditional array
+    /// \param[in]    b    replacement scalar value
+    ///
+    /// \ingroup data_func_replace
     AFAPI void replace(array &a, const array  &cond, const double &b);
 #endif
 
 #if AF_API_VERSION >= 37
-    /**
-       \param[in] in is the input array to be padded
-       \param[in] beginPadding informs the number of elements to be
-                  padded at beginning of each dimension
-       \param[in] endPadding informs the number of elements to be
-                  padded at end of each dimension
-       \param[in] padFillType is indicates what values should fill padded region
-
-       \return the padded array
-
-       \ingroup data_func_pad
-    */
+    /// C++ Interface to pad an array.
+    ///
+    /// \param[in] in           input array
+    /// \param[in] beginPadding number of elements to be padded at the start of
+    ///                         each dimension
+    /// \param[in] endPadding   number of elements to be padded at the end of
+    ///                         each dimension
+    /// \param[in] padFillType  values to fill into the padded region
+    /// \return                 padded array
+    ///
+    /// \ingroup data_func_pad
     AFAPI array pad(const array &in, const dim4 &beginPadding,
                     const dim4 &endPadding, const borderType padFillType);
 #endif
 
 #if AF_API_VERSION >= 39
-    /**
-       \param[inout]  a is the input array
-       \param[in]  cond is the conditional array.
-       \param[in]  b is the replacement scalar value.
-
-       \note Values of \p a are replaced with value \p b, when \p cond is false.
-
-       \ingroup data_func_replace
-    */
+    /// C++ Interface to replace elements of an array with a scalar value.
+    ///
+    /// Elements of `a` are replaced with a scalar value when `cond` is false.
+    ///
+    /// \param[inout] a    input array
+    /// \param[in]    cond conditional array
+    /// \param[in]    b    replacement scalar value
+    ///
+    /// \ingroup data_func_replace
     AFAPI void replace(array &a, const array &cond, const long long b);
 
-    /**
-       \param[inout]  a is the input array
-       \param[in]  cond is the conditional array.
-       \param[in]  b is the replacement scalar value.
-
-       \note Values of \p a are replaced with value \p b, when \p cond is false.
-
-       \ingroup data_func_replace
-    */
+    /// C++ Interface to replace elements of an array with a scalar value.
+    ///
+    /// Elements of `a` are replaced with a scalar value when `cond` is false.
+    ///
+    /// \param[inout] a    input array
+    /// \param[in]    cond conditional array
+    /// \param[in]    b    replacement scalar value
+    ///
+    /// \ingroup data_func_replace
     AFAPI void replace(array &a, const array &cond,
                        const unsigned long long b);
 
-    /**
-       \param[in]  cond is the conditional array
-       \param[in]  a is the array containing elements from the true part of the
-                   condition
-       \param[in]  b is a scalar assigned to \p out when \p cond is false
-       \return  the output containing elements of \p a when \p cond is true
-                else the value \p b
-
-       \ingroup data_func_select
-    */
+    /// C++ Interface to select elements based on a conditional array.
+    ///
+    /// \param[in] cond conditional array
+    /// \param[in] a    when true, select array element
+    /// \param[in] b    when false, select scalar value
+    /// \return         `a` when `cond` is true, else `b`
+    ///
+    /// \ingroup data_func_select
     AFAPI array select(const array &cond, const array &a, const long long b);
 
-    /**
-       \param[in]  cond is the conditional array
-       \param[in]  a is the array containing elements from the true part of the
-                   condition
-       \param[in]  b is a scalar assigned to \p out when \p cond is false
-       \return  the output containing elements of \p a when \p cond is true
-                else the value \p b
-
-       \ingroup data_func_select
-    */
+    /// C++ Interface to select elements based on a conditional array.
+    ///
+    /// \param[in] cond conditional array
+    /// \param[in] a    when true, select array element
+    /// \param[in] b    when false, select scalar value
+    /// \return         `a` when `cond` is true, else `b`
+    ///
+    /// \ingroup data_func_select
     AFAPI array select(const array &cond, const array &a,
                        const unsigned long long b);
 
-    /**
-       \param[in]  cond is the conditional array
-       \param[in]  a is a scalar assigned to \p out when \p cond is true
-       \param[in]  b is the array containing elements from the false part of the
-                   condition
-       \return  the output containing the value \p a when \p cond is true else
-                elements from \p b
-
-       \ingroup data_func_select
-    */
+    /// C++ Interface to select elements based on a conditional array.
+    ///
+    /// \param[in] cond conditional array
+    /// \param[in] a    when true, select scalar value
+    /// \param[in] b    when false, select array element
+    /// \return         `a` when `cond` is true, else `b`
+    ///
+    /// \ingroup data_func_select
     AFAPI array select(const array &cond, const long long a, const array &b);
 
-    /**
-       \param[in]  cond is the conditional array
-       \param[in]  a is a scalar assigned to \p out when \p cond is true
-       \param[in]  b is the array containing elements from the false part of the
-                   condition
-       \return  the output containing the value \p a when \p cond is true else
-                elements from \p b
-
-       \ingroup data_func_select
-    */
+    /// C++ Interface to select elements based on a conditional array.
+    ///
+    /// \param[in] cond conditional array
+    /// \param[in] a    when true, select scalar value
+    /// \param[in] b    when false, select array element
+    /// \return         `a` when `cond` is true, else `b`
+    ///
+    /// \ingroup data_func_select
     AFAPI array select(const array &cond, const unsigned long long a,
                        const array &b);
 #endif
@@ -530,46 +500,65 @@ namespace af
 extern "C" {
 #endif
     /**
-        \param[out] arr is the generated array of given type
-        \param[in] val is the value of each element in the generated array
-        \param[in] ndims is size of dimension array \p dims
-        \param[in] dims is the array containing sizes of the dimension
-        \param[in] type is the type of array to generate
+       C Interface to generate an array with elements set to a specified value.
+
+       \param[out] arr   constant array
+       \param[in]  val   constant value
+       \param[in]  ndims size of the dimension array
+       \param[in]  dims  dimensions of the array to be generated
+       \param[in]  type  type
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_constant
     */
     AFAPI af_err af_constant(af_array *arr, const double val, const unsigned ndims, const dim_t * const dims, const af_dtype type);
 
     /**
-        \param[out] arr is the generated array of type \ref c32 or \ref c64
-        \param[in] real is the real value of each element in the generated array
-        \param[in] imag is the imaginary value of each element in the generated array
-        \param[in] ndims is size of dimension array \p dims
-        \param[in] dims is the array containing sizes of the dimension
-        \param[in] type is the type of array to generate
+       C Interface to generate a complex array with elements set to a specified
+       value.
+
+       \param[out] arr   constant complex array
+       \param[in]  real  real constant value
+       \param[in]  imag  imaginary constant value
+       \param[in]  ndims size of the dimension array
+       \param[in]  dims  dimensions of the array to be generated
+       \param[in]  type  type, \ref c32 or \ref c64
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_constant
     */
-
     AFAPI af_err af_constant_complex(af_array *arr, const double real, const double imag,
                                      const unsigned ndims, const dim_t * const dims, const af_dtype type);
 
     /**
-        \param[out] arr is the generated array of type \ref s64
-        \param[in] val is a complex value of each element in the generated array
-        \param[in] ndims is size of dimension array \p dims
-        \param[in] dims is the array containing sizes of the dimension
+       C Interface to generate an array with elements set to a specified value.
+
+       Output type is \ref s64.
+
+       \param[out] arr   constant array
+       \param[in]  val   constant value
+       \param[in]  ndims size of the dimension array
+       \param[in]  dims  dimensions of the array to be generated
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_constant
     */
-
     AFAPI af_err af_constant_long (af_array *arr, const long long val, const unsigned ndims, const dim_t * const dims);
 
     /**
-        \param[out] arr is the generated array of type \ref u64
-        \param[in] val is a complex value of each element in the generated array
-        \param[in] ndims is size of dimension array \p dims
-        \param[in] dims is the array containing sizes of the dimension
+       C Interface to generate an array with elements set to a specified value.
+
+       Output type is \ref u64.
+
+       \param[out] arr   constant array
+       \param[in]  val   constant value
+       \param[in]  ndims size of the dimension array
+       \param[in]  dims  dimensions of the array to be generated
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_constant
     */
@@ -577,186 +566,246 @@ extern "C" {
     AFAPI af_err af_constant_ulong(af_array *arr, const unsigned long long val, const unsigned ndims, const dim_t * const dims);
 
     /**
-    * C Interface for creating an array with `[0, n-1]` values along the `seq_dim` dimension and tiled across other dimensions specified by an array of `ndims` dimensions.
-    *
-        \param[out] out the generated array
-        \param[in] ndims the size of dimension array `dims`
-        \param[in] dims the array containing the dimension sizes
-        \param[in] seq_dim the dimension along which `[0, dim[seq_dim] - 1]` is created
-        \param[in] type the type of the generated array
-
-        \ingroup data_func_range
+       C Interface to generate an identity array.
+
+       \param[out] out   identity array
+       \param[in]  ndims number of dimensions
+       \param[in]  dims  size
+       \param[in]  type  type
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup data_func_identity
+    */
+    AFAPI af_err af_identity(af_array* out, const unsigned ndims, const dim_t* const dims, const af_dtype type);
+
+    /**
+       C Interface to generate an array with `[0, n-1]` values along the
+       `seq_dim` dimension and tiled across other dimensions of shape `dim4`.
+
+       \param[out] out     range array
+       \param[in]  ndims   number of dimensions, specified by the size of `dims`
+       \param[in]  dims    size
+       \param[in]  seq_dim dimension along which the range is created
+       \param[in]  type    type
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup data_func_range
     */
     AFAPI af_err af_range(af_array *out, const unsigned ndims, const dim_t * const dims,
                           const int seq_dim, const af_dtype type);
 
     /**
-        \param[out] out is the generated array
-        \param[in] ndims is size of dimension array \p dims
-        \param[in] dims is the array containing sizes of the dimension
-        \param[in] t_ndims is size of tile array \p tdims
-        \param[in] tdims is array containing the number of repetitions of the unit dimensions
-        \param[in] type is the type of array to generate
-
-        \ingroup data_func_iota
+       C Interface to generate an array with `[0, n-1]` values modified to
+       specified dimensions and tiling.
+
+       \param[out] out     iota array
+       \param[in]  ndims   number of dimensions
+       \param[in]  dims    size
+       \param[in]  t_ndims number of dimensions of tiled array
+       \param[in]  tdims   number of tiled repetitions in each dimension
+       \param[in]  type    type
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup data_func_iota
     */
     AFAPI af_err af_iota(af_array *out, const unsigned ndims, const dim_t * const dims,
                          const unsigned t_ndims, const dim_t * const tdims, const af_dtype type);
 
-
     /**
-        \param[out] out is the generated array
-        \param[in] ndims is size of dimension array \p dims
-        \param[in] dims is the array containing sizes of the dimension
-        \param[in] type is the type of array to generate
+       C Interface to create a diagonal matrix from an extracted diagonal
+       array.
 
-        \ingroup data_func_identity
-    */
-    AFAPI af_err af_identity(af_array *out, const unsigned ndims, const dim_t * const dims, const af_dtype type);
+       See also, \ref af_diag_extract.
 
-    /**
-        \param[out] out is the array created from the input array \p in
-        \param[in] in is the input array which is the diagonal
-        \param[in] num is the diagonal index
+       \param[out] out diagonal matrix
+       \param[in]  in  diagonal array
+       \param[in]  num diagonal index
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup data_func_diag
+       \ingroup data_func_diag
     */
     AFAPI af_err af_diag_create(af_array *out, const af_array in, const int num);
 
     /**
-        \param[out] out is the \p num -th diagonal of \p in
-        \param[in] in is the input matrix
-        \param[in] num is the diagonal index
+       C Interface to extract the diagonal from an array.
 
-        \ingroup data_func_diag
+       See also, \ref af_diag_create.
+
+       \param[out] out `num`-th diagonal array
+       \param[in]  in  input array
+       \param[in]  num diagonal index
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup data_func_diag
     */
     AFAPI af_err af_diag_extract(af_array *out, const af_array in, const int num);
 
     /**
-        \brief Join 2 arrays along \p dim
+       C Interface to join 2 arrays along a dimension.
 
-        \param[out] out is the generated array
-        \param[in] dim is the dimension along which join occurs
-        \param[in] first is the first input array
-        \param[in] second is the second input array
+       Empty arrays are ignored.
 
-        \note empty arrays will be ignored
+       \param[out] out    joined array
+       \param[in]  dim    dimension along which the join occurs
+       \param[in]  first  input array
+       \param[in]  second input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup manip_func_join
+       \ingroup manip_func_join
     */
     AFAPI af_err af_join(af_array *out, const int dim, const af_array first, const af_array second);
 
     /**
-        \brief Join many arrays along \p dim
-
-        Current limit is set to 10 arrays.
+       C Interface to join many arrays along a dimension.
 
-        \param[out] out is the generated array
-        \param[in] dim is the dimension along which join occurs
-        \param[in] n_arrays number of arrays to join
-        \param[in] inputs is an array of af_arrays containing handles to the arrays to be joined
+       Limited to 10 arrays. Empty arrays are ignored.
 
-        \note empty arrays will be ignored
+       \param[out] out      joined array
+       \param[in]  dim      dimension along which the join occurs
+       \param[in]  n_arrays number of arrays to join
+       \param[in]  inputs   array of af_arrays containing handles to the
+                             arrays to be joined
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup manip_func_join
+       \ingroup manip_func_join
     */
     AFAPI af_err af_join_many(af_array *out, const int dim, const unsigned n_arrays, const af_array *inputs);
 
     /**
-        \param[out] out is the tiled version of the input array
-        \param[in] in is the input matrix
-        \param[in] x is the number of times \p in is copied along the first dimension
-        \param[in] y is the number of times \p in is copied along the the second dimension
-        \param[in] z is the number of times \p in is copied along the third dimension
-        \param[in] w is the number of times \p in is copied along the fourth dimension
-
-        \note \p x, \p y, \p z, and \p w includes the original in the count as
-              well. Thus, if no duplicates are needed in a certain dimension,
-              leave it as 1 (the default value for just one copy)
-
-        \ingroup manip_func_tile
+       C Interface to generate a tiled array.
+
+       Note, `x`, `y`, `z`, and `w` include the original in the count.
+
+       \param[out] out tiled array
+       \param[in]  in  input array
+       \param[in]  x   number of tiles along the first dimension
+       \param[in]  y   number of tiles along the second dimension
+       \param[in]  z   number of tiles along the third dimension
+       \param[in]  w   number of tiles along the fourth dimension
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup manip_func_tile
     */
     AFAPI af_err af_tile(af_array *out, const af_array in,
                          const unsigned x, const unsigned y, const unsigned z, const unsigned w);
 
     /**
-        \param[out] out is the reordered array
-        \param[in] in is the input matrix
-        \param[in] x specifies which dimension should be first
-        \param[in] y specifies which dimension should be second
-        \param[in] z specifies which dimension should be third
-        \param[in] w specifies which dimension should be fourth
-
-        \ingroup manip_func_reorder
+       C Interface to reorder an array.
+
+       \param[out] out reordered array
+       \param[in]  in  input array
+       \param[in]  x   specifies which dimension should be first
+       \param[in]  y   specifies which dimension should be second
+       \param[in]  z   specifies which dimension should be third
+       \param[in]  w   specifies which dimension should be fourth
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup manip_func_reorder
     */
     AFAPI af_err af_reorder(af_array *out, const af_array in,
                             const unsigned x, const unsigned y, const unsigned z, const unsigned w);
 
     /**
-        \param[in] out is the shifted array
-        \param[in] in is the input array
-        \param[in] x specifies the shift along first dimension
-        \param[in] y specifies the shift along second dimension
-        \param[in] z specifies the shift along third dimension
-        \param[in] w specifies the shift along fourth dimension
-
-        \ingroup manip_func_shift
+       C Interface to shift an array.
+
+       \param[out] out shifted array
+       \param[in]  in  input array
+       \param[in]  x   specifies the shift along first dimension
+       \param[in]  y   specifies the shift along second dimension
+       \param[in]  z   specifies the shift along third dimension
+       \param[in]  w   specifies the shift along fourth dimension
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup manip_func_shift
     */
     AFAPI af_err af_shift(af_array *out, const af_array in, const int x, const int y, const int z, const int w);
 
     /**
-    * C Interface for modifying the dimensions of an input array to the shape specified by an array of `ndims` dimensions
-    *
-        \param[out] out the modded output
-        \param[in] in the input array
-        \param[in] ndims the number of dimensions
-        \param[in] dims the array of new dimension sizes
-
-        \ingroup manip_func_moddims
+       C Interface to modify the dimensions of an input array to a specified
+       shape.
+
+       \param[out] out   modded output
+       \param[in]  in    input array
+       \param[in]  ndims number of dimensions
+       \param[in]  dims  new dimension sizes
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup manip_func_moddims
     */
     AFAPI af_err af_moddims(af_array *out, const af_array in, const unsigned ndims, const dim_t * const dims);
 
     /**
-        \param[out] out is the flat array
-        \param[in] in is the input array
+       C Interface to flatten an array.
+
+       \param[out] out flat array
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup manip_func_flat
+       \ingroup manip_func_flat
     */
     AFAPI af_err af_flat(af_array *out, const af_array in);
 
     /**
-        \param[out] out is the flipped array
-        \param[in] in is the input array
-        \param[in] dim is the dimensions to flip the array
+       C Interface to flip an array.
 
-        \ingroup manip_func_flip
+       \param[out] out flipped array
+       \param[in]  in  input array
+       \param[in]  dim dimension to flip
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup manip_func_flip
     */
     AFAPI af_err af_flip(af_array *out, const af_array in, const unsigned dim);
 
     /**
-        \param[out] out is the lower traingle matrix
-        \param[in] in is the input matrix
-        \param[in] is_unit_diag is a boolean parameter specifying if the diagonal elements should be 1
+       C Interface to return the lower triangle array.
+
+       \param[out] out          lower traingle array
+       \param[in]  in           input array
+       \param[in]  is_unit_diag boolean specifying if diagonal elements are 1's
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup data_func_lower
+       \ingroup data_func_lower
     */
     AFAPI af_err af_lower(af_array *out, const af_array in, bool is_unit_diag);
 
     /**
-        \param[out] out is the upper triangle matrix
-        \param[in] in is the input matrix
-        \param[in] is_unit_diag is a boolean parameter specifying if the diagonal elements should be 1
+       C Interface to return the upper triangle array.
+
+       \param[out] out          upper triangle array
+       \param[in]  in           input array
+       \param[in]  is_unit_diag boolean specifying if diagonal elements are 1's
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-        \ingroup data_func_upper
+       \ingroup data_func_upper
     */
     AFAPI af_err af_upper(af_array *out, const af_array in, bool is_unit_diag);
 
 #if AF_API_VERSION >= 31
     /**
-       \param[out] out is the output containing elements of \p a when \p cond is true else elements from \p b
-       \param[in]  cond is the conditional array
-       \param[in]  a is the array containing elements from the true part of the condition
-       \param[in]  b is the array containing elements from the false part of the condition
+       C Interface to select elements based on a conditional array.
+
+       \param[out] out  `a` when `cond` is true, else `b`
+       \param[in]  cond conditional array
+       \param[in]  a    when true, select array element
+       \param[in]  b    when false, select array element
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_select
     */
@@ -765,10 +814,14 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       \param[out] out is the output containing elements of \p a when \p cond is true else elements from \p b
-       \param[in]  cond is the conditional array
-       \param[in]  a is the array containing elements from the true part of the condition
-       \param[in]  b is a scalar assigned to \p out when \p cond is false
+       C Interface to select elements based on a conditional array.
+
+       \param[out] out  `a` when `cond` is true, else `b`
+       \param[in]  cond conditional array
+       \param[in]  a    when true, select array element
+       \param[in]  b    when false, select scalar value
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_select
     */
@@ -777,10 +830,14 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       \param[out] out is the output containing elements of \p a when \p cond is true else elements from \p b
-       \param[in]  cond is the conditional array
-       \param[in]  a is a scalar assigned to \p out when \p cond is true
-       \param[in]  b is the array containing elements from the false part of the condition
+       C Interface to select elements based on a conditional array.
+
+       \param[out] out  `a` when `cond` is true, else `b`
+       \param[in]  cond conditional array
+       \param[in]  a    when true, select scalar value
+       \param[in]  b    when false, select array element
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_select
     */
@@ -789,11 +846,17 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       \param[inout]  a is the input array
-       \param[in]  cond is the conditional array.
-       \param[in]  b is the replacement array.
+       C Interface to replace elements of an array with elements of another
+       array.
 
-       \note Values of \p a are replaced with corresponding values of \p b, when \p cond is false.
+       Elements of `a` are replaced with corresponding elements of `b` when
+       `cond` is false.
+
+       \param[inout]  a    input array
+       \param[in]     cond conditional array
+       \param[in]     b    replacement array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_replace
     */
@@ -802,11 +865,15 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       \param[inout]  a is the input array
-       \param[in]  cond is the conditional array.
-       \param[in]  b is the replacement array.
+       C Interface to replace elements of an array with a scalar value.
+
+       Elements of `a` are replaced with a scalar value when `cond` is false.
 
-       \note Values of \p a are replaced with corresponding values of \p b, when \p cond is false.
+       \param[inout] a    input array
+       \param[in]    cond conditional array
+       \param[in]    b    replacement scalar value
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_replace
     */
@@ -815,15 +882,19 @@ extern "C" {
 
 #if AF_API_VERSION >= 37
     /**
-       \param[out] out is the padded array
-       \param[in] in is the input array to be padded
-       \param[in] begin_ndims is size of \p l_dims array
-       \param[in] begin_dims array contains padding size at beginning of each
-                  dimension
-       \param[in] end_ndims is size of \p u_dims array
-       \param[in] end_dims array contains padding sizes at end of each dimension
-       \param[in] pad_fill_type is indicates what values should fill
-                  padded region
+       C Interface to pad an array.
+
+       \param[out] out           padded array
+       \param[in]  in            input array
+       \param[in]  begin_ndims   number of dimensions for start padding
+       \param[in]  begin_dims    number of elements to be padded at the start
+                                 of each dimension
+       \param[in]  end_ndims     number of dimensions for end padding
+       \param[in]  end_dims      number of elements to be padded at the end of
+                                 each dimension
+       \param[in]  pad_fill_type values to fill into the padded region
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_pad
     */
@@ -836,12 +907,15 @@ extern "C" {
 
 #if AF_API_VERSION >= 39
     /**
-       \param[inout]  a is the input array
-       \param[in]  cond is the conditional array.
-       \param[in]  b is the replacement array.
+       C Interface to replace elements of an array with a scalar value.
 
-       \note Values of \p a are replaced with corresponding values of \p b, when
-       \p cond is false.
+       Elements of `a` are replaced with a scalar value when `cond` is false.
+
+       \param[inout] a    input array
+       \param[in]    cond conditional array
+       \param[in]    b    replacement scalar value
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_replace
     */
@@ -849,12 +923,15 @@ extern "C" {
                                         const long long b);
 
     /**
-       \param[inout]  a is the input array
-       \param[in]  cond is the conditional array.
-       \param[in]  b is the replacement array.
+       C Interface to replace elements of an array with a scalar value.
+
+       Elements of `a` are replaced with a scalar value when `cond` is false.
 
-       \note Values of \p a are replaced with corresponding values of \p b, when
-       \p cond is false.
+       \param[inout] a    input array
+       \param[in]    cond conditional array
+       \param[in]    b    replacement scalar value
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_replace
     */
@@ -862,13 +939,14 @@ extern "C" {
                                          const unsigned long long b);
 
     /**
-       \param[out] out is the output containing elements of \p a when \p cond is
-       true else elements from \p b
-       \param[in]  cond is the conditional array
-       \param[in]  a is the array containing elements from the true part of the
-       condition
-       \param[in]  b is a scalar assigned to \p out when \p cond is
-       false
+       C Interface to select elements based on a conditional array.
+
+       \param[out] out  `a` when `cond` is true, else `b`
+       \param[in]  cond conditional array
+       \param[in]  a    when true, select array element
+       \param[in]  b    when false, select scalar value
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_select
     */
@@ -876,13 +954,14 @@ extern "C" {
                                          const af_array a, const long long b);
 
     /**
-       \param[out] out is the output containing elements of \p a when \p cond is
-       true else elements from \p b
-       \param[in]  cond is the conditional array
-       \param[in]  a is the array containing elements from the true part of the
-       condition
-       \param[in]  b is a scalar assigned to \p out when \p cond is
-       false
+       C Interface to select elements based on a conditional array.
+
+       \param[out] out  `a` when `cond` is true, else `b`
+       \param[in]  cond conditional array
+       \param[in]  a    when true, select array element
+       \param[in]  b    when false, select scalar value
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_select
     */
@@ -891,12 +970,14 @@ extern "C" {
                                           const unsigned long long b);
 
     /**
-       \param[out] out is the output containing elements of \p a when \p cond is
-       true else elements from \p b
-       \param[in]  cond is the conditional array
-       \param[in]  a is a scalar assigned to \p out when \p cond is true
-       \param[in]  b is the array containing elements from the false part of the
-       condition
+       C Interface to select elements based on a conditional array.
+
+       \param[out] out  `a` when `cond` is true, else `b`
+       \param[in]  cond conditional array
+       \param[in]  a    when true, select scalar value
+       \param[in]  b    when false, select array element
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_select
     */
@@ -904,12 +985,14 @@ extern "C" {
                                          const long long a, const af_array b);
 
     /**
-       \param[out] out is the output containing elements of \p a when \p cond is
-       true else elements from \p b
-       \param[in]  cond is the conditional array
-       \param[in]  a is a scalar assigned to \p out when \p cond is true
-       \param[in]  b is the array containing elements from the false part of the
-       condition
+       C Interface to select elements based on a conditional array.
+
+       \param[out] out  `a` when `cond` is true, else `b`
+       \param[in]  cond conditional array
+       \param[in]  a    when true, select scalar value
+       \param[in]  b    when false, select array element
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup data_func_select
     */
diff --git a/include/af/lapack.h b/include/af/lapack.h
index 271d99cf4c..be30cd5900 100644
--- a/include/af/lapack.h
+++ b/include/af/lapack.h
@@ -16,12 +16,13 @@ namespace af
 {
 #if AF_API_VERSION >= 31
     /**
-       C++ Interface for SVD decomposition
+       C++ Interface to perform singular value decomposition.
 
-       \param[out] u is the output array containing U
-       \param[out] s is the output array containing the diagonal values of sigma, (singular values of the input matrix))
-       \param[out] vt is the output array containing V^H
-       \param[in] in is the input matrix
+       \param[out] u  U
+       \param[out] s  diagonal values of sigma (singular values of the input 
+                      matrix)
+       \param[out] vt V^H
+       \param[in]  in input array
 
        \ingroup lapack_factor_func_svd
     */
@@ -30,18 +31,16 @@ namespace af
 
 #if AF_API_VERSION >= 31
     /**
-       C++ Interface for SVD decomposition (in-place)
+       C++ Interface to perform in-place singular value decomposition.
 
-       \param[out]    u is the output array containing U
-       \param[out]    s is the output array containing the diagonal values of sigma,
-                        (singular values of the input matrix))
-       \param[out]    vt is the output array containing V^H
-       \param[in,out] in is the input matrix and will contain random data after
-                         this operation
+       This function minimizes memory usage if `in` is dispensable. Input array
+       `in` is limited to arrays where `dim0` \f$\geq\f$ `dim1`.
 
-       \note Currently, \p in is limited to arrays where `dim0` \f$\geq\f$ `dim1`
-       \note This is best used when minimizing memory usage and \p in is
-             dispensable
+       \param[out]   u  U
+       \param[out]   s  diagonal values of sigma (singular values of the input
+                        matrix)
+       \param[out]   vt V^H
+       \param[inout] in input array; contains random data after the operation                       this operation
 
        \ingroup lapack_factor_func_svd
     */
@@ -49,158 +48,176 @@ namespace af
 #endif
 
     /**
-       C++ Interface for LU decomposition in packed format
+       C++ Interface to perform LU decomposition in packed format.
 
-       \param[out] out is the output array containing the packed LU decomposition
-       \param[out] pivot will contain the permutation indices to map the input to the decomposition
-       \param[in] in is the input matrix
-       \param[in] is_lapack_piv specifies if the pivot is returned in original LAPACK compliant format
+       This function is not supported in GFOR.
 
-       \note This function is not supported in GFOR
+       \param[out] out           packed LU decomposition
+       \param[out] pivot         permutation indices mapping the input to the
+                                 decomposition
+       \param[in]  in            input array
+       \param[in]  is_lapack_piv specifies if the pivot is returned in original
+                                 LAPACK compliant format
 
        \ingroup lapack_factor_func_lu
     */
     AFAPI void lu(array &out, array &pivot, const array &in, const bool is_lapack_piv=true);
 
     /**
-       C++ Interface for LU decomposition
+       C++ Interface to perform LU decomposition.
 
-       \param[out] lower will contain the lower triangular matrix of the LU decomposition
-       \param[out] upper will contain the upper triangular matrix of the LU decomposition
-       \param[out] pivot will contain the permutation indices to map the input to the decomposition
-       \param[in] in is the input matrix
+       This function is not supported in GFOR.
 
-       \note This function is not supported in GFOR
+       \param[out] lower lower triangular matrix of the LU decomposition
+       \param[out] upper upper triangular matrix of the LU decomposition
+       \param[out] pivot permutation indices mapping the input to the
+                         decomposition
+       \param[in]  in    input array
 
        \ingroup lapack_factor_func_lu
     */
     AFAPI void lu(array &lower, array &upper, array &pivot, const array &in);
 
     /**
-      C++ Interface for in place LU decomposition
+       C++ Interface to perform in-place LU decomposition.
 
-      \param[out] pivot will contain the permutation indices to map the input to the decomposition
-      \param[inout] in contains the input on entry, the packed LU decomposition on exit
-      \param[in] is_lapack_piv specifies if the pivot is returned in original LAPACK compliant format
+       This function is not supported in GFOR.
 
-      \note This function is not supported in GFOR
+       \param[out]   pivot         permutation indices mapping the input to the
+                                   decomposition
+       \param[inout] in            input array on entry; packed LU
+                                   decomposition on exit
+       \param[in]    is_lapack_piv specifies if the pivot is returned in
+                                   original LAPACK-compliant format
 
-      \ingroup lapack_factor_func_lu
+       \ingroup lapack_factor_func_lu
     */
     AFAPI void luInPlace(array &pivot, array &in, const bool is_lapack_piv=true);
 
     /**
-       C++ Interface for QR decomposition in packed format
+       C++ Interface to perform QR decomposition in packed format.
 
-       \param[out] out is the output array containing the packed QR decomposition
-       \param[out] tau will contain additional information needed for unpacking the data
-       \param[in] in is the input matrix
+       This function is not supported in GFOR.
 
-       \note This function is not supported in GFOR
+       \param[out] out packed QR decomposition
+       \param[out] tau additional information needed for unpacking the data
+       \param[in]  in  input array
 
        \ingroup lapack_factor_func_qr
     */
     AFAPI void qr(array &out, array &tau, const array &in);
 
     /**
-       C++ Interface for QR decomposition
+       C++ Interface to perform QR decomposition.
 
-       \param[out] q is the orthogonal matrix from QR decomposition
-       \param[out] r is the upper triangular matrix from QR decomposition
-       \param[out] tau will contain additional information needed for solving a least squares problem using \p q and \p r
-       \param[in] in is the input matrix
+       This function is not supported in GFOR.
 
-       \note This function is not supported in GFOR
+       \param[out] q   orthogonal matrix from QR decomposition
+       \param[out] r   upper triangular matrix from QR decomposition
+       \param[out] tau additional information needed for solving a
+                       least-squares problem using `q` and `r`
+       \param[in]  in  input array
 
        \ingroup lapack_factor_func_qr
     */
     AFAPI void qr(array &q, array &r, array &tau, const array &in);
 
     /**
-       C++ Interface for QR decomposition
+       C++ Interface to perform QR decomposition.
 
-       \param[out] tau will contain additional information needed for unpacking the data
-       \param[inout] in is the input matrix on entry. It contains packed QR decomposition on exit
+       This function is not supported in GFOR.
 
-       \note This function is not supported in GFOR
+       \param[out]   tau additional information needed for unpacking the data
+       \param[inout] in  input array on entry; packed QR decomposition on exit
 
        \ingroup lapack_factor_func_qr
     */
     AFAPI void qrInPlace(array &tau, array &in);
 
     /**
-       C++ Interface for cholesky decomposition
-
-       \param[out] out contains the triangular matrix. Multiply \p out with its conjugate transpose reproduces the input \p in.
-       \param[in] in is the input matrix
-       \param[in] is_upper a boolean determining if \p out is upper or lower triangular
+       C++ Interface to perform Cholesky decomposition.
 
-       \returns \p 0 if cholesky decomposition passes, if not it returns the rank at which the decomposition failed.
+       Multiplying `out` with its conjugate transpose reproduces the input
+       `in`.
+       
+       The input must be positive definite.
+       
+       This function is not supported in GFOR.
 
-       \note The input matrix \b has to be a positive definite matrix, if it is not zero, the cholesky decomposition functions return a non-zero output.
-       \note This function is not supported in GFOR
+       \param[out] out      triangular matrix; 
+       \param[in]  in       input matrix
+       \param[in]  is_upper boolean determining if `out` is upper or lower
+                            triangular
+       \returns    `0` if cholesky decomposition passes; if not, it returns the
+                   rank at which the decomposition fails
 
        \ingroup lapack_factor_func_cholesky
     */
     AFAPI int cholesky(array &out, const array &in, const bool is_upper = true);
 
     /**
-       C++ Interface for in place cholesky decomposition
+       C++ Interface to perform in-place Cholesky decomposition.
 
-       \param[inout] in is the input matrix on entry. It contains the triangular matrix on exit.
-       \param[in] is_upper a boolean determining if \p in is upper or lower triangular
+       The input must be positive definite.
 
-       \returns \p 0 if cholesky decomposition passes, if not it returns the rank at which the decomposition failed.
+       This function is not supported in GFOR.
 
-       \note The input matrix \b has to be a positive definite matrix, if it is not zero, the cholesky decomposition functions return a non-zero output.
-       \note This function is not supported in GFOR
+       \param[inout] in       input matrix on entry; triangular matrix on exit
+       \param[in]    is_upper boolean determining if `in` is upper or lower
+                              triangular
+       \returns      `0` if cholesky decomposition passes; if not, it returns
+                     the rank at which the decomposition fails
 
        \ingroup lapack_factor_func_cholesky
     */
     AFAPI int choleskyInPlace(array &in, const bool is_upper = true);
 
     /**
-       C++ Interface for solving a system of equations
+       C++ Interface to solve a system of equations.
 
-       \param[in] a is the coefficient matrix
-       \param[in] b is the measured values
-       \param[in] options determining various properties of matrix \p a
-       \returns \p x, the matrix of unknown variables
+       The `options` parameter must be one of \ref AF_MAT_NONE,
+       \ref AF_MAT_LOWER or \ref AF_MAT_UPPER.
 
-       \note \p options needs to be one of \ref AF_MAT_NONE, \ref AF_MAT_LOWER or \ref AF_MAT_UPPER
-       \note This function is not supported in GFOR
+       This function is not supported in GFOR.
+
+       \param[in] a       coefficient matrix
+       \param[in] b       measured values
+       \param[in] options determines various properties of matrix `a`
+       \returns   `x`, the matrix of unknown variables
 
        \ingroup lapack_solve_func_gen
     */
     AFAPI array solve(const array &a, const array &b, const matProp options = AF_MAT_NONE);
 
-
     /**
-       C++ Interface for solving a system of equations
+       C++ Interface to solve a system of equations.
 
-       \param[in] a is the output matrix from packed LU decomposition of the coefficient matrix
-       \param[in] piv is the pivot array from packed LU decomposition of the coefficient matrix
-       \param[in] b is the matrix of measured values
-       \param[in] options determining various properties of matrix \p a
-       \returns \p x, the matrix of unknown variables
+       The `options` parameter currently must be \ref AF_MAT_NONE.
 
-       \ingroup lapack_solve_lu_func_gen
+       This function is not supported in GFOR.
+
+       \param[in] a       packed LU decomposition of the coefficient matrix
+       \param[in] piv     pivot array from the packed LU decomposition of the
+                          coefficient matrix
+       \param[in] b       measured values
+       \param[in] options determines various properties of matrix `a`
+       \returns   `x`, the matrix of unknown variables
 
-       \note \p options currently needs to be \ref AF_MAT_NONE
-       \note This function is not supported in GFOR
+       \ingroup lapack_solve_lu_func_gen
     */
     AFAPI array solveLU(const array &a, const array &piv,
                         const array &b, const matProp options = AF_MAT_NONE);
 
     /**
-       C++ Interface for inverting a matrix
+       C++ Interface to invert a matrix.
+
+       The `options` parameter currently must be \ref AF_MAT_NONE.
 
-       \param[in] in is input matrix
-       \param[in] options determining various properties of matrix \p in
-       \returns \p x, the inverse of the input matrix
+       This function is not supported in GFOR.
 
-       \note \p options currently needs to be \ref AF_MAT_NONE
-       \note This function is not supported in GFOR
+       \param[in] in      input matrix
+       \param[in] options determines various properties of matrix `in`
+       \returns   inverse matrix
 
        \ingroup lapack_ops_func_inv
     */
@@ -208,19 +225,22 @@ namespace af
 
 #if AF_API_VERSION >= 37
     /**
-       C++ Interface for pseudo-inverting (Moore-Penrose) a matrix.
+       C++ Interface to pseudo-invert (Moore-Penrose) a matrix.
+
        Currently uses the SVD-based approach.
 
-       \param[in] in is the input matrix
-       \param[in] tol defines the lower threshold for singular values from SVD
-       \param[in] options must be AF_MAT_NONE (more options might be supported
-                  in the future)
-       \returns the pseudo-inverse of the input matrix
+       Parameter `tol` is not the actual lower threshold, but it is passed in
+       as a parameter to the calculation of the actual threshold relative to
+       the shape and contents of `in`.
+       
+       This function is not supported in GFOR.
 
-       \note \p tol is not the actual lower threshold, but it is passed in as
-             a parameter to the calculation of the actual threshold relative to
-             the shape and contents of \p in.
-       \note This function is not supported in GFOR
+       \param[in] in      input matrix
+       \param[in] tol     defines the lower threshold for singular values from
+                          SVD
+       \param[in] options must be AF_MAT_NONE (more options might be supported
+                          in the future)
+       \returns   pseudo-inverse matrix
 
        \ingroup lapack_ops_func_pinv
     */
@@ -229,37 +249,36 @@ namespace af
 #endif
 
     /**
-       C++ Interface for finding the rank of a matrix
-
-       \param[in] in is input matrix
-       \param[in] tol is the tolerance value
+       C++ Interface to find the rank of a matrix.
 
-       \returns the rank of the matrix
+       \param[in] in  input matrix
+       \param[in] tol tolerance value
+       \returns   rank
 
        \ingroup lapack_ops_func_rank
     */
     AFAPI unsigned rank(const array &in, const double tol=1E-5);
 
     /**
-       C++ Interface for finding the determinant of a matrix
+       C++ Interface to find the determinant of a matrix.
 
-       \param[in] in is input matrix
-
-       \returns the determinant of the matrix
+       \param[in] in input matrix
+       \returns   determinant
 
        \ingroup lapack_ops_func_det
     */
     template<typename T> T det(const array &in);
 
     /**
-       C++ Interface for norm of a matrix
-
-       \param[in] in is the input matrix
-       \param[in] type specifies the \ref af::normType. Default: \ref AF_NORM_VECTOR_1
-       \param[in] p specifies the value of P when \p type is one of \ref AF_NORM_VECTOR_P, AF_NORM_MATRIX_L_PQ is used. It is ignored for other values of \p type
-       \param[in] q specifies the value of Q when \p type is AF_NORM_MATRIX_L_PQ. This parameter is ignored if \p type is anything else
+       C++ Interface to find the norm of a matrix.
 
-       \returns the norm of \p inbased on \p type
+       \param[in] in   input matrix
+       \param[in] type \ref af::normType. Default: \ref AF_NORM_VECTOR_1
+       \param[in] p    value of P when `type` is \ref AF_NORM_VECTOR_P or
+                       \ref AF_NORM_MATRIX_L_PQ, else ignored
+       \param[in] q    value of Q when `type` is \ref AF_NORM_MATRIX_L_PQ, else
+                       ignored
+       \returns   norm
 
        \ingroup lapack_ops_func_norm
     */
@@ -268,9 +287,9 @@ namespace af
 
 #if AF_API_VERSION >= 33
     /**
-       Returns true is ArrayFire is compiled with LAPACK support
+       Returns true if ArrayFire is compiled with LAPACK support.
 
-       \returns true is LAPACK support is available, false otherwise
+       \returns true if LAPACK support is available; false otherwise
 
        \ingroup lapack_helper_func_available
     */
@@ -286,12 +305,15 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       C Interface for SVD decomposition
+       C Interface to perform singular value decomposition.
 
-       \param[out] u is the output array containing U
-       \param[out] s is the output array containing the diagonal values of sigma, (singular values of the input matrix))
-       \param[out] vt is the output array containing V^H
-       \param[in] in is the input matrix
+       \param[out] u  U
+       \param[out] s  diagonal values of sigma (singular values of the input
+                      matrix)
+       \param[out] vt V^H
+       \param[in]  in input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup lapack_factor_func_svd
     */
@@ -300,18 +322,18 @@ extern "C" {
 
 #if AF_API_VERSION >= 31
     /**
-       C Interface for SVD decomposition (in-place)
+       C Interface to perform in-place singular value decomposition.
 
-       \param[out]    u  is the output array containing U
-       \param[out]    s  is the output array containing the diagonal values of
-                         sigma, (singular values of the input matrix))
-       \param[out]    vt is the output array containing V^H
-       \param[in,out] in is the input matrix that will contain random data after
-                         this operation
+       This function minimizes memory usage if `in` is dispensable. Input array
+       `in` is limited to arrays where `dim0` \f$\geq\f$ `dim1`.
 
-       \note Currently, \p in is limited to arrays where `dim0` \f$\geq\f$ `dim1`
-       \note This is best used when minimizing memory usage and \p in is
-             dispensable
+       \param[out]   u  U
+       \param[out]   s  diagonal values of sigma (singular values of the input
+                        matrix)
+       \param[out]   vt V^H
+       \param[inout] in input array; contains random data after the operation                       this operation
+       \return       \ref AF_SUCCESS, if function returns successfully, else
+                     an \ref af_err code is given
 
        \ingroup lapack_factor_func_svd
     */
@@ -319,139 +341,182 @@ extern "C" {
 #endif
 
     /**
-       C Interface for LU decomposition
+       C Interface to perform LU decomposition.
 
-       \param[out] lower will contain the lower triangular matrix of the LU decomposition
-       \param[out] upper will contain the upper triangular matrix of the LU decomposition
-       \param[out] pivot will contain the permutation indices to map the input to the decomposition
-       \param[in] in is the input matrix
+       \param[out] lower lower triangular matrix of the LU decomposition
+       \param[out] upper upper triangular matrix of the LU decomposition
+       \param[out] pivot permutation indices mapping the input to the
+                         decomposition
+       \param[in]  in    input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup lapack_factor_func_lu
     */
     AFAPI af_err af_lu(af_array *lower, af_array *upper, af_array *pivot, const af_array in);
 
     /**
-       C Interface for in place LU decomposition
+       C Interface to perform in-place LU decomposition.
+
+       This function is not supported in GFOR.
 
-       \param[out] pivot will contain the permutation indices to map the input to the decomposition
-       \param[inout] in contains the input on entry, the packed LU decomposition on exit
-       \param[in] is_lapack_piv specifies if the pivot is returned in original LAPACK compliant format
+       \param[out]   pivot         permutation indices mapping the input to the
+                                   decomposition
+       \param[inout] in            input array on entry; packed LU
+                                   decomposition on exit
+       \param[in]    is_lapack_piv specifies if the pivot is returned in
+                                   original LAPACK-compliant format
+       \return       \ref AF_SUCCESS, if function returns successfully, else
+                     an \ref af_err code is given
 
        \ingroup lapack_factor_func_lu
     */
     AFAPI af_err af_lu_inplace(af_array *pivot, af_array in, const bool is_lapack_piv);
 
     /**
-       C Interface for QR decomposition
+       C Interface to perform QR decomposition.
 
-       \param[out] q is the orthogonal matrix from QR decomposition
-       \param[out] r is the upper triangular matrix from QR decomposition
-       \param[out] tau will contain additional information needed for solving a least squares problem using \p q and \p r
-       \param[in] in is the input matrix
+       This function is not supported in GFOR.
+
+       \param[out] q   orthogonal matrix from QR decomposition
+       \param[out] r   upper triangular matrix from QR decomposition
+       \param[out] tau additional information needed for solving a
+                       least-squares problem using `q` and `r`
+       \param[in]  in  input array
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup lapack_factor_func_qr
     */
     AFAPI af_err af_qr(af_array *q, af_array *r, af_array *tau, const af_array in);
 
     /**
-       C Interface for QR decomposition
+       C Interface to perform QR decomposition.
+
+       This function is not supported in GFOR.
 
-       \param[out] tau will contain additional information needed for unpacking the data
-       \param[inout] in is the input matrix on entry. It contains packed QR decomposition on exit
+       \param[out]   tau additional information needed for unpacking the data
+       \param[inout] in  input array on entry; packed QR decomposition on exit
+       \return       \ref AF_SUCCESS, if function returns successfully, else
+                     an \ref af_err code is given
 
        \ingroup lapack_factor_func_qr
     */
     AFAPI af_err af_qr_inplace(af_array *tau, af_array in);
 
     /**
-       C++ Interface for cholesky decomposition
+       C Interface to perform Cholesky decomposition.
 
-       \param[out] out contains the triangular matrix. Multiply \p out with it conjugate transpose reproduces the input \p in.
-       \param[out] info is \p 0 if cholesky decomposition passes, if not it returns the rank at which the decomposition failed.
-       \param[in] in is the input matrix
-       \param[in] is_upper a boolean determining if \p out is upper or lower triangular
+       Multiplying `out` with its conjugate transpose reproduces the input
+       `in`.
 
-       \note The input matrix \b has to be a positive definite matrix, if it is not zero, the cholesky decomposition functions return a non zero output.
+       The input must be positive definite.
+
+       \param[out] out      triangular matrix;
+       \param[out] info     `0` if cholesky decomposition passes; if not, it
+                            returns the rank at which the decomposition fails
+       \param[in]  in       input matrix
+       \param[in]  is_upper boolean determining if `out` is upper or lower
+                            triangular
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup lapack_factor_func_cholesky
     */
     AFAPI af_err af_cholesky(af_array *out, int *info, const af_array in, const bool is_upper);
 
     /**
-       C Interface for in place cholesky decomposition
+       C Interface to perform in-place Cholesky decomposition.
 
-       \param[out] info is \p 0 if cholesky decomposition passes, if not it returns the rank at which the decomposition failed.
-       \param[inout] in is the input matrix on entry. It contains the triangular matrix on exit.
-       \param[in] is_upper a boolean determining if \p in is upper or lower triangular
+       The input must be positive definite.
 
-       \note The input matrix \b has to be a positive definite matrix, if it is not zero, the cholesky decomposition functions return a non zero output.
+       \param[out]   info     `0` if cholesky decomposition passes; if not, it
+                              returns the rank at which the decomposition fails
+       \param[inout] in       input matrix on entry; triangular matrix on exit
+       \param[in]    is_upper boolean determining if `in` is upper or lower
+                              triangular
+       \return       \ref AF_SUCCESS, if function returns successfully, else
+                     an \ref af_err code is given
 
        \ingroup lapack_factor_func_cholesky
     */
     AFAPI af_err af_cholesky_inplace(int *info, af_array in, const bool is_upper);
 
     /**
-       C Interface for solving a system of equations
+       C Interface to solve a system of equations.
 
-       \param[out] x is the matrix of unknown variables
-       \param[in] a is the coefficient matrix
-       \param[in] b is the measured values
-       \param[in] options determining various properties of matrix \p a
+       The `options` parameter must be one of \ref AF_MAT_NONE,
+       \ref AF_MAT_LOWER or \ref AF_MAT_UPPER.
 
-       \ingroup lapack_solve_func_gen
+       This function is not supported in GFOR.
 
-       \note \p options needs to be one of \ref AF_MAT_NONE, \ref AF_MAT_LOWER or \ref AF_MAT_UPPER
+       \param[out] x       matrix of unknown variables
+       \param[in]  a       coefficient matrix
+       \param[in]  b       measured values
+       \param[in]  options determines various properties of matrix `a`
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
+
+       \ingroup lapack_solve_func_gen
     */
     AFAPI af_err af_solve(af_array *x, const af_array a, const af_array b,
                           const af_mat_prop options);
 
     /**
-       C Interface for solving a system of equations
+       C Interface to solve a system of equations.
 
-       \param[out] x will contain the matrix of unknown variables
-       \param[in] a is the output matrix from packed LU decomposition of the coefficient matrix
-       \param[in] piv is the pivot array from packed LU decomposition of the coefficient matrix
-       \param[in] b is the matrix of measured values
-       \param[in] options determining various properties of matrix \p a
+       The `options` parameter currently must be \ref AF_MAT_NONE.
 
-       \ingroup lapack_solve_lu_func_gen
+       \param[out] x       matrix of unknown variables
+       \param[in]  a       packed LU decomposition of the coefficient matrix
+       \param[in]  piv     pivot array from the packed LU decomposition of the
+                           coefficient matrix
+       \param[in]  b       measured values
+       \param[in]  options determines various properties of matrix `a`
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-       \note \p options currently needs to be \ref AF_MAT_NONE
-       \note This function is not supported in GFOR
+       \ingroup lapack_solve_lu_func_gen
     */
     AFAPI af_err af_solve_lu(af_array *x, const af_array a, const af_array piv,
                              const af_array b, const af_mat_prop options);
 
     /**
-       C Interface for inverting a matrix
+       C Interface to invert a matrix.
 
-       \param[out] out will contain the inverse of matrix \p in
-       \param[in] in is input matrix
-       \param[in] options determining various properties of matrix \p in
+       The `options` parameter currently must be \ref AF_MAT_NONE.
 
-       \ingroup lapack_ops_func_inv
+       \param[out] out     inverse matrix
+       \param[in]  in      input matrix
+       \param[in]  options determines various properties of matrix `in`
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
-       \note currently options needs to be \ref AF_MAT_NONE
+       \ingroup lapack_ops_func_inv
     */
     AFAPI af_err af_inverse(af_array *out, const af_array in, const af_mat_prop options);
 
 #if AF_API_VERSION >= 37
     /**
-       C Interface for pseudo-inverting (Moore-Penrose) a matrix.
+       C Interface to pseudo-invert (Moore-Penrose) a matrix.
+
        Currently uses the SVD-based approach.
 
-       \param[out] out will contain the pseudo-inverse of matrix \p in
-       \param[in] in is the input matrix
-       \param[in] tol defines the lower threshold for singular values from SVD
-       \param[in] options must be AF_MAT_NONE (more options might be supported
-       in the future)
+       Parameter `tol` is not the actual lower threshold, but it is passed in
+       as a parameter to the calculation of the actual threshold relative to
+       the shape and contents of `in`.
 
-       \note \p tol is not the actual lower threshold, but it is passed in as a
-             parameter to the calculation of the actual threshold relative to the
-             shape and contents of \p in.
-       \note At first, try setting \p tol to 1e-6 for single precision and 1e-12
-             for double.
-       \note This function is not supported in GFOR
+       Suggested parameters for `tol`:  1e-6 for single precision and 1e-12 for
+       double precision.
+
+       \param[out] out     pseudo-inverse matrix
+       \param[in]  in      input matrix
+       \param[in]  tol     defines the lower threshold for singular values from
+                           SVD
+       \param[in]  options must be AF_MAT_NONE (more options might be supported
+                           in the future)
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup lapack_ops_func_pinv
     */
@@ -460,36 +525,43 @@ extern "C" {
 #endif
 
     /**
-       C Interface for finding the rank of a matrix
+       C Interface to find the rank of a matrix.
 
-       \param[out] rank will contain the rank of \p in
-       \param[in] in is input matrix
-       \param[in] tol is the tolerance value
+       \param[out] rank rank
+       \param[in]  in   input matrix
+       \param[in]  tol  tolerance value
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup lapack_ops_func_rank
     */
     AFAPI af_err af_rank(unsigned *rank, const af_array in, const double tol);
 
     /**
-       C Interface for finding the determinant of a matrix
+       C Interface to find the determinant of a matrix.
 
-       \param[out] det_real will contain the real part of the determinant of \p in
-       \param[out] det_imag will contain the imaginary part of the determinant of \p in
-       \param[in] in is input matrix
+       \param[out] det_real real part of the determinant
+       \param[out] det_imag imaginary part of the determinant
+       \param[in]  in       input matrix
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup lapack_ops_func_det
     */
     AFAPI af_err af_det(double *det_real, double *det_imag, const af_array in);
 
     /**
-       C Interface for norm of a matrix
-
-       \param[out] out will contain the norm of \p in
-       \param[in] in is the input matrix
-       \param[in] type specifies the \ref af::normType. Default: \ref AF_NORM_VECTOR_1
-       \param[in] p specifies the value of P when \p type is one of \ref AF_NORM_VECTOR_P,  AF_NORM_MATRIX_L_PQ is used. It is ignored for other values of \p type
-       \param[in] q specifies the value of Q when \p type is AF_NORM_MATRIX_L_PQ. This parameter is ignored if \p type is anything else
+       C Interface to find the norm of a matrix.
 
+       \param[out] out  norm
+       \param[in]  in   input matrix
+       \param[in]  type \ref af::normType. Default: \ref AF_NORM_VECTOR_1
+       \param[in]  p    value of P when `type` is \ref AF_NORM_VECTOR_P or
+                        \ref AF_NORM_MATRIX_L_PQ, else ignored
+       \param[in]  q    value of Q when `type` is \ref AF_NORM_MATRIX_L_PQ, else
+                        ignored
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup lapack_ops_func_norm
     */
@@ -497,11 +569,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 33
     /**
-       Returns true is ArrayFire is compiled with LAPACK support
-
-       \param[out] out is true if LAPACK support is available, false otherwise
+       Returns true if ArrayFire is compiled with LAPACK support.
 
-       \returns AF_SUCCESS if successful (does not depend on the value of out)
+       \param[out] out true if LAPACK support is available; false otherwise
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given; does not depend on the value
+                   of `out`
 
        \ingroup lapack_helper_func_available
     */
diff --git a/include/af/random.h b/include/af/random.h
index bf81e9218e..53939be226 100644
--- a/include/af/random.h
+++ b/include/af/random.h
@@ -11,7 +11,7 @@
 #include <af/defines.h>
 
 ///
-/// \brief Handle for random engine
+/// \brief Handle for a random engine object.
 ///
 /// This handle is used to reference the internal random engine object.
 ///
@@ -24,7 +24,7 @@ namespace af
     class array;
     class dim4;
 #if AF_API_VERSION >= 34
-    /// \brief Random Number Generation Engine Class
+    /// C++ Interface - Random Number Generation Engine Class
     ///
     /// The \ref af::randomEngine class is used to set the type and seed of
     /// random number generation engine based on \ref af::randomEngineType.
@@ -39,79 +39,79 @@ namespace af
 
     public:
       /**
-          This function creates a \ref af::randomEngine object with a
-          \ref af::randomEngineType and a seed.
+          C++ Interface to create a \ref af::randomEngine object with a \ref
+          af::randomEngineType and a seed.
 
           \code
-          // creates random engine of default type with seed = 1
-          randomEngine r(AF_RANDOM_ENGINE_DEFAULT, 1);
-         \endcode
+            // create a random engine of default type with seed = 1
+            randomEngine r(AF_RANDOM_ENGINE_DEFAULT, 1);
+          \endcode
       */
       explicit randomEngine(randomEngineType typeIn = AF_RANDOM_ENGINE_DEFAULT,
                             unsigned long long seedIn = 0);
 
       /**
-          Copy constructor for \ref af::randomEngine.
+          C++ Interface copy constructor for a \ref af::randomEngine.
 
-          \param[in] other The input random engine object
+          \param[in] other input random engine object
       */
       randomEngine(const randomEngine &other);
 
       /**
-          Creates a copy of the random engine object from a \ref
-          af_random_engine handle.
+          C++ Interface to create a copy of the random engine object from a
+          \ref af_random_engine handle.
 
           \param[in] engine The input random engine object
       */
       randomEngine(af_random_engine engine);
 
       /**
-          \brief Destructor for \ref af::randomEngine
+          C++ Interface destructor for a \ref af::randomEngine.
       */
       ~randomEngine();
 
       /**
-          \brief Assigns the internal state of randome engine
+          C++ Interface to assign the internal state of randome engine.
 
-          \param[in] other The object to be assigned to the random engine
+          \param[in] other object to be assigned to the random engine
 
-          \returns the reference to this
+          \return the reference to this
       */
       randomEngine &operator=(const randomEngine &other);
 
       /**
-          \brief Sets the random type of the random engine
+          C++ Interface to set the random type of the random engine.
 
-          \param[in] type The type of the random number generator
+          \param[in] type type of the random number generator
       */
       void setType(const randomEngineType type);
 
       /**
-          \brief Return the random type of the random engine
+          C++ Interface to get the random type of the random engine.
 
-          \returns the \ref af::randomEngineType associated with random engine
+          \return \ref af::randomEngineType associated with random engine
       */
       randomEngineType getType(void);
 
       /**
-          \brief Sets the seed of the random engine
+          C++ Interface to set the seed of the random engine.
 
-          \param[in] seed The initializing seed of the random number generator
+          \param[in] seed initializing seed of the random number generator
       */
       void setSeed(const unsigned long long seed);
 
       /**
-          \brief Returns the seed of the random engine
+          C++ Interface to return the seed of the random engine.
 
-          \returns the seed associated with random engine
+          \return seed associated with random engine
       */
       unsigned long long getSeed(void) const;
 
       /**
-          \brief Returns the af_random_engine handle of this object
+          C++ Interface to return the af_random_engine handle of this object.
 
-          \returns the handle to the af_random_engine associated with this
-                   random engine
+          \return handle to the af_random_engine associated with this random
+                  engine
       */
       af_random_engine get(void) const;
     };
@@ -119,11 +119,13 @@ namespace af
 
 #if AF_API_VERSION >= 34
     /**
-        \param[in] dims The dimensions of the array to be generated
-        \param[in] ty The type of the array
-        \param[in] r The random engine object
+        C++ Interface to create an array of random numbers uniformly
+        distributed.
 
-        \return array of size \p dims
+        \param[in] dims dimensions of the array to be generated
+        \param[in] ty   type of the array
+        \param[in] r    random engine object
+        \return    random number array of size `dims`
 
         \ingroup random_func_randu
     */
@@ -132,11 +134,13 @@ namespace af
 
 #if AF_API_VERSION >= 34
     /**
-        \param[in] dims The dimensions of the array to be generated
-        \param[in] ty The type of the array
-        \param[in] r The random engine object
+        C++ Interface to create an array of random numbers normally
+        distributed.
 
-        \return array of size \p dims
+        \param[in] dims dimensions of the array to be generated
+        \param[in] ty   type of the array
+        \param[in] r    random engine object
+        \return    random number array of size `dims`
 
         \ingroup random_func_randn
     */
@@ -144,31 +148,36 @@ namespace af
 #endif
 
     /**
-        \param[in] dims The dimensions of the array to be generated
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers uniformly
+        distributed.
 
-        \return array of size \p dims
+        \param[in] dims dimensions of the array to be generated
+        \param[in] ty   type of the array
 
         \ingroup random_func_randu
     */
     AFAPI array randu(const dim4 &dims, const dtype ty=f32);
 
     /**
-        \param[in] d0 The size of the first dimension
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers uniformly
+        distributed.
 
-        \return array of size \p d0
+        \param[in] d0 size of the first dimension
+        \param[in] ty type of the array
+        \return    random number array of size `d0`
 
         \ingroup random_func_randu
     */
     AFAPI array randu(const dim_t d0, const dtype ty=f32);
 
     /**
-        \param[in] d0 The size of the first dimension
-        \param[in] d1 The size of the second dimension
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers uniformly
+        distributed.
 
-        \return array of size \p d0 x \p d1
+        \param[in] d0 size of the first dimension
+        \param[in] d1 size of the second dimension
+        \param[in] ty type of the array
+        \return    random number array of size `d0` x `d1`
 
         \ingroup random_func_randu
     */
@@ -176,12 +185,14 @@ namespace af
                       const dim_t d1, const dtype ty=f32);
 
     /**
-        \param[in] d0 The size of the first dimension
-        \param[in] d1 The size of the second dimension
-        \param[in] d2 The size of the third dimension
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers uniformly
+        distributed.
 
-        \return array of size \p d0 x \p d1 x \p d2
+        \param[in] d0 size of the first dimension
+        \param[in] d1 size of the second dimension
+        \param[in] d2 size of the third dimension
+        \param[in] ty type of the array
+        \return    random number array of size `d0` x `d1` x `d2`
 
         \ingroup random_func_randu
     */
@@ -189,13 +200,15 @@ namespace af
                       const dim_t d1, const dim_t d2, const dtype ty=f32);
 
     /**
-        \param[in] d0 The size of the first dimension
-        \param[in] d1 The size of the second dimension
-        \param[in] d2 The size of the third dimension
-        \param[in] d3 The size of the fourth dimension
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers uniformly
+        distributed.
 
-        \return array of size \p d0 x \p d1 x \p d2 x \p d3
+        \param[in] d0 size of the first dimension
+        \param[in] d1 size of the second dimension
+        \param[in] d2 size of the third dimension
+        \param[in] d3 size of the fourth dimension
+        \param[in] ty type of the array
+        \return    random number array of size `d0` x `d1` x `d2` x `d3`
 
         \ingroup random_func_randu
     */
@@ -204,42 +217,50 @@ namespace af
                       const dim_t d3, const dtype ty=f32);
 
     /**
-        \param[in] dims The dimensions of the array to be generated
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers normally
+        distributed.
 
-        \return array of size \p dims
+        \param[in] dims dimensions of the array to be generated
+        \param[in] ty   type of the array
+        \return    random number array of size `dims`
 
         \ingroup random_func_randn
     */
     AFAPI array randn(const dim4 &dims, const dtype ty=f32);
 
     /**
-        \param[in] d0 The size of the first dimension
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers normally
+        distributed.
 
-        \return array of size \p d0
+        \param[in] d0 size of the first dimension
+        \param[in] ty type of the array
+        \return    random number array of size `d0`
 
         \ingroup random_func_randn
     */
     AFAPI array randn(const dim_t d0, const dtype ty=f32);
     /**
-        \param[in] d0 The size of the first dimension
-        \param[in] d1 The size of the second dimension
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers normally
+        distributed.
 
-        \return array of size \p d0 x \p d1
+        \param[in] d0 size of the first dimension
+        \param[in] d1 size of the second dimension
+        \param[in] ty type of the array
+        \return    random number array of size `d0` x `d1`
 
         \ingroup random_func_randn
     */
     AFAPI array randn(const dim_t d0,
                       const dim_t d1, const dtype ty=f32);
     /**
-        \param[in] d0 The size of the first dimension
-        \param[in] d1 The size of the second dimension
-        \param[in] d2 The size of the third dimension
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers normally
+        distributed.
 
-        \return array of size \p d0 x \p d1 x \p d2
+        \param[in] d0 size of the first dimension
+        \param[in] d1 size of the second dimension
+        \param[in] d2 size of the third dimension
+        \param[in] ty type of the array
+        \return    random number array of size `d0` x `d1` x `d2`
 
         \ingroup random_func_randn
     */
@@ -247,13 +268,15 @@ namespace af
                       const dim_t d1, const dim_t d2, const dtype ty=f32);
 
     /**
-        \param[in] d0 The size of the first dimension
-        \param[in] d1 The size of the second dimension
-        \param[in] d2 The size of the third dimension
-        \param[in] d3 The size of the fourth dimension
-        \param[in] ty The type of the array
+        C++ Interface to create an array of random numbers normally
+        distributed.
 
-        \return array of size \p d0 x \p d1 x \p d2 x \p d3
+        \param[in] d0 size of the first dimension
+        \param[in] d1 size of the second dimension
+        \param[in] d2 size of the third dimension
+        \param[in] d3 size of the fourth dimension
+        \param[in] ty type of the array
+        \return    random number array of size `d0` x `d1` x `d2` x `d3`
 
         \ingroup random_func_randn
     */
@@ -263,7 +286,9 @@ namespace af
 
 #if AF_API_VERSION >= 34
     /**
-        \param[in] rtype The type of the random number generator
+        C++ Interface to set the default random engine type.
+
+        \param[in] rtype type of the random number generator
 
         \ingroup random_func_set_default_engine
     */
@@ -272,7 +297,9 @@ namespace af
 
 #if AF_API_VERSION >= 34
     /**
-        \returns the \ref af::randomEngine object for the default random engine
+        C++ Interface to get the default random engine type.
+
+        \return \ref af::randomEngine object for the default random engine
 
         \ingroup random_func_get_default_engine
     */
@@ -280,17 +307,19 @@ namespace af
 #endif
 
     /**
-        \brief Sets the seed of the default random number generator
+        C++ Interface to set the seed of the default random number generator.
+
+        \param[in] seed 64-bit unsigned integer
 
-        \param[in] seed A 64 bit unsigned integer
         \ingroup random_func_set_seed
     */
     AFAPI void setSeed(const unsigned long long seed);
 
     /**
-        \brief Gets the seed of the default random number generator
+        C++ Interface to get the seed of the default random number generator.
+
+        \return seed 64-bit unsigned integer
 
-        \returns seed A 64 bit unsigned integer
         \ingroup random_func_get_seed
     */
     AFAPI unsigned long long getSeed();
@@ -304,13 +333,13 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for creating random engine
+       C Interface to create a random engine.
 
-       \param[out]  engine The pointer to the returned random engine object
-       \param[in]   rtype The type of the random number generator
-       \param[in]   seed The initializing seed of the random number generator
-
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[out] engine pointer to the returned random engine object
+       \param[in]  rtype  type of the random number generator
+       \param[in]  seed   initializing seed of the random number generator
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_random_engine
     */
@@ -321,12 +350,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for retaining random engine
-
-       \param[out]  out The pointer to the returned random engine object
-       \param[in]   engine The random engine object
+       C Interface to retain a random engine.
 
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[out] out    pointer to the returned random engine object
+       \param[in]  engine random engine object
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_random_engine
     */
@@ -336,12 +365,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for changing random engine type
-
-       \param[in]   engine The random engine object
-       \param[in]   rtype The type of the random number generator
+       C Interface to change random engine type.
 
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[in]  engine random engine object
+       \param[in]  rtype  type of the random number generator
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_random_engine
     */
@@ -351,12 +380,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for getting random engine type
+       C Interface to get random engine type.
 
-       \param[out]  rtype The type of the random number generator
-       \param[in]   engine The random engine object
-
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[out] rtype  type of the random number generator
+       \param[in]  engine random engine object
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_random_engine
     */
@@ -366,18 +395,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for creating an array of uniform numbers using a random
-       engine
-
-       \param[out]  out The pointer to the returned object.
-       \param[in]   ndims The number of dimensions read from the \p dims
-                    parameter
-       \param[in]   dims A C pointer with \p ndims elements. Each value
-                    represents the size of that dimension
-       \param[in]   type The type of the \ref af_array object
-       \param[in]   engine The random engine object
+       C Interface to create an array of uniform numbers using a random engine.
 
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[out] out    pointer to the returned object
+       \param[in]  ndims  number of dimensions
+       \param[in]  dims   C pointer with `ndims` elements; each value
+                          represents the size of that dimension
+       \param[in]  type   type of the \ref af_array object
+       \param[in]  engine random engine object
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_randu
     */
@@ -388,17 +415,16 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for creating an array of normal numbers using a random engine
+       C Interface to create an array of normal numbers using a random engine.
 
-       \param[out]  out The pointer to the returned object.
-       \param[in]   ndims The number of dimensions read from the \p dims
-                    parameter
-       \param[in]   dims A C pointer with \p ndims elements. Each value
-                    represents the size of that dimension
-       \param[in]   type The type of the \ref af_array object
-       \param[in]   engine The random engine object
-
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[out] out    pointer to the returned object
+       \param[in]  ndims  number of dimensions
+       \param[in]  dims   C pointer with `ndims` elements; each value
+                          represents the size of that dimension
+       \param[in]  type   type of the \ref af_array object
+       \param[in]  engine random engine object
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_randn
     */
@@ -409,12 +435,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for setting the seed of a random engine
-
-       \param[out]  engine The pointer to the returned random engine object
-       \param[in]   seed The initializing seed of the random number generator
+       C Interface to set the seed of a random engine.
 
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[out] engine pointer to the returned random engine object
+       \param[in]  seed   initializing seed of the random number generator
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_random_engine
     */
@@ -424,11 +450,11 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for getting the default random engine
+       C Interface to get the default random engine.
 
-       \param[out]  engine The pointer to returned default random engine object
-
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[out] engine pointer to the returned default random engine object
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_get_default_engine
     */
@@ -437,11 +463,11 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for setting the type of the default random engine
-
-       \param[in]   rtype The type of the random number generator
+       C Interface to set the type of the default random engine.
 
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[in]  rtype type of the random number generator
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_set_default_engine
     */
@@ -450,12 +476,12 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for getting the seed of a random engine
-
-       \param[out]  seed The pointer to the returned seed.
-       \param[in]   engine The random engine object
+       C Interface to get the seed of a random engine.
 
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[out] seed   pointer to the returned seed
+       \param[in]  engine random engine object
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_random_engine
     */
@@ -465,10 +491,11 @@ extern "C" {
 
 #if AF_API_VERSION >= 34
     /**
-       C Interface for releasing random engine
+       C Interface to release a random engine.
 
-       \param[in] engine The random engine object
-       \returns \ref AF_SUCCESS if the execution completes properly
+       \param[in] engine random engine object
+       \return    \ref AF_SUCCESS, if function returns successfully, else
+                  an \ref af_err code is given
 
        \ingroup random_func_random_engine
     */
@@ -476,10 +503,12 @@ extern "C" {
 #endif
 
     /**
-        \param[out] out The generated array
-        \param[in] ndims Size of dimension array \p dims
-        \param[in] dims The array containing sizes of the dimension
-        \param[in] type The type of array to generate
+       \param[out] out   generated array
+       \param[in]  ndims number of dimensions
+       \param[in]  dims  array containing sizes of the dimension
+       \param[in]  type  type of array to generate
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_randu
     */
@@ -487,10 +516,12 @@ extern "C" {
                           const dim_t * const dims, const af_dtype type);
 
     /**
-        \param[out] out The generated array
-        \param[in] ndims Size of dimension array \p dims
-        \param[in] dims The array containing sizes of the dimension
-        \param[in] type The type of array to generate
+       \param[out] out   generated array
+       \param[in]  ndims number of dimensions
+       \param[in]  dims  array containing sizes of the dimension
+       \param[in]  type  type of array to generate
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
        \ingroup random_func_randn
     */
@@ -498,14 +529,18 @@ extern "C" {
                           const dim_t * const dims, const af_dtype type);
 
     /**
-        \param[in] seed A 64 bit unsigned integer
+       \param[in] seed a 64-bit unsigned integer
+       \return    \ref AF_SUCCESS, if function returns successfully, else
+                  an \ref af_err code is given
 
         \ingroup random_func_set_seed
     */
     AFAPI af_err af_set_seed(const unsigned long long seed);
 
     /**
-        \param[out] seed A 64 bit unsigned integer
+       \param[out] seed a 64-bit unsigned integer
+       \return     \ref AF_SUCCESS, if function returns successfully, else
+                   an \ref af_err code is given
 
         \ingroup random_func_get_seed
     */

From 4e4a4145e5a6305366835700fef44dd9b3cceab1 Mon Sep 17 00:00:00 2001
From: pv-pterab-s <75991366+pv-pterab-s@users.noreply.github.com>
Date: Thu, 17 Aug 2023 23:54:01 -0400
Subject: [PATCH 367/473] unified: backend id fix (#3424)

* fix: incorrect backend id bitshift. incorrect number of backends
* fix unified: convert backend_id to index with backend_index()


---------

Co-authored-by: Gallagher Donovan Pryor <gallagher@arrayfire.com>
---
 src/api/unified/symbol_manager.cpp | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/api/unified/symbol_manager.cpp b/src/api/unified/symbol_manager.cpp
index d3aed5f498..93ca06938f 100644
--- a/src/api/unified/symbol_manager.cpp
+++ b/src/api/unified/symbol_manager.cpp
@@ -193,16 +193,15 @@ AFSymbolManager::AFSymbolManager()
     // In order of priority.
     static const af_backend order[] = {AF_BACKEND_CUDA, AF_BACKEND_ONEAPI,
                                        AF_BACKEND_OPENCL, AF_BACKEND_CPU};
-
-    LibHandle handle    = nullptr;
-    af::Backend backend = AF_BACKEND_DEFAULT;
+    LibHandle handle                = nullptr;
+    af::Backend backend             = AF_BACKEND_DEFAULT;
     // Decremeting loop. The last successful backend loaded will be the most
     // prefered one.
     for (int i = NUM_BACKENDS - 1; i >= 0; i--) {
-        int backend_index          = order[i] >> 1U;  // 2 4 1 -> 1 2 0
-        bkndHandles[backend_index] = openDynLibrary(order[i]);
-        if (bkndHandles[backend_index]) {
-            handle  = bkndHandles[backend_index];
+        int bknd_idx          = backend_index(order[i]);
+        bkndHandles[bknd_idx] = openDynLibrary(order[i]);
+        if (bkndHandles[bknd_idx]) {
+            handle  = bkndHandles[bknd_idx];
             backend = order[i];
             numBackends++;
             backendsAvailable += order[i];
@@ -242,7 +241,7 @@ af_err setBackend(af::Backend bknd) {
             UNIFIED_ERROR_LOAD_LIB();
         }
     }
-    int idx = bknd >> 1U;  // Convert 1, 2, 4 -> 0, 1, 2
+    int idx = backend_index(bknd);
     if (instance.getHandle(idx)) {
         getActiveHandle()  = instance.getHandle(idx);
         getActiveBackend() = bknd;

From b2f18400bc0510ed43af4da8e4bb1370ef57809f Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 18 Aug 2023 20:30:05 -0400
Subject: [PATCH 368/473] Fix bug in Shift JIT kernels because of empty
 dimensions in Arrays

---
 src/backend/common/jit/Node.cpp          |  6 ++
 src/backend/common/jit/Node.hpp          | 71 ++++++++++++++++++++++++
 src/backend/common/jit/ShiftNodeBase.hpp |  3 +
 src/backend/cuda/CMakeLists.txt          |  1 +
 src/backend/cuda/Param.hpp               |  3 +
 src/backend/cuda/jit.cpp                 | 25 +++------
 src/backend/cuda/jit/ShiftNode.hpp       | 22 ++++++++
 src/backend/cuda/shift.cpp               |  6 +-
 src/backend/oneapi/CMakeLists.txt        |  1 +
 src/backend/oneapi/Param.hpp             |  6 ++
 src/backend/oneapi/jit.cpp               | 19 ++-----
 src/backend/oneapi/jit/ShiftNode.hpp     | 22 ++++++++
 src/backend/opencl/CMakeLists.txt        |  1 +
 src/backend/opencl/Param.hpp             |  3 +
 src/backend/opencl/jit.cpp               | 19 ++-----
 src/backend/opencl/jit/ShiftNode.hpp     | 21 +++++++
 src/backend/opencl/kernel/KParam.hpp     |  6 ++
 src/backend/opencl/shift.cpp             |  4 +-
 test/shift.cpp                           |  9 +++
 19 files changed, 195 insertions(+), 53 deletions(-)
 create mode 100644 src/backend/cuda/jit/ShiftNode.hpp
 create mode 100644 src/backend/oneapi/jit/ShiftNode.hpp
 create mode 100644 src/backend/opencl/jit/ShiftNode.hpp

diff --git a/src/backend/common/jit/Node.cpp b/src/backend/common/jit/Node.cpp
index 0e67228f91..f77d68e260 100644
--- a/src/backend/common/jit/Node.cpp
+++ b/src/backend/common/jit/Node.cpp
@@ -76,6 +76,12 @@ auto isScalar(const Node &ptr) -> bool { return ptr.isScalar(); }
 
 bool Node::isLinear(const dim_t dims[4]) const { return true; }
 
+/// This function returns true if the \p node is a Shift node or a Buffer node
+auto isBufferOrShift(const Node_ptr &node) -> bool {
+    return node->getNodeType() == kNodeType::Buffer ||
+           node->getNodeType() == kNodeType::Shift;
+}
+
 }  // namespace common
 }  // namespace arrayfire
 
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 42da5a09d3..8f2e0183b6 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -14,6 +14,7 @@
 #include <types.hpp>
 #include <af/defines.h>
 
+#include <nonstd/span.hpp>
 #include <algorithm>
 #include <array>
 #include <functional>
@@ -330,9 +331,79 @@ std::string getFuncName(const std::vector<Node *> &output_nodes,
                         const bool is_linear, const bool loop0,
                         const bool loop1, const bool loop2, const bool loop3);
 
+/// Returns true if the \p ptr is a Buffer Node
 auto isBuffer(const Node &ptr) -> bool;
 
+/// Returns true if the \p ptr is a Scalar Node
 auto isScalar(const Node &ptr) -> bool;
 
+/// Returns true if \p node is a Buffer or a Shift node
+auto isBufferOrShift(const Node_ptr &node) -> bool;
+
+template<typename T>
+inline void applyShifts(std::array<int, 4> &shifts, nonstd::span<T> dims) {
+    std::array<T, 4> out;
+    for (size_t i = 0; i < shifts.size(); i++) { out[i] = dims[shifts[i]]; }
+    std::copy(begin(out), std::end(out), std::begin(dims));
+}
+
+template<typename ArrayT>
+inline std::array<int, 4> compressArray(ArrayT dims) {
+    std::array<int, 4> shifts{0, 1, 2, 3};
+    bool changed;
+    do {
+        changed = false;
+        for (int i = 0; i < AF_MAX_DIMS - 1; i++) {
+            if (dims[i] == 1 && dims[i + 1] != 1) {
+                std::swap(dims[i], dims[i + 1]);
+                std::swap(shifts[i], shifts[i + 1]);
+                changed = true;
+            }
+        }
+    } while (changed);
+    return shifts;
+}
+
+/// Removes empty columns from output and the other node pointers in \p nodes
+template<typename ParamT, typename BufferNodeT, typename ShiftNodeT>
+void removeEmptyDimensions(nonstd::span<ParamT> outputs,
+                           nonstd::span<Node_ptr> nodes) {
+    dim_t *outDims{outputs[0].dims_ptr()};
+    dim_t *outStrides{outputs[0].strides_ptr()};
+    auto shifts = compressArray(outDims);
+    applyShifts<dim_t>(shifts, {outStrides, AF_MAX_DIMS});
+    for (auto nodeIt{begin(nodes)}, endIt{end(nodes)};
+         (nodeIt = find_if(nodeIt, endIt, isBufferOrShift)) != endIt;
+         ++nodeIt) {
+        switch ((*nodeIt)->getNodeType()) {
+            case kNodeType::Buffer: {
+                BufferNodeT *buf{static_cast<BufferNodeT *>(nodeIt->get())};
+                applyShifts<dim_t>(shifts,
+                                   {buf->m_param.dims_ptr(), AF_MAX_DIMS});
+                applyShifts<dim_t>(shifts,
+                                   {buf->m_param.strides_ptr(), AF_MAX_DIMS});
+            } break;
+            case kNodeType::Shift: {
+                ShiftNodeT &shiftNode{
+                    *static_cast<ShiftNodeT *>(nodeIt->get())};
+                BufferNodeT &buf{shiftNode.getBufferNode()};
+                applyShifts<dim_t>(shifts,
+                                   {buf.m_param.dims_ptr(), AF_MAX_DIMS});
+                applyShifts<dim_t>(shifts,
+                                   {buf.m_param.strides_ptr(), AF_MAX_DIMS});
+
+                auto &node_shifts = shiftNode.getShifts();
+                applyShifts<int>(shifts, node_shifts);
+            } break;
+            default: break;
+        }
+    }
+    std::for_each(
+        std::begin(outputs) + 1, std::end(outputs), [&shifts](ParamT &output) {
+            applyShifts<dim_t>(shifts, {output.dims_ptr(), AF_MAX_DIMS});
+            applyShifts<dim_t>(shifts, {output.strides_ptr(), AF_MAX_DIMS});
+        });
+}
+
 }  // namespace common
 }  // namespace arrayfire
diff --git a/src/backend/common/jit/ShiftNodeBase.hpp b/src/backend/common/jit/ShiftNodeBase.hpp
index 106040f693..553f4a16a1 100644
--- a/src/backend/common/jit/ShiftNodeBase.hpp
+++ b/src/backend/common/jit/ShiftNodeBase.hpp
@@ -53,6 +53,8 @@ class ShiftNodeBase : public Node {
         return *this;
     }
 
+    std::array<int, 4> &getShifts() { return m_shifts; }
+
     std::unique_ptr<Node> clone() final {
         return std::make_unique<ShiftNodeBase>(*this);
     }
@@ -65,6 +67,7 @@ class ShiftNodeBase : public Node {
         swap(m_shifts, other.m_shifts);
     }
 
+    BufferNode &getBufferNode() { return *m_buffer_node; }
     const BufferNode &getBufferNode() const { return *m_buffer_node; }
 
     bool isLinear(const dim_t dims[4]) const final {
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 1f6e819b2f..5ffb28dafd 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -553,6 +553,7 @@ add_library(afcuda
     wrap.hpp
 
     jit/BufferNode.hpp
+    jit/ShiftNode.hpp
     jit/kernel_generators.hpp
 
     ${scan_by_key_sources}
diff --git a/src/backend/cuda/Param.hpp b/src/backend/cuda/Param.hpp
index 817d601eaa..496d4eea68 100644
--- a/src/backend/cuda/Param.hpp
+++ b/src/backend/cuda/Param.hpp
@@ -35,6 +35,9 @@ class Param {
         return dims[0] * dims[1] * dims[2] * dims[3];
     }
 
+    dim_t *dims_ptr() { return dims; }
+    dim_t *strides_ptr() { return strides; }
+
     Param(const Param<T> &other) noexcept               = default;
     Param(Param<T> &&other) noexcept                    = default;
     Param<T> &operator=(const Param<T> &other) noexcept = default;
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 903c47fe9f..146cb07db2 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -20,6 +20,7 @@
 #include <debug_cuda.hpp>
 #include <device_manager.hpp>
 #include <err_cuda.hpp>
+#include <jit/ShiftNode.hpp>
 #include <kernel_headers/jit_cuh.hpp>
 #include <math.hpp>
 #include <platform.hpp>
@@ -38,6 +39,8 @@ using arrayfire::common::findModule;
 using arrayfire::common::getEnvVar;
 using arrayfire::common::getFuncName;
 using arrayfire::common::half;
+using arrayfire::common::isBufferOrShift;
+using arrayfire::common::kNodeType;
 using arrayfire::common::ModdimNode;
 using arrayfire::common::Node;
 using arrayfire::common::Node_ids;
@@ -45,6 +48,8 @@ using arrayfire::common::Node_map_t;
 using arrayfire::common::Node_ptr;
 using arrayfire::common::NodeIterator;
 using arrayfire::common::saveKernel;
+using arrayfire::cuda::jit::BufferNode;
+using arrayfire::cuda::jit::ShiftNode;
 
 using std::array;
 using std::equal;
@@ -58,7 +63,6 @@ using std::vector;
 
 namespace arrayfire {
 namespace cuda {
-using jit::BufferNode;
 
 static string getKernelString(const string& funcName,
                               const vector<Node*>& full_nodes,
@@ -474,22 +478,9 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
                 }
             }
             if (emptyColumnsFound) {
-                const auto isBuffer{
-                    [](const Node_ptr& node) { return node->isBuffer(); }};
-                for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
-                     (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt;
-                     ++nodeIt) {
-                    BufferNode<T>* buf{
-                        static_cast<BufferNode<T>*>(nodeIt->get())};
-                    removeEmptyColumns(outDims, ndims, buf->m_param.dims,
-                                       buf->m_param.strides);
-                }
-                for_each(++begin(outputs), end(outputs),
-                         [outDims, ndims](Param<T>& output) {
-                             removeEmptyColumns(outDims, ndims, output.dims,
-                                                output.strides);
-                         });
-                ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
+                common::removeEmptyDimensions<Param<T>, BufferNode<T>,
+                                              ShiftNode<T>>(outputs,
+                                                            node_clones);
             }
 
             full_nodes.clear();
diff --git a/src/backend/cuda/jit/ShiftNode.hpp b/src/backend/cuda/jit/ShiftNode.hpp
new file mode 100644
index 0000000000..16bdf5d0f9
--- /dev/null
+++ b/src/backend/cuda/jit/ShiftNode.hpp
@@ -0,0 +1,22 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/jit/ShiftNodeBase.hpp>
+#include <jit/BufferNode.hpp>
+
+namespace arrayfire {
+namespace cuda {
+namespace jit {
+
+template<typename T>
+using ShiftNode = common::ShiftNodeBase<BufferNode<T>>;
+
+}  // namespace jit
+}  // namespace cuda
+}  // namespace arrayfire
diff --git a/src/backend/cuda/shift.cpp b/src/backend/cuda/shift.cpp
index 82aab5e1fe..6f88a38472 100644
--- a/src/backend/cuda/shift.cpp
+++ b/src/backend/cuda/shift.cpp
@@ -11,6 +11,7 @@
 #include <common/jit/ShiftNodeBase.hpp>
 #include <err_cuda.hpp>
 #include <jit/BufferNode.hpp>
+#include <jit/ShiftNode.hpp>
 #include <shift.hpp>
 
 #include <memory>
@@ -18,9 +19,8 @@
 using af::dim4;
 
 using arrayfire::common::Node_ptr;
-using arrayfire::common::ShiftNodeBase;
-
 using arrayfire::cuda::jit::BufferNode;
+using arrayfire::cuda::jit::ShiftNode;
 
 using std::array;
 using std::make_shared;
@@ -29,8 +29,6 @@ using std::string;
 
 namespace arrayfire {
 namespace cuda {
-template<typename T>
-using ShiftNode = ShiftNodeBase<BufferNode<T>>;
 
 template<typename T>
 Array<T> shift(const Array<T> &in, const int sdims[4]) {
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 4ecb470ef9..9bd7e0850a 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -99,6 +99,7 @@ add_library(afoneapi
   ireduce.hpp
   jit.cpp
   jit/BufferNode.hpp
+  jit/ShiftNode.hpp
   jit/kernel_generators.hpp
   join.cpp
   join.hpp
diff --git a/src/backend/oneapi/Param.hpp b/src/backend/oneapi/Param.hpp
index 752a6f7039..4a935c5e2c 100644
--- a/src/backend/oneapi/Param.hpp
+++ b/src/backend/oneapi/Param.hpp
@@ -27,6 +27,9 @@ struct Param {
     Param(const Param& other)            = default;
     Param(Param&& other)                 = default;
 
+    dim_t* dims_ptr() { return info.dims; }
+    dim_t* strides_ptr() { return info.strides; }
+
     // AF_DEPRECATED("Use Array<T>")
     Param() : data(nullptr), info{{0, 0, 0, 0}, {0, 0, 0, 0}, 0} {}
 
@@ -54,6 +57,9 @@ struct AParam {
     AParam(const AParam& other)            = default;
     AParam(AParam&& other)                 = default;
 
+    dim_t* dims_ptr() { return dims.get(); }
+    dim_t* strides_ptr() { return strides.get(); }
+
     // AF_DEPRECATED("Use Array<T>")
     AParam() : data(), dims{0, 0, 0, 0}, strides{0, 0, 0, 0}, offset(0) {}
 
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 3e317b68e2..ecd5bc04b9 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include <CL/cl.h>
+#include <jit/ShiftNode.hpp>
 #include <jit/kernel_generators.hpp>
 
 #include <kernel_headers/KParam.hpp>
@@ -55,6 +56,7 @@ using arrayfire::common::NodeIterator;
 using arrayfire::common::ShiftNodeBase;
 using arrayfire::oneapi::getActiveDeviceBaseBuildFlags;
 using arrayfire::oneapi::jit::BufferNode;
+using arrayfire::oneapi::jit::ShiftNode;
 
 using std::array;
 using std::begin;
@@ -468,21 +470,8 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
             }
         }
         if (emptyColumnsFound) {
-            const auto isBuffer{
-                [](const Node_ptr& ptr) { return ptr->isBuffer(); }};
-            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
-                 (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt;
-                 ++nodeIt) {
-                BufferNode<T>* buf{static_cast<BufferNode<T>*>(nodeIt->get())};
-                removeEmptyColumns(outDims, ndims, buf->m_param.dims.get(),
-                                   buf->m_param.strides.get());
-            }
-            for_each(++begin(outputs), end(outputs),
-                     [outDims, ndims](Param<T>& output) {
-                         removeEmptyColumns(outDims, ndims, output.info.dims,
-                                            output.info.strides);
-                     });
-            ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
+            common::removeEmptyDimensions<Param<T>, BufferNode<T>,
+                                          ShiftNode<T>>(outputs, node_clones);
         }
     }
 
diff --git a/src/backend/oneapi/jit/ShiftNode.hpp b/src/backend/oneapi/jit/ShiftNode.hpp
new file mode 100644
index 0000000000..6a87b28729
--- /dev/null
+++ b/src/backend/oneapi/jit/ShiftNode.hpp
@@ -0,0 +1,22 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/jit/ShiftNodeBase.hpp>
+#include <jit/BufferNode.hpp>
+
+namespace arrayfire {
+namespace oneapi {
+namespace jit {
+
+template<typename T>
+using ShiftNode = common::ShiftNodeBase<BufferNode<T>>;
+
+}  // namespace jit
+}  // namespace oneapi
+}  // namespace arrayfire
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 8a0e55d2e4..5c920f44f8 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -468,6 +468,7 @@ target_sources(afopencl
 target_sources(afopencl
   PRIVATE
     jit/BufferNode.hpp
+    jit/ShiftNode.hpp
     jit/kernel_generators.hpp
   )
 
diff --git a/src/backend/opencl/Param.hpp b/src/backend/opencl/Param.hpp
index aaf19dea62..879c92c677 100644
--- a/src/backend/opencl/Param.hpp
+++ b/src/backend/opencl/Param.hpp
@@ -22,6 +22,9 @@ struct Param {
     Param(const Param& other)            = default;
     Param(Param&& other)                 = default;
 
+    dim_t* dims_ptr() { return info.dims; }
+    dim_t* strides_ptr() { return info.strides; }
+
     // AF_DEPRECATED("Use Array<T>")
     Param();
     // AF_DEPRECATED("Use Array<T>")
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 727724cc85..7ace33cd96 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -19,6 +19,7 @@
 #include <device_manager.hpp>
 #include <err_opencl.hpp>
 #include <jit/BufferNode.hpp>
+#include <jit/ShiftNode.hpp>
 #include <kernel_headers/jit.hpp>
 #include <threadsMgt.hpp>
 #include <type_util.hpp>
@@ -42,6 +43,7 @@ using arrayfire::common::Node_map_t;
 using arrayfire::common::Node_ptr;
 using arrayfire::common::NodeIterator;
 using arrayfire::common::saveKernel;
+using arrayfire::opencl::jit::ShiftNode;
 
 using cl::Kernel;
 using cl::NDRange;
@@ -418,21 +420,8 @@ void evalNodes(vector<Param>& outputs, const vector<Node*>& output_nodes) {
             }
         }
         if (emptyColumnsFound) {
-            const auto isBuffer{
-                [](const Node_ptr& ptr) { return ptr->isBuffer(); }};
-            for (auto nodeIt{begin(node_clones)}, endIt{end(node_clones)};
-                 (nodeIt = find_if(nodeIt, endIt, isBuffer)) != endIt;
-                 ++nodeIt) {
-                BufferNode* buf{static_cast<BufferNode*>(nodeIt->get())};
-                removeEmptyColumns(outDims, ndims, buf->m_param.dims,
-                                   buf->m_param.strides);
-            }
-            for_each(++begin(outputs), end(outputs),
-                     [outDims, ndims](Param& output) {
-                         removeEmptyColumns(outDims, ndims, output.info.dims,
-                                            output.info.strides);
-                     });
-            ndims = removeEmptyColumns(outDims, ndims, outDims, outStrides);
+            common::removeEmptyDimensions<Param, BufferNode, ShiftNode>(
+                outputs, node_clones);
         }
 
         full_nodes.clear();
diff --git a/src/backend/opencl/jit/ShiftNode.hpp b/src/backend/opencl/jit/ShiftNode.hpp
new file mode 100644
index 0000000000..8132105faf
--- /dev/null
+++ b/src/backend/opencl/jit/ShiftNode.hpp
@@ -0,0 +1,21 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <common/jit/ShiftNodeBase.hpp>
+#include <jit/BufferNode.hpp>
+
+namespace arrayfire {
+namespace opencl {
+namespace jit {
+
+using ShiftNode = common::ShiftNodeBase<BufferNode>;
+
+}  // namespace jit
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/KParam.hpp b/src/backend/opencl/kernel/KParam.hpp
index 38a3752760..165bec9b02 100644
--- a/src/backend/opencl/kernel/KParam.hpp
+++ b/src/backend/opencl/kernel/KParam.hpp
@@ -21,6 +21,12 @@ typedef struct {
     dim_t dims[4];
     dim_t strides[4];
     dim_t offset;
+
+#ifndef __OPENCL_VERSION__
+    dim_t *dims_ptr() { return dims; }
+    dim_t *strides_ptr() { return strides; }
+#endif
+
 } KParam;
 
 #endif
diff --git a/src/backend/opencl/shift.cpp b/src/backend/opencl/shift.cpp
index 512c113ed1..8b257f2c97 100644
--- a/src/backend/opencl/shift.cpp
+++ b/src/backend/opencl/shift.cpp
@@ -9,14 +9,15 @@
 
 #include <shift.hpp>
 
-#include <common/jit/ShiftNodeBase.hpp>
 #include <err_opencl.hpp>
+#include <jit/ShiftNode.hpp>
 #include <traits.hpp>
 
 using af::dim4;
 using arrayfire::common::Node_ptr;
 using arrayfire::common::ShiftNodeBase;
 using arrayfire::opencl::jit::BufferNode;
+using arrayfire::opencl::jit::ShiftNode;
 using std::array;
 using std::make_shared;
 using std::static_pointer_cast;
@@ -24,7 +25,6 @@ using std::string;
 
 namespace arrayfire {
 namespace opencl {
-using ShiftNode = ShiftNodeBase<BufferNode>;
 
 template<typename T>
 Array<T> shift(const Array<T> &in, const int sdims[4]) {
diff --git a/test/shift.cpp b/test/shift.cpp
index b37385a6f8..2de341b3bc 100644
--- a/test/shift.cpp
+++ b/test/shift.cpp
@@ -146,3 +146,12 @@ TEST(Shift, MaxDim) {
     output = abs(input - output);
     ASSERT_EQ(1.f, product<float>(output));
 }
+
+TEST(Shift, RowVector) {
+    const unsigned shift_x = 1;
+    const unsigned shift_y = 1;
+    array input            = iota(dim4(1, 4));
+    array output           = shift(input, shift_x, shift_y);
+    vector<float> gold{3.f, 0.f, 1.f, 2.f};
+    EXPECT_VEC_ARRAY_EQ(gold, dim4(1, 4), output);
+}

From 23ee0650e034e33a70015d98f71deb350238189d Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 18 Aug 2023 20:34:31 -0400
Subject: [PATCH 369/473] Fix reorder to avoid eval on copied array instead of
 input array

The reorder funciton was copying the Array object internally and
then the other operations were performed on the copy. This causes
the eval to be performed on the copied array instead of the input
array.
---
 src/api/c/reorder.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/api/c/reorder.cpp b/src/api/c/reorder.cpp
index b283c800bf..556e1f0e20 100644
--- a/src/api/c/reorder.cpp
+++ b/src/api/c/reorder.cpp
@@ -33,12 +33,14 @@ using std::swap;
 
 template<typename T>
 static inline af_array reorder(const af_array in, const af::dim4 &rdims0) {
-    Array<T> In = getArray<T>(in);
+    Array<T> In = detail::createEmptyArray<T>(af::dim4(0));
     dim4 rdims  = rdims0;
 
     if (rdims[0] == 1 && rdims[1] == 0) {
-        In = transpose(In, false);
+        In = transpose(getArray<T>(in), false);
         std::swap(rdims[0], rdims[1]);
+    } else {
+        In = getArray<T>(in);
     }
     const dim4 idims    = In.dims();
     const dim4 istrides = In.strides();
@@ -48,8 +50,7 @@ static inline af_array reorder(const af_array in, const af::dim4 &rdims0) {
 
     af_array out;
     if (rdims[0] == 0 && rdims[1] == 1 && rdims[2] == 2 && rdims[3] == 3) {
-        const Array<T> &Out = In;
-        out                 = getHandle(Out);
+        out = getHandle(In);
     } else if (rdims[0] == 0) {
         dim4 odims    = dim4(1, 1, 1, 1);
         dim4 ostrides = dim4(1, 1, 1, 1);

From 5583b899d2d402a25afe7008809615abd0cca0e9 Mon Sep 17 00:00:00 2001
From: John Melonakos <john.melonakos@intel.com>
Date: Mon, 28 Aug 2023 17:35:19 -0400
Subject: [PATCH 370/473] updated the README, timing, install, gfor, and added
 the jit pages (#3490)

* updated the README, timing, install, gfor, and added the jit pages
* minor example code tweaks, adds jit to tutorials page

---------

Co-authored-by: syurkevi <stefan@arrayfire.com>
---
 docs/pages/README.md    | 101 ++++++++++++----------
 docs/pages/gfor.md      |  61 +++++++------
 docs/pages/install.md   |  55 ++++++------
 docs/pages/jit.md       | 102 ++++++++++++++++++++++
 docs/pages/timing.md    | 185 +++++++++++++++++++++++++++++-----------
 docs/pages/tutorials.md |   1 +
 6 files changed, 351 insertions(+), 154 deletions(-)
 create mode 100644 docs/pages/jit.md

diff --git a/docs/pages/README.md b/docs/pages/README.md
index d20dc6b246..08cc17578d 100644
--- a/docs/pages/README.md
+++ b/docs/pages/README.md
@@ -5,12 +5,14 @@ Overview {#mainpage}
 
 ## About ArrayFire
 
-ArrayFire is a high performance software library for parallel computing with an easy-to-use API. Its array based function set makes parallel programming more accessible.
+ArrayFire is a high performance software library for parallel computing with
+an easy-to-use API. Its array based function set makes parallel programming
+more accessible.
 
 ## Installing ArrayFire
 
-You can install ArrayFire using either a binary installer for Windows, OSX,
-or Linux or download it from source:
+Install ArrayFire using either a binary installer for Windows, OSX, or Linux
+or download it from source:
 
 * [Binary installers for Windows, OSX, and Linux](\ref installing)
 * [Build from source](https://github.com/arrayfire/arrayfire)
@@ -20,18 +22,18 @@ or Linux or download it from source:
 The [array](\ref af::array) object is beautifully simple.
 
 Array-based notation effectively expresses computational algorithms in
-readable math-resembling notation. You _do not_ need expertise in
-parallel programming to use ArrayFire.
+readable math-resembling notation. Expertise in parallel programming _is not_
+required to use ArrayFire.
 
-A few lines of ArrayFire code
-accomplishes what can take 100s of complicated lines in CUDA or OpenCL
-kernels.
+A few lines of ArrayFire code accomplishes what can take 100s of complicated
+lines in CUDA, oneAPI, or OpenCL kernels.
 
 ## ArrayFire is extensive!
 
 #### Support for multiple domains
 
-ArrayFire contains [hundreds of functions](\ref arrayfire_func) across various domains including:
+ArrayFire contains [hundreds of functions](\ref arrayfire_func) across various
+domains including:
 - [Vector Algorithms](\ref vector_mat)
 - [Image Processing](\ref image_mat)
 - [Computer Vision](\ref cv_mat)
@@ -40,61 +42,67 @@ ArrayFire contains [hundreds of functions](\ref arrayfire_func) across various d
 - [Statistics](\ref stats_mat)
 - and more.
 
-Each function is hand-tuned by ArrayFire
-developers with all possible low-level optimizations.
+Each function is hand-tuned by ArrayFire developers with all possible
+low-level optimizations.
 
 #### Support for various data types and sizes
 
-ArrayFire operates on common [data shapes and sizes](\ref indexing),
-including vectors, matrices, volumes, and
+ArrayFire operates on common [data shapes and sizes](\ref indexing), including
+vectors, matrices, volumes, and
 
-It supports common [data types](\ref gettingstarted_datatypes),
-including single and double precision floating
-point values, complex numbers, booleans, and 32-bit signed and
-unsigned integers.
+It supports common [data types](\ref gettingstarted_datatypes), including
+single and double precision floating point values, complex numbers, booleans,
+and 32-bit signed and unsigned integers.
 
 #### Extending ArrayFire
 
-ArrayFire can be used as a stand-alone application or integrated with
-existing CUDA or OpenCL code. All ArrayFire `arrays` can be
-interchanged with other CUDA or OpenCL data structures.
+ArrayFire can be used as a stand-alone application or integrated with existing
+CUDA, oneAPI, or OpenCL code. All ArrayFire `arrays` can be interchanged with
+other CUDA, oneAPI, or OpenCL data structures.
 
 ## Code once, run anywhere!
 
-With support for x86, ARM, CUDA, and OpenCL devices, ArrayFire supports for a comprehensive list of devices.
+With support for x86, ARM, CUDA, oneAPI, and OpenCL devices, ArrayFire
+supports for a comprehensive list of devices.
 
 Each ArrayFire installation comes with:
- - a CUDA version (named 'libafcuda') for [NVIDIA
- GPUs](https://developer.nvidia.com/cuda-gpus),
- - an OpenCL version (named 'libafopencl') for [OpenCL devices](http://www.khronos.org/conformance/adopters/conformant-products#opencl)
- - a CPU version (named 'libafcpu') to fall back to when CUDA or OpenCL devices are not available.
+- a CUDA backend (named 'libafcuda') for [NVIDIA
+  GPUs](https://developer.nvidia.com/cuda-gpus),
+- a oneAPI backend (named 'libafoneapi') for [oneAPI
+  devices](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/intel-oneapi-base-toolkit-system-requirements.html),
+- an OpenCL backend (named 'libafopencl') for [OpenCL
+  devices](http://www.khronos.org/conformance/adopters/conformant-products#opencl),
+- a CPU backend (named 'libafcpu') to fall back to when CUDA, oneAPI, or
+  OpenCL devices are unavailable.
 
 ## ArrayFire is highly efficient
 
 #### Vectorized and Batched Operations
 
-ArrayFire supports batched operations on N-dimensional arrays.
-Batch operations in ArrayFire are run in parallel ensuring an optimal usage of your CUDA or OpenCL device.
+ArrayFire supports batched operations on N-dimensional arrays. Batch
+operations in ArrayFire are run in parallel ensuring an optimal usage of CUDA,
+oneAPI, or OpenCL devices.
 
-You can get the best performance out of ArrayFire using [vectorization techniques](\ref vectorization).
+Best performance with ArrayFire is achieved using
+[vectorization techniques](\ref vectorization).
 
 ArrayFire can also execute loop iterations in parallel with
 [the gfor function](\ref gfor).
 
 #### Just in Time compilation
 
-ArrayFire performs run-time analysis of your code to increase
-arithmetic intensity and memory throughput, while avoiding unnecessary
-temporary allocations. It has an awesome internal JIT compiler to make
-optimizations for you.
+ArrayFire performs run-time analysis of code to increase arithmetic intensity
+and memory throughput, while avoiding unnecessary temporary allocations. It
+has an awesome internal JIT compiler to make important optimizations.
 
-Read more about how [ArrayFire JIT](http://arrayfire.com/performance-of-arrayfire-jit-code-generation/) can improve the performance in your application.
+Read more about how [ArrayFire JIT](\ref jit).  can improve the performance in
+your application.
 
 ## Simple Example
 
-Here's a live example to let you see ArrayFire code. You create [arrays](\ref af::array)
-which reside on CUDA or OpenCL devices. Then you can use
-[ArrayFire functions](modules.htm) on those [arrays](\ref af::array).
+Here is an example of ArrayFire code. First, [arrays](\ref af::array) are
+created which reside on CUDA, oneAPI, or OpenCL devices. Then
+[ArrayFire functions](modules.htm) are used on those [arrays](\ref af::array).
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
 // sample 40 million points on the GPU
@@ -111,17 +119,19 @@ af_print(pi);
 
 #### Free Community Options
 
-* [ArrayFire mailing list](https://groups.google.com/forum/#!forum/arrayfire-users) (recommended)
+* [ArrayFire mailing
+  list](https://groups.google.com/forum/#!forum/arrayfire-users) (recommended)
 * [StackOverflow](http://stackoverflow.com/questions/tagged/arrayfire)
 
 #### Premium Support
 
-* Phone Support - available for purchase ([request a quote](mailto:sales@arrayfire.com))
+* Phone Support - available for purchase ([request a
+  quote](mailto:sales@arrayfire.com))
 
 #### Contact Us
 
-* If you need to contact us, visit our
-[contact us page](http://arrayfire.com/company/#contact).
+* If you need to contact us, visit our [contact us
+  page](http://arrayfire.com/company/#contact).
 
 #### Email
 
@@ -130,9 +140,10 @@ af_print(pi);
 
 ## Citations and Acknowledgements
 
-If you redistribute ArrayFire, please follow the terms established in <a href="https://github.com/arrayfire/arrayfire/blob/master/LICENSE">the license</a>.
-If you wish to cite ArrayFire in an academic publication, please use the
-following reference:
+If you redistribute ArrayFire, please follow the terms established in <a
+href="https://github.com/arrayfire/arrayfire/blob/master/LICENSE">the
+license</a>. If you wish to cite ArrayFire in an academic publication, please
+use the following reference:
 
 Formatted:
 
@@ -153,4 +164,6 @@ BibTeX:
     year = {2015}
     }
 
-ArrayFire development is funded by ArrayFire LLC and several third parties, please see the list of <a href="https://github.com/arrayfire/arrayfire/blob/master/ACKNOWLEDGEMENTS.md">acknowledgements</a>.
+ArrayFire development is funded by AccelerEyes LLC (dba ArrayFire) and several
+third parties, please see the list of <a
+href="https://github.com/arrayfire/arrayfire/blob/master/ACKNOWLEDGEMENTS.md">acknowledgements</a>.
diff --git a/docs/pages/gfor.md b/docs/pages/gfor.md
index e6886b5bb4..bbced5d14b 100644
--- a/docs/pages/gfor.md
+++ b/docs/pages/gfor.md
@@ -8,18 +8,17 @@ Run many independent loops simultaneously on the GPU or device.
 Introduction {#gfor_intro}
 ============
 
-The gfor-loop construct may be used to simultaneously launch all of
-the iterations of a for-loop on the GPU or device, as long as the
-iterations are independent. While the standard for-loop performs each
-iteration sequentially, ArrayFire's gfor-loop performs each iteration
-at the same time (in parallel). ArrayFire does this by tiling out the
-values of all loop iterations and then performing computation on those
-tiles in one pass.
-
-You can think of `gfor` as performing auto-vectorization of your
-code, e.g. you write a gfor-loop that increments every element of a
-vector but behind the scenes ArrayFire rewrites it to operate on
-the entire vector in parallel.
+The gfor-loop construct may be used to simultaneously launch all of the
+iterations of a for-loop on the GPU or device, as long as the iterations are
+independent. While the standard for-loop performs each iteration sequentially,
+ArrayFire's gfor-loop performs each iteration at the same time (in
+parallel). ArrayFire does this by tiling out the values of all loop iterations
+and then performing computation on those tiles in one pass.
+
+You can think of `gfor` as performing auto-vectorization of your code,
+e.g. you write a gfor-loop that increments every element of a vector but
+behind the scenes ArrayFire rewrites it to operate on the entire vector in
+parallel.
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
 for (int i = 0; i < n; ++i)
@@ -29,19 +28,19 @@ gfor (seq i, n)
    A(i) = A(i) + 1;
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Behind the scenes, ArrayFire rewrites your code into this
-equivalent and faster version:
+Behind the scenes, ArrayFire rewrites your code into this equivalent and
+faster version:
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
 A = A + 1;
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-It is best to vectorize computation as much as possible to avoid
-the overhead in both for-loops and gfor-loops.
+It is best to vectorize computation as much as possible to avoid the overhead
+in both for-loops and gfor-loops.
 
-To see another example, you could run an FFT on every 2D slice of a
-volume in a for-loop, or you could "vectorize" and simply do it all
-in one gfor-loop operation:
+To see another example, you could run an FFT on every 2D slice of a volume in
+a for-loop, or you could "vectorize" and simply do it all in one gfor-loop
+operation:
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
 for (int i = 0; i < N; ++i)
@@ -89,11 +88,11 @@ User Functions called within GFOR {#gfor_user_functions}
 ---------------------------------
 
 If you have defined a function that you want to call within a GFOR loop, then
-that function has to meet all the conditions described in this page in
-order to be able to work as expected.
+that function has to meet all the conditions described in this page in order
+to be able to work as expected.
 
-Consider the (trivial) example below. The function compute() has to satisfy all
-requirements for GFOR Usage, so you cannot use if-else conditions inside
+Consider the (trivial) example below. The function compute() has to satisfy
+all requirements for GFOR Usage, so you cannot use if-else conditions inside
 it.
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
@@ -384,7 +383,8 @@ gfor (seq i, n) {
 }
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The problem is that every GFOR tile has a different number of elements, something which GFOR cannot yet handle.
+The problem is that every GFOR tile has a different number of elements,
+something which GFOR cannot yet handle.
 
 Similar to the workaround for conditional statements, it might work to use
 masked arithmetic:
@@ -410,14 +410,13 @@ gfor (seq i, n) {
 Memory considerations {#gfor_memory}
 =====================
 
-Since each computation is done in parallel for all iterator values,
-you need to have enough card memory available to do all iterations
-simultaneously. If the problem exceeds memory, it will trigger "out of
-memory" errors.
+Since each computation is done in parallel for all iterator values, you need
+to have enough card memory available to do all iterations simultaneously. If
+the problem exceeds memory, it will trigger "out of memory" errors.
 
-You can work around the memory limitations of your GPU or device by
-breaking the GFOR loop up into segments; however, you might want to
-consider using a larger memory GPU or device.
+You can work around the memory limitations of your GPU or device by breaking
+the GFOR loop up into segments; however, you might want to consider using a
+larger memory GPU or device.
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
 // BEFORE
diff --git a/docs/pages/install.md b/docs/pages/install.md
index 7a78b95f71..a0b3af61b3 100644
--- a/docs/pages/install.md
+++ b/docs/pages/install.md
@@ -1,24 +1,15 @@
 # ArrayFire Installer {#installing}
 
 Installing ArrayFire couldn't be easier. Navigate to
-https://arrayfire.com/download and download the installer for your architecture
-and operating system. Although you could [build ArrayFire from
-source](https://github.com/arrayfire/arrayfire), we recommend using our
-installers as we have packaged together all of the necessary dependencies to
-give you the best performance.
-
-We provide installers for Windows, Linux, and macOS. There are two installers
-for each operating system: one with graphics support and the other without
-graphics support. Download the installer with graphics support if you would like
-to be able to do high performance visualizations using our
-[Forge](https://github.com/arrayfire/forge) library. Otherwise, download the
-installer without graphics support.
-
-Make sure you have the latest device drivers installed on your system before
-using ArrayFire. If you are going to be targeting the CPU using ArrayFire’s
-OpenCL backend, you will need to have the OpenCL **runtime** installed on your
-system. Drivers and runtimes should be downloaded and installed from your device
-vendor’s website.
+https://arrayfire.com/download and download the appropriate installer for the
+target architecture and operating system. Although ArrayFire can be [built
+from source](https://github.com/arrayfire/arrayfire), the installers
+conveniently package necessary dependencies.
+
+Install the latest device drivers before using ArrayFire. If you are going to
+target the CPU using ArrayFire’s OpenCL backend, install the OpenCL
+runtime. Drivers and runtimes should be downloaded and installed from the
+device vendor’s website.
 
 # Install Instructions {#InstallInstructions}
 
@@ -29,14 +20,14 @@ vendor’s website.
 ## Windows {#Windows}
 
 Prior to installing ArrayFire on Windows,
-[download](https://www.microsoft.com/en-in/download/details.aspx?id=48145)
+[download](https://www.microsoft.com/en-in/download/details.aspx?id=48145) and
 install the Visual Studio 2015 (x64) runtime libraries.
 
-Once you have downloaded the ArrayFire installer, execute the installer as you
-normally would on Windows. If you choose not to modify the path during the
-installation procedure, you'll need to manually add ArrayFire to the path for
-all users. Simply append `%%AF_PATH%/lib` to the PATH variable so that the loader
-can find ArrayFire DLLs.
+Once the ArrayFire installer has been downloaded, run the installer. If you
+choose not to modify the path during the installation procedure, you'll need
+to manually add ArrayFire to the path for all users. Simply append
+`%%AF_PATH%/lib` to the PATH variable so that the loader can find ArrayFire
+DLLs.
 
 For more information on using ArrayFire on Windows, visit the following
 [page](http://arrayfire.org/docs/using_on_windows.htm).
@@ -47,13 +38,14 @@ There are two ways to install ArrayFire on Linux.
 1. Package Manager
 2. Using ArrayFire Linux Installer
 
-As of today, approach (1) is only supported for Ubuntu 18.04 and 20.04. Please go
-through [our GitHub wiki page](https://github.com/arrayfire/arrayfire/wiki/Install-ArrayFire-From-Linux-Package-Managers)
+As of today, approach (1) is only supported for Ubuntu 18.04 and 20.04. Please
+go through [our GitHub wiki
+page](https://github.com/arrayfire/arrayfire/wiki/Install-ArrayFire-From-Linux-Package-Managers)
 for the detailed instructions.
 
-For approach (2), once you have downloaded the ArrayFire installer, execute the
-installer from the terminal as shown below. Set the `--prefix` argument to the
-directory you would like to install ArrayFire to - we recommend `/opt`.
+For approach (2), once you have downloaded the ArrayFire installer, execute
+the installer from the terminal as shown below. Set the `--prefix` argument to
+the directory you would like to install ArrayFire to - we recommend `/opt`.
 
     ./Arrayfire_*_Linux_x86_64.sh --include-subdir --prefix=/opt
 
@@ -131,8 +123,9 @@ On Unix-like systems:
     ./helloworld/helloworld_{cpu,cuda,opencl}
 
 On Windows, open the CMakeLists.txt file from CMake-GUI and set `ASSETS_DIR`
-variable to the parent folder of examples folder. Once the project is configured
-and generated, you can build and run the examples from Visual Studio.
+variable to the parent folder of examples folder. Once the project is
+configured and generated, you can build and run the examples from Visual
+Studio.
 
 ## <a name="GettingHelp"></a> Getting help
 
diff --git a/docs/pages/jit.md b/docs/pages/jit.md
new file mode 100644
index 0000000000..8b5c783755
--- /dev/null
+++ b/docs/pages/jit.md
@@ -0,0 +1,102 @@
+ArrayFire JIT Code Generation {#jit}
+================
+
+The ArrayFire library offers JIT (Just In Time) compiling for elementwise
+arithmetic operations. This includes trigonometric functions, comparisons, and
+element-wise operations.
+
+At runtime, ArrayFire aggregates these function calls using an Abstract Syntax
+Tree (AST) data structure such that whenever a JIT-supported function is
+called, it is added into the AST for a given variable instance. The AST of the
+variable is computed if one of the following conditions is met:
+
+* an explication evaluation is required by the programmer using the
+  [eval](\ref af::eval) function, or
+* the variable is required to compute a different variable that is not
+  JIT-supported.
+
+When the above occurs, and the variable needs to be evaluated, the functions
+and variables in the AST data structure are used to create a single
+kernel. This is done by creating a customized kernel on-the-fly that is made
+up of all the functions in the AST. The customized function is then executed.
+
+This JIT compilation technique has multiple benefits:
+
+* A reduced number of kernel calls – a kernel call can be a significant
+  overhead for small data sets.
+* Better cache performance – there are many instances in which the memory
+  required by a single element in the array can be reused multiple times, or
+  the temporary value of a computation can be stored in the cache and reused
+  by future computations.
+* Temporary memory allocation and write-back can be reduced – when multiple
+  expressions are evaluated and stored into temporary arrays, these arrays
+  need to be allocated and the results written back to main memory.
+* Avoid computing elements that are not used – there are cases in which the
+  AST is created for a variable; however, the expression is not used later in
+  the computation. Thus, its evaluation can be avoided.
+* Better performance – all the above can help reduce the total execution time.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
+// As JIT is automatically enabled in ArrayFire, this version of the function
+// forces each expression to be evaluated. If the eval() function calls are
+// removed, then the execution of this code would be equivalent to the
+// following function.
+
+static double pi_no_jit(array x, array y, array temp, int samples) {
+        temp = x * x;
+        temp.eval();
+        temp += y * y;
+        temp.eval();
+        temp = sqrt(temp);
+        temp.eval();
+        temp = temp < 1;
+        temp.eval();
+        return 4.0 sum(temp)/samples;
+}
+
+static double pi_jit(array x, array y, array temp,int samples){
+        temp = sqrt(x*x + y*y) < 1;
+        temp.eval();
+        return 4.0 * sum(temp) / samples;
+}
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The above code computes the value of π using a Monte-Carlo simulation where
+points are randomly generated within the unit square. Each point is tested to
+see if it is within the unit circle. The ratio of points within the circle and
+square approximate the value π. The accuracy of π improves as the number of
+samples is increased, which motivates using additional samples.
+
+There are two implementations above:
+1. an implementation that does not benefit from the JIT (pi\_no\_jit), and
+2. an implementation that takes advantage of the JIT feature (pi\_jit).
+
+Specifically, as JIT is an integral feature of the ArrayFire library, it
+cannot simply be turned on and off. The only way for a programmer to sidestep
+the JIT operations is to manually force the evaluation of expressions. This is
+done in the non-JIT-supported implementation.
+
+Timing these two implementations results in the following performance
+benchmark:
+
+<img src="jit_cuda1.webp" alt="Performance of JIT and Non-JIT implementations"
+width="100%" />
+
+
+The above figure depicts the execution time (abscissa) as a function of the
+number of samples (ordinate) for the two implementations discussed above.
+
+When the number of samples is small, the execution time of pi\_no\_jit is
+dominated by the launch of multiple kernels and the execution time pi\_jit is
+dominated by on-the-fly compilation of the JIT code required to launch a
+single kernel. Even with this JIT compilation time, pi\_jit outperforms
+pi_no_jit by 1.4-2.0X for smaller sample sizes.
+
+When the number of samples is large, both the kernel launch overhead and the
+JIT code creation are no longer the limiting factors – the kernel’s
+computational load dominates the execution time. Here, the pi\_jit outperforms
+pi\_no\_jit by 2.0-2.7X.
+
+The number of applications that benefit from the JIT code generation is
+significant. The actual performance benefits are also application-dependent.
+
diff --git a/docs/pages/timing.md b/docs/pages/timing.md
index fc9b1a725f..8c43808a5c 100644
--- a/docs/pages/timing.md
+++ b/docs/pages/timing.md
@@ -1,64 +1,153 @@
-Timing Your Code {#timing}
+Timing ArrayFire Code {#timing}
 ================
 
-timer() : A platform-independent timer with microsecond accuracy:
-* [timer::start()](\ref af::timer::start) starts a timer
+In performance-sensitive applications, it is vital to profile and measure the
+execution time of operations. ArrayFire provides mechanisms to achieve this.
 
-* [timer::start()](\ref af::timer::stop) seconds since last \ref af::timer::start "start"
+ArrayFire employs an asynchronous evaluation model for all of its
+functions. This means that operations are queued to execute but do not
+necessarily complete prior to function return. Hence, directly measuring the
+time taken for an ArrayFire function could be misleading. To accurately
+measure time, one must ensure the operations are evaluated and synchronize the
+ArrayFire stream.
 
-* \ref af::timer::stop(af::timer start) "timer::stop(timer start)" seconds since 'start'
+ArrayFire also employs a lazy evaluation model for its elementwise arithmetic
+operations. This means operations are not queued for execution until the
+result is needed by downstream operations blocking until the operations are
+complete.
 
-Example: single timer
+The following describes how to time ArrayFire code using the eval and sync
+functions along with the timer and timeit functions. A final note on kernel
+caching also provides helpful details about ArrayFire runtimes.
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
-   // start timer
-   timer::start();
-   // run your code
-   printf("elapsed seconds: %g\n", timer::stop());
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+## Using ArrayFire eval and sync functions
 
-Example: multiple timers
+ArrayFire provides functions to force the evaluation of lazy functions and to
+block until all asynchoronous operations complete.
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
-   // start timers
-   timer start1 = timer::start();
-   timer start2 = timer::start();
-   // run some code
-   printf("elapsed seconds: %g\n", timer::stop(start1));
-   // run more code
-   printf("elapsed seconds: %g\n", timer::stop(start2));
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+1. The [eval](\ref af::eval) function:
 
-Accurate and reliable measurement of performance involves several factors:
-* Executing enough iterations to achieve peak performance.
-* Executing enough repetitions to amortize any overhead from system timers.
+   Forces the evaluation of an ArrayFire array. It ensures the execution of
+   operations queued up for a specific array.
 
-To take care of much of this boilerplate, [timeit](\ref af::timeit) provides
-accurate and reliable estimates of both CPU or GPU code.
+   It is only required for timing purposes if elementwise arithmetic functions
+   are called on the array, since these are handled by the ArrayFire JIT.
 
-Here`s a stripped down example of
-[Monte-Carlo estimation of PI](\ref benchmarks/pi.cpp) making use
-of [timeit](\ref af::timeit).  Notice how it expects a `void` function pointer.
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
+   af::array A = af::randu(1000, 1000);
+   af::array B = A + A;                 // Elementwise arithmetic operation.
+   B.eval();                            // Forces evaluation of B.
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
-#include <stdio.h>
-#include <arrayfire.h>
-using namespace af;
+   The function initializes the evaluation of the JIT-tree for that array and
+   may return prior to the completion of those operations. To ensure proper
+   timing, combine with a [sync](\ref af::sync) function.
 
-void pi_function() {
-  int n = 20e6; // 20 million random samples
-  array x = randu(n,f32), y = randu(n,f32);
-  // how many fell inside unit circle?
-  float pi = 4.0 * sum<float>(sqrt(x*x + y*y)) < 1) / n;
-}
+2. The [sync](\ref af::sync) function:
 
-int main() {
-  printf("pi_function took %g seconds\n", timeit(pi_function));
-  return 0;
-}
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+   Synchronizes the ArrayFire stream. It waits for all the previous operations
+   in the stream to finish. It is often used after [eval](\ref af::eval) to
+   ensure that operations have indeed been completed.
 
-This produces:
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
+   af::sync();  // Waits for all previous operations to complete.
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    pi_function took 0.007252 seconds
-    (test machine: Core i7 920 @ 2.67GHz with a Tesla C2070)
+## Using ArrayFire timer and timeit functions
+
+ArrayFire provides a simple timer functions that returns the current time in
+seconds.
+
+1. The [timer](\ref af::timer) function:
+
+   timer() : A platform-independent timer with microsecond accuracy:
+   * [timer::start()](\ref af::timer::start) starts a timer
+
+   * [timer::start()](\ref af::timer::stop) seconds since last \ref
+     af::timer::start "start"
+
+   * \ref af::timer::stop(af::timer start) "timer::stop(timer start)" seconds
+     since 'start'
+
+   Example: single timer
+
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
+       // start timer
+       // - be sure to use the eval and sync functions so that previous code
+       //   does not get timed as part of the execution segment being measured
+       timer::start();
+       // run a code segment
+       // - be sure to use the eval and sync functions to ensure the code
+       //   segment operations have been completed
+       // stop timer
+       printf("elapsed seconds: %g\n", timer::stop());
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+   Example: multiple timers
+
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
+       // start timers
+       // - be sure to use the eval and sync functions so that previous code
+       //   does not get timed as part of the execution segment being measured
+       timer start1 = timer::start();
+       timer start2 = timer::start();
+       // run a code segment
+       // - be sure to use the eval and sync functions to ensure the code
+       //   segment operations have been completed
+       // stop timer1
+       printf("elapsed seconds: %g\n", timer::stop(start1));
+       // run another code segment
+       // - be sure to use the eval and sync functions to ensure the code
+       //   segment operations have been completed
+       // stop timer2
+       printf("elapsed seconds: %g\n", timer::stop(start2));
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+   Accurate and reliable measurement of performance involves several factors:
+   * Executing enough iterations to achieve peak performance.
+   * Executing enough repetitions to amortize any overhead from system timers.
+
+2. The [timeit](\ref af::timeit) function:
+
+   To take care of much of this boilerplate, [timeit](\ref af::timeit) provides
+   accurate and reliable estimates of both CPU or GPU code.
+
+   Here is a stripped down example of [Monte-Carlo estimation of PI](\ref
+   benchmarks/pi.cpp) making use of [timeit](\ref af::timeit). Notice how it
+   expects a `void` function pointer.
+
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
+   #include <stdio.h>
+   #include <arrayfire.h>
+   using namespace af;
+
+   void pi_function() {
+     int n = 20e6; // 20 million random samples
+     array x = randu(n, f32), y = randu(n, f32);
+     // how many fell inside unit circle?
+     float pi = 4.0 * sum<float>(sqrt(x*x + y*y)) < 1) / n;
+   }
+
+   int main() {
+     printf("pi_function took %g seconds\n", timeit(pi_function));
+     return 0;
+   }
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+   This produces:
+
+       pi_function took 0.007252 seconds
+       (test machine: Core i7 920 @ 2.67GHz with a Tesla C2070)
+
+
+## A note on kernel caching
+
+The first run of ArrayFire code exercises any JIT compilation in the
+application, automatically saving a cache of the compilation to
+disk. Subsequent runs load the cache from disk, executing without
+compilation. Therefore, it is typically best to "warm up" the code with one
+run to initiate the application's kernel cache. Afterwards, subsequent runs do
+not include the compile time and are tend to be faster than the first run.
+
+Averaging the time taken is always the best approach and one reason why the
+[timeit](\ref af::timeit) function is helpful.
diff --git a/docs/pages/tutorials.md b/docs/pages/tutorials.md
index f6056b8e19..34b65be12c 100644
--- a/docs/pages/tutorials.md
+++ b/docs/pages/tutorials.md
@@ -15,4 +15,5 @@
 * [Timing ArrayFire](\ref timing)
 * [Configuring ArrayFire Environment](\ref configuring_environment)
 * [Debugging ArrayFire Code](\ref debugging)
+* [ArrayFire JIT Code Generation](\ref jit)
 * [GFOR Usage](\ref page_gfor)

From 02ce5cb169762effe6aa793441227503167b3f61 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 28 Aug 2023 19:36:57 -0400
Subject: [PATCH 371/473] Update release notes for v3.9 v3.8.3 and update docs
 to include oneAPI

---
 README.md                                     |  3 +-
 docs/pages/README.md                          |  8 +-
 .../configuring_arrayfire_environment.md      | 10 +++
 docs/pages/getting_started.md                 | 13 ++--
 docs/pages/install.md                         |  2 +-
 docs/pages/release_notes.md                   | 69 ++++++++++++++++++
 docs/pages/unified_backend.md                 | 73 ++++++++++++-------
 docs/pages/using_on_linux.md                  |  7 +-
 docs/pages/using_on_windows.md                |  1 +
 9 files changed, 145 insertions(+), 41 deletions(-)

diff --git a/README.md b/README.md
index c56f29623f..fed0820455 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,8 @@ Several of ArrayFire's benefits include:
 * [Easy to use](http://arrayfire.org/docs/gettingstarted.htm), stable,
   [well-documented](http://arrayfire.org/docs) API
 * Rigorous benchmarks and tests ensuring top performance and numerical accuracy
-* Cross-platform compatibility with support for CUDA, OpenCL, and native CPU on Windows, Mac, and Linux
+* Cross-platform compatibility with support for CUDA, oneAPI, OpenCL, and
+   native CPU on Windows, Mac, and Linux
 * Built-in visualization functions through [Forge](https://github.com/arrayfire/forge)
 * Commercially friendly open-source licensing
 * Enterprise support from [ArrayFire](http://arrayfire.com)
diff --git a/docs/pages/README.md b/docs/pages/README.md
index 08cc17578d..7c22adf87c 100644
--- a/docs/pages/README.md
+++ b/docs/pages/README.md
@@ -57,8 +57,7 @@ and 32-bit signed and unsigned integers.
 #### Extending ArrayFire
 
 ArrayFire can be used as a stand-alone application or integrated with existing
-CUDA, oneAPI, or OpenCL code. All ArrayFire `arrays` can be interchanged with
-other CUDA, oneAPI, or OpenCL data structures.
+CUDA, oneAPI, or OpenCL code.
 
 ## Code once, run anywhere!
 
@@ -100,9 +99,8 @@ your application.
 
 ## Simple Example
 
-Here is an example of ArrayFire code. First, [arrays](\ref af::array) are
-created which reside on CUDA, oneAPI, or OpenCL devices. Then
-[ArrayFire functions](modules.htm) are used on those [arrays](\ref af::array).
+Here is an example of ArrayFire code that performs a Monte Carlo estimation of
+PI.
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.cpp}
 // sample 40 million points on the GPU
diff --git a/docs/pages/configuring_arrayfire_environment.md b/docs/pages/configuring_arrayfire_environment.md
index fd11628105..7b20be9b4a 100644
--- a/docs/pages/configuring_arrayfire_environment.md
+++ b/docs/pages/configuring_arrayfire_environment.md
@@ -38,6 +38,16 @@ variable are the device identifiers shown when af::info is run.
 AF_CUDA_DEFAULT_DEVICE=1 ./myprogram_cuda
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+AF_ONEAPI_DEFAULT_DEVICE {#af_oneapi_default_device}
+-------------------------------------------------------------------------------
+
+Use this variable to set the default oneAPI device. Valid values for this
+variable are the device identifiers shown when af::info is run.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+AF_ONEAPI_DEFAULT_DEVICE=1 ./myprogram_oneapi
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Note: af::setDevice call in the source code will take precedence over this
 variable.
 
diff --git a/docs/pages/getting_started.md b/docs/pages/getting_started.md
index d958892c2e..19660f8cc8 100644
--- a/docs/pages/getting_started.md
+++ b/docs/pages/getting_started.md
@@ -24,6 +24,7 @@ can represent one of many different [basic data types](\ref af_dtype):
 * [c32](\ref c32) complex single-precision (`cfloat`)
 * [f64](\ref f64) real double-precision (`double`)
 * [c64](\ref c64) complex double-precision (`cdouble`)
+* [f16](\ref f16) real half-precision (`half_float::half`)
 * [b8](\ref b8) 8-bit boolean values (`bool`)
 * [s32](\ref s32) 32-bit signed integer (`int`)
 * [u32](\ref u32) 32-bit unsigned integer (`unsigned`)
@@ -153,11 +154,11 @@ using the `af::` namespace.
 
 # Indexing {#getting_started_indexing}
 
-Like all functions in ArrayFire, indexing is also executed in parallel on
-the OpenCL/CUDA device.
-Because of this, indexing becomes part of a JIT operation and is accomplished
-using parentheses instead of square brackets (i.e. as `A(0)` instead of `A[0]`).
-To index `af::array`s you may use one or a combination of the following functions:
+Like all functions in ArrayFire, indexing is also executed in parallel on the
+OpenCL/CUDA devices. Because of this, indexing becomes part of a JIT operation
+and is accomplished using parentheses instead of square brackets (i.e. as `A(0)`
+instead of `A[0]`). To index `af::array`s you may use one or a combination of
+the following functions:
 
 * integer scalars
 * [seq()](\ref af::seq) representing a linear sequence
@@ -223,7 +224,7 @@ simply include the `arrayfire.h` header file and start coding!
         double result;
         af_sum_all(&result, 0, a);
         printf("sum: %g\n", result);
-        
+
         return 0;
     }
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/pages/install.md b/docs/pages/install.md
index a0b3af61b3..555e702a1b 100644
--- a/docs/pages/install.md
+++ b/docs/pages/install.md
@@ -20,7 +20,7 @@ device vendor’s website.
 ## Windows {#Windows}
 
 Prior to installing ArrayFire on Windows,
-[download](https://www.microsoft.com/en-in/download/details.aspx?id=48145) and
+[download](https://www.microsoft.com/download/details.aspx?id=48145)
 install the Visual Studio 2015 (x64) runtime libraries.
 
 Once the ArrayFire installer has been downloaded, run the installer. If you
diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index bc40f2a7b7..464eba664d 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -1,6 +1,75 @@
 Release Notes {#releasenotes}
 ==============
 
+v3.9.0
+======
+
+## Improvements
+- Add oneAPI backend \PR{3296}
+- Add support to directly access arrays on other devices \PR{3447}
+- Add broadcast support \PR{2871}
+- Improve OpenCL CPU JIT performance \PR{3257} \PR{3392}
+- Optimize thread/block calculations of several kernels \PR{3144}
+- Add support for fast math compiliation when building ArrayFire \PR{3334 \PR{3337}
+- Optimize performance of fftconvolve when using floats \PR{3338}
+- Add support for CUDA 12.1 and 12.2
+- Better handling of empty arrays \PR{3398}
+- Better handling of memory in linear algebra functions in OpenCL \PR{3423}
+- Better logging with JIT kernels \PR{3468}
+- Optimize memory manager/JIT interactions for small number of buffers \PR{3468}
+- Documentation improvements \PR{3485}
+- Optimize reorder function \PR{3488}
+
+## Fixes
+- Improve Errors when creating OpenCL contexts from devices \PR{3257}
+- Improvements to vcpkg builds \PR{3376 \PR{3476}
+- Fix reduce by key when nan's are present \PR{3261}
+- Fix error in convolve where the ndims parameter was forced to be equal to 2  \PR{3277}
+- Make constructors that accept dim_t to be explicit to avoid invalid conversions  \PR{3259} 
+- Fix error in randu when compiling against clang 14 \PR{3333} 
+- Fix bug in OpenCL linear algebra functions  \PR{3398}
+- Fix bug with thread local variables when device was changed \PR{3420} \PR{3421} 
+- Fix bug in qr related to uninitialized memory \PR{3422} 
+- Fix bug in shift where the array had an empty middle dimension \PR{3488}
+
+
+## Contributions
+
+Special thanks to our contributors:
+[Willy Born](https://github.com/willyborn)
+[Mike Mullen](https://github.com/mfzmullen)
+
+v3.8.3
+======
+
+## Improvements
+
+- Add support for CUDA 12 \PR{3352}
+- Modernize documentation style and content \PR{3351}
+- memcpy performance improvements \PR{3144}
+- JIT performance improvements \PR{3144}
+- join performance improvements \PR{3144}
+- Improve support for Intel and newer Clang compilers \PR{3334}
+- CCache support on Windows \PR{3257}
+
+## Fixes
+
+- Fix issue with some locales with OpenCL kernel generation \PR{3294}
+- Internal improvements
+- Fix leak in clfft on exit.
+- Fix some cases where ndims was incorrectly used ot calculate shape \PR{3277}
+- Fix issue when setDevice was not called in new threads \PR{3269}
+- Restrict initializer list to just fundamental types \PR{3264}
+
+## Contributions
+
+Special thanks to our contributors:
+[Carlo Cabrera](https://github.com/carlocab)
+[Guillaume Schmid](https://github.com/GuillaumeSchmid)
+[Willy Born](https://github.com/willyborn)
+[ktdq](https://github.com/ktdq)
+
+
 v3.8.2
 ======
 
diff --git a/docs/pages/unified_backend.md b/docs/pages/unified_backend.md
index 6924f92707..5a99bff8f4 100644
--- a/docs/pages/unified_backend.md
+++ b/docs/pages/unified_backend.md
@@ -7,7 +7,7 @@ Unified Backend {#unifiedbackend}
 
 The Unified backend was introduced in ArrayFire with version 3.2.
 While this is not an independent backend, it allows the user to switch between
-the different ArrayFire backends (CPU, CUDA and OpenCL) at runtime.
+the different ArrayFire backends (CPU, CUDA, oneAPI and OpenCL) at runtime.
 
 # Compiling with Unified
 
@@ -24,7 +24,7 @@ To use with CMake, use the __ArrayFire_Unified_LIBRARIES__ variable.
 # Using the Unified Backend
 
 The Unified backend will try to dynamically load the backend libraries. The
-priority of backends is __CUDA -> OpenCL -> CPU__
+priority of backends is __CUDA -> oneAPI -> OpenCL -> CPU__
 
 The most important aspect to note here is that all the libraries the ArrayFire
 libs depend on need to be in the environment paths
@@ -78,6 +78,15 @@ int main()
         fprintf(stderr, "%s\n", e.what());
     }
 
+    try {
+        printf("Trying oneAPI Backend\n");
+        af::setBackend(AF_BACKEND_ONEAPI);
+        testBackend();
+    } catch (af::exception& e) {
+        printf("Caught exception when trying oneAPI backend\n");
+        fprintf(stderr, "%s\n", e.what());
+    }
+
     try {
         printf("Trying CUDA Backend\n");
         af::setBackend(AF_BACKEND_CUDA);
@@ -103,39 +112,53 @@ int main()
 This output would be:
 
     Trying CPU Backend
-    ArrayFire v3.2.0 (CPU, 64-bit Linux, build fc7630f)
-    [0] Intel: Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz Max threads(8)
+    ArrayFire v3.9.0 (CPU, 64-bit Linux, build 23ee0650e)
+    [0] AMD: AMD Ryzen Threadripper PRO 3955WX 16-Cores     af::randu(5, 4)
+    [5 4 1 1]
+        0.6010     0.5497     0.1583     0.3636
+        0.0278     0.2864     0.3712     0.4165
+        0.9806     0.3410     0.3543     0.5814
+        0.2126     0.7509     0.6450     0.8962
+        0.0655     0.4105     0.9675     0.3712
+
+    Trying oneAPI Backend
+    ArrayFire v3.9.0 (oneAPI, 64-bit Linux, build 23ee0650e)
+    [0] Intel(R) OpenCL: AMD Ryzen Threadripper PRO 3955WX 16-Cores     , 128650 MB (fp64)
     af::randu(5, 4)
     [5 4 1 1]
-        0.0000     0.2190     0.3835     0.5297
-        0.1315     0.0470     0.5194     0.6711
-        0.7556     0.6789     0.8310     0.0077
-        0.4587     0.6793     0.0346     0.3834
-        0.5328     0.9347     0.0535     0.0668
+        0.6010     0.5497     0.1583     0.3636
+        0.0278     0.2864     0.3712     0.4165
+        0.9806     0.3410     0.3543     0.5814
+        0.2126     0.7509     0.6450     0.8962
+        0.0655     0.4105     0.9675     0.3712
 
     Trying CUDA Backend
-    ArrayFire v3.2.0 (CUDA, 64-bit Linux, build fc7630f)
-    Platform: CUDA Toolkit 7.5, Driver: 355.11
-    [0] Quadro K5000, 4093 MB, CUDA Compute 3.0
+    ArrayFire v3.9.0 (CUDA, 64-bit Linux, build 23ee0650e)
+    Platform: CUDA Runtime 12.2, Driver: 535.104.05
+    [0] NVIDIA RTX A5500, 22721 MB, CUDA Compute 8.6
+    -1- NVIDIA RTX A5500, 22719 MB, CUDA Compute 8.6
     af::randu(5, 4)
     [5 4 1 1]
-        0.7402     0.4464     0.7762     0.2920
-        0.9210     0.6673     0.2948     0.3194
-        0.0390     0.1099     0.7140     0.8109
-        0.9690     0.4702     0.3585     0.1541
-        0.9251     0.5132     0.6814     0.4452
+        0.6010     0.5497     0.1583     0.3636
+        0.0278     0.2864     0.3712     0.4165
+        0.9806     0.3410     0.3543     0.5814
+        0.2126     0.7509     0.6450     0.8962
+        0.0655     0.4105     0.9675     0.3712
 
     Trying OpenCL Backend
-    ArrayFire v3.2.0 (OpenCL, 64-bit Linux, build fc7630f)
-    [0] NVIDIA  : Quadro K5000
-    -1- INTEL   : Intel(R) Core(TM) i7-4770K CPU @ 3.50GHz
+    ArrayFire v3.9.0 (OpenCL, 64-bit Linux, build 23ee0650e)
+    [0] NVIDIA: NVIDIA RTX A5500, 22720 MB
+    -1- NVIDIA: NVIDIA RTX A5500, 22718 MB
+    -2- Intel(R) FPGA Emulation Platform for OpenCL(TM): Intel(R) FPGA Emulation Device, 128650 MB
+    -3- INTEL: AMD Ryzen Threadripper PRO 3955WX 16-Cores     , 128650 MB
     af::randu(5, 4)
     [5 4 1 1]
-        0.4107     0.0081     0.6600     0.1046
-        0.8224     0.3775     0.0764     0.8827
-        0.9518     0.3027     0.0901     0.1647
-        0.1794     0.6456     0.5933     0.8060
-        0.4198     0.5591     0.1098     0.5938
+        0.6010     0.5497     0.1583     0.3636
+        0.0278     0.2864     0.3712     0.4165
+        0.9806     0.3410     0.3543     0.5814
+        0.2126     0.7509     0.6450     0.8962
+        0.0655     0.4105     0.9675     0.3712
+
 
 # Dos and Don'ts
 
diff --git a/docs/pages/using_on_linux.md b/docs/pages/using_on_linux.md
index 0fcd23bba1..7dbff74d2a 100644
--- a/docs/pages/using_on_linux.md
+++ b/docs/pages/using_on_linux.md
@@ -15,7 +15,7 @@ installer will populate files in the following sub-directories:
 
     include/arrayfire.h         - Primary ArrayFire include file
     include/af/*.h              - Additional include files
-    lib/libaf*                  - CPU, CUDA, and OpenCL libraries (.a, .so)
+    lib/libaf*                  - CPU, CUDA, oneAPI and OpenCL libraries (.a, .so)
     lib/libforge*               - Visualization library
     lib/libcu*                  - CUDA backend dependencies
     lib/libOpenCL.so            - OpenCL ICD Loader library
@@ -81,6 +81,7 @@ how to use CMake. To link with a specific backend directly, replace the
 
 * `ArrayFire::afcpu` for CPU backend.
 * `ArrayFire::afcuda` for CUDA backend.
+* `ArrayFire::afoneapi` for oneAPI backend.
 * `ArrayFire::afopencl` for OpenCL backend.
 
 Next we need to instruct CMake to create build instructions and then compile. We
@@ -116,8 +117,8 @@ directory containing `arrayfire.h` file. This should be `-I
 Similarly, you will need to specify the path to the ArrayFire library using the
 `-L` option (e.g. `-L/opt/arrayfire/lib`) followed by the specific ArrayFire
 library you wish to use using the `-l` option (for example `-lafcpu`,
-`-lafopencl`, `-lafcuda`, or `-laf` for the CPU, OpenCL, CUDA, and unified
-backends, respectively.
+`-lafopencl`, `-lafoneapi`, `-lafcuda`, or `-laf` for the CPU, OpenCL, oneAPI
+and CUDA, and unified backends, respectively.
 
 Here is a minimal example Makefile which uses ArrayFire's CPU backend:
 
diff --git a/docs/pages/using_on_windows.md b/docs/pages/using_on_windows.md
index b178ad9c86..072445a4ae 100644
--- a/docs/pages/using_on_windows.md
+++ b/docs/pages/using_on_windows.md
@@ -141,6 +141,7 @@ how to use CMake. To link with a specific backend directly, replace the
 
 * `ArrayFire::afcpu` for CPU backend.
 * `ArrayFire::afcuda` for CUDA backend.
+* `ArrayFire::afoneapi` for oneAPI backend.
 * `ArrayFire::afopencl` for OpenCL backend.
 
 Next we need to instruct CMake to create build instructions and then compile. We

From 9b9acea3aee7273e1bcde53fcb3da20fe55933f7 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 28 Aug 2023 19:37:38 -0400
Subject: [PATCH 372/473] Add FMT_HEADER_ONLY definition when SPDLOG is set to
 header only

---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index deafa7a759..12d6e557c9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -279,6 +279,7 @@ else()
   if(AF_WITH_SPDLOG_HEADER_ONLY)
     set_target_properties(af_spdlog
       PROPERTIES
+        INTERFACE_COMPILE_DEFINITIONS "FMT_HEADER_ONLY=1"
         INTERFACE_LINK_LIBRARIES "spdlog_header_only")
   else()
     target_compile_options(spdlog

From 67bd7499d6ba9eadaa7c95d139d6f05bed7d5430 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 28 Aug 2023 19:39:43 -0400
Subject: [PATCH 373/473] Fix namespace for isnan in oneAPI math header

---
 src/backend/oneapi/math.hpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp
index b6aba91663..4a3c8b41b2 100644
--- a/src/backend/oneapi/math.hpp
+++ b/src/backend/oneapi/math.hpp
@@ -83,22 +83,22 @@ inline auto is_nan<sycl::half>(const sycl::half &val) -> bool {
 
 template<>
 inline auto is_nan<float>(const float &val) -> bool {
-    return std::isnan(val);
+    return sycl::isnan(val);
 }
 
 template<>
 inline auto is_nan<double>(const double &val) -> bool {
-    return std::isnan(val);
+    return sycl::isnan(val);
 }
 
 template<>
 inline auto is_nan<cfloat>(const cfloat &in) -> bool {
-    return std::isnan(real(in)) || std::isnan(imag(in));
+    return sycl::isnan(real(in)) || sycl::isnan(imag(in));
 }
 
 template<>
 inline auto is_nan<cdouble>(const cdouble &in) -> bool {
-    return std::isnan(real(in)) || std::isnan(imag(in));
+    return sycl::isnan(real(in)) || sycl::isnan(imag(in));
 }
 
 template<typename T>

From bada0467a3158f50b8941f0845e690b0262e9469 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 28 Aug 2023 19:43:07 -0400
Subject: [PATCH 374/473] Fix LIBRARY_SUFFIXES in FindAF_MKL to find MKL kernel
 libraries

---
 CMakeModules/FindAF_MKL.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeModules/FindAF_MKL.cmake b/CMakeModules/FindAF_MKL.cmake
index 662f0046da..a58809d495 100644
--- a/CMakeModules/FindAF_MKL.cmake
+++ b/CMakeModules/FindAF_MKL.cmake
@@ -221,7 +221,7 @@ function(find_mkl_library)
   add_library(MKL::${mkl_args_NAME}_STATIC STATIC IMPORTED)
 
   if(NOT (WIN32 AND mkl_args_DLL_ONLY))
-    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".so.1")
+    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".so.1;.so.2;.so.3;.so.4;.so.12")
     find_library(MKL_${mkl_args_NAME}_LINK_LIBRARY
       NAMES
         ${mkl_args_LIBRARY_NAME}${shared_suffix}

From b59a1ae535da369db86451e5b28a7bc0eaf3e84a Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Mon, 28 Aug 2023 19:49:55 -0400
Subject: [PATCH 375/473] Pass OPENCL_LIBRARIES to CLBlast ExternalProject
 command

---
 CMakeModules/build_CLBlast.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeModules/build_CLBlast.cmake b/CMakeModules/build_CLBlast.cmake
index 0f67d3fdee..933531cdf2 100644
--- a/CMakeModules/build_CLBlast.cmake
+++ b/CMakeModules/build_CLBlast.cmake
@@ -75,6 +75,7 @@ else()
         -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}
         "-DCMAKE_C_FLAGS:STRING=${CMAKE_C_FLAGS}"
         -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+        -DOPENCL_LIBRARIES="${OPENCL_LIBRARIES}"
         ${extproj_build_type_option}
         -DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
         -DCMAKE_INSTALL_LIBDIR:PATH=lib

From 4061db86e66306995175a14cf906c55c35373918 Mon Sep 17 00:00:00 2001
From: John Melonakos <john.melonakos@intel.com>
Date: Wed, 4 Oct 2023 16:25:56 -0400
Subject: [PATCH 376/473] Update README.md to add Intel to GPUs and to fix word
 wrapping

---
 README.md | 54 +++++++++++++++++++++++++++++++-----------------------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index fed0820455..eb6dc6a5f6 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,15 @@
-
 <p align="center"><a href="http://arrayfire.com/"><img src="http://arrayfire.com/logos/arrayfire_logo_whitebkgnd.png" width="800"></a></p>
 
-ArrayFire is a general-purpose tensor library that simplifies the process of
-software development for the parallel architectures found in CPUs, GPUs, and
-other hardware acceleration devices. The library serves users in every technical
-computing market.
+ArrayFire is a general-purpose tensor library that simplifies the software
+development process for the parallel architectures found in CPUs, GPUs, and
+other hardware acceleration devices. The library serves users in every
+technical computing market.
 
 Several of ArrayFire's benefits include:
 
-* Hundreds of accelerated [tensor computing functions](https://arrayfire.org/docs/group__arrayfire__func.htm), in the following areas:
+* Hundreds of accelerated [tensor computing
+  functions](https://arrayfire.org/docs/group__arrayfire__func.htm), in the
+  following areas:
     * Array handling
     * Computer vision
     * Image processing
@@ -22,8 +23,9 @@ Several of ArrayFire's benefits include:
   [well-documented](http://arrayfire.org/docs) API
 * Rigorous benchmarks and tests ensuring top performance and numerical accuracy
 * Cross-platform compatibility with support for CUDA, oneAPI, OpenCL, and
-   native CPU on Windows, Mac, and Linux
-* Built-in visualization functions through [Forge](https://github.com/arrayfire/forge)
+  native CPU on Windows, Mac, and Linux
+* Built-in visualization functions through
+  [Forge](https://github.com/arrayfire/forge)
 * Commercially friendly open-source licensing
 * Enterprise support from [ArrayFire](http://arrayfire.com)
 
@@ -34,19 +36,22 @@ translated into near-optimal kernels that execute on the computational device.
 
 ArrayFire runs on devices ranging from low-power mobile phones to high-power
 GPU-enabled supercomputers. ArrayFire runs on CPUs from all major vendors
-(Intel, AMD, ARM), GPUs from the prominent manufacturers (NVIDIA, AMD, and
-Qualcomm), as well as a variety of other accelerator devices on Windows, Mac,
-and Linux.
+(Intel, AMD, ARM), GPUs from the prominent manufacturers (AMD, Intel, NVIDIA,
+and Qualcomm), as well as a variety of other accelerator devices on Windows,
+Mac, and Linux.
 
 # Getting ArrayFire
 
-Instructions to [install][32] or to build ArrayFire from source can be found on the [wiki][1].
+Instructions to [install][32] or to build ArrayFire from source can be found on
+the [wiki][1].
 
 ### Conway's Game of Life Using ArrayFire
 
 Visit the [Wikipedia page][2] for a description of Conway's Game of Life.
 
-<img align="left" src="https://github.com/arrayfire/assets/blob/master/gifs/conway.gif" alt="Conway's Game of Life" height="256" width="256">
+<img align="left"
+src="https://github.com/arrayfire/assets/blob/master/gifs/conway.gif"
+alt="Conway's Game of Life" height="256" width="256">
 
 ```cpp
 static const float h_kernel[] = { 1, 1, 1, 1, 0, 1, 1, 1, 1 };
@@ -66,7 +71,9 @@ The complete source code can be found [here][3].
 
 ### Perceptron
 
-<img align="left" src="https://github.com/arrayfire/assets/blob/imgs_readme_improv/gifs/perceptron.gif" alt="Perceptron" height="400" width="300">
+<img align="left"
+src="https://github.com/arrayfire/assets/blob/imgs_readme_improv/gifs/perceptron.gif"
+alt="Perceptron" height="400" width="300">
 
 ```cpp
 array predict(const array &X, const array &W) {
@@ -132,9 +139,10 @@ Mission](https://github.com/arrayfire/arrayfire/wiki/The-ArrayFire-Mission-State
 for fast scientific computing for all.
 
 Contributions of any kind are welcome! Please refer to [the
-wiki](https://github.com/arrayfire/arrayfire/wiki) and our [Code of Conduct](33)
-to learn more about how you can get involved with the ArrayFire Community
-through [Sponsorship](https://github.com/arrayfire/arrayfire/wiki/Sponsorship),
+wiki](https://github.com/arrayfire/arrayfire/wiki) and our [Code of
+Conduct](33) to learn more about how you can get involved with the ArrayFire
+Community through
+[Sponsorship](https://github.com/arrayfire/arrayfire/wiki/Sponsorship),
 [Developer
 Commits](https://github.com/arrayfire/arrayfire/wiki/Contributing-Code-to-ArrayFire),
 or [Governance](https://github.com/arrayfire/arrayfire/wiki/Governance).
@@ -146,8 +154,8 @@ license](LICENSE). If you wish to cite ArrayFire in an academic publication,
 please use the following [citation document](.github/CITATION.md).
 
 ArrayFire development is funded by AccelerEyes LLC and several third parties,
-please see the list of [acknowledgements](ACKNOWLEDGEMENTS.md) for an expression
-of our gratitude.
+please see the list of [acknowledgements](ACKNOWLEDGEMENTS.md) for an
+expression of our gratitude.
 
 # Support and Contact Info
 
@@ -157,10 +165,10 @@ of our gratitude.
 
 # Trademark Policy
 
-The literal mark "ArrayFire" and ArrayFire logos are trademarks of
-AccelerEyes LLC (dba ArrayFire).
-If you wish to use either of these marks in your own project, please consult
-[ArrayFire's Trademark Policy](http://arrayfire.com/trademark-policy/)
+The literal mark "ArrayFire" and ArrayFire logos are trademarks of AccelerEyes
+LLC (dba ArrayFire). If you wish to use either of these marks in your own
+project, please consult [ArrayFire's Trademark
+Policy](http://arrayfire.com/trademark-policy/)
 
 [1]: https://github.com/arrayfire/arrayfire/wiki
 [2]: https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life

From f4db00f2cc57272f16f0e7e8534f65c2a895cfda Mon Sep 17 00:00:00 2001
From: Filip Matzner <floop@floop.cz>
Date: Thu, 7 Mar 2024 13:54:29 +0100
Subject: [PATCH 377/473] Update toolkit driver versions for CUDA 12.4

---
 src/backend/cuda/device_manager.cpp | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 9e7cc2d68b..e3faf0376d 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -101,6 +101,8 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {12040, 9, 0, 0},
+    {12030, 9, 0, 0},
     {12020, 9, 0, 0},
     {12010, 9, 0, 0},
     {12000, 9, 0, 0},
@@ -140,9 +142,11 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
-        {12020, 525.60f, 527.41f},
-        {12010, 525.60f, 527.41f},
-        {12000, 525.60f, 527.41f},
+        {12040, 525.60f, 528.33f},
+        {12030, 525.60f, 528.33f},
+        {12020, 525.60f, 528.33f},
+        {12010, 525.60f, 528.33f},
+        {12000, 525.60f, 528.33f},
         {11080, 450.80f, 452.39f},
         {11070, 450.80f, 452.39f},
         {11060, 450.80f, 452.39f},

From 48a97be35f3892044094172d8fe9db586ccb601a Mon Sep 17 00:00:00 2001
From: Edwin <edwinsolisf12@gmail.com>
Date: Wed, 12 Jun 2024 15:55:58 -0500
Subject: [PATCH 378/473] Fixed incompatibility issues with newer opencl hpp
 headers

---
 src/backend/opencl/device_manager.cpp |  9 ++++++---
 src/backend/opencl/platform.cpp       | 13 ++++++++++---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/backend/opencl/device_manager.cpp b/src/backend/opencl/device_manager.cpp
index 1e628af521..62c06a21a5 100644
--- a/src/backend/opencl/device_manager.cpp
+++ b/src/backend/opencl/device_manager.cpp
@@ -261,8 +261,11 @@ DeviceManager::DeviceManager()
 
     // Create contexts and queues once the sort is done
     for (int i = 0; i < nDevices; i++) {
-        cl_platform_id device_platform =
-            devices[i]->getInfo<CL_DEVICE_PLATFORM>();
+        // For OpenCL-HPP >= v2023.12.14 type is cl::Platform instead of
+        // cl_platform_id
+        cl::Platform device_platform;
+        device_platform = devices[i]->getInfo<CL_DEVICE_PLATFORM>();
+
         try {
             mContexts.emplace_back(
                 make_unique<cl::Context>(mDeviceContextMap[*devices[i]]));
@@ -272,7 +275,7 @@ DeviceManager::DeviceManager()
             mDeviceTypes.push_back(getDeviceTypeEnum(*devices[i]));
             mPlatforms.push_back(
                 std::make_pair<std::unique_ptr<cl::Platform>, afcl_platform>(
-                    make_unique<cl::Platform>(device_platform, true),
+                    make_unique<cl::Platform>(device_platform(), true),
                     getPlatformEnum(*devices[i])));
             mDevices.emplace_back(std::move(devices[i]));
 
diff --git a/src/backend/opencl/platform.cpp b/src/backend/opencl/platform.cpp
index d6406a32e1..b6886c97bb 100644
--- a/src/backend/opencl/platform.cpp
+++ b/src/backend/opencl/platform.cpp
@@ -337,7 +337,11 @@ const std::string& getActiveDeviceBaseBuildFlags() {
 }
 
 vector<Version> getOpenCLCDeviceVersion(const Device& device) {
-    Platform device_platform(device.getInfo<CL_DEVICE_PLATFORM>(), false);
+    // For OpenCL-HPP >= v2023.12.14 type is cl::Platform instead of
+    // cl_platform_id
+    Platform device_platform;
+    device_platform = device.getInfo<CL_DEVICE_PLATFORM>();
+
     auto platform_version = device_platform.getInfo<CL_PLATFORM_VERSION>();
     vector<Version> out;
 
@@ -540,10 +544,13 @@ void addDeviceContext(cl_device_id dev, cl_context ctx, cl_command_queue que) {
         devMngr.mDeviceTypes.push_back(
             static_cast<int>(tDevice.getInfo<CL_DEVICE_TYPE>()));
 
-        auto device_platform = tDevice.getInfo<CL_DEVICE_PLATFORM>();
+        // For OpenCL-HPP >= v2023.12.14 type is cl::Platform instead of
+        // cl_platform_id
+        cl::Platform device_platform;
+        device_platform = tDevice.getInfo<CL_DEVICE_PLATFORM>();
         devMngr.mPlatforms.push_back(
             std::make_pair<std::unique_ptr<cl::Platform>, afcl_platform>(
-                make_unique<cl::Platform>(device_platform, true),
+                make_unique<cl::Platform>(device_platform(), true),
                 getPlatformEnum(tDevice)));
 
         devMngr.mDevices.emplace_back(make_unique<cl::Device>(move(tDevice)));

From cb09bfc5457489d6da434a7841b9098dded58cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Thu, 29 Aug 2024 19:29:59 -0500
Subject: [PATCH 379/473] Fix issue 3543: Explicitly added <climits> for C
 limits constants (#3592)

---
 src/backend/cpu/math.hpp    | 1 +
 src/backend/cuda/math.hpp   | 1 +
 src/backend/oneapi/math.hpp | 1 +
 src/backend/opencl/math.hpp | 1 +
 4 files changed, 4 insertions(+)

diff --git a/src/backend/cpu/math.hpp b/src/backend/cpu/math.hpp
index 16a4e2abbf..06c1027edf 100644
--- a/src/backend/cpu/math.hpp
+++ b/src/backend/cpu/math.hpp
@@ -15,6 +15,7 @@
 #include <af/defines.h>
 
 #include <algorithm>
+#include <climits>
 #include <limits>
 #include <numeric>
 
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index f7b11347cc..6986bcb445 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -18,6 +18,7 @@
 #endif  //__CUDACC__
 
 #include <algorithm>
+#include <climits>
 #include <limits>
 
 #endif  //__CUDACC_RTC__
diff --git a/src/backend/oneapi/math.hpp b/src/backend/oneapi/math.hpp
index 4a3c8b41b2..7362874442 100644
--- a/src/backend/oneapi/math.hpp
+++ b/src/backend/oneapi/math.hpp
@@ -18,6 +18,7 @@
 
 #include <algorithm>
 #include <complex>
+#include <climits>
 #include <limits>
 
 #if defined(__GNUC__) || defined(__GNUG__)
diff --git a/src/backend/opencl/math.hpp b/src/backend/opencl/math.hpp
index e4745d9e92..f164c3002c 100644
--- a/src/backend/opencl/math.hpp
+++ b/src/backend/opencl/math.hpp
@@ -18,6 +18,7 @@
 
 #include <algorithm>
 #include <complex>
+#include <climits>
 #include <limits>
 
 #if defined(__GNUC__) || defined(__GNUG__)

From bf233f381b46183c51dce66af8917f6ae70d7330 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Thu, 29 Aug 2024 19:31:25 -0500
Subject: [PATCH 380/473] Specified version for jasper, a subdependecy of
 freeimage, due to upstream build error (#3591)

---
 vcpkg.json | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/vcpkg.json b/vcpkg.json
index db3318eb47..6ca4ec32be 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -19,6 +19,10 @@
         {
             "name": "spdlog",
             "version": "1.9.2"
+        },
+        {
+            "name": "jasper",
+            "version": "4.2.0"
         }
     ],
     "features": {

From d3a6e2afcbbb26c23062531517e95d52eb7b7d84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Thu, 29 Aug 2024 19:34:06 -0500
Subject: [PATCH 381/473] Update toolkit driver version for cuda 12.6 (#3586)

Note that this will be superseded by #3588
---
 src/backend/cuda/device_manager.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index e3faf0376d..80f00f614a 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -101,6 +101,7 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {12060, 9, 0, 0},
     {12040, 9, 0, 0},
     {12030, 9, 0, 0},
     {12020, 9, 0, 0},
@@ -142,6 +143,7 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
+        {12060, 525.60f, 528.33f},
         {12040, 525.60f, 528.33f},
         {12030, 525.60f, 528.33f},
         {12020, 525.60f, 528.33f},

From eefbb7c8c3df9e1ad11a55989d3a6a1e63113336 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Fri, 30 Aug 2024 10:07:43 -0500
Subject: [PATCH 382/473] Fix issue 3563: added message to dependency_check
 (#3564)

---
 test/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index cf7e66255f..92c4d90acd 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -20,7 +20,7 @@ if(AF_TEST_WITH_MTX_FILES)
 endif()
 
 if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
-    dependency_check(GTest_FOUND)
+  dependency_check(GTest_FOUND "Google Tests not found.")
 elseif(NOT TARGET GTest::gtest)
   af_dep_check_and_populate(${gtest_prefix}
     URI https://github.com/google/googletest.git

From 231a300eb11d770151ffdda3a6aa1d91d342cbe0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Fri, 30 Aug 2024 10:10:13 -0500
Subject: [PATCH 383/473] Fix issue 3556: cassert not being included for assert
 macro (#3557)

---
 src/backend/common/err_common.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/backend/common/err_common.hpp b/src/backend/common/err_common.hpp
index 3936cee77c..e1e4a6d118 100644
--- a/src/backend/common/err_common.hpp
+++ b/src/backend/common/err_common.hpp
@@ -17,6 +17,7 @@
 #include <common/defines.hpp>
 #include <af/defines.h>
 
+#include <cassert>
 #include <sstream>
 #include <stdexcept>
 #include <string>

From c9476633e82423645a83f6e1bca6027b356680a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Fri, 30 Aug 2024 14:23:38 -0500
Subject: [PATCH 384/473] Fix for issue 3551: Implemented event_impl class for
 AF_DISABLE_CPU_ASYNC (#3555)

* Fix issue 3551: implemented empty event_impl class

* Applied clang format
---
 src/backend/cpu/queue.hpp | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/src/backend/cpu/queue.hpp b/src/backend/cpu/queue.hpp
index 594396a78e..cdcfb8092f 100644
--- a/src/backend/cpu/queue.hpp
+++ b/src/backend/cpu/queue.hpp
@@ -38,6 +38,42 @@ class queue_impl {
     }
 };
 
+class event_impl {
+   public:
+    event_impl() noexcept                              = default;
+    ~event_impl() noexcept                             = default;
+    explicit event_impl(const event_impl &other)       = default;
+    event_impl(event_impl &&other) noexcept            = default;
+    event_impl &operator=(event_impl &&other) noexcept = default;
+    event_impl &operator=(event_impl &other) noexcept  = default;
+
+    explicit event_impl(const int val) {}
+
+    event_impl &operator=(int val) noexcept { return *this; }
+
+    int create() {
+        AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL);
+        return 0;
+    }
+
+    int mark(queue_impl &queue) {
+        AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL);
+        return 0;
+    }
+
+    int wait(queue_impl &queue) const {
+        AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL);
+        return 0;
+    }
+
+    int sync() const noexcept {
+        AF_ERROR("Incorrectly configured", AF_ERR_INTERNAL);
+        return 0;
+    }
+
+    operator bool() const noexcept { return false; }
+};
+
 #else
 
 #include <threads/async_queue.hpp>

From 773c96b18726a323c34617dbd2f29683f230157c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Fri, 30 Aug 2024 15:08:39 -0500
Subject: [PATCH 385/473] Fix for issue 3528: cmake generator expression space
 removed (#3554)

---
 src/api/unified/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/api/unified/CMakeLists.txt b/src/api/unified/CMakeLists.txt
index ca6805c7a4..bd373acab8 100644
--- a/src/api/unified/CMakeLists.txt
+++ b/src/api/unified/CMakeLists.txt
@@ -82,8 +82,8 @@ target_include_directories(af
 target_include_directories(af
   SYSTEM PRIVATE
     $<TARGET_PROPERTY:afcommon_interface,INTERFACE_INCLUDE_DIRECTORIES>
-    $<$<BOOL:${OpenCL_FOUND}>: $<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>>
-    $<$<BOOL:${CUDA_FOUND}>:  ${CUDA_INCLUDE_DIRS}>
+    $<$<BOOL:${OpenCL_FOUND}>:$<TARGET_PROPERTY:OpenCL::OpenCL,INTERFACE_INCLUDE_DIRECTORIES>>
+    $<$<BOOL:${CUDA_FOUND}>:${CUDA_INCLUDE_DIRS}>
   )
 
 target_link_libraries(af

From 41771248d2739a8388219fffc826b3f07104ca23 Mon Sep 17 00:00:00 2001
From: errata-c <77643526+errata-c@users.noreply.github.com>
Date: Fri, 30 Aug 2024 16:28:28 -0400
Subject: [PATCH 386/473] Implement simple fix for AF_JIT_KERNEL_TRACE on
 windows (#3517)

* Implement simple fix for AF_JIT_KERNEL_TRACE on windows

* Replaced tabs with spaces for consistency

---------

Co-authored-by: errata-c <erratachem@gmail.com>
---
 src/backend/common/util.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp
index 2d4a8e5ea0..f0b24bba65 100644
--- a/src/backend/common/util.cpp
+++ b/src/backend/common/util.cpp
@@ -125,7 +125,13 @@ void saveKernel(const string& funcName, const string& jit_ker,
     // Path to a folder
     const string ffp =
         string(jitKernelsOutput) + AF_PATH_SEPARATOR + funcName + ext;
+
+#if defined(OS_WIN)
+    FILE* f = fopen(ffp.c_str(), "w");
+#else
     FILE* f = fopen(ffp.c_str(), "we");
+#endif
+
     if (!f) {
         fprintf(stderr, "Cannot open file %s\n", ffp.c_str());
         return;

From 7978352aa858605ec7ebe333426475d130445d3d Mon Sep 17 00:00:00 2001
From: errata-c <77643526+errata-c@users.noreply.github.com>
Date: Fri, 30 Aug 2024 16:52:28 -0400
Subject: [PATCH 387/473] Fix build failure of cuda backend when cudnn is used.
 (#3521)

* Fixes formatting issue with cudnnStatus_t

* Fixed call to dependency_check with too few arguments

* Reformat according to the repository clang-format

---------

Co-authored-by: errata-c <erratachem@gmail.com>
Co-authored-by: Filip Matzner <floop@floop.cz>
Co-authored-by: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
---
 src/backend/cuda/platform.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/backend/cuda/platform.cpp b/src/backend/cuda/platform.cpp
index 52a22cdbaf..0de2451c4d 100644
--- a/src/backend/cuda/platform.cpp
+++ b/src/backend/cuda/platform.cpp
@@ -49,6 +49,7 @@
 #include <stdexcept>
 #include <string>
 #include <thread>
+#include <type_traits>
 
 using std::call_once;
 using std::make_unique;
@@ -123,8 +124,10 @@ unique_handle<cudnnHandle_t> *nnManager(const int deviceId) {
         if (!(*handle)) { getLogger()->error("Error initalizing cuDNN"); }
     });
     if (error) {
-        string error_msg = fmt::format("Error initializing cuDNN({}): {}.",
-                                       error, errorString(error));
+        string error_msg = fmt::format(
+            "Error initializing cuDNN({}): {}.",
+            static_cast<std::underlying_type<cudnnStatus_t>::type>(error),
+            errorString(error));
         AF_ERROR(error_msg, AF_ERR_RUNTIME);
     }
     CUDNN_CHECK(getCudnnPlugin().cudnnSetStream(cudnnHandles[deviceId],

From ebb13ff40edc173fe4d09742a315c32c3a9f050c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Fri, 30 Aug 2024 20:11:27 -0500
Subject: [PATCH 388/473] Fix missing installation of spdlog library (#3567)

---
 CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 12d6e557c9..f3a1484a72 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -285,6 +285,9 @@ else()
     target_compile_options(spdlog
       PRIVATE
         $<$<BOOL:${has_cxx_fp_model}>:-fp-model precise>)
+    install(TARGETS spdlog
+      COMPONENT common_backend_dependencies
+      DESTINATION ${AF_INSTALL_BIN_DIR})
     set_target_properties(af_spdlog
       PROPERTIES
         INTERFACE_LINK_LIBRARIES "spdlog")

From f4157374e73a2140293e68de7a00174fa675da6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Tue, 3 Sep 2024 14:39:21 -0500
Subject: [PATCH 389/473] Fix for issue 3349: added cmake cuda version check
 (#3552)

---
 src/backend/cuda/CMakeLists.txt | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 5ffb28dafd..0c4563ed40 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -99,7 +99,12 @@ if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
     # contains GPU accelerated stedc and bdsqr. The user has to link
     # libcusolver_static.a with liblapack_static.a in order to build
     # successfully.
-    af_find_static_cuda_libs(lapack_static)
+    # Cuda Versions >= 12.0 changed lib name to libcusolver_lapack_static.a
+    if (CUDA_VERSION VERSION_GREATER_EQUAL 12.0)
+      af_find_static_cuda_libs(cusolver_lapack_static)
+    else()
+      af_find_static_cuda_libs(lapack_static)
+    endif()
 
     set(af_cuda_static_flags "${af_cuda_static_flags};-lcusolver_static")
   else()

From ec66afdf79fb10fc8c103ea2e1296d795f1f14a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Mon, 30 Sep 2024 10:32:30 -0700
Subject: [PATCH 390/473] Fix issue 3378: Added back classify in naive bayes
 example (#3577)

---
 examples/machine_learning/mnist_common.h  | 2 +-
 examples/machine_learning/naive_bayes.cpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/machine_learning/mnist_common.h b/examples/machine_learning/mnist_common.h
index a32d21932c..11531f3ffb 100644
--- a/examples/machine_learning/mnist_common.h
+++ b/examples/machine_learning/mnist_common.h
@@ -145,7 +145,7 @@ static void display_results(const af::array &test_images,
             (test_images(span, span, i) > 0.1f).as(u8).host<unsigned char>();
         for (int j = 0; j < 28; j++) {
             for (int k = 0; k < 28; k++) {
-                std::cout << (img[j * 28 + k] ? "\u2588" : " ") << " ";
+                std::cout << (img[k * 28 + j] ? "\u2588" : " ") << " ";
             }
             std::cout << std::endl;
         }
diff --git a/examples/machine_learning/naive_bayes.cpp b/examples/machine_learning/naive_bayes.cpp
index 9fe6456f0e..aadca32bc0 100644
--- a/examples/machine_learning/naive_bayes.cpp
+++ b/examples/machine_learning/naive_bayes.cpp
@@ -135,8 +135,8 @@ void naive_bayes_demo(bool console, int perc) {
     if (!console) {
         test_images = test_images.T();
         test_labels = test_labels.T();
-        // FIXME: Crashing in mnist_common.h::classify
-        // display_results<false>(test_images, res_labels, test_labels , 20);
+
+        display_results<false>(test_images, res_labels, test_labels, 20);
     }
 }
 

From 90b27acf30d84437bd3672528d69fde092aa5519 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Mon, 30 Sep 2024 17:21:48 -0700
Subject: [PATCH 391/473] Fixed issue 3578: added correct compilation define
 for coo2dense kernel (#3579)

---
 src/backend/opencl/kernel/sparse.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/opencl/kernel/sparse.hpp b/src/backend/opencl/kernel/sparse.hpp
index e1b29c986c..9005265710 100644
--- a/src/backend/opencl/kernel/sparse.hpp
+++ b/src/backend/opencl/kernel/sparse.hpp
@@ -39,7 +39,7 @@ void coo2dense(Param out, const Param values, const Param rowIdx,
     };
     std::vector<std::string> compileOpts = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
-        DefineKeyValue(resp, REPEAT),
+        DefineKeyValue(reps, REPEAT),
     };
     compileOpts.emplace_back(getTypeBuildDefinition<T>());
 

From 989b71b0e88a59bd57a24d19d3fe981aa780c866 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Mon, 30 Sep 2024 17:22:02 -0700
Subject: [PATCH 392/473] Fix issue 3542: fixed opencl, oneapi, and cuda sparse
 coo2dense launch dimensions (#3583)

---
 src/backend/cuda/kernel/sparse.cuh     | 14 ++++++--------
 src/backend/cuda/kernel/sparse.hpp     |  2 +-
 src/backend/oneapi/kernel/sparse.hpp   | 17 +++++++----------
 src/backend/opencl/kernel/coo2dense.cl | 13 +++++--------
 src/backend/opencl/kernel/sparse.hpp   |  3 ++-
 5 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/src/backend/cuda/kernel/sparse.cuh b/src/backend/cuda/kernel/sparse.cuh
index bdf0e20884..84825bdd24 100644
--- a/src/backend/cuda/kernel/sparse.cuh
+++ b/src/backend/cuda/kernel/sparse.cuh
@@ -17,15 +17,13 @@ namespace cuda {
 template<typename T>
 __global__ void coo2Dense(Param<T> output, CParam<T> values, CParam<int> rowIdx,
                           CParam<int> colIdx) {
-    int id = blockIdx.x * blockDim.x * reps + threadIdx.x;
-    if (id >= values.dims[0]) return;
+    for (int i = threadIdx.x; i < reps * blockDim.x; i += blockDim.x) {
+        int id = i + blockIdx.x * blockDim.x * reps;
+        if (id >= values.dims[0]) return;
 
-    for (int i = threadIdx.x; i <= reps * blockDim.x; i += blockDim.x) {
-        if (i >= values.dims[0]) return;
-
-        T v   = values.ptr[i];
-        int r = rowIdx.ptr[i];
-        int c = colIdx.ptr[i];
+        T v   = values.ptr[id];
+        int r = rowIdx.ptr[id];
+        int c = colIdx.ptr[id];
 
         int offset = r + c * output.strides[1];
 
diff --git a/src/backend/cuda/kernel/sparse.hpp b/src/backend/cuda/kernel/sparse.hpp
index 6629d0fec6..60068d3e20 100644
--- a/src/backend/cuda/kernel/sparse.hpp
+++ b/src/backend/cuda/kernel/sparse.hpp
@@ -30,7 +30,7 @@ void coo2dense(Param<T> output, CParam<T> values, CParam<int> rowIdx,
 
     dim3 threads(256, 1, 1);
 
-    dim3 blocks(divup(output.dims[0], threads.x * reps), 1, 1);
+    dim3 blocks(divup(values.dims[0], threads.x * reps), 1, 1);
 
     EnqueueArgs qArgs(blocks, threads, getActiveStream());
 
diff --git a/src/backend/oneapi/kernel/sparse.hpp b/src/backend/oneapi/kernel/sparse.hpp
index 70bf051868..8cc7f99fcc 100644
--- a/src/backend/oneapi/kernel/sparse.hpp
+++ b/src/backend/oneapi/kernel/sparse.hpp
@@ -47,19 +47,16 @@ class coo2DenseCreateKernel {
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g = it.get_group();
 
-        const int id = g.get_group_id(0) * g.get_local_range(0) * REPEAT +
-                       it.get_local_id(0);
-
-        if (id >= values_.dims[0]) return;
-
         const int dimSize = g.get_local_range(0);
 
         for (int i = it.get_local_id(0); i < REPEAT * dimSize; i += dimSize) {
-            if (i >= values_.dims[0]) return;
+            const int id =
+                g.get_group_id(0) * g.get_local_range(0) * REPEAT + i;
+            if (id >= values_.dims[0]) return;
 
-            T v   = vPtr_[i];
-            int r = rPtr_[i];
-            int c = cPtr_[i];
+            T v   = vPtr_[id];
+            int r = rPtr_[id];
+            int c = cPtr_[id];
 
             int offset = r + c * output_.strides[1];
 
@@ -83,7 +80,7 @@ void coo2dense(Param<T> out, const Param<T> values, const Param<int> rowIdx,
                const Param<int> colIdx) {
     auto local  = sycl::range(THREADS_PER_BLOCK, 1);
     auto global = sycl::range(
-        divup(out.info.dims[0], local[0] * REPEAT) * THREADS_PER_BLOCK, 1);
+        divup(values.info.dims[0], local[0] * REPEAT) * THREADS_PER_BLOCK, 1);
 
     getQueue().submit([&](auto &h) {
         sycl::accessor d_rowIdx{*rowIdx.data, h, sycl::read_only};
diff --git a/src/backend/opencl/kernel/coo2dense.cl b/src/backend/opencl/kernel/coo2dense.cl
index f86c073621..539c98ada1 100644
--- a/src/backend/opencl/kernel/coo2dense.cl
+++ b/src/backend/opencl/kernel/coo2dense.cl
@@ -11,18 +11,15 @@ kernel void coo2Dense(global T *oPtr, const KParam output, global const T *vPtr,
                       const KParam values, global const int *rPtr,
                       const KParam rowIdx, global const int *cPtr,
                       const KParam colIdx) {
-    const int id = get_group_id(0) * get_local_size(0) * reps + get_local_id(0);
-
-    if (id >= values.dims[0]) return;
-
     const int dimSize = get_local_size(0);
 
     for (int i = get_local_id(0); i < reps * dimSize; i += dimSize) {
-        if (i >= values.dims[0]) return;
+        const int id = i + get_group_id(0) * dimSize * reps;
+        if (id >= values.dims[0]) return;
 
-        T v   = vPtr[i];
-        int r = rPtr[i];
-        int c = cPtr[i];
+        T v   = vPtr[id];
+        int r = rPtr[id];
+        int c = cPtr[id];
 
         int offset = r + c * output.strides[1];
 
diff --git a/src/backend/opencl/kernel/sparse.hpp b/src/backend/opencl/kernel/sparse.hpp
index 9005265710..13a4a9c5fb 100644
--- a/src/backend/opencl/kernel/sparse.hpp
+++ b/src/backend/opencl/kernel/sparse.hpp
@@ -49,7 +49,8 @@ void coo2dense(Param out, const Param values, const Param rowIdx,
     cl::NDRange local(THREADS_PER_GROUP, 1, 1);
 
     cl::NDRange global(
-        divup(out.info.dims[0], local[0] * REPEAT) * THREADS_PER_GROUP, 1, 1);
+        divup(values.info.dims[0], local[0] * REPEAT) * THREADS_PER_GROUP, 1,
+        1);
 
     coo2dense(cl::EnqueueArgs(getQueue(), global, local), *out.data, out.info,
               *values.data, values.info, *rowIdx.data, rowIdx.info,

From 01c89484256b1bdc07c645e7e2e6c382d95e8aa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Mon, 30 Sep 2024 17:40:25 -0700
Subject: [PATCH 393/473] Added sparse dense conversion test (#3589)

---
 test/sparse.cpp | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/test/sparse.cpp b/test/sparse.cpp
index a130a6bb58..3142a3735a 100644
--- a/test/sparse.cpp
+++ b/test/sparse.cpp
@@ -416,3 +416,24 @@ TEST(Sparse, CPPDenseToSparseToDenseUsage) {
     ASSERT_ARRAYS_EQ(in, gold);
     ASSERT_ARRAYS_EQ(dense, gold);
 }
+
+TEST(Sparse, CPPDenseToSparseConversions) {
+    array in      = af::randu(200, 200);
+    in(in < 0.75) = 0;
+
+    array coo_sparse_arr = af::sparse(in, AF_STORAGE_COO);
+    array csr_sparse_arr = af::sparse(in, AF_STORAGE_CSR);
+
+    array coo_dense_arr = af::dense(coo_sparse_arr);
+    array csr_dense_arr = af::dense(csr_sparse_arr);
+
+    ASSERT_ARRAYS_EQ(in, coo_dense_arr);
+    ASSERT_ARRAYS_EQ(in, csr_dense_arr);
+
+    array non_zero   = af::flat(in)(af::where(in));
+    array non_zero_T = af::flat(in.T())(af::where(in.T()));
+    ASSERT_ARRAYS_EQ(non_zero, af::sparseGetValues(coo_sparse_arr));
+    ASSERT_ARRAYS_EQ(
+        non_zero_T,
+        af::sparseGetValues(csr_sparse_arr));  // csr values are transposed
+}
\ No newline at end of file

From a4420e1a8a480323a07761e79bf4e1c8f7951bbf Mon Sep 17 00:00:00 2001
From: Tyler Hilbert <tylerjoshuahilbert@gmail.com>
Date: Tue, 1 Oct 2024 13:56:31 -0400
Subject: [PATCH 394/473] Fixed N and D comment for KMeans example (#3584)

---
 examples/machine_learning/kmeans.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/machine_learning/kmeans.cpp b/examples/machine_learning/kmeans.cpp
index e40cc34368..963d6a609f 100644
--- a/examples/machine_learning/kmeans.cpp
+++ b/examples/machine_learning/kmeans.cpp
@@ -17,7 +17,7 @@
 using namespace af;
 
 array distance(array data, array means) {
-    int n = data.dims(0);   // Number of features
+    int n = data.dims(0);   // Number of data points
     int k = means.dims(1);  // Number of means
 
     array data2  = tile(data, 1, k, 1);
@@ -60,8 +60,8 @@ array new_means(array data, array clusters, int k) {
 // means: output, vector of means
 void kmeans(array &means, array &clusters, const array in, int k,
             int iter = 100) {
-    unsigned n = in.dims(0);  // Num features
-    unsigned d = in.dims(2);  // feature length
+    unsigned n = in.dims(0);  // Num of data points
+    unsigned d = in.dims(2);  // Num of features (will only be 1 in spider image example)
 
     // reshape input
     array data = in * 0;

From f7e965183c35f6be21b2dce224e5587c91e832b7 Mon Sep 17 00:00:00 2001
From: j-bo <j-bo@users.noreply.github.com>
Date: Tue, 1 Oct 2024 20:41:28 +0200
Subject: [PATCH 395/473] Fix interop_cuda.md issue related to afcu::getStream
 (#3572)

afcu::getStream() documentation specify that the required id for it's call is the ArrayFire device id. There is not need to retrieve the cuda native id in the example as it may lead to sync issues when using a device with multiple GPUs
---
 docs/pages/interop_cuda.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/pages/interop_cuda.md b/docs/pages/interop_cuda.md
index dae46ae027..2132dfcb2c 100644
--- a/docs/pages/interop_cuda.md
+++ b/docs/pages/interop_cuda.md
@@ -80,8 +80,7 @@ int main() {
 
     // 5. Determine ArrayFire's CUDA stream
     int af_id = af::getDevice();
-    int cuda_id = afcu::getNativeId(af_id);
-    cudaStream_t af_cuda_stream = afcu::getStream(cuda_id);
+    cudaStream_t af_cuda_stream = afcu::getStream(af_id);
 
     // 6. Set arguments and run your kernel in ArrayFire's stream
     //    Here launch with 1 block of 10 threads

From a8576577af19a9b20d634080ed1cf4ea0b300f13 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 1 Oct 2024 19:08:30 -0400
Subject: [PATCH 396/473] Loosen indexing assertions for af_assign_gen (#3514)

* Loosen indexing assertions for af_assign_gen

* Add tests for assignment argument loosening
---
 include/af/index.h   |  2 +-
 src/api/c/assign.cpp |  3 ---
 test/gen_assign.cpp  | 43 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/include/af/index.h b/include/af/index.h
index 3bceb96cbf..8eaaeaa0a5 100644
--- a/include/af/index.h
+++ b/include/af/index.h
@@ -274,7 +274,7 @@ extern "C" {
     ///                     the sequences
     /// \param[in] lhs      is the input array
     /// \param[in] ndims    is the number of \ref af_index_t provided
-    /// \param[in] indices  is an af_array of \ref af_index_t objects
+    /// \param[in] indices  is a C array of \ref af_index_t objects
     /// \param[in] rhs      is the array whose values will be assigned to \p lhs
     ///
     /// \ingroup index_func_assign
diff --git a/src/api/c/assign.cpp b/src/api/c/assign.cpp
index e53b43a6c5..22f11255e9 100644
--- a/src/api/c/assign.cpp
+++ b/src/api/c/assign.cpp
@@ -260,8 +260,6 @@ af_err af_assign_gen(af_array* out, const af_array lhs, const dim_t ndims,
             return af_create_handle(out, 0, nullptr, lhsType);
         }
 
-        ARG_ASSERT(2, (ndims == 1) || (ndims == (dim_t)lInfo.ndims()));
-
         if (ndims == 1 && ndims != static_cast<dim_t>(lInfo.ndims())) {
             af_array tmp_in  = 0;
             af_array tmp_out = 0;
@@ -279,7 +277,6 @@ af_err af_assign_gen(af_array* out, const af_array lhs, const dim_t ndims,
 
         ARG_ASSERT(1, (lhsType == rhsType));
         ARG_ASSERT(1, (lhsDims.ndims() >= rhsDims.ndims()));
-        ARG_ASSERT(2, (lhsDims.ndims() >= ndims));
 
         af_array output = 0;
         if (*out != lhs) {
diff --git a/test/gen_assign.cpp b/test/gen_assign.cpp
index 7cfd78ae62..07685108c4 100644
--- a/test/gen_assign.cpp
+++ b/test/gen_assign.cpp
@@ -455,3 +455,46 @@ TEST(GeneralAssign, CPP_AANN) {
     freeHost(hIdx0);
     freeHost(hIdx1);
 }
+
+TEST(GeneralAssign, NDimsDoesNotMatchLDims) {
+    af_err err;
+    af_array zeros, l1, l2, sevens;
+    dim_t sevens_size[3] = {5, 1, 1};
+    short hsevens[5]     = {7, 7, 7, 7, 7};
+
+    dim_t zeros_size[3] = {5, 6, 1};
+    short hzeros[5 * 6] = {0};
+
+    dim_t hone[1] = {1};
+
+    ASSERT_SUCCESS(af_create_array(&zeros, hzeros, 3, zeros_size, s16));
+    ASSERT_SUCCESS(af_create_array(&sevens, hsevens, 3, sevens_size, s16));
+    ASSERT_SUCCESS(af_create_array(&l2, hone, 1, hone, s64));
+
+    af_index_t *ix;
+    ASSERT_SUCCESS(af_create_indexers(&ix));
+    ASSERT_SUCCESS(af_set_array_indexer(ix, l2, 1));
+
+    // clang-format off
+    vector<short> gold = {
+            0, 0, 0, 0, 0,
+            7, 7, 7, 7, 7,
+            0, 0, 0, 0, 0,
+            0, 0, 0, 0, 0,
+            0, 0, 0, 0, 0,
+            0, 0, 0, 0, 0,
+        };
+    // clang-format on
+    for (int number_of_indices = 2; number_of_indices < 4;
+         number_of_indices++) {
+        af_array result = 0;
+        ASSERT_SUCCESS(
+            af_assign_gen(&result, zeros, number_of_indices, ix, sevens));
+
+        ASSERT_VEC_ARRAY_EQ(gold, dim4(3, zeros_size), af::array(result));
+    }
+    ASSERT_SUCCESS(af_release_array(zeros));
+    ASSERT_SUCCESS(af_release_array(sevens));
+    ASSERT_SUCCESS(af_release_array(l2));
+    ASSERT_SUCCESS(af_release_indexers(ix));
+}

From cc996ad36341ad191bd58e2c1327b6913322cf66 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 1 Oct 2024 19:44:17 -0400
Subject: [PATCH 397/473] Fix OpenCL memory migration on devices with different
 contexts (#3510)

---
 src/backend/opencl/Array.cpp |  6 ++++--
 test/array.cpp               | 23 +++++++++++++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index 21dec5166c..b4b6bcd5a9 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -201,10 +201,12 @@ void checkAndMigrate(Array<T> &arr) {
         AF_TRACE("Migrating array from {} to {}.", arr_id, cur_id);
         auto migrated_data           = memAlloc<T>(arr.elements());
         void *mapped_migrated_buffer = getQueue().enqueueMapBuffer(
-            *migrated_data, CL_TRUE, CL_MAP_READ, 0, arr.elements());
+            *migrated_data, CL_TRUE, CL_MAP_WRITE_INVALIDATE_REGION, 0,
+            sizeof(T) * arr.elements());
         setDevice(arr_id);
         Buffer &buf = *arr.get();
-        getQueue().enqueueReadBuffer(buf, CL_TRUE, 0, arr.elements(),
+        getQueue().enqueueReadBuffer(buf, CL_TRUE, 0,
+                                     sizeof(T) * arr.elements(),
                                      mapped_migrated_buffer);
         setDevice(cur_id);
         getQueue().enqueueUnmapMemObject(*migrated_data,
diff --git a/test/array.cpp b/test/array.cpp
index bcf6fa997e..b68f06820a 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -501,6 +501,29 @@ TEST(DeviceId, Different) {
     deviceGC();
 }
 
+TEST(Device, MigrateAllDevicesToAllDevices) {
+    int ndevices = getDeviceCount();
+    if (ndevices < 2) GTEST_SKIP() << "Skipping mult-GPU test";
+
+    for (int i = 0; i < ndevices; i++) {
+        for (int j = 0; j < ndevices; j++) {
+            setDevice(i);
+            array a = constant(i * 255, 10, 10);
+            a.eval();
+
+            setDevice(j);
+            array b = constant(j * 256, 10, 10);
+            b.eval();
+
+            array c = a + b;
+
+            std::vector<float> gold(10 * 10, i * 255 + j * 256);
+
+            ASSERT_VEC_ARRAY_EQ(gold, dim4(10, 10), c);
+        }
+    }
+}
+
 TEST(Device, empty) {
     array a = array();
     ASSERT_EQ(a.device<float>(), nullptr);

From a6f18278c39f4466a341846043c4773c3ee09471 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Tue, 1 Oct 2024 19:47:53 -0400
Subject: [PATCH 398/473] Fix source tarball GitHub workflow (#3498)

---
 .github/workflows/release_src_artifact.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release_src_artifact.yml b/.github/workflows/release_src_artifact.yml
index c616c8db5b..41b01d4f72 100644
--- a/.github/workflows/release_src_artifact.yml
+++ b/.github/workflows/release_src_artifact.yml
@@ -9,7 +9,7 @@ name: ci
 jobs:
     upload_src_tarball:
         name: Upload release source tarball
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-latest
         steps:
             - name: Fetch Repo Info
               run: |
@@ -40,7 +40,7 @@ jobs:
                                           libopenblas-dev \
                                           ocl-icd-opencl-dev \
                                           nvidia-cuda-toolkit \
-                                          libboost1.68-dev
+                                          libboost-dev
 
             - name: CMake Configure
               run: |

From 4d5954d455a9aed0da58627740e7c27e2aaea464 Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Wed, 23 Oct 2024 01:12:55 +0200
Subject: [PATCH 399/473] unified reports filled af::exception errors. (#3617)

---
 src/api/unified/error.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/api/unified/error.cpp b/src/api/unified/error.cpp
index 9fd89c0166..24a2dbfac9 100644
--- a/src/api/unified/error.cpp
+++ b/src/api/unified/error.cpp
@@ -42,7 +42,7 @@ void af_get_last_error(char **str, dim_t *len) {
         typedef void (*af_func)(char **, dim_t *);
         void *vfn    = LOAD_SYMBOL();
         af_func func = nullptr;
-        memcpy(&func, vfn, sizeof(void *));
+        memcpy(&func, &vfn, sizeof(void *));
         func(str, len);
     }
 }

From bdda3b3ea454030099871319c724eec4ee205bed Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Wed, 23 Oct 2024 02:34:11 +0200
Subject: [PATCH 400/473] Changed compare function to Strict Weak Ordering
 criteria. (#3612)

---
 examples/machine_learning/mnist_common.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/machine_learning/mnist_common.h b/examples/machine_learning/mnist_common.h
index 11531f3ffb..8d079df75a 100644
--- a/examples/machine_learning/mnist_common.h
+++ b/examples/machine_learning/mnist_common.h
@@ -13,7 +13,7 @@
 #include "../common/idxio.h"
 
 bool compare(const std::pair<float, int> l, const std::pair<float, int> r) {
-    return l.first >= r.first;
+    return l.first > r.first;
 }
 
 typedef std::pair<float, int> sort_type;

From 5c18fb3d73f3fe898b97bd2e26b34eba6df50aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:59:13 -0800
Subject: [PATCH 401/473] Workaround fix for issue with inline namespace thrust
 (#3566)

* Fix issue with inline namespace thrust

* applied clang format

* Update ThrustArrayFirePolicy.hpp

Hi Edwin, there seems to be an issue caused by the clang format apply. I propose these changes.

---------

Co-authored-by: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
---
 src/backend/cuda/ThrustArrayFirePolicy.hpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/backend/cuda/ThrustArrayFirePolicy.hpp b/src/backend/cuda/ThrustArrayFirePolicy.hpp
index 189ee558b3..339d3ea088 100644
--- a/src/backend/cuda/ThrustArrayFirePolicy.hpp
+++ b/src/backend/cuda/ThrustArrayFirePolicy.hpp
@@ -37,7 +37,11 @@ inline void return_temporary_buffer(ThrustArrayFirePolicy, Pointer p) {
 }  // namespace cuda
 }  // namespace arrayfire
 
+#if defined(_WIN32)
+THRUST_NAMESPACE_BEGIN
+#else
 namespace thrust {
+#endif
 namespace cuda_cub {
 template<>
 __DH__ inline cudaStream_t get_stream<arrayfire::cuda::ThrustArrayFirePolicy>(
@@ -60,4 +64,8 @@ inline cudaError_t synchronize_stream(
 }
 
 }  // namespace cuda_cub
+#if defined(_WIN32)
+THRUST_NAMESPACE_END
+#else
 }  // namespace thrust
+#endif

From efec9b0822c47ab3c50b3610486ce094ed6011be Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Wed, 6 Nov 2024 21:27:19 +0100
Subject: [PATCH 402/473] Solves C7626 error when compiling with MSVC 2019
 16.6+ (#3512)

---
 src/backend/opencl/kernel/KParam.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/backend/opencl/kernel/KParam.hpp b/src/backend/opencl/kernel/KParam.hpp
index 165bec9b02..1f4f1d5ba4 100644
--- a/src/backend/opencl/kernel/KParam.hpp
+++ b/src/backend/opencl/kernel/KParam.hpp
@@ -17,7 +17,7 @@
 #endif
 
 // Defines the size and shape of the data in the OpenCL buffer
-typedef struct {
+typedef struct KParam_t {
     dim_t dims[4];
     dim_t strides[4];
     dim_t offset;

From 77cd027cf8a4360530cf5b762fd88dd2dd7f9604 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Thu, 12 Dec 2024 11:32:29 -0800
Subject: [PATCH 403/473] Override boost build helper version to 1.84.0#3 which
 fixes a bug that prevents building in Visual Studio 2022 (#3626)

---
 vcpkg.json | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/vcpkg.json b/vcpkg.json
index 6ca4ec32be..fe16a0aa6d 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -23,6 +23,10 @@
         {
             "name": "jasper",
             "version": "4.2.0"
+        },
+        {
+            "name": "boost-modular-build-helper",
+            "version": "1.84.0#3"
         }
     ],
     "features": {

From 41b67015641388ce00878d54e9d248fab8a75171 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <christophe@arrayfire.com>
Date: Wed, 21 Aug 2024 14:53:00 -0700
Subject: [PATCH 404/473] Fix for incorrect x axis values for histogram.
 Rounding was being applied to x axis min and max values but this should not
 be done for a histogram where the values are in fact bin labels.

---
 src/api/c/hist.cpp | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/api/c/hist.cpp b/src/api/c/hist.cpp
index f37ba5cea1..1e250b5df4 100644
--- a/src/api/c/hist.cpp
+++ b/src/api/c/hist.cpp
@@ -68,19 +68,21 @@ fg_chart setup_histogram(fg_window const window, const af_array in,
         T freqMax =
             getScalar<T>(detail::reduce_all<af_max_t, T, T>(histogramInput));
 
+	// For histogram, xMin and xMax should always be the first
+	// and last bin respectively and should not be rounded
         if (xMin == 0 && xMax == 0 && yMin == 0 && yMax == 0) {
             // No previous limits. Set without checking
-            xMin = static_cast<float>(step_round(minval, false));
-            xMax = static_cast<float>(step_round(maxval, true));
+            xMin = static_cast<float>(minval);
+            xMax = static_cast<float>(maxval);
             yMax = static_cast<float>(step_round(freqMax, true));
             // For histogram, always set yMin to 0.
             yMin = 0;
         } else {
             if (xMin > minval) {
-                xMin = static_cast<float>(step_round(minval, false));
+                xMin = static_cast<float>(minval);
             }
             if (xMax < maxval) {
-                xMax = static_cast<float>(step_round(maxval, true));
+                xMax = static_cast<float>(maxval);
             }
             if (yMax < freqMax) {
                 yMax = static_cast<float>(step_round(freqMax, true));

From d8a176f95007c2fab2b83f56b6f3839ae9e2e10b Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Wed, 8 Jan 2025 00:39:15 +0100
Subject: [PATCH 405/473] Corrected field example (#3369) (#3375)

* Extended field example to include 2D points and 2D coordinates

* Fix buffer overflow in vector_field #3369
---
 examples/graphics/field.cpp | 13 +++++++++++--
 src/api/c/vector_field.cpp  | 27 ++++++++++++++-------------
 2 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/examples/graphics/field.cpp b/examples/graphics/field.cpp
index a723791fc8..f493c7ecd6 100644
--- a/examples/graphics/field.cpp
+++ b/examples/graphics/field.cpp
@@ -22,7 +22,7 @@ int main(int, char**) {
         af::info();
         af::Window myWindow(1024, 1024, "2D Vector Field example: ArrayFire");
 
-        myWindow.grid(1, 2);
+        myWindow.grid(2, 2);
 
         array dataRange = seq(MINIMUM, MAXIMUM, STEP);
 
@@ -38,12 +38,21 @@ int main(int, char**) {
             array saddle = join(1, flat(x), -1.0f * flat(y));
 
             array bvals = sin(scale * (x * x + y * y));
-            array hbowl = join(1, constant(1, x.elements()), flat(bvals));
+            array hbowl = join(1, constant(1., x.elements()), flat(bvals));
             hbowl.eval();
 
+            // 2D points
             myWindow(0, 0).vectorField(points, saddle, "Saddle point");
             myWindow(0, 1).vectorField(
                 points, hbowl, "hilly bowl (in a loop with varying amplitude)");
+
+            // 2D coordinates
+            myWindow(1, 0).vectorField(2.0 * flat(x), flat(y), flat(x),
+                                       -flat(y), "Saddle point");
+            myWindow(1, 1).vectorField(
+                2.0 * flat(x), flat(y), constant(1., x.elements()), flat(bvals),
+                "hilly bowl (in a loop with varying amplitude)");
+
             myWindow.show();
 
             scale -= 0.0010f;
diff --git a/src/api/c/vector_field.cpp b/src/api/c/vector_field.cpp
index a46d1eed47..701db6fc12 100644
--- a/src/api/c/vector_field.cpp
+++ b/src/api/c/vector_field.cpp
@@ -50,20 +50,21 @@ fg_chart setup_vector_field(fg_window window, const vector<af_array>& points,
     vector<Array<T>> pnts;
     vector<Array<T>> dirs;
 
-    for (unsigned i = 0; i < points.size(); ++i) {
-        pnts.push_back(getArray<T>(points[i]));
-        dirs.push_back(getArray<T>(directions[i]));
-    }
-
-    // Join for set up vector
-    dim4 odims(3, points.size());
-    Array<T> out_pnts = createEmptyArray<T>(odims);
-    Array<T> out_dirs = createEmptyArray<T>(odims);
-    detail::join(out_pnts, 1, pnts);
-    detail::join(out_dirs, 1, dirs);
-    Array<T> pIn = out_pnts;
-    Array<T> dIn = out_dirs;
+    Array<T> pIn = getArray<T>(points[0]);
+    Array<T> dIn = getArray<T>(directions[0]);
+    if (points.size() > 1) {
+        for (unsigned i = 0; i < points.size(); ++i) {
+            pnts.push_back(getArray<T>(points[i]));
+            dirs.push_back(getArray<T>(directions[i]));
+        }
 
+        // Join for set up vector
+        const dim4 odims(pIn.dims()[0], points.size());
+        pIn = createEmptyArray<T>(odims);
+        dIn = createEmptyArray<T>(odims);
+        detail::join<T>(pIn, 1, pnts);
+        detail::join<T>(dIn, 1, dirs);
+    }
     // do transpose if required
     if (transpose_) {
         pIn = transpose<T>(pIn, false);

From ab6978c54e975e039763b8c641b9594af49dd146 Mon Sep 17 00:00:00 2001
From: Tyler Hilbert <tylerjoshuahilbert@gmail.com>
Date: Tue, 7 Jan 2025 15:44:33 -0800
Subject: [PATCH 406/473] Added REQUIRED to CMake find_package for easier build
 debugging (#3581)

---
 examples/benchmarks/CMakeLists.txt       | 2 +-
 examples/computer_vision/CMakeLists.txt  | 2 +-
 examples/financial/CMakeLists.txt        | 2 +-
 examples/getting_started/CMakeLists.txt  | 2 +-
 examples/graphics/CMakeLists.txt         | 2 +-
 examples/helloworld/CMakeLists.txt       | 2 +-
 examples/image_processing/CMakeLists.txt | 2 +-
 examples/lin_algebra/CMakeLists.txt      | 2 +-
 examples/machine_learning/CMakeLists.txt | 2 +-
 examples/pde/CMakeLists.txt              | 2 +-
 examples/unified/CMakeLists.txt          | 2 +-
 11 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/examples/benchmarks/CMakeLists.txt b/examples/benchmarks/CMakeLists.txt
index 9cf8197317..4fd0853e58 100644
--- a/examples/benchmarks/CMakeLists.txt
+++ b/examples/benchmarks/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Benchmarks
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 if(ArrayFire_CPU_FOUND)
   add_executable(blas_cpu blas.cpp)
diff --git a/examples/computer_vision/CMakeLists.txt b/examples/computer_vision/CMakeLists.txt
index 7113816566..2683eb1931 100644
--- a/examples/computer_vision/CMakeLists.txt
+++ b/examples/computer_vision/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Computer-Vision
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"")
 
diff --git a/examples/financial/CMakeLists.txt b/examples/financial/CMakeLists.txt
index f2b82d4de8..f365f88b47 100644
--- a/examples/financial/CMakeLists.txt
+++ b/examples/financial/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Financial
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 if(ArrayFire_CPU_FOUND)
   # Black-Scholes Options
diff --git a/examples/getting_started/CMakeLists.txt b/examples/getting_started/CMakeLists.txt
index 790afd3d1f..a9d1ce4bcb 100644
--- a/examples/getting_started/CMakeLists.txt
+++ b/examples/getting_started/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Getting-Started
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 if(ArrayFire_CPU_FOUND)
   # Convolve examples
diff --git a/examples/graphics/CMakeLists.txt b/examples/graphics/CMakeLists.txt
index dd2918b641..6140142343 100644
--- a/examples/graphics/CMakeLists.txt
+++ b/examples/graphics/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Graphics
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"")
 
diff --git a/examples/helloworld/CMakeLists.txt b/examples/helloworld/CMakeLists.txt
index 0aa58ca2c9..b3a02e9fc6 100644
--- a/examples/helloworld/CMakeLists.txt
+++ b/examples/helloworld/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-HelloWorld
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 if(ArrayFire_CPU_FOUND)
   # Hello World example
diff --git a/examples/image_processing/CMakeLists.txt b/examples/image_processing/CMakeLists.txt
index cfcd109922..e4ab1d3d8a 100644
--- a/examples/image_processing/CMakeLists.txt
+++ b/examples/image_processing/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Image-Processing
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"")
 
diff --git a/examples/lin_algebra/CMakeLists.txt b/examples/lin_algebra/CMakeLists.txt
index b08aceeeee..89b9c89600 100644
--- a/examples/lin_algebra/CMakeLists.txt
+++ b/examples/lin_algebra/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Linear-Algebra
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 if(ArrayFire_CPU_FOUND)
   # Cholesky example
diff --git a/examples/machine_learning/CMakeLists.txt b/examples/machine_learning/CMakeLists.txt
index d1cbcc9541..480f3f7f12 100644
--- a/examples/machine_learning/CMakeLists.txt
+++ b/examples/machine_learning/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Linear-Algebra
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"")
 
diff --git a/examples/pde/CMakeLists.txt b/examples/pde/CMakeLists.txt
index 23a89ace31..bceb38665a 100644
--- a/examples/pde/CMakeLists.txt
+++ b/examples/pde/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-PDE
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 if(ArrayFire_CPU_FOUND)
   # Shallow Water simulation example
diff --git a/examples/unified/CMakeLists.txt b/examples/unified/CMakeLists.txt
index 42ab6432f0..a399f58c00 100644
--- a/examples/unified/CMakeLists.txt
+++ b/examples/unified/CMakeLists.txt
@@ -10,7 +10,7 @@ project(ArrayFire-Example-Unified
   VERSION 3.5.0
   LANGUAGES CXX)
 
-find_package(ArrayFire)
+find_package(ArrayFire REQUIRED)
 
 if(ArrayFire_Unified_FOUND)
   # Simple unified backend example

From 279d0ea683bb928104041761d2a6a749ae231dd3 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 7 Jan 2025 16:25:39 -0800
Subject: [PATCH 407/473] 1918 sparse matrix not updated in for loop in opencl
 (#3602)

* Fix for issue in the opencl backend where array offsets for the values and/or row/cols arrays are not accounted for when converting a sparse array to a dense one. This can happen when a sparse matrix is constructed using values and/or row/cols arrays that have been indexed using the seq method.

* Add test case to verify fix for sparse to dense conversion bug in the opencl backend.

* Fix uninitialized array in test reference.
---
 src/backend/opencl/kernel/csr2dense.cl | 11 +++++++----
 src/backend/opencl/kernel/sparse.hpp   |  5 ++++-
 test/sparse.cpp                        | 21 +++++++++++++++++++++
 3 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/src/backend/opencl/kernel/csr2dense.cl b/src/backend/opencl/kernel/csr2dense.cl
index 15a7c0c60d..e15ef014f3 100644
--- a/src/backend/opencl/kernel/csr2dense.cl
+++ b/src/backend/opencl/kernel/csr2dense.cl
@@ -9,13 +9,16 @@
 
 kernel void csr2Dense(global T *output, global const T *values,
                       global const int *rowidx, global const int *colidx,
-                      const int M) {
+                      const int M, const int v_off, const int r_off, const int c_off) {
+    T *v = values + v_off;
+    int *r = rowidx + r_off;
+    int *c = colidx + c_off;
     int lid = get_local_id(0);
     for (int rowId = get_group_id(0); rowId < M; rowId += get_num_groups(0)) {
-        int colStart = rowidx[rowId];
-        int colEnd   = rowidx[rowId + 1];
+        int colStart = r[rowId];
+        int colEnd   = r[rowId + 1];
         for (int colId = colStart + lid; colId < colEnd; colId += THREADS) {
-            output[rowId + colidx[colId] * M] = values[colId];
+            output[rowId + c[colId] * M] = v[colId];
         }
     }
 }
diff --git a/src/backend/opencl/kernel/sparse.hpp b/src/backend/opencl/kernel/sparse.hpp
index 13a4a9c5fb..4d3a33d14a 100644
--- a/src/backend/opencl/kernel/sparse.hpp
+++ b/src/backend/opencl/kernel/sparse.hpp
@@ -85,7 +85,10 @@ void csr2dense(Param output, const Param values, const Param rowIdx,
     cl::NDRange global(local[0] * groups_x, 1);
 
     csr2dense(cl::EnqueueArgs(getQueue(), global, local), *output.data,
-              *values.data, *rowIdx.data, *colIdx.data, M);
+              *values.data, *rowIdx.data, *colIdx.data, M,
+	      static_cast<int>(values.info.offset),
+	      static_cast<int>(rowIdx.info.offset),
+	      static_cast<int>(colIdx.info.offset));
     CL_DEBUG_FINISH(getQueue());
 }
 
diff --git a/test/sparse.cpp b/test/sparse.cpp
index 3142a3735a..9e3f29ae35 100644
--- a/test/sparse.cpp
+++ b/test/sparse.cpp
@@ -19,6 +19,7 @@ using af::dtype_traits;
 using af::identity;
 using af::randu;
 using af::span;
+using af::seq;
 
 #define SPARSE_TESTS(T, eps)                                                \
     TEST(Sparse, T##Square) { sparseTester<T>(1000, 1000, 100, 5, eps); }   \
@@ -109,6 +110,26 @@ TEST(Sparse, ISSUE_1745) {
                               row_idx.get(), col_idx.get(), AF_STORAGE_CSR));
 }
 
+TEST(Sparse, ISSUE_1918) {
+    array reference(2,2);
+    reference(0, span) = 0;
+    reference(1, span) = 2;
+    array output;
+    float value[] = { 1, 1, 2, 2 };
+    int index[] = { -1, 1, 2 };
+    int row[] = { 0, 2, 2, 0, 0, 2 };
+    int col[] = { 0, 1, 0, 1 };
+    array values(4, 1, value, afHost);
+    array rows(6, 1, row, afHost);
+    array cols(4, 1, col, afHost);
+    array S;
+  
+    S = sparse(2, 2, values(seq(2, 3)), rows(seq(3, 5)), cols(seq(2, 3)));
+    output = dense(S);
+
+    ASSERT_ARRAYS_EQ(reference, output);
+}
+
 TEST(Sparse, ISSUE_2134_COO) {
     int rows[]     = {0, 0, 0, 1, 1, 2, 2};
     int cols[]     = {0, 1, 2, 0, 1, 0, 2};

From 374bf9761cb3e5f1d6ad327249d009f2a37c4d0e Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 7 Jan 2025 17:19:54 -0800
Subject: [PATCH 408/473] Add note to write() array class method documentation
 about copy on write behavior. (#3613)

---
 include/af/array.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/af/array.h b/include/af/array.h
index 0edb9558e1..4186b95d08 100644
--- a/include/af/array.h
+++ b/include/af/array.h
@@ -655,6 +655,7 @@ namespace af
 
         /**
            Perform deep copy from host/device pointer to an existing array
+           \note Unlike all other assignment operations, this does NOT result in a copy on write.
         */
         template<typename T> void write(const T *ptr, const size_t bytes, af::source src = afHost);
 

From b25ff740afbf7e70e5f659d0fb29b172658c46ca Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 7 Jan 2025 17:21:08 -0800
Subject: [PATCH 409/473] Add condition so that cuda_* tests (including
 cuda_unified) are only built if AF_BUILD_CUDA is true (#3598)

---
 test/CMakeLists.txt | 88 +++++++++++++++++++++++----------------------
 1 file changed, 45 insertions(+), 43 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 92c4d90acd..95bab411bc 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -372,51 +372,53 @@ if(OpenCL_FOUND)
             CXX11)
 endif()
 
-if(CUDA_FOUND)
-  include(AFcuda_helpers)
-  foreach(backend ${enabled_backends})
-    set(cuda_test_backends "cuda" "unified")
-    if(${backend} IN_LIST cuda_test_backends)
-      set(target test_cuda_${backend})
-      add_executable(${target} cuda.cu)
-      target_include_directories(${target}
-        PRIVATE
-        ${CMAKE_SOURCE_DIR}
-        ${CMAKE_CURRENT_SOURCE_DIR})
-      target_include_directories(${target}
-        SYSTEM PRIVATE
-          ${ArrayFire_SOURCE_DIR}/extern/half/include)
-      if(${backend} STREQUAL "unified")
-        target_link_libraries(${target}
-          ArrayFire::af)
-      else()
+if(AF_BUILD_CUDA)
+  if(CUDA_FOUND)
+    include(AFcuda_helpers)
+    foreach(backend ${enabled_backends})
+      set(cuda_test_backends "cuda" "unified")
+      if(${backend} IN_LIST cuda_test_backends)
+        set(target test_cuda_${backend})
+        add_executable(${target} cuda.cu)
+        target_include_directories(${target}
+          PRIVATE
+          ${CMAKE_SOURCE_DIR}
+          ${CMAKE_CURRENT_SOURCE_DIR})
+        target_include_directories(${target}
+          SYSTEM PRIVATE
+            ${ArrayFire_SOURCE_DIR}/extern/half/include)
+        if(${backend} STREQUAL "unified")
+          target_link_libraries(${target}
+            ArrayFire::af)
+        else()
+          target_link_libraries(${target}
+            ArrayFire::af${backend})
+        endif()
         target_link_libraries(${target}
-          ArrayFire::af${backend})
-      endif()
-      target_link_libraries(${target}
-        mmio
-        arrayfire_test)
-
-      # Couldn't get Threads::Threads to work with this cuda binary. The import
-      # target would not add the -pthread flag which is required for this
-      # executable (on Ubuntu 18.04 anyway)
-      check_cxx_compiler_flag(-pthread pthread_flag)
-      if(pthread_flag)
-        target_link_libraries(${target} -pthread)
+          mmio
+          arrayfire_test)
+  
+        # Couldn't get Threads::Threads to work with this cuda binary. The import
+        # target would not add the -pthread flag which is required for this
+        # executable (on Ubuntu 18.04 anyway)
+        check_cxx_compiler_flag(-pthread pthread_flag)
+        if(pthread_flag)
+          target_link_libraries(${target} -pthread)
+        endif()
+  
+        af_detect_and_set_cuda_architectures(${target})
+  
+        set_target_properties(${target}
+          PROPERTIES
+            FOLDER "Tests"
+            OUTPUT_NAME "cuda_${backend}")
+  
+        if(NOT ${backend} STREQUAL "unified")
+          af_add_test(${target} ${backend} ON)
+        endif()
       endif()
-
-      af_detect_and_set_cuda_architectures(${target})
-
-      set_target_properties(${target}
-        PROPERTIES
-          FOLDER "Tests"
-          OUTPUT_NAME "cuda_${backend}")
-
-      if(NOT ${backend} STREQUAL "unified")
-        af_add_test(${target} ${backend} ON)
-      endif()
-    endif()
-  endforeach()
+    endforeach()
+  endif()
 endif()
 
 

From 23a990a40d3f43035bcbed0805987f8f40a664e9 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Wed, 8 Jan 2025 09:59:43 -0800
Subject: [PATCH 410/473] The shfl_instrinsics header file contains wrapper
 routines for the warp primitives and calls the new primitives for CUDA
 versions greater than 9 and the old ones for older CUDA versions. The new
 primitives have an additional argument which is a mask of the warp threads
 that are participating in the operation. The old primitives always involve
 all the threads in a warp. The wrapper routines originally allowed you to
 specify the mask which was ignored for the old primitives but this has now
 been removed. This is because if an old version of CUDA is being used then
 all threads must enter the wrapper routine and if a new version of CUDA is
 being used only the threads corresponding to the mask must enter. If threads
 outside the mask enter the routine then the behavior is undefined. In CUDA
 versions <=12.2 the primitive executes without any errors given however in
 later versions of CUDA a warp illegal instruction exception will be thrown.
 In order to preserve the same behavior of these wrapper functions for old and
 new versions of CUDA, the mask is always set to all threads in a warp for the
 new primitives. The specific new primitive can always be called with a custom
 mask which is already done elsewhere in the reduce_by_key routine. (#3576)

---
 src/backend/cuda/kernel/reduce_by_key.hpp   | 70 ++++++++-------------
 src/backend/cuda/kernel/shfl_intrinsics.hpp | 46 +++++++-------
 2 files changed, 49 insertions(+), 67 deletions(-)

diff --git a/src/backend/cuda/kernel/reduce_by_key.hpp b/src/backend/cuda/kernel/reduce_by_key.hpp
index ea015aaff2..1e04a123ec 100644
--- a/src/backend/cuda/kernel/reduce_by_key.hpp
+++ b/src/backend/cuda/kernel/reduce_by_key.hpp
@@ -25,8 +25,6 @@
 
 using std::unique_ptr;
 
-const static unsigned int FULL_MASK = 0xFFFFFFFF;
-
 namespace arrayfire {
 namespace cuda {
 namespace kernel {
@@ -68,9 +66,9 @@ __global__ void test_needs_reduction(int *needs_another_reduction,
 
     if (tid < n) { k = keys_in.ptr[tid]; }
 
-    int update_key = (k == shfl_down_sync(FULL_MASK, k, 1)) &&
+    int update_key = (k == shfl_down_sync(k, 1)) &&
                      (tid < (n - 1)) && ((threadIdx.x % 32) < 31);
-    int remaining_updates = any_sync(FULL_MASK, update_key);
+    int remaining_updates = any_sync(update_key);
 
     __syncthreads();
 
@@ -83,7 +81,7 @@ __global__ void test_needs_reduction(int *needs_another_reduction,
          && (threadIdx.x < (blockDim.x - 1))  // not last thread in block
          // next value valid and equal
          && ((tid + 1) < n) && (k == keys_in.ptr[tid + 1]));
-    remaining_updates = any_sync(FULL_MASK, update_key);
+    remaining_updates = any_sync(update_key);
 
     // TODO: single per warp? change to assignment rather than atomicOr
     if (remaining_updates) atomicOr(needs_another_reduction, remaining_updates);
@@ -243,7 +241,7 @@ __global__ static void reduce_blocks_by_key(int *reduced_block_sizes,
         v = common::Binary<compute_t<To>, op>::init();
     }
 
-    compute_t<Tk> eq_check = (k != shfl_up_sync(FULL_MASK, k, 1));
+    compute_t<Tk> eq_check = (k != shfl_up_sync(k, 1));
     // mark threads containing unique keys
     char unique_flag = (eq_check || (laneid == 0)) && (tidx < n);
 
@@ -251,42 +249,33 @@ __global__ static void reduce_blocks_by_key(int *reduced_block_sizes,
     char unique_id = unique_flag;
 #pragma unroll
     for (int offset = 1; offset < 32; offset <<= 1) {
-        char y = shfl_up_sync(FULL_MASK, unique_id, offset);
+        char y = shfl_up_sync(unique_id, offset);
         if (laneid >= offset) unique_id += y;
     }
 
     //
     // Reduce each warp by key
-    char all_eq = (k == shfl_down_sync(FULL_MASK, k, 1));
-    if (all_sync(FULL_MASK,
-                 all_eq)) {  // check special case of single key per warp
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 1));
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 2));
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 4));
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 8));
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 16));
+    char all_eq = (k == shfl_down_sync(k, 1));
+    if (all_sync(all_eq)) {  // check special case of single key per warp
+        v = reduce(v, shfl_down_sync(v, 1));
+        v = reduce(v, shfl_down_sync(v, 2));
+        v = reduce(v, shfl_down_sync(v, 4));
+        v = reduce(v, shfl_down_sync(v, 8));
+        v = reduce(v, shfl_down_sync(v, 16));
     } else {
         compute_t<To> init = common::Binary<compute_t<To>, op>::init();
         int eq_check, update_key;
-        unsigned shflmask;
 #pragma unroll
         for (int delta = 1; delta < 32; delta <<= 1) {
             eq_check =
-                (unique_id == shfl_down_sync(FULL_MASK, unique_id, delta));
+                (unique_id == shfl_down_sync(unique_id, delta));
 
             // checks if this thread should perform a reduction
             update_key =
                 eq_check && (laneid < (32 - delta)) && ((tidx + delta) < n);
 
-            // obtains mask of all threads that should be reduced
-            shflmask = ballot_sync(FULL_MASK, update_key);
-
-            // shifts mask to include source threads that should participate in
-            // _shfl
-            shflmask |= (shflmask << delta);
-
             // shfls data from neighboring threads
-            compute_t<To> uval = shfl_down_sync(shflmask, v, delta);
+            compute_t<To> uval = shfl_down_sync(v, delta);
 
             // update if thread requires it
             v = reduce(v, (update_key ? uval : init));
@@ -479,7 +468,7 @@ __global__ static void reduce_blocks_dim_by_key(
         v = init;
     }
 
-    Tk eq_check = (k != shfl_up_sync(FULL_MASK, k, 1));
+    Tk eq_check = (k != shfl_up_sync(k, 1));
     // mark threads containing unique keys
     char unique_flag = (eq_check || (laneid == 0)) && (tidx < n);
 
@@ -487,42 +476,33 @@ __global__ static void reduce_blocks_dim_by_key(
     char unique_id = unique_flag;
 #pragma unroll
     for (int offset = 1; offset < 32; offset <<= 1) {
-        char y = shfl_up_sync(FULL_MASK, unique_id, offset);
+        char y = shfl_up_sync(unique_id, offset);
         if (laneid >= offset) unique_id += y;
     }
 
     //
     // Reduce each warp by key
-    char all_eq = (k == shfl_down_sync(FULL_MASK, k, 1));
-    if (all_sync(FULL_MASK,
-                 all_eq)) {  // check special case of single key per warp
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 1));
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 2));
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 4));
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 8));
-        v = reduce(v, shfl_down_sync(FULL_MASK, v, 16));
+    char all_eq = (k == shfl_down_sync(k, 1));
+    if (all_sync(all_eq)) {  // check special case of single key per warp
+        v = reduce(v, shfl_down_sync(v, 1));
+        v = reduce(v, shfl_down_sync(v, 2));
+        v = reduce(v, shfl_down_sync(v, 4));
+        v = reduce(v, shfl_down_sync(v, 8));
+        v = reduce(v, shfl_down_sync(v, 16));
     } else {
         compute_t<To> init = common::Binary<compute_t<To>, op>::init();
         int eq_check, update_key;
-        unsigned shflmask;
 #pragma unroll
         for (int delta = 1; delta < 32; delta <<= 1) {
             eq_check =
-                (unique_id == shfl_down_sync(FULL_MASK, unique_id, delta));
+                (unique_id == shfl_down_sync(unique_id, delta));
 
             // checks if this thread should perform a reduction
             update_key =
                 eq_check && (laneid < (32 - delta)) && ((tidx + delta) < n);
 
-            // obtains mask of all threads that should be reduced
-            shflmask = ballot_sync(FULL_MASK, update_key);
-
-            // shifts mask to include source threads that should participate in
-            // _shfl
-            shflmask |= (shflmask << delta);
-
             // shfls data from neighboring threads
-            compute_t<To> uval = shfl_down_sync(shflmask, v, delta);
+            compute_t<To> uval = shfl_down_sync(v, delta);
 
             // update if thread requires it
             v = reduce(v, (update_key ? uval : init));
diff --git a/src/backend/cuda/kernel/shfl_intrinsics.hpp b/src/backend/cuda/kernel/shfl_intrinsics.hpp
index 687abf5144..a91dc74148 100644
--- a/src/backend/cuda/kernel/shfl_intrinsics.hpp
+++ b/src/backend/cuda/kernel/shfl_intrinsics.hpp
@@ -11,11 +11,13 @@ namespace arrayfire {
 namespace cuda {
 namespace kernel {
 
+constexpr unsigned int FULL_MASK = 0xffffffff;
+
 //__all_sync wrapper
 template<typename T>
-__device__ T all_sync(unsigned mask, T var) {
+__device__ T all_sync(T var) {
 #if (CUDA_VERSION >= 9000)
-    return __all_sync(mask, var);
+    return __all_sync(FULL_MASK, var);
 #else
     return __all(var);
 #endif
@@ -23,9 +25,9 @@ __device__ T all_sync(unsigned mask, T var) {
 
 //__all_sync wrapper
 template<typename T>
-__device__ T any_sync(unsigned mask, T var) {
+__device__ T any_sync(T var) {
 #if (CUDA_VERSION >= 9000)
-    return __any_sync(mask, var);
+    return __any_sync(FULL_MASK, var);
 #else
     return __any(var);
 #endif
@@ -33,9 +35,9 @@ __device__ T any_sync(unsigned mask, T var) {
 
 //__shfl_down_sync wrapper
 template<typename T>
-__device__ T ballot_sync(unsigned mask, T var) {
+__device__ T ballot_sync(T var) {
 #if (CUDA_VERSION >= 9000)
-    return __ballot_sync(mask, var);
+    return __ballot_sync(FULL_MASK, var);
 #else
     return __ballot(var);
 #endif
@@ -43,19 +45,19 @@ __device__ T ballot_sync(unsigned mask, T var) {
 
 //__shfl_down_sync wrapper
 template<typename T>
-__device__ T shfl_down_sync(unsigned mask, T var, int delta) {
+__device__ T shfl_down_sync(T var, int delta) {
 #if (CUDA_VERSION >= 9000)
-    return __shfl_down_sync(mask, var, delta);
+    return __shfl_down_sync(FULL_MASK, var, delta);
 #else
     return __shfl_down(var, delta);
 #endif
 }
 // specialization for cfloat
 template<>
-inline __device__ cfloat shfl_down_sync(unsigned mask, cfloat var, int delta) {
+inline __device__ cfloat shfl_down_sync(cfloat var, int delta) {
 #if (CUDA_VERSION >= 9000)
-    cfloat res = {__shfl_down_sync(mask, var.x, delta),
-                  __shfl_down_sync(mask, var.y, delta)};
+    cfloat res = {__shfl_down_sync(FULL_MASK, var.x, delta),
+                  __shfl_down_sync(FULL_MASK, var.y, delta)};
 #else
     cfloat res  = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)};
 #endif
@@ -63,11 +65,11 @@ inline __device__ cfloat shfl_down_sync(unsigned mask, cfloat var, int delta) {
 }
 // specialization for cdouble
 template<>
-inline __device__ cdouble shfl_down_sync(unsigned mask, cdouble var,
+inline __device__ cdouble shfl_down_sync(cdouble var,
                                          int delta) {
 #if (CUDA_VERSION >= 9000)
-    cdouble res = {__shfl_down_sync(mask, var.x, delta),
-                   __shfl_down_sync(mask, var.y, delta)};
+    cdouble res = {__shfl_down_sync(FULL_MASK, var.x, delta),
+                   __shfl_down_sync(FULL_MASK, var.y, delta)};
 #else
     cdouble res = {__shfl_down(var.x, delta), __shfl_down(var.y, delta)};
 #endif
@@ -76,19 +78,19 @@ inline __device__ cdouble shfl_down_sync(unsigned mask, cdouble var,
 
 //__shfl_up_sync wrapper
 template<typename T>
-__device__ T shfl_up_sync(unsigned mask, T var, int delta) {
+__device__ T shfl_up_sync(T var, int delta) {
 #if (CUDA_VERSION >= 9000)
-    return __shfl_up_sync(mask, var, delta);
+    return __shfl_up_sync(FULL_MASK, var, delta);
 #else
     return __shfl_up(var, delta);
 #endif
 }
 // specialization for cfloat
 template<>
-inline __device__ cfloat shfl_up_sync(unsigned mask, cfloat var, int delta) {
+inline __device__ cfloat shfl_up_sync(cfloat var, int delta) {
 #if (CUDA_VERSION >= 9000)
-    cfloat res = {__shfl_up_sync(mask, var.x, delta),
-                  __shfl_up_sync(mask, var.y, delta)};
+    cfloat res = {__shfl_up_sync(FULL_MASK, var.x, delta),
+                  __shfl_up_sync(FULL_MASK, var.y, delta)};
 #else
     cfloat res  = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)};
 #endif
@@ -96,10 +98,10 @@ inline __device__ cfloat shfl_up_sync(unsigned mask, cfloat var, int delta) {
 }
 // specialization for cdouble
 template<>
-inline __device__ cdouble shfl_up_sync(unsigned mask, cdouble var, int delta) {
+inline __device__ cdouble shfl_up_sync(cdouble var, int delta) {
 #if (CUDA_VERSION >= 9000)
-    cdouble res = {__shfl_up_sync(mask, var.x, delta),
-                   __shfl_up_sync(mask, var.y, delta)};
+    cdouble res = {__shfl_up_sync(FULL_MASK, var.x, delta),
+                   __shfl_up_sync(FULL_MASK, var.y, delta)};
 #else
     cdouble res = {__shfl_up(var.x, delta), __shfl_up(var.y, delta)};
 #endif

From 424f1d6dd2f08dafb36e6a72653c150537de0d21 Mon Sep 17 00:00:00 2001
From: errata-c <77643526+errata-c@users.noreply.github.com>
Date: Wed, 8 Jan 2025 17:57:38 -0500
Subject: [PATCH 411/473] Added CMakeUserPresets.json to .gitignore, allow for
 local cmake configuration (#3520)

Co-authored-by: errata-c <erratachem@gmail.com>
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index d56dd8ccf0..933736dba0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 #CMakeCache.txt
 #./CMakeFiles/
+CMakeUserPresets.json
 build*/
 Release/
 #Makefile

From f6559a5c4db0d9e486347c6d004b8d4032186564 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Wed, 8 Jan 2025 15:16:59 -0800
Subject: [PATCH 412/473] 3545 bug fp16 types not allowed for atan2 method
 (#3559)

* Add cases for float16 arguments to atan2 and hypot functions

* Added test cases for half precision atan2 and hypot functions
---
 src/api/c/binary.cpp |  6 ++++--
 test/binary.cpp      | 13 +++++++++++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/api/c/binary.cpp b/src/api/c/binary.cpp
index 50590568f8..ee727c264a 100644
--- a/src/api/c/binary.cpp
+++ b/src/api/c/binary.cpp
@@ -476,7 +476,7 @@ af_err af_atan2(af_array *out, const af_array lhs, const af_array rhs,
     try {
         const af_dtype type = implicit(lhs, rhs);
 
-        if (type != f32 && type != f64) {
+        if (type != f16 && type != f32 && type != f64) {
             AF_ERROR("Only floating point arrays are supported for atan2 ",
                      AF_ERR_NOT_SUPPORTED);
         }
@@ -491,6 +491,7 @@ af_err af_atan2(af_array *out, const af_array lhs, const af_array rhs,
 
         af_array res;
         switch (type) {
+            case f16: res = arithOp<half, af_atan2_t>(lhs, rhs, odims); break;
             case f32: res = arithOp<float, af_atan2_t>(lhs, rhs, odims); break;
             case f64: res = arithOp<double, af_atan2_t>(lhs, rhs, odims); break;
             default: TYPE_ERROR(0, type);
@@ -507,7 +508,7 @@ af_err af_hypot(af_array *out, const af_array lhs, const af_array rhs,
     try {
         const af_dtype type = implicit(lhs, rhs);
 
-        if (type != f32 && type != f64) {
+        if (type != f16 && type != f32 && type != f64) {
             AF_ERROR("Only floating point arrays are supported for hypot ",
                      AF_ERR_NOT_SUPPORTED);
         }
@@ -523,6 +524,7 @@ af_err af_hypot(af_array *out, const af_array lhs, const af_array rhs,
 
         af_array res;
         switch (type) {
+            case f16: res = arithOp<half, af_hypot_t>(lhs, rhs, odims); break;
             case f32: res = arithOp<float, af_hypot_t>(lhs, rhs, odims); break;
             case f64: res = arithOp<double, af_hypot_t>(lhs, rhs, odims); break;
             default: TYPE_ERROR(0, type);
diff --git a/test/binary.cpp b/test/binary.cpp
index dafc3b8bff..c029a19da5 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -14,6 +14,8 @@
 #include <af/data.h>
 #include <af/device.h>
 #include <af/random.h>
+#include <af/half.h>
+#include "half.hpp"  //note: NOT common. From extern/half/include/half.hpp
 
 #include <cfenv>
 #include <cmath>
@@ -21,6 +23,8 @@
 using namespace std;
 using namespace af;
 
+using half_float_half = half_float::half;
+
 const int num = 10000;
 
 #define add(left, right) (left) + (right)
@@ -122,7 +126,7 @@ af::array randgen(const int num, dtype ty) {
                                                                       \
         af_dtype ta = (af_dtype)dtype_traits<Ta>::af_type;            \
         af::array a = randgen(num, ta);                               \
-        Tb h_b      = 0.3;                                            \
+        Tb h_b      = (Tb)0.3;                                            \
         af::array c = func(a, h_b);                                   \
         Ta *h_a     = a.host<Ta>();                                   \
         Td *h_d     = c.host<Td>();                                   \
@@ -139,7 +143,7 @@ af::array randgen(const int num, dtype ty) {
         SUPPORTED_TYPE_CHECK(Tc);                                     \
                                                                       \
         af_dtype tb = (af_dtype)dtype_traits<Tb>::af_type;            \
-        Ta h_a      = 0.3;                                            \
+        Ta h_a      = (Ta)0.3;                                            \
         af::array b = randgen(num, tb);                               \
         af::array c = func(h_a, b);                                   \
         Tb *h_b     = b.host<Tb>();                                   \
@@ -163,6 +167,8 @@ af::array randgen(const int num, dtype ty) {
 #define BINARY_TESTS_UINT(func) BINARY_TESTS(uint, uint, uint, func)
 #define BINARY_TESTS_INTL(func) BINARY_TESTS(intl, intl, intl, func)
 #define BINARY_TESTS_UINTL(func) BINARY_TESTS(uintl, uintl, uintl, func)
+#define BINARY_TESTS_NEAR_HALF(func) \
+    BINARY_TESTS_NEAR(half_float_half, half_float_half, half_float_half, func, 1e-3)
 #define BINARY_TESTS_NEAR_FLOAT(func) \
     BINARY_TESTS_NEAR(float, float, float, func, 1e-5)
 #define BINARY_TESTS_NEAR_DOUBLE(func) \
@@ -188,6 +194,9 @@ BINARY_TESTS_NEAR_FLOAT(atan2)
 BINARY_TESTS_NEAR_FLOAT(pow)
 BINARY_TESTS_NEAR_FLOAT(hypot)
 
+BINARY_TESTS_NEAR_HALF(atan2)
+BINARY_TESTS_NEAR_HALF(hypot)
+
 BINARY_TESTS_NEAR_DOUBLE(atan2)
 BINARY_TESTS_NEAR_DOUBLE(pow)
 BINARY_TESTS_NEAR_DOUBLE(hypot)

From f4edcf2685067e6e29889da0a0f400b76dc33196 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Thu, 9 Jan 2025 10:15:05 -0800
Subject: [PATCH 413/473] 3580 bug investigate test failures when running with
 cuda 126 (#3588)

* Update CUDA device manager structs for new versions of CUDA and drivers up to 12.6

* The shfl_instrinsics header file contains wrapper routines for the warp primitives and calls the new primitives for CUDA versions greater than 9 and the old ones for older CUDA versions. The new primitives have an additional argument which is a mask of the warp threads that are participating in the operation. The old primitives always involve all the threads in a warp. The wrapper routines originally allowed you to specify the mask which was ignored for the old primitives but this has now been removed. This is because if an old version of CUDA is being used then all threads must enter the wrapper routine and if a new version of CUDA is being used only the threads corresponding to the mask must enter. If threads outside the mask enter the routine then the behavior is undefined. In CUDA versions <=12.2 the primitive executes without any errors given however in later versions of CUDA a warp illegal instruction exception will be thrown. In order to preserve the same behavior of these wrapper functions for old and new versions of CUDA, the mask is always set to all threads in a warp for the new primitives. The specific new primitive can always be called with a custom mask which is already done elsewhere in the reduce_by_key routine.

* Fix for bug where new workspace size was not being calculated for the cusolver ormqr routine call which was causing memory errors.

* Fix for similar bug in the least squares solve routine where the new workspace size was not being calculated for the cusolver ormqr routine.

* Loosened tolerance for convolution filter tests for the floating point type to ensure all tests pass.

* Update src/backend/cuda/device_manager.cpp

Update driver versions to minimum required.

Co-authored-by: Filip Matzner <FloopCZ@users.noreply.github.com>

---------

Co-authored-by: Filip Matzner <FloopCZ@users.noreply.github.com>
---
 src/backend/cuda/device_manager.cpp |  2 ++
 src/backend/cuda/qr.cpp             | 33 ++++++++++++++++--
 src/backend/cuda/solve.cu           | 54 +++++++++++++++++++++++------
 test/convolve.cpp                   |  2 +-
 4 files changed, 76 insertions(+), 15 deletions(-)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 80f00f614a..05f775a821 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -102,6 +102,7 @@ static const int jetsonComputeCapabilities[] = {
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
     {12060, 9, 0, 0},
+    {12050, 9, 0, 0},
     {12040, 9, 0, 0},
     {12030, 9, 0, 0},
     {12020, 9, 0, 0},
@@ -144,6 +145,7 @@ struct ComputeCapabilityToStreamingProcessors {
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
         {12060, 525.60f, 528.33f},
+        {12050, 525.60f, 528.33f},
         {12040, 525.60f, 528.33f},
         {12030, 525.60f, 528.33f},
         {12020, 525.60f, 528.33f},
diff --git a/src/backend/cuda/qr.cpp b/src/backend/cuda/qr.cpp
index c28a41523f..f388944127 100644
--- a/src/backend/cuda/qr.cpp
+++ b/src/backend/cuda/qr.cpp
@@ -67,6 +67,16 @@ struct mqr_func_def_t {
                                               int, T *, int, int *);
 };
 
+template<typename T>
+struct mqr_buf_func_def_t {
+    using mqr_buf_func_def = cusolverStatus_t (*)(cusolverDnHandle_t,
+                                                  cublasSideMode_t,
+                                                  cublasOperation_t, int, int, int,
+                                                  const T *, int, const T *, T *,
+                                                  int, int *);
+};
+
+
 #define QR_FUNC_DEF(FUNC)                                         \
     template<typename T>                                          \
     typename FUNC##_func_def_t<T>::FUNC##_func_def FUNC##_func(); \
@@ -94,15 +104,25 @@ QR_FUNC(geqrf, double, D)
 QR_FUNC(geqrf, cfloat, C)
 QR_FUNC(geqrf, cdouble, Z)
 
-#define MQR_FUNC_DEF(FUNC) \
-    template<typename T>   \
-    typename FUNC##_func_def_t<T>::FUNC##_func_def FUNC##_func();
+#define MQR_FUNC_DEF(FUNC)                                        \
+    template<typename T>                                          \
+    typename FUNC##_func_def_t<T>::FUNC##_func_def FUNC##_func(); \
+                                                                  \
+    template<typename T>                                          \
+    typename FUNC##_buf_func_def_t<T>::FUNC##_buf_func_def FUNC##_buf_func();
 
 #define MQR_FUNC(FUNC, TYPE, PREFIX)                                        \
     template<>                                                              \
     typename FUNC##_func_def_t<TYPE>::FUNC##_func_def FUNC##_func<TYPE>() { \
         return (FUNC##_func_def_t<TYPE>::FUNC##_func_def) &                 \
                cusolverDn##PREFIX;                                          \
+    }                                                                       \
+                                                                            \
+    template<>                                                              \
+    typename FUNC##_buf_func_def_t<TYPE>::FUNC##_buf_func_def               \
+        FUNC##_buf_func<TYPE>() {                                           \
+        return (FUNC##_buf_func_def_t<TYPE>::FUNC##_buf_func_def) &         \
+               cusolverDn##PREFIX##_bufferSize;                             \
     }
 
 MQR_FUNC_DEF(mqr)
@@ -143,6 +163,13 @@ void qr(Array<T> &q, Array<T> &r, Array<T> &t, const Array<T> &in) {
     dim4 qdims(M, mn);
     q = identity<T>(qdims);
 
+    CUSOLVER_CHECK(mqr_buf_func<T>()(
+        solverDnHandle(), CUBLAS_SIDE_LEFT, CUBLAS_OP_N, q.dims()[0],
+	q.dims()[1], min(M, N), in_copy.get(), in_copy.strides()[1], t.get(),
+        q.get(), q.strides()[1], &lwork));
+
+    workspace = memAlloc<T>(lwork);
+
     CUSOLVER_CHECK(mqr_func<T>()(
         solverDnHandle(), CUBLAS_SIDE_LEFT, CUBLAS_OP_N, q.dims()[0],
         q.dims()[1], min(M, N), in_copy.get(), in_copy.strides()[1], t.get(),
diff --git a/src/backend/cuda/solve.cu b/src/backend/cuda/solve.cu
index 884d7735b1..568e44b136 100644
--- a/src/backend/cuda/solve.cu
+++ b/src/backend/cuda/solve.cu
@@ -164,6 +164,13 @@ struct mqr_solve_func_def_t {
         const T *, int, const T *, T *, int, T *, int, int *);
 };
 
+template<typename T>
+struct mqr_solve_buf_func_def_t {
+    typedef cusolverStatus_t (*mqr_solve_buf_func_def)(
+	cusolverDnHandle_t, cublasSideMode_t, cublasOperation_t, int, int, int,
+        const T *, int, const T *, T *, int, int *);
+};
+
 #define QR_FUNC_DEF(FUNC)                                                     \
     template<typename T>                                                      \
     static typename FUNC##_solve_func_def_t<T>::FUNC##_solve_func_def         \
@@ -195,17 +202,28 @@ QR_FUNC(geqrf, double, D)
 QR_FUNC(geqrf, cfloat, C)
 QR_FUNC(geqrf, cdouble, Z)
 
-#define MQR_FUNC_DEF(FUNC)                                            \
-    template<typename T>                                              \
-    static typename FUNC##_solve_func_def_t<T>::FUNC##_solve_func_def \
-        FUNC##_solve_func();
-
-#define MQR_FUNC(FUNC, TYPE, PREFIX)                                    \
-    template<>                                                          \
-    typename FUNC##_solve_func_def_t<TYPE>::FUNC##_solve_func_def       \
-        FUNC##_solve_func<TYPE>() {                                     \
-        return (FUNC##_solve_func_def_t<TYPE>::FUNC##_solve_func_def) & \
-               cusolverDn##PREFIX;                                      \
+#define MQR_FUNC_DEF(FUNC)                                                    \
+    template<typename T>                                                      \
+    static typename FUNC##_solve_func_def_t<T>::FUNC##_solve_func_def         \
+        FUNC##_solve_func();                                                  \
+	                                                                      \
+    template<typename T>                                                      \
+    static typename FUNC##_solve_buf_func_def_t<T>::FUNC##_solve_buf_func_def \
+       	FUNC##_solve_buf_func();
+
+#define MQR_FUNC(FUNC, TYPE, PREFIX)                                            \
+    template<>                                                                  \
+    typename FUNC##_solve_func_def_t<TYPE>::FUNC##_solve_func_def               \
+        FUNC##_solve_func<TYPE>() {                                             \
+        return (FUNC##_solve_func_def_t<TYPE>::FUNC##_solve_func_def) &         \
+               cusolverDn##PREFIX;                                              \
+    }                                                                           \
+                                                                                \
+    template<>                                                                  \
+    typename FUNC##_solve_buf_func_def_t<TYPE>::FUNC##_solve_buf_func_def       \
+        FUNC##_solve_buf_func<TYPE>() {                                         \
+        return (FUNC##_solve_buf_func_def_t<TYPE>::FUNC##_solve_buf_func_def) & \
+               cusolverDn##PREFIX##_bufferSize;                                 \
     }
 
 MQR_FUNC_DEF(mqr)
@@ -393,6 +411,13 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
         B.resetDims(dim4(N, K));
 
         // matmul(Q, Bpad)
+        CUSOLVER_CHECK(mqr_solve_buf_func<T>()(
+            solverDnHandle(), CUBLAS_SIDE_LEFT, CUBLAS_OP_N, B.dims()[0],
+    	    B.dims()[1], A.dims()[0], A.get(), A.strides()[1], t.get(), B.get(),
+	    B.strides()[1], &lwork));
+    
+        workspace = memAlloc<T>(lwork);
+
         CUSOLVER_CHECK(mqr_solve_func<T>()(
             solverDnHandle(), CUBLAS_SIDE_LEFT, CUBLAS_OP_N, B.dims()[0],
             B.dims()[1], A.dims()[0], A.get(), A.strides()[1], t.get(), B.get(),
@@ -427,10 +452,17 @@ Array<T> leastSquares(const Array<T> &a, const Array<T> &b) {
             t.get(), workspace.get(), lwork, info.get()));
 
         // matmul(Q1, B)
+        CUSOLVER_CHECK(mqr_solve_buf_func<T>()(
+            solverDnHandle(), CUBLAS_SIDE_LEFT, trans<T>(), M, K, N, A.get(),
+	    A.strides()[1], t.get(), B.get(), B.strides()[1], &lwork));
+    
+        workspace = memAlloc<T>(lwork);
+
         CUSOLVER_CHECK(mqr_solve_func<T>()(
             solverDnHandle(), CUBLAS_SIDE_LEFT, trans<T>(), M, K, N, A.get(),
             A.strides()[1], t.get(), B.get(), B.strides()[1], workspace.get(),
             lwork, info.get()));
+
         // tri_solve(R1, Bt)
         A.resetDims(dim4(N, N));
         B.resetDims(dim4(N, K));
diff --git a/test/convolve.cpp b/test/convolve.cpp
index 8adeb40fd8..39daff3373 100644
--- a/test/convolve.cpp
+++ b/test/convolve.cpp
@@ -898,7 +898,7 @@ float tolerance();
 
 template<>
 float tolerance<float>() {
-    return 2e-3;
+    return 4e-3;
 }
 
 template<>

From e770c8875f9d9cb23b76c98a58c1c86dd3a931be Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Mon, 13 Jan 2025 10:14:39 -0800
Subject: [PATCH 414/473] Alternative OpenCL kernel for performing the CSC
 matrix vector multiply using atomic operations. Benchmarking so far has shown
 it to be on par with the CUDA backend on my Nvidia RTX 4060 GPU. Note that
 support has been included for the BLAS style matrix vector multiply with
 alpha and beta parameters however it appears that this is not supported
 elsewhere in the code for sparse matrices so it has not been tested. Existing
 sparse matrix vector multiply tests are all passing for single and double
 precision as well as complex. (#3608)

---
 src/backend/opencl/kernel/cscmv.cl  | 148 ++++++++++++----------------
 src/backend/opencl/kernel/cscmv.hpp |  51 +++++++---
 src/backend/opencl/traits.hpp       |  30 ++++++
 3 files changed, 129 insertions(+), 100 deletions(-)

diff --git a/src/backend/opencl/kernel/cscmv.cl b/src/backend/opencl/kernel/cscmv.cl
index fab18301a1..bc56f57e46 100644
--- a/src/backend/opencl/kernel/cscmv.cl
+++ b/src/backend/opencl/kernel/cscmv.cl
@@ -7,6 +7,10 @@
  * http://arrayfire.com/licenses/BSD-3-Clause
  ********************************************************/
 
+#if IS_DBL || IS_LONG
+#pragma OPENCL EXTENSION cl_khr_int64_base_atomics : enable
+#endif
+
 #if IS_CPLX
 T __cmul(T lhs, T rhs) {
     T out;
@@ -35,100 +39,70 @@ T __ccmul(T lhs, T rhs) {
 #define CMUL(a, b) (a) * (b)
 #endif
 
-int binary_search(global const int *ptr, int len, int val) {
-    int start = 0;
-    int end   = len;
-    while (end > start) {
-        int mid = start + (end - start) / 2;
-        if (val < ptr[mid]) {
-            end = mid;
-        } else if (val > ptr[mid]) {
-            start = mid + 1;
-        } else {
-            return mid;
-        }
-    }
-    return start;
+#if IS_DBL || IS_LONG
+#define U ulong
+#define ATOMIC_FN atom_cmpxchg
+#else
+#define U unsigned
+#define ATOMIC_FN atomic_cmpxchg
+#endif
+
+#if IS_CPLX
+inline void atomicAdd(volatile __global T *ptr, T val) {
+    union {
+        U u[2];
+        T t;
+    } next, expected, current;
+    current.t = *ptr;
+
+    do {
+        expected.t.x = current.t.x;
+        next.t.x = expected.t.x + val.x;
+        current.u[0] = ATOMIC_FN((volatile __global U *) ptr, expected.u[0], next.u[0]);
+    } while(current.u[0] != expected.u[0]);
+    do {
+        expected.t.y = current.t.y;
+        next.t.y = expected.t.y + val.y;
+        current.u[1] = ATOMIC_FN(((volatile __global U *) ptr) + 1, expected.u[1], next.u[1]);
+    } while(current.u[1] != expected.u[1]);
+}
+#else
+inline void atomicAdd(volatile __global T *ptr, T val) {
+    union {
+        U u;
+        T t;
+    } next, expected, current;
+    current.t = *ptr;
+
+    do {
+        expected.t = current.t;
+        next.t = expected.t + val;
+        current.u = ATOMIC_FN((volatile __global U *) ptr, expected.u, next.u);
+    } while(current.u != expected.u);
+}
+#endif
+
+kernel void cscmv_beta(global T *output, const int M, const T beta) {
+    for(unsigned j = get_global_id(0); j < M; j += THREADS * get_num_groups(0))
+        output[j] *= beta;
 }
 
-// Each thread performs Matrix Vector multiplications for ROWS_PER_GROUP rows
-// and (K / THREAD) columns. This generates a local output buffer of size
-// ROWS_PER_THREAD for each thread. The outputs from each thread are added up to
-// generate the final result.
-kernel void cscmv_block(
-    global T *output, __global const T *values,
-    global const int *colidx,  // rowidx from csr is colidx in csc
-    global const int *rowidx,  // colidx from csr is rowidx in csc
-    const int M,                 // K from csr is M in csc
+kernel void cscmv_atomic(
+    global T *output, __global T *values,
+    global int *colidx,  // rowidx from csr is colidx in csc
+    global int *rowidx,  // colidx from csr is rowidx in csc
     const int K,                 // M from csr is K in csc
-    global const T *rhs, const KParam rinfo, const T alpha, const T beta) {
-    int lid = get_local_id(0);
+    global const T *rhs, const KParam rinfo, const T alpha) {
 
-    // Get the row offset for the current group in the uncompressed matrix
-    int rowOff = get_group_id(0) * ROWS_PER_GROUP;
-    int rowLim = min(ROWS_PER_GROUP, M - rowOff);
     rhs += rinfo.offset;
 
-    T l_outvals[ROWS_PER_GROUP];
-    for (int i = 0; i < rowLim; i++) { l_outvals[i] = 0; }
-
-    for (int colId = lid; colId < K; colId += THREADS) {
-        int rowStart     = colidx[colId];
-        int rowEnd       = colidx[colId + 1];
-        int nonZeroCount = rowEnd - rowStart;
-
-        // Find the location of the next non zero element after rowOff
-        int rowPos = binary_search(rowidx + rowStart, nonZeroCount, rowOff);
-        T rhsval   = rhs[colId];
-
-        // Traversing through nonzero elements in the current chunk
-        for (int id = rowPos + rowStart; id < rowEnd; id++) {
-            int rowId = rowidx[id];
-
-            // Exit if moving past current chunk
-            if (rowId >= rowOff + ROWS_PER_GROUP) break;
-
-            l_outvals[rowId - rowOff] += CMUL(values[id], rhsval);
-        }
-    }
-
-    // s_outvals is used for reduction
-    local T s_outvals[THREADS];
-
-    // s_output is used to store the final output into local memory
-    local T s_output[ROWS_PER_GROUP];
-
-    // For each row of output, copy registers to local memory, add results,
-    // write to output.
-    for (int i = 0; i < rowLim; i++) {
-        // Copying to local memory
-        s_outvals[lid] = l_outvals[i];
-        barrier(CLK_LOCAL_MEM_FENCE);
-
-        // Adding the results through reduction
-        for (int n = THREADS / 2; n > 0; n /= 2) {
-            if (lid < n) s_outvals[lid] += s_outvals[lid + n];
-            barrier(CLK_LOCAL_MEM_FENCE);
-        }
-
-        // Store to another local buffer so it can be written in a coalesced
-        // manner later
-        if (lid == 0) { s_output[i] = s_outvals[0]; }
-    }
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    // For each row in output, write output in coalesced manner
-    for (int i = lid; i < ROWS_PER_GROUP; i += THREADS) {
-        T outval = s_output[i];
-
+    for(unsigned j = get_group_id(0); j < K; j += get_num_groups(0)) {
+        for(unsigned i = get_local_id(0) + colidx[j]; i < colidx[j + 1]; i += THREADS) {
+            T outval = CMUL(values[i], rhs[j]);
 #if USE_ALPHA
-        outval = MUL(alpha, outval);
-#endif
-
-#if USE_BETA
-        output[rowOff + i] = outval + MUL(beta, output[j * M + rowOff + i]);
-#else
-        output[rowOff + i] = outval;
+            outval = MUL(alpha, outval);
 #endif
+            atomicAdd(output + rowidx[i], outval);
+        }
     }
 }
diff --git a/src/backend/opencl/kernel/cscmv.hpp b/src/backend/opencl/kernel/cscmv.hpp
index 88008480f8..2ab88b202c 100644
--- a/src/backend/opencl/kernel/cscmv.hpp
+++ b/src/backend/opencl/kernel/cscmv.hpp
@@ -32,39 +32,64 @@ void cscmv(Param out, const Param &values, const Param &colIdx,
            bool is_conj) {
     // TODO: rows_per_group limited by register pressure. Find better way to
     // handle this.
+    constexpr int threads_per_g = 64;
     constexpr int rows_per_group = 64;
 
     const bool use_alpha = (alpha != scalar<T>(1.0));
     const bool use_beta  = (beta != scalar<T>(0.0));
 
-    cl::NDRange local(THREADS_PER_GROUP);
+    cl::NDRange local(threads_per_g);
 
-    std::array<TemplateArg, 6> targs = {
+    int K        = colIdx.info.dims[0] - 1;
+    int M        = out.info.dims[0];
+
+    std::array<TemplateArg, 5> targs = {
         TemplateTypename<T>(),       TemplateArg(use_alpha),
-        TemplateArg(use_beta),       TemplateArg(is_conj),
-        TemplateArg(rows_per_group), TemplateArg(local[0]),
+        TemplateArg(is_conj), TemplateArg(rows_per_group),
+        TemplateArg(local[0]),
     };
-    std::array<std::string, 8> options = {
+    std::array<std::string, 9> options = {
         DefineKeyValue(T, dtype_traits<T>::getName()),
         DefineKeyValue(USE_ALPHA, use_alpha),
-        DefineKeyValue(USE_BETA, use_beta),
         DefineKeyValue(IS_CONJ, is_conj),
         DefineKeyValue(THREADS, local[0]),
         DefineKeyValue(ROWS_PER_GROUP, rows_per_group),
         DefineKeyValue(IS_CPLX, (iscplx<T>() ? 1 : 0)),
+        DefineKeyValue(IS_DBL, (isdbl<T>() ? 1 : 0)),
+        DefineKeyValue(IS_LONG, (islong<T>() ? 1 : 0)),
         getTypeBuildDefinition<T>()};
 
-    auto cscmvBlock =
-        common::getKernel("cscmv_block", {{cscmv_cl_src}}, targs, options);
+    if(use_beta) {
+        std::array<TemplateArg, 4> targs_beta = {
+            TemplateTypename<T>(), TemplateArg(is_conj),
+            TemplateArg(rows_per_group), TemplateArg(local[0])};
+        std::array<std::string, 8> options_beta = {
+            DefineKeyValue(T, dtype_traits<T>::getName()),
+            DefineKeyValue(IS_CONJ, is_conj),
+            DefineKeyValue(THREADS, local[0]),
+            DefineKeyValue(ROWS_PER_GROUP, rows_per_group),
+            DefineKeyValue(IS_CPLX, (iscplx<T>() ? 1 : 0)),
+            DefineKeyValue(IS_DBL, (isdbl<T>() ? 1 : 0)),
+            DefineKeyValue(IS_LONG, (islong<T>() ? 1 : 0)),
+            getTypeBuildDefinition<T>()};
+
+        int groups_x = divup(M, rows_per_group * threads_per_g);
+        cl::NDRange global(local[0] * groups_x, 1);
+        auto cscmvBeta = common::getKernel("cscmv_beta", {{cscmv_cl_src}}, targs_beta, options_beta);
+        cscmvBeta(cl::EnqueueArgs(getQueue(), global, local), *out.data, M, beta);
+
+    } else {
+        getQueue().enqueueFillBuffer(*out.data, 0, 0, M * sizeof(T));
+    }
 
-    int K        = colIdx.info.dims[0] - 1;
-    int M        = out.info.dims[0];
     int groups_x = divup(M, rows_per_group);
     cl::NDRange global(local[0] * groups_x, 1);
 
-    cscmvBlock(cl::EnqueueArgs(getQueue(), global, local), *out.data,
-               *values.data, *colIdx.data, *rowIdx.data, M, K, *rhs.data,
-               rhs.info, alpha, beta);
+    auto cscmvAtomic =
+        common::getKernel("cscmv_atomic", {{cscmv_cl_src}}, targs, options);
+    cscmvAtomic(cl::EnqueueArgs(getQueue(), global, local), *out.data,
+                *values.data, *colIdx.data, *rowIdx.data, K, *rhs.data,
+                rhs.info, alpha);
     CL_DEBUG_FINISH(getQueue());
 }
 }  // namespace kernel
diff --git a/src/backend/opencl/traits.hpp b/src/backend/opencl/traits.hpp
index 00af1d17b0..2af7257b76 100644
--- a/src/backend/opencl/traits.hpp
+++ b/src/backend/opencl/traits.hpp
@@ -49,6 +49,36 @@ inline bool iscplx<cdouble>() {
     return true;
 }
 
+template<typename T>
+static bool isdbl() {
+    return false;
+}
+
+template<>
+inline bool isdbl<double>() {
+    return true;
+}
+
+template<>
+inline bool isdbl<cdouble>() {
+    return true;
+}
+
+template<typename T>
+static bool islong() {
+    return false;
+}
+
+template<>
+inline bool islong<long>() {
+    return true;
+}
+
+template<>
+inline bool islong<unsigned long>() {
+    return true;
+}
+
 template<typename T>
 inline std::string scalar_to_option(const T &val) {
     using namespace arrayfire::common;

From 7127a0babfa3a05ab2d166a2708d9bd6533569db Mon Sep 17 00:00:00 2001
From: errata-c <77643526+errata-c@users.noreply.github.com>
Date: Mon, 13 Jan 2025 15:26:22 -0500
Subject: [PATCH 415/473] Fixed padding comparison in convolve2GradientNN
 (#3519)

Co-authored-by: errata-c <erratachem@gmail.com>
---
 src/api/c/convolve.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/api/c/convolve.cpp b/src/api/c/convolve.cpp
index abbcd2f71b..61af7b1b16 100644
--- a/src/api/c/convolve.cpp
+++ b/src/api/c/convolve.cpp
@@ -437,7 +437,7 @@ af_err af_convolve2_gradient_nn(
         size_t padding_ndims  = padding.ndims();
         size_t dilation_ndims = dilation.ndims();
         ARG_ASSERT(3, stride_ndims > 0 && stride_ndims <= 2);
-        ARG_ASSERT(5, padding_ndims > 0 && padding_ndims <= 2);
+        ARG_ASSERT(5, padding_ndims >= 0 && padding_ndims <= 2);
         ARG_ASSERT(7, dilation_ndims > 0 && dilation_ndims <= 2);
 
         af_dtype type = oinfo.getType();

From 5c2ea2998573ddbbb9da3885e385faa40552567e Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 14 Jan 2025 10:20:35 -0800
Subject: [PATCH 416/473] Extend test for convolve2GradientNN function to
 verify zero padding fix in PR 3519 (#3631)

---
 test/convolve.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/convolve.cpp b/test/convolve.cpp
index 39daff3373..ac731ef31c 100644
--- a/test/convolve.cpp
+++ b/test/convolve.cpp
@@ -1176,4 +1176,10 @@ TEST(ConvolveNN, ZeroPadding_Issue2817) {
     array convolved = convolve2NN(signal, filter, strides, padding, dilation);
     ASSERT_EQ(sum<float>(abs(signal(seq(1, 3), seq(1, 3)) - convolved)) < 1E-5,
               true);
+
+    array incoming_gradient = constant(1 / 9.f, 3, 3);
+    array convolved_grad = convolve2GradientNN(incoming_gradient, signal, filter,
+                                               convolved, strides, padding, dilation,
+                                               AF_CONV_GRADIENT_FILTER);
+    ASSERT_EQ(sum<float>(abs(convolved - convolved_grad)) < 1E-5, true);
 }

From eef57732c94b29c5afc834eb111b5924dc232adb Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Thu, 16 Jan 2025 02:47:15 +0100
Subject: [PATCH 417/473] Correct the conversion from float/double to half on
 CUDA (#3627)

---
 src/backend/common/half.hpp | 44 +++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index b6585dc905..3f966c6f81 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -87,6 +87,7 @@ using uint16_t = unsigned short;
 #define AF_CONSTEXPR constexpr
 #else
 #include <af/compilers.h>
+#include <algorithm>
 #include <cmath>
 #include <cstdint>
 #include <cstring>
@@ -245,9 +246,9 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(T value) noexcept {
 /// \return binary representation of half-precision value
 template<std::float_round_style R = std::round_to_nearest>
 __DH__ native_half_t float2half_impl(float value) noexcept {
-    uint32_t bits = 0;  // = *reinterpret_cast<uint32*>(&value);
-                        // //violating strict aliasing!
-    std::memcpy(&bits, &value, sizeof(float));
+    alignas(std::max(alignof(uint32_t), alignof(float))) float _value = value;
+    uint32_t bits = *reinterpret_cast<uint32_t*>(&_value);
+
     constexpr uint16_t base_table[512] = {
         0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
         0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
@@ -337,9 +338,10 @@ __DH__ native_half_t float2half_impl(float value) noexcept {
         24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
         24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
         24, 24, 24, 24, 24, 24, 24, 13};
-    uint16_t hbits =
-        base_table[bits >> 23] +
-        static_cast<uint16_t>((bits & 0x7FFFFF) >> shift_table[bits >> 23]);
+    alignas(std::max(alignof(uint16_t), alignof(native_half_t)))
+        uint16_t hbits =
+            base_table[bits >> 23] +
+            static_cast<uint16_t>((bits & 0x7FFFFF) >> shift_table[bits >> 23]);
     AF_IF_CONSTEXPR(R == std::round_to_nearest)
     hbits +=
         (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) |
@@ -367,7 +369,8 @@ __DH__ native_half_t float2half_impl(float value) noexcept {
           (((bits >> 23) <= 358) & ((bits >> 23) != 256))) &
          (hbits < 0xFC00) & (hbits >> 15)) -
         ((hbits == 0x7C00) & ((bits >> 23) != 255));
-    return hbits;
+
+    return *reinterpret_cast<native_half_t*>(&hbits);
 }
 
 /// Convert IEEE double-precision to half-precision.
@@ -379,11 +382,11 @@ __DH__ native_half_t float2half_impl(float value) noexcept {
 /// \return binary representation of half-precision value
 template<std::float_round_style R>
 __DH__ native_half_t float2half_impl(double value) {
-    uint64_t bits{0};  // = *reinterpret_cast<uint64*>(&value);		//violating
-                       // strict aliasing!
-    std::memcpy(&bits, &value, sizeof(double));
+    alignas(std::max(alignof(uint64_t), alignof(double))) double _value = value;
+    uint64_t bits = *reinterpret_cast<uint64_t*>(&_value);
     uint32_t hi = bits >> 32, lo = bits & 0xFFFFFFFF;
-    uint16_t hbits = (hi >> 16) & 0x8000;
+    alignas(std::max(alignof(uint16_t), alignof(native_half_t)))
+        uint16_t hbits = (hi >> 16) & 0x8000;
     hi &= 0x7FFFFFFF;
     int exp = hi >> 20;
     if (exp == 2047)
@@ -420,7 +423,8 @@ __DH__ native_half_t float2half_impl(double value) {
         ~(hbits >> 15) & (s | g);
     else AF_IF_CONSTEXPR(R == std::round_toward_neg_infinity) hbits +=
         (hbits >> 15) & (g | s);
-    return hbits;
+
+    return *reinterpret_cast<native_half_t*>(&hbits);
 }
 
 __DH__ inline float half2float_impl(native_half_t value) noexcept {
@@ -790,14 +794,14 @@ __DH__ inline float half2float_impl(native_half_t value) noexcept {
         1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,
         1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024};
 
-    uint16_t value_bits = 0;
-    std::memcpy(&value_bits, &value, sizeof(uint16_t));
-    uint32_t bits =
+    alignas(std::max(alignof(uint16_t), alignof(native_half_t)))
+        native_half_t _value = value;
+    uint16_t value_bits      = *reinterpret_cast<uint16_t*>(&_value);
+
+    alignas(std::max(alignof(uint32_t), alignof(float))) uint32_t bits =
         mantissa_table[offset_table[value_bits >> 10] + (value_bits & 0x3FF)] +
         exponent_table[value_bits >> 10];
-    float out = 0.0f;
-    std::memcpy(&out, &bits, sizeof(float));
-    return out;
+    return *reinterpret_cast<float*>(&bits);
 }
 
 #endif  // __CUDACC_RTC__
@@ -872,7 +876,9 @@ AF_CONSTEXPR T half2int(native_half_t value) {
     else AF_IF_CONSTEXPR(std::is_same<T, int>::value) {
         return __half2int_rn(value);
     }
-    else { return __half2uint_rn(value); }
+    else {
+        return __half2uint_rn(value);
+    }
 #elif defined(AF_ONEAPI)
     return static_cast<T>(value);
 #else

From 6e5dca46957e3c894c807f269d8f90aa28614160 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Wed, 15 Jan 2025 17:54:05 -0800
Subject: [PATCH 418/473] Reverted an error in the interop_cuda example code
 where the cuda stream id was being used instead of the arrayfire stream id.
 (#3594)


From ffda1b6e241ceb4d6a1d8718afd8be6a10ad7e26 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Thu, 30 Jan 2025 10:31:38 -0800
Subject: [PATCH 419/473] 3560 bug incorrect results when using the pow
 function with float16 arguments with cuda backend (#3561)

* Add cases for float16 arguments to atan2 and hypot functions

* Added test cases for half precision atan2 and hypot functions

* Fix for incorrect result when using the pow function with float16 arguments with the CUDA backend. Since the half precision CUDA library doesn't have a pow function, the default pow function is used, casting the arguments to double precision.
---
 src/backend/cuda/kernel/jit.cuh | 5 -----
 test/binary.cpp                 | 1 +
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/backend/cuda/kernel/jit.cuh b/src/backend/cuda/kernel/jit.cuh
index cfb5837719..76fd344010 100644
--- a/src/backend/cuda/kernel/jit.cuh
+++ b/src/backend/cuda/kernel/jit.cuh
@@ -59,14 +59,9 @@ typedef cuDoubleComplex cdouble;
 #define __rem(lhs, rhs) ((lhs) % (rhs))
 #define __mod(lhs, rhs) ((lhs) % (rhs))
 
-#ifdef AF_WITH_FAST_MATH
 #define __pow(lhs, rhs)  \
     static_cast<double>( \
         pow(static_cast<double>(lhs), static_cast<double>(rhs)));
-#else
-#define __pow(lhs, rhs) \
-    __float2int_rn(powf(__int2float_rn((int)lhs), __int2float_rn((int)rhs)))
-#endif
 #define __powll(lhs, rhs) \
     __double2ll_rn(pow(__ll2double_rn(lhs), __ll2double_rn(rhs)))
 #define __powul(lhs, rhs) \
diff --git a/test/binary.cpp b/test/binary.cpp
index c029a19da5..a274c11346 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -195,6 +195,7 @@ BINARY_TESTS_NEAR_FLOAT(pow)
 BINARY_TESTS_NEAR_FLOAT(hypot)
 
 BINARY_TESTS_NEAR_HALF(atan2)
+BINARY_TESTS_NEAR_HALF(pow)
 BINARY_TESTS_NEAR_HALF(hypot)
 
 BINARY_TESTS_NEAR_DOUBLE(atan2)

From c644e4f5a375e41608b36339f444d49ab2967287 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Mon, 10 Feb 2025 18:37:42 -0800
Subject: [PATCH 420/473] Added testing for sequence indexing with non-unitary
 steps mixed in with array indexing (#3587)

---
 test/gen_index.cpp | 50 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/test/gen_index.cpp b/test/gen_index.cpp
index e65d4e48e5..0716751fa0 100644
--- a/test/gen_index.cpp
+++ b/test/gen_index.cpp
@@ -253,6 +253,56 @@ TEST(GeneralIndex, AASS) {
     ASSERT_SUCCESS(af_release_array(outArray));
 }
 
+TEST(GeneralIndex, SSAS_LinearSteps) {
+    vector<dim4> numDims;
+    vector<vector<float>> in;
+    vector<vector<float>> tests;  // Read tests from file
+
+    readTestsFromFile<float, float>(
+        TEST_DIR "/gen_index/s29_9__3s0_9_2as0_n.test", numDims, in, tests);
+
+    af_array outArray  = 0;
+    af_array inArray   = 0;
+    af_array idxArray0 = 0;
+    dim4 dims0         = numDims[0];
+    dim4 dims1         = numDims[1];
+
+    ASSERT_SUCCESS(af_create_array(&inArray, &(in[0].front()), dims0.ndims(),
+                                   dims0.get(),
+                                   (af_dtype)dtype_traits<float>::af_type));
+
+    ASSERT_SUCCESS(af_create_array(&idxArray0, &(in[1].front()), dims1.ndims(),
+                                   dims1.get(),
+                                   (af_dtype)dtype_traits<float>::af_type));
+
+    af_index_t indexs[4];
+    indexs[0].idx.seq = af_make_seq(29, 9, -3);
+    indexs[1].idx.seq = af_make_seq(0, 9, 2);
+    indexs[2].idx.arr = idxArray0;
+    indexs[3].idx.seq = af_span;
+
+    indexs[0].isSeq = true;
+    indexs[1].isSeq = true;
+    indexs[2].isSeq = false;
+    indexs[3].isSeq = true;
+
+    ASSERT_SUCCESS(af_index_gen(&outArray, inArray, 4, indexs));
+
+    vector<float> currGoldBar = tests[0];
+    size_t nElems             = currGoldBar.size();
+    vector<float> outData(nElems);
+
+    ASSERT_SUCCESS(af_get_data_ptr((void *)outData.data(), outArray));
+
+    for (size_t elIter = 0; elIter < nElems; ++elIter) {
+        ASSERT_EQ(currGoldBar[elIter], outData[elIter])
+            << "at: " << elIter << endl;
+    }
+
+    ASSERT_SUCCESS(af_release_array(inArray));
+    ASSERT_SUCCESS(af_release_array(outArray));
+}
+
 using af::array;
 using af::freeHost;
 using af::randu;

From 18028c090be521414a9e0c4495c2c29b1d4436f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Mon, 10 Feb 2025 18:38:21 -0800
Subject: [PATCH 421/473] Fix issue 3525: correct handling of array indexing
 with sequence (#3585)

* Fix issue 3525: implement correct handling of steps in array indexing with sequence

* Implement handling of af_span sequence
---
 src/backend/cpu/index.cpp                     | 11 +++++++++-
 src/backend/cpu/kernel/index.hpp              | 22 ++++++++++---------
 src/backend/cuda/assign_kernel_param.hpp      |  1 +
 src/backend/cuda/index.cpp                    | 10 +++++++++
 src/backend/cuda/kernel/index.cuh             | 12 ++++++----
 src/backend/oneapi/index.cpp                  | 10 +++++++++
 .../oneapi/kernel/assign_kernel_param.hpp     |  1 +
 src/backend/oneapi/kernel/index.hpp           | 12 ++++++----
 src/backend/opencl/index.cpp                  | 10 +++++++++
 src/backend/opencl/kernel/index.cl            | 21 +++++++++++-------
 src/backend/opencl/kernel/index.hpp           |  1 +
 11 files changed, 84 insertions(+), 27 deletions(-)

diff --git a/src/backend/cpu/index.cpp b/src/backend/cpu/index.cpp
index 315406b46d..850239acfe 100644
--- a/src/backend/cpu/index.cpp
+++ b/src/backend/cpu/index.cpp
@@ -35,7 +35,16 @@ Array<T> index(const Array<T>& in, const af_index_t idxrs[]) {
     // create seq vector to retrieve output
     // dimensions, offsets & offsets
     for (unsigned x = 0; x < isSeq.size(); ++x) {
-        if (idxrs[x].isSeq) { seqs[x] = idxrs[x].idx.seq; }
+        if (idxrs[x].isSeq) {
+            af_seq seq = idxrs[x].idx.seq;
+            // Handle af_span as a sequence that covers the complete axis
+            if (seq.begin == af_span.begin && seq.end == af_span.end &&
+                seq.step == af_span.step) {
+                seqs[x] = af_seq{0, (double)(in.dims()[x] - 1), 1};
+            } else {
+                seqs[x] = seq;
+            }
+        }
         isSeq[x] = idxrs[x].isSeq;
     }
 
diff --git a/src/backend/cpu/kernel/index.hpp b/src/backend/cpu/kernel/index.hpp
index 2a6a6d9bc4..962b0713dc 100644
--- a/src/backend/cpu/kernel/index.hpp
+++ b/src/backend/cpu/kernel/index.hpp
@@ -34,25 +34,27 @@ void index(Param<T> out, CParam<T> in, const af::dim4 dDims,
 
     for (dim_t l = 0; l < oDims[3]; ++l) {
         dim_t lOff   = l * oStrides[3];
-        dim_t inIdx3 = trimIndex(isSeq[3] ? l + iOffs[3] : ptr3[l], iDims[3]);
+        dim_t inIdx3 = trimIndex(
+            isSeq[3] ? l * seqs[3].step + iOffs[3] : ptr3[l], iDims[3]);
         dim_t inOff3 = inIdx3 * iStrds[3];
 
         for (dim_t k = 0; k < oDims[2]; ++k) {
-            dim_t kOff = k * oStrides[2];
-            dim_t inIdx2 =
-                trimIndex(isSeq[2] ? k + iOffs[2] : ptr2[k], iDims[2]);
+            dim_t kOff   = k * oStrides[2];
+            dim_t inIdx2 = trimIndex(
+                isSeq[2] ? k * seqs[2].step + iOffs[2] : ptr2[k], iDims[2]);
             dim_t inOff2 = inIdx2 * iStrds[2];
 
             for (dim_t j = 0; j < oDims[1]; ++j) {
-                dim_t jOff = j * oStrides[1];
-                dim_t inIdx1 =
-                    trimIndex(isSeq[1] ? j + iOffs[1] : ptr1[j], iDims[1]);
+                dim_t jOff   = j * oStrides[1];
+                dim_t inIdx1 = trimIndex(
+                    isSeq[1] ? j * seqs[1].step + iOffs[1] : ptr1[j], iDims[1]);
                 dim_t inOff1 = inIdx1 * iStrds[1];
 
                 for (dim_t i = 0; i < oDims[0]; ++i) {
-                    dim_t iOff = i * oStrides[0];
-                    dim_t inIdx0 =
-                        trimIndex(isSeq[0] ? i + iOffs[0] : ptr0[i], iDims[0]);
+                    dim_t iOff   = i * oStrides[0];
+                    dim_t inIdx0 = trimIndex(
+                        isSeq[0] ? i * seqs[0].step + iOffs[0] : ptr0[i],
+                        iDims[0]);
                     dim_t inOff0 = inIdx0 * iStrds[0];
 
                     dst[lOff + kOff + jOff + iOff] =
diff --git a/src/backend/cuda/assign_kernel_param.hpp b/src/backend/cuda/assign_kernel_param.hpp
index 0591ca80ad..350893f911 100644
--- a/src/backend/cuda/assign_kernel_param.hpp
+++ b/src/backend/cuda/assign_kernel_param.hpp
@@ -15,6 +15,7 @@ namespace cuda {
 typedef struct {
     int offs[4];
     int strds[4];
+    int steps[4];
     bool isSeq[4];
     unsigned int* ptr[4];
 } AssignKernelParam;
diff --git a/src/backend/cuda/index.cpp b/src/backend/cuda/index.cpp
index 88a95da73b..d8acf90c12 100644
--- a/src/backend/cuda/index.cpp
+++ b/src/backend/cuda/index.cpp
@@ -44,6 +44,16 @@ Array<T> index(const Array<T>& in, const af_index_t idxrs[]) {
         p.isSeq[i] = idxrs[i].isSeq;
         p.offs[i]  = iOffs[i];
         p.strds[i] = iStrds[i];
+        p.steps[i] = 0;
+        if (idxrs[i].isSeq) {
+            af_seq seq = idxrs[i].idx.seq;
+            // The step for af_span used in the kernel must be 1
+            if (seq.begin == af_span.begin && seq.end == af_span.end &&
+                seq.step == af_span.step)
+                p.steps[i] = 1;
+            else
+                p.steps[i] = seq.step;
+        }
     }
 
     std::vector<Array<uint>> idxArrs(4, createEmptyArray<uint>(dim4()));
diff --git a/src/backend/cuda/kernel/index.cuh b/src/backend/cuda/kernel/index.cuh
index 37b6b63d46..968e9ae0c6 100644
--- a/src/backend/cuda/kernel/index.cuh
+++ b/src/backend/cuda/kernel/index.cuh
@@ -43,13 +43,17 @@ __global__ void index(Param<T> out, CParam<T> in, const IndexKernelParam p,
         gw < out.dims[3]) {
         // calculate pointer offsets for input
         int i =
-            p.strds[0] * trimIndex(s0 ? gx + p.offs[0] : ptr0[gx], in.dims[0]);
+            p.strds[0] *
+            trimIndex(s0 ? gx * p.steps[0] + p.offs[0] : ptr0[gx], in.dims[0]);
         int j =
-            p.strds[1] * trimIndex(s1 ? gy + p.offs[1] : ptr1[gy], in.dims[1]);
+            p.strds[1] *
+            trimIndex(s1 ? gy * p.steps[1] + p.offs[1] : ptr1[gy], in.dims[1]);
         int k =
-            p.strds[2] * trimIndex(s2 ? gz + p.offs[2] : ptr2[gz], in.dims[2]);
+            p.strds[2] *
+            trimIndex(s2 ? gz * p.steps[2] + p.offs[2] : ptr2[gz], in.dims[2]);
         int l =
-            p.strds[3] * trimIndex(s3 ? gw + p.offs[3] : ptr3[gw], in.dims[3]);
+            p.strds[3] *
+            trimIndex(s3 ? gw * p.steps[3] + p.offs[3] : ptr3[gw], in.dims[3]);
         // offset input and output pointers
         const T* src = (const T*)in.ptr + (i + j + k + l);
         T* dst = (T*)out.ptr + (gx * out.strides[0] + gy * out.strides[1] +
diff --git a/src/backend/oneapi/index.cpp b/src/backend/oneapi/index.cpp
index bec65902d8..2548df2011 100644
--- a/src/backend/oneapi/index.cpp
+++ b/src/backend/oneapi/index.cpp
@@ -44,6 +44,16 @@ Array<T> index(const Array<T>& in, const af_index_t idxrs[]) {
         p.isSeq[i] = idxrs[i].isSeq;
         p.offs[i]  = iOffs[i];
         p.strds[i] = iStrds[i];
+        p.steps[i] = 0;
+        if (idxrs[i].isSeq) {
+            af_seq seq = idxrs[i].idx.seq;
+            // The step for af_span used in the kernel must be 1
+            if (seq.begin == af_span.begin && seq.end == af_span.end &&
+                seq.step == af_span.step)
+                p.steps[i] = 1;
+            else
+                p.steps[i] = seq.step;
+        }
     }
 
     std::vector<Array<uint>> idxArrs(4, createEmptyArray<uint>(dim4(1)));
diff --git a/src/backend/oneapi/kernel/assign_kernel_param.hpp b/src/backend/oneapi/kernel/assign_kernel_param.hpp
index e2539ed2b3..e2eec56d18 100644
--- a/src/backend/oneapi/kernel/assign_kernel_param.hpp
+++ b/src/backend/oneapi/kernel/assign_kernel_param.hpp
@@ -19,6 +19,7 @@ namespace oneapi {
 typedef struct {
     int offs[4];
     int strds[4];
+    int steps[4];
     bool isSeq[4];
     std::array<sycl::accessor<unsigned int, 1, sycl::access::mode::read,
                               sycl::access::target::device>,
diff --git a/src/backend/oneapi/kernel/index.hpp b/src/backend/oneapi/kernel/index.hpp
index 857b299aef..c7bb591953 100644
--- a/src/backend/oneapi/kernel/index.hpp
+++ b/src/backend/oneapi/kernel/index.hpp
@@ -88,13 +88,17 @@ class indexKernel {
         if (gx < odims0 && gy < odims1 && gz < odims2 && gw < odims3) {
             // calculate pointer offsets for input
             int i = p.strds[0] *
-                    trimIndex(s0 ? gx + p.offs[0] : ptr0[gx], inp.dims[0]);
+                    trimIndex(s0 ? gx * p.steps[0] + p.offs[0] : ptr0[gx],
+                              inp.dims[0]);
             int j = p.strds[1] *
-                    trimIndex(s1 ? gy + p.offs[1] : ptr1[gy], inp.dims[1]);
+                    trimIndex(s1 ? gy * p.steps[1] + p.offs[1] : ptr1[gy],
+                              inp.dims[1]);
             int k = p.strds[2] *
-                    trimIndex(s2 ? gz + p.offs[2] : ptr2[gz], inp.dims[2]);
+                    trimIndex(s2 ? gz * p.steps[2] + p.offs[2] : ptr2[gz],
+                              inp.dims[2]);
             int l = p.strds[3] *
-                    trimIndex(s3 ? gw + p.offs[3] : ptr3[gw], inp.dims[3]);
+                    trimIndex(s3 ? gw * p.steps[3] + p.offs[3] : ptr3[gw],
+                              inp.dims[3]);
             // offset input and output pointers
             const T* src = (const T*)in.get_pointer() + (i + j + k + l);
             T* dst       = (T*)out.get_pointer() +
diff --git a/src/backend/opencl/index.cpp b/src/backend/opencl/index.cpp
index 0911229936..d2864e6a81 100644
--- a/src/backend/opencl/index.cpp
+++ b/src/backend/opencl/index.cpp
@@ -42,6 +42,16 @@ Array<T> index(const Array<T>& in, const af_index_t idxrs[]) {
         p.isSeq[i] = idxrs[i].isSeq ? 1 : 0;
         p.offs[i]  = iOffs[i];
         p.strds[i] = iStrds[i];
+        p.steps[i] = 0;
+        if (idxrs[i].isSeq) {
+            af_seq seq = idxrs[i].idx.seq;
+            // The step for af_span used in the kernel must be 1
+            if (seq.begin == af_span.begin && seq.end == af_span.end &&
+                seq.step == af_span.step)
+                p.steps[i] = 1;
+            else
+                p.steps[i] = seq.step;
+        }
     }
 
     cl::Buffer* bPtrs[4];
diff --git a/src/backend/opencl/kernel/index.cl b/src/backend/opencl/kernel/index.cl
index 85e6e10cc0..2cc3cb57fe 100644
--- a/src/backend/opencl/kernel/index.cl
+++ b/src/backend/opencl/kernel/index.cl
@@ -10,6 +10,7 @@
 typedef struct {
     int offs[4];
     int strds[4];
+    int steps[4];
     char isSeq[4];
 } IndexKernelParam_t;
 
@@ -47,14 +48,18 @@ kernel void indexKernel(global T* optr, KParam oInfo, global const T* iptr,
     if (gx < oInfo.dims[0] && gy < oInfo.dims[1] && gz < oInfo.dims[2] &&
         gw < oInfo.dims[3]) {
         // calculate pointer offsets for input
-        int i = p.strds[0] *
-                trimIndex(s0 ? gx + p.offs[0] : ptr0[gx], iInfo.dims[0]);
-        int j = p.strds[1] *
-                trimIndex(s1 ? gy + p.offs[1] : ptr1[gy], iInfo.dims[1]);
-        int k = p.strds[2] *
-                trimIndex(s2 ? gz + p.offs[2] : ptr2[gz], iInfo.dims[2]);
-        int l = p.strds[3] *
-                trimIndex(s3 ? gw + p.offs[3] : ptr3[gw], iInfo.dims[3]);
+        int i =
+            p.strds[0] * trimIndex(s0 ? gx * p.steps[0] + p.offs[0] : ptr0[gx],
+                                   iInfo.dims[0]);
+        int j =
+            p.strds[1] * trimIndex(s1 ? gy * p.steps[1] + p.offs[1] : ptr1[gy],
+                                   iInfo.dims[1]);
+        int k =
+            p.strds[2] * trimIndex(s2 ? gz * p.steps[2] + p.offs[2] : ptr2[gz],
+                                   iInfo.dims[2]);
+        int l =
+            p.strds[3] * trimIndex(s3 ? gw * p.steps[3] + p.offs[3] : ptr3[gw],
+                                   iInfo.dims[3]);
         // offset input and output pointers
         global const T* src = iptr + (i + j + k + l) + iInfo.offset;
         global T* dst = optr + (gx * oInfo.strides[0] + gy * oInfo.strides[1] +
diff --git a/src/backend/opencl/kernel/index.hpp b/src/backend/opencl/kernel/index.hpp
index 9433893b96..5362a8e78b 100644
--- a/src/backend/opencl/kernel/index.hpp
+++ b/src/backend/opencl/kernel/index.hpp
@@ -26,6 +26,7 @@ namespace kernel {
 typedef struct {
     int offs[4];
     int strds[4];
+    int steps[4];
     char isSeq[4];
 } IndexKernelParam_t;
 

From b1e85d3d59a97def95ab705598e78f890ec4b295 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Wed, 12 Feb 2025 20:32:06 -0500
Subject: [PATCH 422/473] 3539 build oneapi version 2024 incompatible for mkl
 (#3573)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Updated cmake files to support oneAPI version 2024.1. Currently doesn't support compiling on Windows, this will be added later.

* Some tests were showing oneAPI errors due to nested calls of submit to the SYCL queue which is not supported. This has been fixed by moving calls to get() out of the submit calls. If a get call is made to a node that has not yet been evaluated, it will need to submit work to the SYCL queue.

* Modify test cases to check if library functions are supported on the current backend. If a function is not supported the test is skipped. This works with both the C API which returns an error flag and the C++ API which throws an exception.

* Modified half support check to check the native vector width for half precision as well as the fp16 aspect for the oneAPI backend. Some devices advertise the fp16 aspect but their native vector width for half precision is zero which results in errors when calling OpenCL routines with half precision arguments.

* Fix for bug in index function introduced when implementing a fix for nested oneAPI queue submissions.

* Fixed bug in irreduce_dim_launcher where incorrect templated calls were made to ireduceDimKernelSMEM for 1, 2 and 4 y threads

* Fix bug in wrap function where one dimension of the output array was missing from the global problem size.

* Check for failure of asserts in apiWrapper and return from test to prevent segfaults in subsequent asserts.

* Modified ASSERT_SUCCESS macro to skip unsupported tests rather than failing when AF_SKIP_UNSUPPORTED_TESTS CMake option is enabled

* Fixed cmake policy issue for oneapi fix (#3569)

* Cmake function CMakeDetermineCompileFeatures has been changed to CMakeDetermineCompilerSupport in version 3.30. Added support for this.

* Removed unsupported compute capabilites from all architectures list for CUDA 12

* Rename variables for input and output arrays of join method

* Fix issue where NOT_SUPPORTED errors were storing the error message as the back end name

* Added macro for asserting success of C++ API functions that throw exceptions. If the not supported exception is thrown, the test can be skipped if the AF_SKIP_UNSUPPORTED_TESTS variable is on.

* Remove macros that check for unsupported exceptions and skip tests. A SKIP_BACKEND macro will be made instead that will need to be explicitly added to each test that calls a function unsupported for a given backend.

* The UNSUPPORTED_BACKEND macro has now been added to all tests that are not supported by the oneAPI back end. If AF_SKIP_UNSUPPORTED_TESTS is set to ON then all tests with this macro will be skipped. These will need to be removed as oneAPI support is added for each feature.

* Update getBackendName test helper function to support all back ends. It is now used for the UNSUPPORTED_BACKEND macro.

---------

Co-authored-by: Edwin Lester Solís Fuentes <68087165+edwinsolisf@users.noreply.github.com>
---
 CMakeLists.txt                                |   9 +-
 CMakeModules/CMakeTestSYCLCompiler.cmake      |   9 +-
 CMakeModules/CPackProjectConfig.cmake         |  44 ++++++
 CMakeModules/FindAF_MKL.cmake                 |   8 ++
 CMakeModules/nsis/NSIS.definitions.nsh.in     |   2 +-
 CMakeModules/nsis/NSIS.template.in            |   2 +-
 CMakeModules/select_compute_arch.cmake        |  24 +++-
 LICENSE                                       |   2 +-
 docs/doxygen.mk                               | 129 +++++++++++-------
 src/backend/common/err_common.cpp             |   4 +-
 src/backend/common/err_common.hpp             |   2 +-
 src/backend/cpu/err_cpu.hpp                   |   4 +-
 src/backend/cuda/CMakeLists.txt               |   3 +
 src/backend/cuda/err_cuda.hpp                 |   4 +-
 src/backend/oneapi/Array.cpp                  |   6 +-
 src/backend/oneapi/CMakeLists.txt             |   2 +-
 src/backend/oneapi/copy.cpp                   |   3 +-
 src/backend/oneapi/err_oneapi.hpp             |   4 +-
 src/backend/oneapi/join.cpp                   |  15 +-
 src/backend/oneapi/kernel/index.hpp           |   5 +-
 src/backend/oneapi/kernel/ireduce.hpp         |   6 +-
 src/backend/oneapi/kernel/mean.hpp            |  12 +-
 src/backend/oneapi/kernel/reduce_all.hpp      |  10 +-
 .../oneapi/kernel/sort_by_key_impl.hpp        |   9 +-
 src/backend/oneapi/kernel/sparse_arith.hpp    |   3 +-
 src/backend/oneapi/kernel/wrap.hpp            |   2 +-
 src/backend/oneapi/platform.cpp               |   7 +-
 src/backend/oneapi/reduce_impl.hpp            |  63 ++++++---
 src/backend/opencl/err_opencl.hpp             |   4 +-
 test/CMakeLists.txt                           |   5 +
 test/anisotropic_diffusion.cpp                |  12 +-
 test/arrayfire_test.cpp                       |  20 ++-
 test/canny.cpp                                |  13 +-
 test/confidence_connected.cpp                 |  29 ++--
 test/fast.cpp                                 |   3 +
 test/gloh.cpp                                 |   7 +-
 test/hamming.cpp                              |   6 +
 test/harris.cpp                               |   2 +
 test/homography.cpp                           |  23 ++--
 test/hsv_rgb.cpp                              |   4 +
 test/imageio.cpp                              |   8 +-
 test/match_template.cpp                       |   3 +
 test/medfilt.cpp                              |  13 ++
 test/moments.cpp                              |   5 +
 test/morph.cpp                                |  39 ++++--
 test/nearest_neighbour.cpp                    |  16 +++
 test/orb.cpp                                  |   7 +-
 test/regions.cpp                              |   8 +-
 test/scan_by_key.cpp                          |   4 +
 test/sift.cpp                                 |   8 +-
 test/sobel.cpp                                |   2 +
 test/susan.cpp                                |   1 +
 test/testHelpers.hpp                          |  11 +-
 test/threading.cpp                            |   1 +
 54 files changed, 474 insertions(+), 173 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f3a1484a72..8e0c37c19f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -41,6 +41,9 @@ set_policies(
            CMP0074
            CMP0077
            CMP0079)
+if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.27")
+  cmake_policy(SET CMP0146 OLD)
+endif()
 arrayfire_set_cmake_default_variables()
 
 option(AF_WITH_EXTERNAL_PACKAGES_ONLY "Build ArrayFire with External packages only" OFF)
@@ -107,6 +110,7 @@ option(AF_WITH_SPDLOG_HEADER_ONLY "Build ArrayFire with header only version of s
 option(AF_WITH_FMT_HEADER_ONLY "Build ArrayFire with header only version of fmt" OFF)
 option(AF_WITH_FAST_MATH "Use lower precision but high performance numeric optimizations" OFF)
 option(AF_CTEST_SEPARATED "Run tests separately when called from ctest(increases test times)" OFF)
+option(AF_SKIP_UNSUPPORTED_TESTS "Skip tests where functions are unsupported by the backend instead of failing" OFF)
 
 if(AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   option(AF_WITH_PRUNE_STATIC_CUDA_NUMERIC_LIBS "Prune CUDA static libraries to reduce binary size.(WARNING: May break some libs on older CUDA toolkits for some compute arch)" OFF)
@@ -138,10 +142,10 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
   if(DEFINED ENV{MKLROOT} AND NOT DEFINED MKL_ROOT)
     set(MKL_ROOT "$ENV{MKLROOT}")
   endif()
-  set(DPCPP_COMPILER ON)
+  set(SYCL_COMPILER ON)
   set(MKL_THREADING "tbb_thread")
   set(MKL_INTERFACE "ilp64")
-  find_package(MKL 2023.1)
+  find_package(MKL 2024.1)
 endif()
 
 af_multiple_option(NAME        AF_COMPUTE_LIBRARY
@@ -554,6 +558,7 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE)
     get_filename_component(mkl_shd ${MKL_Core_LINK_LIBRARY} REALPATH)
     get_filename_component(mkl_tly ${MKL_ThreadLayer_LINK_LIBRARY} REALPATH)
     install(FILES
+      ${mkl_sycl}
       ${mkl_rnt}
       ${mkl_shd}
       ${mkl_tly}
diff --git a/CMakeModules/CMakeTestSYCLCompiler.cmake b/CMakeModules/CMakeTestSYCLCompiler.cmake
index e2f37a2da0..ef38081b37 100644
--- a/CMakeModules/CMakeTestSYCLCompiler.cmake
+++ b/CMakeModules/CMakeTestSYCLCompiler.cmake
@@ -66,8 +66,13 @@ if(NOT CMAKE_SYCL_COMPILER_WORKS)
 endif()
 
 # Try to identify the compiler features
-include(CMakeDetermineCompileFeatures)
-CMAKE_DETERMINE_COMPILE_FEATURES(SYCL)
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.30.0)
+    include(CMakeDetermineCompilerSupport)
+    CMAKE_DETERMINE_COMPILER_SUPPORT(CXX)
+else()
+    include(CMakeDetermineCompileFeatures)
+    CMAKE_DETERMINE_COMPILE_FEATURES(CXX)
+endif()
 
 set(CMAKE_TRY_COMPILE_CONFIGURATION "")
 # Re-configure to save learned information.
diff --git a/CMakeModules/CPackProjectConfig.cmake b/CMakeModules/CPackProjectConfig.cmake
index 6cd6e20088..f85dcaa556 100644
--- a/CMakeModules/CPackProjectConfig.cmake
+++ b/CMakeModules/CPackProjectConfig.cmake
@@ -343,6 +343,42 @@ af_component(
   DEB_OPTIONAL "cmake (>= 3.0)"
 )
 
+af_component(
+  COMPONENT oneapi
+  DISPLAY_NAME "oneAPI Runtime"
+  SUMMARY "ArrayFire oneAPI backend shared libraries"
+  DESCRIPTION "ArrayFire oneAPI backend shared libraries"
+  REQUIRES ${oneapi_deps_comps} licenses
+  OPTIONAL forge
+  GROUP afruntime
+  INSTALL_TYPES All Runtime
+
+  DEB_PACKAGE_NAME ${deb_oneapi_runtime_package_name}
+  DEB_PROVIDES "arrayfire-oneapi (= ${CPACK_PACKAGE_VERSION}), arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION}), libarrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-oneapi (<< ${CPACK_PACKAGE_VERSION}), arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_REQUIRES ${deb_oneapi_runtime_requirements}
+  DEB_USE_SHLIBDEPS
+  DEB_ADD_POSTINST
+  DEB_OPTIONAL forge libfreeimage3
+)
+
+af_component(
+  COMPONENT oneapi_dev
+  DISPLAY_NAME "oneAPI Dev"
+  SUMMARY  "ArrayFire oneAPI backend development files"
+  DESCRIPTION  "ArrayFire oneAPI backend development files"
+  REQUIRES oneapi headers cmake
+  GROUP afdevelopment
+  INSTALL_TYPES All Development
+
+  DEB_PACKAGE_NAME arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR}-dev
+  DEB_PROVIDES "arrayfire-oneapi-dev (= ${CPACK_PACKAGE_VERSION}), arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR}-dev (= ${CPACK_PACKAGE_VERSION}), libarrayfire-oneapi-dev (= ${CPACK_PACKAGE_VERSION})"
+  DEB_REPLACES "arrayfire-oneapi-dev (<< ${CPACK_PACKAGE_VERSION}), arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR}-dev (<< ${CPACK_PACKAGE_VERSION}), libarrayfire-oneapi-dev (<< ${CPACK_PACKAGE_VERSION})"
+  DEB_REQUIRES "arrayfire-oneapi${CPACK_PACKAGE_VERSION_MAJOR} (>= ${CPACK_PACKAGE_VERSION}), arrayfire-headers (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_RECOMMENDS "arrayfire-cmake (>= ${CPACK_PACKAGE_VERSION})"
+  DEB_OPTIONAL "cmake (>= 3.0)"
+)
+
 af_component(
   COMPONENT unified
   DISPLAY_NAME "Unified Runtime"
@@ -437,6 +473,14 @@ endif()
 # Debug symbols in debian installers are created using the DEBINFO property
 if(NOT APPLE AND
    NOT CPACK_GENERATOR MATCHES "DEB")
+  af_component(
+    COMPONENT afoneapi_debug_symbols
+    DISPLAY_NAME "oneAPI Debug Symbols"
+    DESCRIPTION "Debug symbols for the oneAPI backend."
+    GROUP debug
+    DISABLED
+    INSTALL_TYPES Development)
+
   af_component(
     COMPONENT afopencl_debug_symbols
     DISPLAY_NAME "OpenCL Debug Symbols"
diff --git a/CMakeModules/FindAF_MKL.cmake b/CMakeModules/FindAF_MKL.cmake
index a58809d495..88037c4519 100644
--- a/CMakeModules/FindAF_MKL.cmake
+++ b/CMakeModules/FindAF_MKL.cmake
@@ -321,6 +321,14 @@ endfunction()
 find_mkl_library(NAME Core LIBRARY_NAME mkl_core SEARCH_STATIC)
 find_mkl_library(NAME RT LIBRARY_NAME mkl_rt)
 
+if(AF_BUILD_ONEAPI)
+    find_mkl_library(NAME Sycl LIBRARY_NAME sycl DLL_ONLY)
+	find_mkl_library(NAME SyclLapack LIBRARY_NAME sycl_lapack DLL_ONLY)
+	find_mkl_library(NAME SyclDft LIBRARY_NAME sycl_dft DLL_ONLY)
+	find_mkl_library(NAME SyclBlas LIBRARY_NAME sycl_blas DLL_ONLY)
+	find_mkl_library(NAME SyclSparse LIBRARY_NAME sycl_sparse DLL_ONLY)
+endif()
+
 # MKL can link against Intel OpenMP, GNU OpenMP, TBB, and Sequential
 if(MKL_THREAD_LAYER STREQUAL "Intel OpenMP")
   find_mkl_library(NAME ThreadLayer LIBRARY_NAME mkl_intel_thread SEARCH_STATIC)
diff --git a/CMakeModules/nsis/NSIS.definitions.nsh.in b/CMakeModules/nsis/NSIS.definitions.nsh.in
index 4c6e8998b7..feedbd7c8d 100644
--- a/CMakeModules/nsis/NSIS.definitions.nsh.in
+++ b/CMakeModules/nsis/NSIS.definitions.nsh.in
@@ -3,7 +3,7 @@
 !define MUI_WELCOMEPAGE_TEXT    \
 "ArrayFire is a high performance software library for parallel computing with an easy-to-use API.\r\n\r\n\
 Its array based function set makes parallel programming simple.\r\n\r\n\
-ArrayFire's multiple backends (CUDA, OpenCL and native CPU) make it platform independent and highly portable.\r\n\r\n\
+ArrayFire's multiple backends (CUDA, OneAPI, OpenCL, and native CPU) make it platform independent and highly portable.\r\n\r\n\
 A few lines of code in ArrayFire can replace dozens of lines of parallel compute code, \
 saving you valuable time and lowering development costs.\r\n\r\n\
 Follow these steps to install the ArrayFire libraries."
diff --git a/CMakeModules/nsis/NSIS.template.in b/CMakeModules/nsis/NSIS.template.in
index bc3a44f233..971eea59bf 100644
--- a/CMakeModules/nsis/NSIS.template.in
+++ b/CMakeModules/nsis/NSIS.template.in
@@ -714,7 +714,7 @@ Section "-Core installation"
 
   ; make sure windows knows about the change
   SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000
-  MessageBox MB_OK "Added AF_PATH environment variable for all users.$\n$\nIf you chose not to modify PATH in the installer, please manually add $\"%AF_PATH%\lib$\" to the user or system PATH variable for running applications using ArrayFire."
+  MessageBox MB_OK "Added AF_PATH environment variable for all users.$\n$\nIf you chose not to modify PATH in the installer, please manually add $\"%AF_PATH%\lib$\" to the user or system PATH variable for running applications using ArrayFire." /SD IDOK
 
 
   ; Write special uninstall registry entries
diff --git a/CMakeModules/select_compute_arch.cmake b/CMakeModules/select_compute_arch.cmake
index 16abb8e6cd..e09490a7e5 100644
--- a/CMakeModules/select_compute_arch.cmake
+++ b/CMakeModules/select_compute_arch.cmake
@@ -7,7 +7,7 @@
 #      ARCH_AND_PTX : NAME | NUM.NUM | NUM.NUM(NUM.NUM) | NUM.NUM+PTX
 #      NAME: Fermi Kepler Maxwell Kepler+Tegra Kepler+Tesla Maxwell+Tegra Pascal Volta Turing Ampere
 #      NUM: Any number. Only those pairs are currently accepted by NVCC though:
-#            2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 7.0 7.2 7.5 8.0 8.6
+#            2.0 2.1 3.0 3.2 3.5 3.7 5.0 5.2 5.3 6.0 6.2 7.0 7.2 7.5 8.0 8.6 9.0
 #      Returns LIST of flags to be added to CUDA_NVCC_FLAGS in ${out_variable}
 #      Additionally, sets ${out_variable}_readable to the resulting numeric list
 #      Example:
@@ -92,6 +92,25 @@ if(CUDA_VERSION VERSION_GREATER_EQUAL "11.1")
   set(CUDA_LIMIT_GPU_ARCHITECTURE "9.0")
 endif()
 
+if(CUDA_VERSION VERSION_GREATER_EQUAL "11.8")
+  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "8.9")
+  list(APPEND CUDA_ALL_GPU_ARCHITECTURES "8.9")
+
+  set(_CUDA_MAX_COMMON_ARCHITECTURE "8.9+PTX")
+  set(CUDA_LIMIT_GPU_ARCHITECTURE "9.0")
+endif()
+
+if(CUDA_VERSION VERSION_GREATER_EQUAL "12.0")
+  list(APPEND CUDA_KNOWN_GPU_ARCHITECTURES "Hopper")
+  list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "9.0")
+  list(APPEND CUDA_ALL_GPU_ARCHITECTURES "9.0")
+
+  set(_CUDA_MAX_COMMON_ARCHITECTURE "9.0+PTX")
+  set(CUDA_LIMIT_GPU_ARCHITECTURE "9.0")
+
+  list(REMOVE_ITEM CUDA_ALL_GPU_ARCHITECTURES "3.5" "3.7")
+endif()
+
 list(APPEND CUDA_COMMON_GPU_ARCHITECTURES "${_CUDA_MAX_COMMON_ARCHITECTURE}")
 
 # Check with: cmake -DCUDA_VERSION=7.0 -P select_compute_arch.cmake
@@ -246,6 +265,9 @@ function(CUDA_SELECT_NVCC_ARCH_FLAGS out_variable)
       elseif(${arch_name} STREQUAL "Ampere")
         set(arch_bin 8.0)
         set(arch_ptx 8.0)
+      elseif(${arch_name} STREQUAL "Hopper")
+        set(arch_bin 9.0)
+        set(arch_ptx 9.0)
       else()
         message(SEND_ERROR "Unknown CUDA Architecture Name ${arch_name} in CUDA_SELECT_NVCC_ARCH_FLAGS")
       endif()
diff --git a/LICENSE b/LICENSE
index 8f4c645ca1..3d960db185 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2014-2022, ArrayFire
+Copyright (c) 2014-2024, ArrayFire
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
diff --git a/docs/doxygen.mk b/docs/doxygen.mk
index 914ebb35b4..9f46a1e37b 100644
--- a/docs/doxygen.mk
+++ b/docs/doxygen.mk
@@ -1,4 +1,4 @@
-# Doxyfile 1.9.6
+# Doxyfile 1.9.7
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -377,6 +377,17 @@ MARKDOWN_SUPPORT       = YES
 
 TOC_INCLUDE_HEADINGS   = 0
 
+# The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to
+# generate identifiers for the Markdown headings. Note: Every identifier is
+# unique.
+# Possible values are: DOXYGEN Use a fixed 'autotoc_md' string followed by a
+# sequence number starting at 0. and GITHUB Use the lower case version of title
+# with any whitespace replaced by '-' and punctations characters removed..
+# The default value is: DOXYGEN.
+# This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
+
+MARKDOWN_ID_STYLE      = DOXYGEN
+
 # When enabled doxygen tries to link words that correspond to documented
 # classes, or namespaces to their corresponding documentation. Such a link can
 # be prevented in individual cases by putting a % sign in front of the word or
@@ -501,6 +512,14 @@ LOOKUP_CACHE_SIZE      = 0
 
 NUM_PROC_THREADS       = 0
 
+# If the TIMESTAMP tag is set different from NO then each generated page will
+# contain the date or date and time when the page was generated. Setting this to
+# NO can help when comparing the output of multiple runs.
+# Possible values are: YES, NO, DATETIME and DATE.
+# The default value is: NO.
+
+TIMESTAMP              = YES
+
 #---------------------------------------------------------------------------
 # Build related configuration options
 #---------------------------------------------------------------------------
@@ -886,7 +905,14 @@ WARN_IF_UNDOC_ENUM_VAL = NO
 # a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS
 # then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but
 # at the end of the doxygen process doxygen will return with a non-zero status.
-# Possible values are: NO, YES and FAIL_ON_WARNINGS.
+# If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then doxygen behaves
+# like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined doxygen will not
+# write the warning messages in between other messages but write them at the end
+# of a run, in case a WARN_LOGFILE is defined the warning messages will be
+# besides being in the defined file also be shown at the end of a run, unless
+# the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case
+# the behavior will remain as with the setting FAIL_ON_WARNINGS.
+# Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT.
 # The default value is: NO.
 
 WARN_AS_ERROR          = NO
@@ -1012,9 +1038,6 @@ EXCLUDE_PATTERNS       = *.cpp
 # output. The symbol name can be a fully qualified name, a word, or if the
 # wildcard * is used, a substring. Examples: ANamespace, AClass,
 # ANamespace::AClass, ANamespace::*Test
-#
-# Note that the wildcards are matched against the file with absolute path, so to
-# exclude all test directories use the pattern */test/*
 
 EXCLUDE_SYMBOLS        =
 
@@ -1405,15 +1428,6 @@ HTML_COLORSTYLE_SAT    = 219
 
 HTML_COLORSTYLE_GAMMA  = 70
 
-# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
-# page will contain the date and time when the page was generated. Setting this
-# to YES can help to show when doxygen was last run and thus if the
-# documentation is up to date.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_HTML is set to YES.
-
-HTML_TIMESTAMP         = YES
-
 # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
 # documentation will contain a main index with vertical navigation menus that
 # are dynamically created via JavaScript. If disabled, the navigation index will
@@ -1563,6 +1577,16 @@ BINARY_TOC             = NO
 
 TOC_EXPAND             = NO
 
+# The SITEMAP_URL tag is used to specify the full URL of the place where the
+# generated documentation will be placed on the server by the user during the
+# deployment of the documentation. The generated sitemap is called sitemap.xml
+# and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL
+# is specified no sitemap is generated. For information about the sitemap
+# protocol see https://www.sitemaps.org
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SITEMAP_URL            =
+
 # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
 # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
 # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
@@ -2051,9 +2075,16 @@ PDF_HYPERLINKS         = YES
 
 USE_PDFLATEX           = YES
 
-# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
-# command to the generated LaTeX files. This will instruct LaTeX to keep running
-# if errors occur, instead of asking the user for help.
+# The LATEX_BATCHMODE tag ignals the behavior of LaTeX in case of an error.
+# Possible values are: NO same as ERROR_STOP, YES same as BATCH, BATCH In batch
+# mode nothing is printed on the terminal, errors are scrolled as if <return> is
+# hit at every error; missing files that TeX tries to input or request from
+# keyboard input (\read on a not open input stream) cause the job to abort,
+# NON_STOP In nonstop mode the diagnostic message will appear on the terminal,
+# but there is no possibility of user interaction just like in batch mode,
+# SCROLL In scroll mode, TeX will stop only for missing files to input or if
+# keyboard input is necessary and ERROR_STOP In errorstop mode, TeX will stop at
+# each error, asking for user intervention.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
@@ -2074,14 +2105,6 @@ LATEX_HIDE_INDICES     = NO
 
 LATEX_BIB_STYLE        = plain
 
-# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
-# page will contain the date and time when the page was generated. Setting this
-# to NO can help when comparing the output of multiple runs.
-# The default value is: NO.
-# This tag requires that the tag GENERATE_LATEX is set to YES.
-
-LATEX_TIMESTAMP        = NO
-
 # The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
 # path from which the emoji images will be read. If a relative path is entered,
 # it will be relative to the LATEX_OUTPUT directory. If left blank the
@@ -2247,7 +2270,7 @@ DOCBOOK_OUTPUT         = docbook
 #---------------------------------------------------------------------------
 
 # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
-# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
+# AutoGen Definitions (see https://autogen.sourceforge.net/) file that captures
 # the structure of the code including all documentation. Note that this feature
 # is still experimental and incomplete at the moment.
 # The default value is: NO.
@@ -2422,16 +2445,9 @@ EXTERNAL_GROUPS        = YES
 EXTERNAL_PAGES         = YES
 
 #---------------------------------------------------------------------------
-# Configuration options related to the dot tool
+# Configuration options related to diagram generator tools
 #---------------------------------------------------------------------------
 
-# You can include diagrams made with dia in doxygen documentation. Doxygen will
-# then run dia to produce the diagram and insert it in the documentation. The
-# DIA_PATH tag allows you to specify the directory where the dia binary resides.
-# If left empty dia is assumed to be found in the default search path.
-
-DIA_PATH               =
-
 # If set to YES the inheritance and collaboration graphs will hide inheritance
 # and usage relations if the target is undocumented or is not a class.
 # The default value is: YES.
@@ -2440,7 +2456,7 @@ HIDE_UNDOC_RELATIONS   = YES
 
 # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
 # available from the path. This tool is part of Graphviz (see:
-# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
 # Bell Labs. The other options in this section have no effect if this option is
 # set to NO
 # The default value is: NO.
@@ -2493,13 +2509,15 @@ DOT_NODE_ATTR          = "shape=box,height=0.2,width=0.4"
 
 DOT_FONTPATH           =
 
-# If the CLASS_GRAPH tag is set to YES (or GRAPH) then doxygen will generate a
-# graph for each documented class showing the direct and indirect inheritance
-# relations. In case HAVE_DOT is set as well dot will be used to draw the graph,
-# otherwise the built-in generator will be used. If the CLASS_GRAPH tag is set
-# to TEXT the direct and indirect inheritance relations will be shown as texts /
-# links.
-# Possible values are: NO, YES, TEXT and GRAPH.
+# If the CLASS_GRAPH tag is set to YES or GRAPH or BUILTIN then doxygen will
+# generate a graph for each documented class showing the direct and indirect
+# inheritance relations. In case the CLASS_GRAPH tag is set to YES or GRAPH and
+# HAVE_DOT is enabled as well, then dot will be used to draw the graph. In case
+# the CLASS_GRAPH tag is set to YES and HAVE_DOT is disabled or if the
+# CLASS_GRAPH tag is set to BUILTIN, then the built-in generator will be used.
+# If the CLASS_GRAPH tag is set to TEXT the direct and indirect inheritance
+# relations will be shown as texts / links.
+# Possible values are: NO, YES, TEXT, GRAPH and BUILTIN.
 # The default value is: YES.
 
 CLASS_GRAPH            = YES
@@ -2640,7 +2658,7 @@ DIR_GRAPH_MAX_DEPTH    = 1
 # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
 # generated by dot. For an explanation of the image formats see the section
 # output formats in the documentation of the dot tool (Graphviz (see:
-# http://www.graphviz.org/)).
+# https://www.graphviz.org/)).
 # Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
 # to make the SVG files visible in IE 9+ (other browsers do not have this
 # requirement).
@@ -2677,11 +2695,12 @@ DOT_PATH               =
 
 DOTFILE_DIRS           =
 
-# The MSCFILE_DIRS tag can be used to specify one or more directories that
-# contain msc files that are included in the documentation (see the \mscfile
-# command).
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
 
-MSCFILE_DIRS           =
+DIA_PATH               =
 
 # The DIAFILE_DIRS tag can be used to specify one or more directories that
 # contain dia files that are included in the documentation (see the \diafile
@@ -2758,3 +2777,19 @@ GENERATE_LEGEND        = YES
 # The default value is: YES.
 
 DOT_CLEANUP            = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. If the MSCGEN_TOOL tag is left empty (the default), then doxygen will
+# use a built-in version of mscgen tool to produce the charts. Alternatively,
+# the MSCGEN_TOOL tag can also specify the name an external tool. For instance,
+# specifying prog as the value, doxygen will call the tool as prog -T
+# <outfile_format> -o <outputfile> <inputfile>. The external tool should support
+# output file formats "png", "eps", "svg", and "ismap".
+
+MSCGEN_TOOL            =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS           =
diff --git a/src/backend/common/err_common.cpp b/src/backend/common/err_common.cpp
index 60fc207a63..672afe6da0 100644
--- a/src/backend/common/err_common.cpp
+++ b/src/backend/common/err_common.cpp
@@ -92,8 +92,8 @@ int ArgumentError::getArgIndex() const noexcept { return argIndex; }
 
 SupportError::SupportError(const char *const func, const char *const file,
                            const int line, const char *const back,
-                           stacktrace st)
-    : AfError(func, file, line, "Unsupported Error", AF_ERR_NOT_SUPPORTED,
+                           const char *const message, stacktrace st)
+    : AfError(func, file, line, message, AF_ERR_NOT_SUPPORTED,
               std::move(st))
     , backend(back) {}
 
diff --git a/src/backend/common/err_common.hpp b/src/backend/common/err_common.hpp
index e1e4a6d118..846f4b516f 100644
--- a/src/backend/common/err_common.hpp
+++ b/src/backend/common/err_common.hpp
@@ -113,7 +113,7 @@ class SupportError : public AfError {
 
    public:
     SupportError(const char* const func, const char* const file, const int line,
-                 const char* const back,
+                 const char* const back, const char* const message,
                  const boost::stacktrace::stacktrace st);
     SupportError(SupportError&& other) noexcept = default;
 
diff --git a/src/backend/cpu/err_cpu.hpp b/src/backend/cpu/err_cpu.hpp
index d618cecb1e..58c7b59aab 100644
--- a/src/backend/cpu/err_cpu.hpp
+++ b/src/backend/cpu/err_cpu.hpp
@@ -11,6 +11,6 @@
 
 #define CPU_NOT_SUPPORTED(message)                                          \
     do {                                                                    \
-        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \
-                           boost::stacktrace::stacktrace());                \
+        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, "CPU",   \
+                           message, boost::stacktrace::stacktrace());       \
     } while (0)
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 0c4563ed40..6d8731e1e1 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -860,6 +860,9 @@ if(AF_INSTALL_STANDALONE)
     endif()
     afcu_collect_libs(cusolver)
     afcu_collect_libs(cusparse)
+    if(CUDA_VERSION VERSION_GREATER 12.0)
+      afcu_collect_libs(nvJitLink)
+    endif()
   elseif(NOT ${use_static_cuda_lapack})
     afcu_collect_libs(cusolver)
   endif()
diff --git a/src/backend/cuda/err_cuda.hpp b/src/backend/cuda/err_cuda.hpp
index 77926cdd79..f6db7e6822 100644
--- a/src/backend/cuda/err_cuda.hpp
+++ b/src/backend/cuda/err_cuda.hpp
@@ -14,8 +14,8 @@
 
 #define CUDA_NOT_SUPPORTED(message)                                         \
     do {                                                                    \
-        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \
-                           boost::stacktrace::stacktrace());                \
+        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, "CUDA",  \
+                           message, boost::stacktrace::stacktrace());       \
     } while (0)
 
 #define CU_CHECK(fn)                                                          \
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index f2ef09c044..8165e6fb08 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -500,10 +500,11 @@ template<typename T>
 void writeHostDataArray(Array<T> &arr, const T *const data,
                         const size_t bytes) {
     if (!arr.isOwner()) { arr = copyArray<T>(arr); }
+    auto arr_get = arr.get();
     getQueue()
         .submit([&](sycl::handler &h) {
             auto host_acc =
-                arr.get()->template get_access<sycl::access_mode::write>(
+                arr_get->template get_access<sycl::access_mode::write>(
                     h, sycl::range(bytes / sizeof(T)), arr.getOffset());
             h.copy(data, host_acc);
         })
@@ -517,10 +518,11 @@ void writeDeviceDataArray(Array<T> &arr, const void *const data,
 
     sycl::buffer<T> *dataptr =
         static_cast<sycl::buffer<T> *>(const_cast<void *>(data));
+    auto arr_get = arr.get();
     getQueue().submit([&](sycl::handler &h) {
         auto src_acc = dataptr->template get_access<sycl::access_mode::read>(
             h, sycl::range(bytes / sizeof(T)));
-        auto dst_acc = arr.get()->template get_access<sycl::access_mode::write>(
+        auto dst_acc = arr_get->template get_access<sycl::access_mode::write>(
             h, sycl::range(bytes / sizeof(T)), arr.getOffset());
         h.copy(src_acc, dst_acc);
     });
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 9bd7e0850a..054681d812 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -359,7 +359,7 @@ target_link_libraries(afoneapi
     $<$<PLATFORM_ID:Linux>:-fvisibility-inlines-hidden>
     $<$<PLATFORM_ID:Linux>:-fno-sycl-rdc>
     -fsycl-max-parallel-link-jobs=${NumberOfThreads}
-    MKL::MKL_DPCPP
+    MKL::MKL_SYCL
   )
   set_sycl_language(afcommon_interface
     oneapi_sort_by_key
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index f99f79854e..506206b11e 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -216,10 +216,11 @@ template<typename T>
 T getScalar(const Array<T> &in) {
     T retVal{};
 
+    auto in_get = in.get();
     getQueue()
         .submit([&](sycl::handler &h) {
             auto acc_in =
-                in.get()->template get_access<sycl::access::mode::read>(
+                in_get->template get_access<sycl::access::mode::read>(
                     h, sycl::range{1},
                     sycl::id{static_cast<uintl>(in.getOffset())});
             h.copy(acc_in, &retVal);
diff --git a/src/backend/oneapi/err_oneapi.hpp b/src/backend/oneapi/err_oneapi.hpp
index fad7d449c0..4f187b6273 100644
--- a/src/backend/oneapi/err_oneapi.hpp
+++ b/src/backend/oneapi/err_oneapi.hpp
@@ -13,8 +13,8 @@
 
 #define ONEAPI_NOT_SUPPORTED(message)                                       \
     do {                                                                    \
-        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \
-                           boost::stacktrace::stacktrace());                \
+        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, "oneAPI",\
+                           message, boost::stacktrace::stacktrace());       \
     } while (0)
 
 #define CL_CHECK(call)                                                      \
diff --git a/src/backend/oneapi/join.cpp b/src/backend/oneapi/join.cpp
index 37c7c14fc9..e95b63c392 100644
--- a/src/backend/oneapi/join.cpp
+++ b/src/backend/oneapi/join.cpp
@@ -94,15 +94,17 @@ Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     if (first.isReady()) {
         if (1LL + jdim >= first.ndims() && first.isLinear()) {
             // first & out are linear
+            auto first_array = first.get();
+            auto out_array = out.get();
             getQueue().submit([&](sycl::handler &h) {
                 sycl::range sz(first.elements());
                 sycl::id src_offset(first.getOffset());
                 sycl::accessor offset_acc_src =
-                    first.get()->template get_access<sycl::access_mode::read>(
+                    first_array->template get_access<sycl::access_mode::read>(
                         h, sz, src_offset);
                 sycl::id dst_offset(0);
                 sycl::accessor offset_acc_dst =
-                    out.get()->template get_access<sycl::access_mode::write>(
+                    out_array->template get_access<sycl::access_mode::write>(
                         h, sz, dst_offset);
                 h.copy(offset_acc_src, offset_acc_dst);
             });
@@ -125,16 +127,18 @@ Array<T> join(const int jdim, const Array<T> &first, const Array<T> &second) {
     if (second.isReady()) {
         if (1LL + jdim >= second.ndims() && second.isLinear()) {
             // second & out are linear
+            auto second_array = second.get();
+            auto out_array = out.get();
             getQueue().submit([&](sycl::handler &h) {
                 sycl::range sz(second.elements());
                 sycl::id src_offset(second.getOffset());
                 sycl::accessor offset_acc_src =
-                    second.get()->template get_access<sycl::access_mode::read>(
+                    second_array->template get_access<sycl::access_mode::read>(
                         h, sz, src_offset);
                 sycl::id dst_offset(fdims.dims[jdim] *
                                     out.strides().dims[jdim]);
                 sycl::accessor offset_acc_dst =
-                    out.get()->template get_access<sycl::access_mode::write>(
+                    out_array->template get_access<sycl::access_mode::write>(
                         h, sz, dst_offset);
                 h.copy(offset_acc_src, offset_acc_dst);
             });
@@ -216,11 +220,12 @@ void join(Array<T> &out, const int jdim, const vector<Array<T>> &inputs) {
             for (const Array<T> *in : s.ins) {
                 if (in->isReady()) {
                     if (1LL + jdim >= in->ndims() && in->isLinear()) {
+                        auto in_array = in->get();
                         getQueue().submit([&](sycl::handler &h) {
                             sycl::range sz(in->elements());
                             sycl::id src_offset(in->getOffset());
                             sycl::accessor offset_acc_src =
-                                in->get()
+                                in_array
                                     ->template get_access<
                                         sycl::access_mode::read>(h, sz,
                                                                  src_offset);
diff --git a/src/backend/oneapi/kernel/index.hpp b/src/backend/oneapi/kernel/index.hpp
index c7bb591953..e86c0bd808 100644
--- a/src/backend/oneapi/kernel/index.hpp
+++ b/src/backend/oneapi/kernel/index.hpp
@@ -137,11 +137,14 @@ void index(Param<T> out, Param<T> in, IndexKernelParam& p,
     blocks[0] *= threads[0];
 
     sycl::nd_range<3> marange(blocks, threads);
+    sycl::buffer<uint> *idxArrs_get[4];
+    for (dim_t x = 0; x < 4; ++x)
+        idxArrs_get[x] = idxArrs[x].get();
     getQueue().submit([&](sycl::handler& h) {
         auto pp = p;
         for (dim_t x = 0; x < 4; ++x) {
             pp.ptr[x] =
-                idxArrs[x].get()->get_access<sycl::access::mode::read>(h);
+                idxArrs_get[x]->get_access<sycl::access::mode::read>(h);
         }
 
         h.parallel_for(
diff --git a/src/backend/oneapi/kernel/ireduce.hpp b/src/backend/oneapi/kernel/ireduce.hpp
index 5f8f96bfc8..9ba79ed61b 100644
--- a/src/backend/oneapi/kernel/ireduce.hpp
+++ b/src/backend/oneapi/kernel/ireduce.hpp
@@ -258,7 +258,7 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
             case 4:
                 h.parallel_for(
                     sycl::nd_range<2>(global, local),
-                    ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                    ireduceDimKernelSMEM<T, op, dim, is_first, 4>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
                         groups_dim[dim], rlenValid, rlen_acc, rlen.info,
@@ -267,7 +267,7 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
             case 2:
                 h.parallel_for(
                     sycl::nd_range<2>(global, local),
-                    ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                    ireduceDimKernelSMEM<T, op, dim, is_first, 2>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
                         groups_dim[dim], rlenValid, rlen_acc, rlen.info,
@@ -276,7 +276,7 @@ void ireduce_dim_launcher(Param<T> out, Param<uint> oloc, Param<T> in,
             case 1:
                 h.parallel_for(
                     sycl::nd_range<2>(global, local),
-                    ireduceDimKernelSMEM<T, op, dim, is_first, 8>(
+                    ireduceDimKernelSMEM<T, op, dim, is_first, 1>(
                         out_acc, out.info, oloc_acc, oloc.info, in_acc, in.info,
                         iloc_acc, iloc.info, groups_dim[0], groups_dim[1],
                         groups_dim[dim], rlenValid, rlen_acc, rlen.info,
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index 695fb7b375..d6f33209a9 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -609,12 +609,14 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
                                       blocks_y, threads_x);
 
         compute_t<T> val;
+        auto tmpOut_get = tmpOut.get();
+        auto tmpWt_get = tmpWt.get();
         getQueue()
             .submit([&](sycl::handler &h) {
                 auto acc_in =
-                    tmpOut.get()->template get_host_access(h, sycl::read_only);
+                    tmpOut_get->template get_host_access(h, sycl::read_only);
                 auto acc_wt =
-                    tmpWt.get()->template get_host_access(h, sycl::read_only);
+                    tmpWt_get->template get_host_access(h, sycl::read_only);
 
                 h.host_task([acc_in, acc_wt, tmp_elements, &val] {
                     val = static_cast<compute_t<T>>(acc_in[0]);
@@ -693,12 +695,14 @@ To mean_all(Param<Ti> in) {
         uintl tmp_elements = tmpOut.elements();
 
         compute_t<To> val;
+        auto tmpOut_get = tmpOut.get();
+        auto tmpCt_get = tmpCt.get();
         getQueue()
             .submit([&](sycl::handler &h) {
                 auto out =
-                    tmpOut.get()->template get_host_access(h, sycl::read_only);
+                    tmpOut_get->template get_host_access(h, sycl::read_only);
                 auto ct =
-                    tmpCt.get()->template get_host_access(h, sycl::read_only);
+                    tmpCt_get->template get_host_access(h, sycl::read_only);
 
                 h.host_task([out, ct, tmp_elements, &val] {
                     val                  = static_cast<compute_t<To>>(out[0]);
diff --git a/src/backend/oneapi/kernel/reduce_all.hpp b/src/backend/oneapi/kernel/reduce_all.hpp
index 4bc3d5254d..7a1e842425 100644
--- a/src/backend/oneapi/kernel/reduce_all.hpp
+++ b/src/backend/oneapi/kernel/reduce_all.hpp
@@ -249,13 +249,17 @@ void reduce_all_launcher_default(Param<To> out, Param<Ti> in,
             "Too many blocks requested (typeof(retirementCount) == unsigned)",
             AF_ERR_RUNTIME);
     }
-    Array<To> tmp = createEmptyArray<To>(tmp_elements);
 
+    Array<To> tmp = createEmptyArray<To>(tmp_elements);
+    auto tmp_get = tmp.get();
+    
     Array<unsigned> retirementCount = createValueArray<unsigned>(1, 0);
+    auto ret_get = retirementCount.get();
+
     getQueue().submit([&](sycl::handler &h) {
         write_accessor<To> out_acc{*out.data, h};
-        auto retCount_acc = retirementCount.get()->get_access(h);
-        auto tmp_acc      = tmp.get()->get_access(h);
+        auto retCount_acc = ret_get->get_access(h);
+        auto tmp_acc      = tmp_get->get_access(h);
         read_accessor<Ti> in_acc{*in.data, h};
 
         auto shrdMem = sycl::local_accessor<compute_t<To>, 1>(
diff --git a/src/backend/oneapi/kernel/sort_by_key_impl.hpp b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
index 5a05eac58c..6e3a0bd655 100644
--- a/src/backend/oneapi/kernel/sort_by_key_impl.hpp
+++ b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
@@ -114,10 +114,11 @@ void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
     auto val_end   = val_begin + elements;
 
     auto cKey = memAlloc<Tk>(elements);
+    auto cKey_get = cKey.get();
     getQueue().submit([&](sycl::handler &h) {
         h.copy(pKey.data->template reinterpret<compute_t<Tk>>().get_access(
                    h, elements),
-               cKey.get()->template reinterpret<compute_t<Tk>>().get_access(
+               cKey_get->template reinterpret<compute_t<Tk>>().get_access(
                    h, elements));
     });
     auto ckey_begin =
@@ -150,10 +151,12 @@ void sortByKeyBatched(Param<Tk> pKey, Param<Tv> pVal, const int dim,
         }
     }
 
+    auto Seq_get = Seq.get();
     auto cSeq = memAlloc<uint>(elements);
+    auto cSeq_get = cSeq.get();
     getQueue().submit([&](sycl::handler &h) {
-        h.copy(Seq.get()->get_access(h, elements),
-               cSeq.get()->get_access(h, elements));
+        h.copy(Seq_get->get_access(h, elements),
+               cSeq_get->get_access(h, elements));
     });
     auto cseq_begin = ::oneapi::dpl::begin(*cSeq.get());
     auto cseq_end   = cseq_begin + elements;
diff --git a/src/backend/oneapi/kernel/sparse_arith.hpp b/src/backend/oneapi/kernel/sparse_arith.hpp
index 819af6ffce..b46baa69df 100644
--- a/src/backend/oneapi/kernel/sparse_arith.hpp
+++ b/src/backend/oneapi/kernel/sparse_arith.hpp
@@ -427,9 +427,10 @@ static void csrCalcOutNNZ(Param<int> outRowIdx, unsigned &nnzC, const uint M,
     auto global = sycl::range(divup(M, local[0]) * local[0]);
 
     Array<unsigned> out = createValueArray<unsigned>(1, 0);
+    auto out_get = out.get();
 
     getQueue().submit([&](auto &h) {
-        sycl::accessor d_out{*out.get(), h, sycl::write_only};
+        sycl::accessor d_out{*out_get, h, sycl::write_only};
         sycl::accessor d_outRowIdx{*outRowIdx.data, h, sycl::write_only};
         sycl::accessor d_lRowIdx{*lrowIdx.data, h, sycl::read_only};
         sycl::accessor d_lColIdx{*lcolIdx.data, h, sycl::read_only};
diff --git a/src/backend/oneapi/kernel/wrap.hpp b/src/backend/oneapi/kernel/wrap.hpp
index b5e5226035..e29403b604 100644
--- a/src/backend/oneapi/kernel/wrap.hpp
+++ b/src/backend/oneapi/kernel/wrap.hpp
@@ -140,7 +140,7 @@ void wrap(Param<T> out, const Param<T> in, const dim_t wx, const dim_t wy,
     dim_t groups_y = divup(out.info.dims[1], local[1]);
 
     auto global = sycl::range{groups_x * local[0] * out.info.dims[2],
-                              groups_y * local[1]};
+                              groups_y * local[1] * out.info.dims[3]};
 
     auto Q = getQueue();
     Q.submit([&](sycl::handler &h) {
diff --git a/src/backend/oneapi/platform.cpp b/src/backend/oneapi/platform.cpp
index 91e307d56c..3994a907a5 100644
--- a/src/backend/oneapi/platform.cpp
+++ b/src/backend/oneapi/platform.cpp
@@ -164,7 +164,9 @@ string getDeviceInfo() noexcept {
                  << ", " << msize / 1048576 << " MB";
             info << " (";
             if (device->has(aspect::fp64)) { info << "fp64 "; }
-            if (device->has(aspect::fp16)) { info << "fp16 "; }
+            if (device->has(aspect::fp16) &&
+                device->get_info<sycl::info::device::native_vector_width_half>() != 0)
+                { info << "fp16 "; }
             info << "\b)";
 #ifndef NDEBUG
             info << " -- ";
@@ -386,7 +388,8 @@ bool isHalfSupported(unsigned device) {
     DeviceManager& devMngr = DeviceManager::getInstance();
 
     common::lock_guard_t lock(devMngr.deviceMutex);
-    return devMngr.mDevices[device]->has(sycl::aspect::fp16);
+    return devMngr.mDevices[device]->has(sycl::aspect::fp16) &&
+           devMngr.mDevices[device]->get_info<sycl::info::device::native_vector_width_half>() != 0;
 }
 
 void devprop(char* d_name, char* d_platform, char* d_toolkit, char* d_compute) {
diff --git a/src/backend/oneapi/reduce_impl.hpp b/src/backend/oneapi/reduce_impl.hpp
index 698f2f1831..b2c478c71f 100644
--- a/src/backend/oneapi/reduce_impl.hpp
+++ b/src/backend/oneapi/reduce_impl.hpp
@@ -58,12 +58,16 @@ void reduceBlocksByKey(sycl::buffer<int> &reduced_block_sizes,
     sycl::range<3> global(local[0] * numBlocks, vals_out.dims()[1],
                           vals_out.dims()[2] * vals_out.dims()[3]);
 
+    auto keys_out_get = keys_out.get();
+    auto vals_out_get = vals_out.get();
+    auto keys_get = keys.get();
+    auto vals_get = vals.get();
     getQueue().submit([&](sycl::handler &h) {
         sycl::accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
-        write_accessor<Tk> keys_out_acc{*keys_out.get(), h};
-        write_accessor<To> vals_out_acc{*vals_out.get(), h};
-        read_accessor<Tk> keys_acc{*keys.get(), h};
-        read_accessor<Ti> vals_acc{*vals.get(), h};
+        write_accessor<Tk> keys_out_acc{*keys_out_get, h};
+        write_accessor<To> vals_out_acc{*vals_out_get, h};
+        read_accessor<Tk> keys_acc{*keys_get, h};
+        read_accessor<Ti> vals_acc{*vals_get, h};
 
         auto l_keys         = sycl::local_accessor<Tk>(threads_x, h);
         auto l_vals         = sycl::local_accessor<compute_t<To>>(threads_x, h);
@@ -100,12 +104,16 @@ void reduceBlocksByKeyDim(sycl::buffer<int> &reduced_block_sizes,
         local[0] * numBlocks, vals_out.dims()[dim_ordering[1]],
         vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]);
 
+    auto keys_out_get = keys_out.get();
+    auto vals_out_get = vals_out.get();
+    auto keys_get = keys.get();
+    auto vals_get = vals.get();
     getQueue().submit([&](sycl::handler &h) {
         sycl::accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
-        write_accessor<Tk> keys_out_acc{*keys_out.get(), h};
-        write_accessor<To> vals_out_acc{*vals_out.get(), h};
-        read_accessor<Tk> keys_acc{*keys.get(), h};
-        read_accessor<Ti> vals_acc{*vals.get(), h};
+        write_accessor<Tk> keys_out_acc{*keys_out_get, h};
+        write_accessor<To> vals_out_acc{*vals_out_get, h};
+        read_accessor<Tk> keys_acc{*keys_get, h};
+        read_accessor<Ti> vals_acc{*vals_get, h};
 
         auto l_keys         = sycl::local_accessor<Tk>(threads_x, h);
         auto l_vals         = sycl::local_accessor<compute_t<To>>(threads_x, h);
@@ -135,10 +143,12 @@ void finalBoundaryReduce(sycl::buffer<int> &reduced_block_sizes, Array<Tk> keys,
     sycl::range<1> local(threads_x);
     sycl::range<1> global(local[0] * numBlocks);
 
+    auto vals_out_get = vals_out.get();
+    auto keys_get = keys.get();
     getQueue().submit([&](sycl::handler &h) {
         write_accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
-        read_accessor<Tk> keys_acc{*keys.get(), h};
-        sycl::accessor<To> vals_out_acc{*vals_out.get(), h};
+        read_accessor<Tk> keys_acc{*keys_get, h};
+        sycl::accessor<To> vals_out_acc{*vals_out_get, h};
 
         h.parallel_for(sycl::nd_range<1>(global, local),
                        kernel::finalBoundaryReduceKernel<Tk, To, op>(
@@ -158,10 +168,12 @@ void finalBoundaryReduceDim(sycl::buffer<int> &reduced_block_sizes,
         local[0] * numBlocks, vals_out.dims()[dim_ordering[1]],
         vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]);
 
+    auto vals_out_get = vals_out.get();
+    auto keys_get = keys.get();
     getQueue().submit([&](sycl::handler &h) {
         write_accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
-        read_accessor<Tk> keys_acc{*keys.get(), h};
-        sycl::accessor<To> vals_out_acc{*vals_out.get(), h};
+        read_accessor<Tk> keys_acc{*keys_get, h};
+        sycl::accessor<To> vals_out_acc{*vals_out_get, h};
 
         // TODO: fold 3,4 dimensions
         h.parallel_for(
@@ -181,12 +193,16 @@ void compact(sycl::buffer<int> reduced_block_sizes, Array<Tk> &keys_out,
     sycl::range<3> global(local[0] * numBlocks, vals_out.dims()[1],
                           vals_out.dims()[2] * vals_out.dims()[3]);
 
+    auto keys_out_get = keys_out.get();
+    auto vals_out_get = vals_out.get();
+    auto keys_get = keys.get();
+    auto vals_get = vals.get();
     getQueue().submit([&](sycl::handler &h) {
         read_accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
-        write_accessor<Tk> keys_out_acc{*keys_out.get(), h};
-        write_accessor<To> vals_out_acc{*vals_out.get(), h};
-        read_accessor<Tk> keys_acc{*keys.get(), h};
-        read_accessor<To> vals_acc{*vals.get(), h};
+        write_accessor<Tk> keys_out_acc{*keys_out_get, h};
+        write_accessor<To> vals_out_acc{*vals_out_get, h};
+        read_accessor<Tk> keys_acc{*keys_get, h};
+        read_accessor<To> vals_acc{*vals_get, h};
 
         h.parallel_for(sycl::nd_range<3>(global, local),
                        kernel::compactKernel<Tk, To>(
@@ -207,12 +223,16 @@ void compactDim(sycl::buffer<int> &reduced_block_sizes, Array<Tk> &keys_out,
         local[0] * numBlocks, vals_out.dims()[dim_ordering[1]],
         vals_out.dims()[dim_ordering[2]] * vals_out.dims()[dim_ordering[3]]);
 
+    auto keys_out_get = keys_out.get();
+    auto vals_out_get = vals_out.get();
+    auto keys_get = keys.get();
+    auto vals_get = vals.get();
     getQueue().submit([&](sycl::handler &h) {
         read_accessor<int> reduced_block_sizes_acc{reduced_block_sizes, h};
-        write_accessor<Tk> keys_out_acc{*keys_out.get(), h};
-        write_accessor<To> vals_out_acc{*vals_out.get(), h};
-        read_accessor<Tk> keys_acc{*keys.get(), h};
-        read_accessor<To> vals_acc{*vals.get(), h};
+        write_accessor<Tk> keys_out_acc{*keys_out_get, h};
+        write_accessor<To> vals_out_acc{*vals_out_get, h};
+        read_accessor<Tk> keys_acc{*keys_get, h};
+        read_accessor<To> vals_acc{*vals_get, h};
 
         h.parallel_for(
             sycl::nd_range<3>(global, local),
@@ -231,10 +251,11 @@ void testNeedsReduction(sycl::buffer<int> needs_reduction,
     sycl::range<1> local(threads_x);
     sycl::range<1> global(local[0] * numBlocks);
 
+    auto keys_get = keys.get();
     getQueue().submit([&](sycl::handler &h) {
         sycl::accessor<int> needs_reduction_acc{needs_reduction, h};
         sycl::accessor<int> needs_boundary_acc{needs_boundary, h};
-        read_accessor<Tk> keys_acc{*keys.get(), h};
+        read_accessor<Tk> keys_acc{*keys_get, h};
         auto l_keys = sycl::local_accessor<Tk>(threads_x, h);
 
         h.parallel_for(sycl::nd_range<1>(global, local),
diff --git a/src/backend/opencl/err_opencl.hpp b/src/backend/opencl/err_opencl.hpp
index 2c1187c569..9a24bc2789 100644
--- a/src/backend/opencl/err_opencl.hpp
+++ b/src/backend/opencl/err_opencl.hpp
@@ -27,6 +27,6 @@ std::string getProgramBuildLog(const cl::Program &prog);
 
 #define OPENCL_NOT_SUPPORTED(message)                                       \
     do {                                                                    \
-        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, message, \
-                           boost::stacktrace::stacktrace());                \
+        throw SupportError(__AF_FUNC__, __AF_FILENAME__, __LINE__, "OpenCL",\
+                           message, boost::stacktrace::stacktrace());       \
     } while (0)
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 95bab411bc..3fae5d68ec 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -259,6 +259,11 @@ function(make_test)
         MTX_TEST_DIR="${ArrayFire_BINARY_DIR}/extern/matrixmarket/"
         )
     endif()
+    if(AF_SKIP_UNSUPPORTED_TESTS)
+      target_compile_definitions(${target}
+        PRIVATE
+          SKIP_UNSUPPORTED_TESTS)
+    endif()
     if(WIN32)
       target_compile_definitions(${target}
         PRIVATE
diff --git a/test/anisotropic_diffusion.cpp b/test/anisotropic_diffusion.cpp
index afeda45d52..60e3c75324 100644
--- a/test/anisotropic_diffusion.cpp
+++ b/test/anisotropic_diffusion.cpp
@@ -98,12 +98,12 @@ void imageTest(string pTestFile, const float dt, const float K,
 
         if (isCurvatureDiffusion) {
             ASSERT_SUCCESS(af_anisotropic_diffusion(&_outArray, inArray, dt, K,
-                                                    iters, fluxKind,
-                                                    AF_DIFFUSION_MCDE));
+                                                                iters, fluxKind,
+                                                                AF_DIFFUSION_MCDE));
         } else {
             ASSERT_SUCCESS(af_anisotropic_diffusion(&_outArray, inArray, dt, K,
-                                                    iters, fluxKind,
-                                                    AF_DIFFUSION_GRAD));
+                                                                iters, fluxKind,
+                                                                AF_DIFFUSION_GRAD));
         }
 
         double maxima, minima, imag;
@@ -142,6 +142,7 @@ void imageTest(string pTestFile, const float dt, const float K,
 }
 
 TYPED_TEST(AnisotropicDiffusion, GradientGrayscale) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     // Numeric values separated by underscore are arguments to fn being tested.
     // Divide first value by 1000 to get time step `dt`
     // Divide second value by 100 to get time step `K`
@@ -153,6 +154,7 @@ TYPED_TEST(AnisotropicDiffusion, GradientGrayscale) {
 }
 
 TYPED_TEST(AnisotropicDiffusion, GradientColorImage) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     imageTest<TypeParam, true>(
         string(TEST_DIR "/gradient_diffusion/color_00125_100_2_exp.test"),
         0.125f, 1.0, 2, AF_FLUX_EXPONENTIAL);
@@ -166,6 +168,7 @@ TEST(AnisotropicDiffusion, GradientInvalidInputArray) {
 }
 
 TYPED_TEST(AnisotropicDiffusion, CurvatureGrayscale) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     // Numeric values separated by underscore are arguments to fn being tested.
     // Divide first value by 1000 to get time step `dt`
     // Divide second value by 100 to get time step `K`
@@ -177,6 +180,7 @@ TYPED_TEST(AnisotropicDiffusion, CurvatureGrayscale) {
 }
 
 TYPED_TEST(AnisotropicDiffusion, CurvatureColorImage) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     imageTest<TypeParam, true>(
         string(TEST_DIR "/curvature_diffusion/color_00125_100_2_mcde.test"),
         0.125f, 1.0, 2, AF_FLUX_EXPONENTIAL, true);
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index 4c6e966220..db1f67a341 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -102,14 +102,20 @@ std::string readNextNonEmptyLine(std::ifstream &file) {
     return result;
 }
 
-std::string getBackendName() {
+std::string getBackendName(bool lower) {
     af::Backend backend = af::getActiveBackend();
-    if (backend == AF_BACKEND_OPENCL)
-        return std::string("opencl");
-    else if (backend == AF_BACKEND_CUDA)
-        return std::string("cuda");
-
-    return std::string("cpu");
+    switch(backend) {
+    case AF_BACKEND_CPU:
+        return lower ? std::string("cpu") : std::string("CPU");
+    case AF_BACKEND_CUDA:
+        return lower ? std::string("cuda") : std::string("CUDA");
+    case AF_BACKEND_OPENCL:
+        return lower ? std::string("opencl") : std::string("OpenCL");
+    case AF_BACKEND_ONEAPI:
+        return lower ? std::string("oneapi") : std::string("oneAPI");
+    default:
+        return lower ? std::string("unknown") : std::string("Unknown");
+    }
 }
 
 std::string getTestName() {
diff --git a/test/canny.cpp b/test/canny.cpp
index 7f2fa2918c..a12ac73965 100644
--- a/test/canny.cpp
+++ b/test/canny.cpp
@@ -53,7 +53,7 @@ void cannyTest(string pTestFile) {
                                    (af_dtype)dtype_traits<T>::af_type));
 
     ASSERT_SUCCESS(af_canny(&outArray, sArray, AF_CANNY_THRESHOLD_MANUAL,
-                            0.4147f, 0.8454f, 3, true));
+                                        0.4147f, 0.8454f, 3, true));
 
     vector<char> outData(sDims.elements());
 
@@ -72,10 +72,12 @@ void cannyTest(string pTestFile) {
 }
 
 TYPED_TEST(CannyEdgeDetector, ArraySizeLessThanBlockSize10x10) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     cannyTest<TypeParam>(string(TEST_DIR "/CannyEdgeDetector/fast10x10.test"));
 }
 
 TYPED_TEST(CannyEdgeDetector, ArraySizeEqualBlockSize16x16) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     cannyTest<TypeParam>(string(TEST_DIR "/CannyEdgeDetector/fast16x16.test"));
 }
 
@@ -129,8 +131,9 @@ void cannyImageOtsuTest(string pTestFile, bool isColor) {
             af_load_image_native(&goldArray, outFiles[testId].c_str()));
 
         ASSERT_SUCCESS(af_canny(&_outArray, inArray,
-                                AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3,
-                                false));
+                                            AF_CANNY_THRESHOLD_AUTO_OTSU,
+                                            0.08, 0.32, 3, false));
+
         unsigned ndims = 0;
         dim_t dims[4];
 
@@ -156,6 +159,7 @@ void cannyImageOtsuTest(string pTestFile, bool isColor) {
 }
 
 TEST(CannyEdgeDetector, OtsuThreshold) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     cannyImageOtsuTest<float>(string(TEST_DIR "/CannyEdgeDetector/gray.test"),
                               false);
 }
@@ -248,7 +252,7 @@ void cannyImageOtsuBatchTest(string pTestFile, const dim_t targetBatchCount) {
         array inputIm  = tile(readImg, 1, 1, targetBatchCount);
 
         array outIm =
-            canny(inputIm, AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3, false);
+              canny(inputIm, AF_CANNY_THRESHOLD_AUTO_OTSU, 0.08, 0.32, 3, false);
         outIm *= 255.0;
 
         ASSERT_IMAGES_NEAR(goldIm, outIm.as(u8), 1.0e-3);
@@ -256,6 +260,7 @@ void cannyImageOtsuBatchTest(string pTestFile, const dim_t targetBatchCount) {
 }
 
 TEST(CannyEdgeDetector, BatchofImagesUsingCPPAPI) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     // DO NOT INCREASE BATCH COUNT BEYOND 4
     // This is a limitation on the test assert macro that is saving
     // images to disk which can't handle a batch of images.
diff --git a/test/confidence_connected.cpp b/test/confidence_connected.cpp
index ac5b0bf2bc..22254e5532 100644
--- a/test/confidence_connected.cpp
+++ b/test/confidence_connected.cpp
@@ -41,17 +41,6 @@ struct CCCTestParams {
     double replace;
 };
 
-void apiWrapper(af_array *out, const af_array in, const af_array seedx,
-                const af_array seedy, const CCCTestParams params) {
-    ASSERT_SUCCESS(af_confidence_cc(out, in, seedx, seedy, params.radius,
-                                    params.multiplier, params.iterations,
-                                    params.replace));
-
-    int device = 0;
-    ASSERT_SUCCESS(af_get_device(&device));
-    ASSERT_SUCCESS(af_sync(device));
-}
-
 template<typename T>
 void testImage(const std::string pTestFile, const size_t numSeeds,
                const unsigned *seedx, const unsigned *seedy,
@@ -103,7 +92,12 @@ void testImage(const std::string pTestFile, const size_t numSeeds,
         params.iterations = iter;
         params.replace    = 255.0;
 
-        apiWrapper(&outArray, inArray, seedxArr, seedyArr, params);
+        ASSERT_SUCCESS(af_confidence_cc(&outArray, inArray, seedxArr, seedyArr, params.radius,
+                                        params.multiplier, params.iterations,
+                                        params.replace));
+        int device = 0;
+        ASSERT_SUCCESS(af_get_device(&device));
+        ASSERT_SUCCESS(af_sync(device));
 
         ASSERT_ARRAYS_EQ(outArray, goldArray);
 
@@ -147,7 +141,12 @@ void testData(CCCTestParams params) {
                                    (af_dtype)af::dtype_traits<T>::af_type));
 
     af_array outArray = 0;
-    apiWrapper(&outArray, inArray, seedxArr, seedyArr, params);
+    ASSERT_SUCCESS(af_confidence_cc(&outArray, inArray, seedxArr, seedyArr, params.radius,
+                                    params.multiplier, params.iterations,
+                                    params.replace));
+    int device = 0;
+    ASSERT_SUCCESS(af_get_device(&device));
+    ASSERT_SUCCESS(af_sync(device));
 
     ASSERT_VEC_ARRAY_EQ(tests[0], dims, outArray);
 
@@ -161,6 +160,7 @@ class ConfidenceConnectedDataTest
     : public testing::TestWithParam<CCCTestParams> {};
 
 TYPED_TEST(ConfidenceConnectedImageTest, DonutBackgroundExtraction) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const unsigned seedx = 10;
     const unsigned seedy = 10;
     testImage<TypeParam>(std::string("donut_background.test"), 1, &seedx,
@@ -168,6 +168,7 @@ TYPED_TEST(ConfidenceConnectedImageTest, DonutBackgroundExtraction) {
 }
 
 TYPED_TEST(ConfidenceConnectedImageTest, DonutRingExtraction) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const unsigned seedx = 132;
     const unsigned seedy = 132;
     testImage<TypeParam>(std::string("donut_ring.test"), 1, &seedx, &seedy, 3,
@@ -175,6 +176,7 @@ TYPED_TEST(ConfidenceConnectedImageTest, DonutRingExtraction) {
 }
 
 TYPED_TEST(ConfidenceConnectedImageTest, DonutKernelExtraction) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const unsigned seedx = 150;
     const unsigned seedy = 150;
     testImage<TypeParam>(std::string("donut_core.test"), 1, &seedx, &seedy, 3,
@@ -182,6 +184,7 @@ TYPED_TEST(ConfidenceConnectedImageTest, DonutKernelExtraction) {
 }
 
 TEST_P(ConfidenceConnectedDataTest, SegmentARegion) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     testData<unsigned char>(GetParam());
 }
 
diff --git a/test/fast.cpp b/test/fast.cpp
index 1d494641ff..693c80db67 100644
--- a/test/fast.cpp
+++ b/test/fast.cpp
@@ -158,12 +158,14 @@ void fastTest(string pTestFile, bool nonmax) {
 
 #define FLOAT_FAST_INIT(desc, image, nonmax)                                \
     TYPED_TEST(FloatFAST, desc) {                                           \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                             \
         fastTest<TypeParam>(string(TEST_DIR "/fast/" #image "_float.test"), \
                             nonmax);                                        \
     }
 
 #define FIXED_FAST_INIT(desc, image, nonmax)                                \
     TYPED_TEST(FixedFAST, desc) {                                           \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                             \
         fastTest<TypeParam>(string(TEST_DIR "/fast/" #image "_fixed.test"), \
                             nonmax);                                        \
     }
@@ -180,6 +182,7 @@ using af::features;
 using af::loadImage;
 
 TEST(FloatFAST, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
diff --git a/test/gloh.cpp b/test/gloh.cpp
index b360ac6a18..4ce2fa547b 100644
--- a/test/gloh.cpp
+++ b/test/gloh.cpp
@@ -161,8 +161,9 @@ void glohTest(string pTestFile) {
             af_load_image(&inArray_f32, inFiles[testId].c_str(), false));
         ASSERT_SUCCESS(conv_image<T>(&inArray, inArray_f32));
 
-        ASSERT_SUCCESS(af_gloh(&feat, &desc, inArray, 3, 0.04f, 10.0f, 1.6f,
-                               true, 1.f / 256.f, 0.05f));
+        ASSERT_SUCCESS(af_gloh(&feat, &desc, inArray, 3,
+                                           0.04f, 10.0f, 1.6f,
+                                           true, 1.f / 256.f, 0.05f));
 
         dim_t n = 0;
         af_array x, y, score, orientation, size;
@@ -253,6 +254,7 @@ void glohTest(string pTestFile) {
 
 #define GLOH_INIT(desc, image)                                         \
     TYPED_TEST(GLOH, desc) {                                           \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                        \
         glohTest<TypeParam>(string(TEST_DIR "/gloh/" #image ".test")); \
     }
 
@@ -261,6 +263,7 @@ GLOH_INIT(man, man);
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(GLOH, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
diff --git a/test/hamming.cpp b/test/hamming.cpp
index b14a33db0a..b8394e36b5 100644
--- a/test/hamming.cpp
+++ b/test/hamming.cpp
@@ -95,21 +95,25 @@ void hammingMatcherTest(string pTestFile, int feat_dim) {
 }
 
 TYPED_TEST(HammingMatcher8, Hamming_500_5000_Dim0) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     hammingMatcherTest<TypeParam>(
         string(TEST_DIR "/hamming/hamming_500_5000_dim0_u8.test"), 0);
 }
 
 TYPED_TEST(HammingMatcher8, Hamming_500_5000_Dim1) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     hammingMatcherTest<TypeParam>(
         string(TEST_DIR "/hamming/hamming_500_5000_dim1_u8.test"), 1);
 }
 
 TYPED_TEST(HammingMatcher32, Hamming_500_5000_Dim0) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     hammingMatcherTest<TypeParam>(
         string(TEST_DIR "/hamming/hamming_500_5000_dim0_u32.test"), 0);
 }
 
 TYPED_TEST(HammingMatcher32, Hamming_500_5000_Dim1) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     hammingMatcherTest<TypeParam>(
         string(TEST_DIR "/hamming/hamming_500_5000_dim1_u32.test"), 1);
 }
@@ -117,6 +121,7 @@ TYPED_TEST(HammingMatcher32, Hamming_500_5000_Dim1) {
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(HammingMatcher, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     using af::array;
     using af::dim4;
 
@@ -155,6 +160,7 @@ TEST(HammingMatcher, CPP) {
 }
 
 TEST(HammingMatcher64bit, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     using af::array;
     using af::dim4;
 
diff --git a/test/harris.cpp b/test/harris.cpp
index 43c0bb6433..f2fd27d47a 100644
--- a/test/harris.cpp
+++ b/test/harris.cpp
@@ -145,6 +145,7 @@ void harrisTest(string pTestFile, float sigma, unsigned block_size) {
 
 #define HARRIS_INIT(desc, image, sigma, block_size)                        \
     TYPED_TEST(Harris, desc) {                                             \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                            \
         harrisTest<TypeParam>(string(TEST_DIR "/harris/" #image "_" #sigma \
                                               "_" #block_size ".test"),    \
                               sigma, block_size);                          \
@@ -167,6 +168,7 @@ using af::harris;
 using af::loadImage;
 
 TEST(FloatHarris, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
diff --git a/test/homography.cpp b/test/homography.cpp
index f4c1c75259..bd4809d428 100644
--- a/test/homography.cpp
+++ b/test/homography.cpp
@@ -69,8 +69,8 @@ void homographyTest(string pTestFile, const af_homography_type htype,
     ASSERT_SUCCESS(af_load_image(&trainArray_f32, inFiles[0].c_str(), false));
     ASSERT_SUCCESS(conv_image<T>(&trainArray, trainArray_f32));
 
-    ASSERT_SUCCESS(af_orb(&train_feat, &train_desc, trainArray, 20.0f, 2000,
-                          1.2f, 8, true));
+    ASSERT_SUCCESS(af_orb(&train_feat, &train_desc, trainArray,
+                               20.0f, 2000, 1.2f, 8, true));
 
     ASSERT_SUCCESS(af_get_features_xpos(&train_feat_x, train_feat));
     ASSERT_SUCCESS(af_get_features_ypos(&train_feat_y, train_feat));
@@ -96,15 +96,16 @@ void homographyTest(string pTestFile, const af_homography_type htype,
     const dim_t test_d0 = inDims[0][0] * size_ratio;
     const dim_t test_d1 = inDims[0][1] * size_ratio;
     const dim_t tDims[] = {test_d0, test_d1};
-    if (rotate)
+    if (rotate) {
         ASSERT_SUCCESS(af_rotate(&queryArray, trainArray, theta, false,
                                  AF_INTERP_NEAREST));
-    else
+    } else {
         ASSERT_SUCCESS(af_resize(&queryArray, trainArray, test_d0, test_d1,
                                  AF_INTERP_BILINEAR));
+    }
 
-    ASSERT_SUCCESS(af_orb(&query_feat, &query_desc, queryArray, 20.0f, 2000,
-                          1.2f, 8, true));
+    ASSERT_SUCCESS(af_orb(&query_feat, &query_desc, queryArray,
+                                      20.0f, 2000, 1.2f, 8, true));
 
     ASSERT_SUCCESS(
         af_hamming_matcher(&idx, &dist, train_desc, query_desc, 0, 1));
@@ -144,9 +145,9 @@ void homographyTest(string pTestFile, const af_homography_type htype,
 
     int inliers = 0;
     ASSERT_SUCCESS(af_homography(&H, &inliers, train_feat_x_idx,
-                                 train_feat_y_idx, query_feat_x_idx,
-                                 query_feat_y_idx, htype, 3.0f, 1000,
-                                 (af_dtype)dtype_traits<T>::af_type));
+                                             train_feat_y_idx, query_feat_x_idx,
+                                             query_feat_y_idx, htype, 3.0f, 1000,
+                                             (af_dtype)dtype_traits<T>::af_type));
 
     array HH(H);
 
@@ -201,6 +202,7 @@ void homographyTest(string pTestFile, const af_homography_type htype,
 
 #define HOMOGRAPHY_INIT(desc, image, htype, rotate, size_ratio)            \
     TYPED_TEST(Homography, desc) {                                         \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                            \
         homographyTest<TypeParam>(                                         \
             string(TEST_DIR "/homography/" #image ".test"), htype, rotate, \
             size_ratio);                                                   \
@@ -220,6 +222,7 @@ using af::features;
 using af::loadImage;
 
 TEST(Homography, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
@@ -262,7 +265,7 @@ TEST(Homography, CPP) {
     array H;
     int inliers = 0;
     homography(H, inliers, feat_train_x, feat_train_y, feat_query_x,
-               feat_query_y, AF_HOMOGRAPHY_RANSAC, 3.0f, 1000, f32);
+                                  feat_query_y, AF_HOMOGRAPHY_RANSAC, 3.0f, 1000, f32);
 
     float* gold_t = new float[8];
     for (int i = 0; i < 8; i++) gold_t[i] = 0.f;
diff --git a/test/hsv_rgb.cpp b/test/hsv_rgb.cpp
index 423fc5fad5..134e56c6c3 100644
--- a/test/hsv_rgb.cpp
+++ b/test/hsv_rgb.cpp
@@ -38,6 +38,7 @@ TEST(hsv_rgb, InvalidArray) {
 }
 
 TEST(hsv2rgb, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     vector<dim4> numDims;
     vector<vector<float>> in;
     vector<vector<float>> tests;
@@ -54,6 +55,7 @@ TEST(hsv2rgb, CPP) {
 }
 
 TEST(rgb2hsv, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     vector<dim4> numDims;
     vector<vector<float>> in;
     vector<vector<float>> tests;
@@ -70,6 +72,7 @@ TEST(rgb2hsv, CPP) {
 }
 
 TEST(rgb2hsv, MaxDim) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     vector<dim4> numDims;
     vector<vector<float>> in;
     vector<vector<float>> tests;
@@ -108,6 +111,7 @@ TEST(rgb2hsv, MaxDim) {
 }
 
 TEST(hsv2rgb, MaxDim) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     vector<dim4> numDims;
     vector<vector<float>> in;
     vector<vector<float>> tests;
diff --git a/test/imageio.cpp b/test/imageio.cpp
index 00834fb693..16cead852c 100644
--- a/test/imageio.cpp
+++ b/test/imageio.cpp
@@ -160,7 +160,7 @@ TEST(ImageIO, SavePNGCPP) {
     input(9, 0, 2)          = 255;
     input(9, 9, span)       = 255;
 
-    std::string testname  = getTestName() + "_" + getBackendName();
+    std::string testname  = getTestName() + "_" + getBackendName(true);
     std::string imagename = "SaveCPP_" + testname + ".png";
 
     saveImage(imagename.c_str(), input);
@@ -180,7 +180,7 @@ TEST(ImageIO, SaveBMPCPP) {
     input(9, 0, 2)          = 255;
     input(9, 9, span)       = 255;
 
-    std::string testname  = getTestName() + "_" + getBackendName();
+    std::string testname  = getTestName() + "_" + getBackendName(true);
     std::string imagename = "SaveCPP_" + testname + ".bmp";
 
     saveImage(imagename.c_str(), input);
@@ -291,7 +291,7 @@ TEST(ImageIO, SaveImage16CPP) {
     array input     = randu(dims, u16);
     array input_255 = floor(input.as(f32) / 257);
 
-    std::string testname  = getTestName() + "_" + getBackendName();
+    std::string testname  = getTestName() + "_" + getBackendName(true);
     std::string imagename = "saveImage16CPP_" + testname + ".png";
 
     saveImage(imagename.c_str(), input);
@@ -366,7 +366,7 @@ void saveLoadImageNativeCPPTest(dim4 dims) {
 
     array input = randu(dims, (af_dtype)dtype_traits<T>::af_type);
 
-    std::string imagename = getTestName() + "_" + getBackendName() + ".png";
+    std::string imagename = getTestName() + "_" + getBackendName(true) + ".png";
 
     saveImageNative(imagename.c_str(), input);
 
diff --git a/test/match_template.cpp b/test/match_template.cpp
index 33b6096815..4ee8fc7e2d 100644
--- a/test/match_template.cpp
+++ b/test/match_template.cpp
@@ -84,16 +84,19 @@ void matchTemplateTest(string pTestFile, af_match_type pMatchType) {
 }
 
 TYPED_TEST(MatchTemplate, Matrix_SAD) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     matchTemplateTest<TypeParam>(
         string(TEST_DIR "/MatchTemplate/matrix_sad.test"), AF_SAD);
 }
 
 TYPED_TEST(MatchTemplate, Matrix_SSD) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     matchTemplateTest<TypeParam>(
         string(TEST_DIR "/MatchTemplate/matrix_ssd.test"), AF_SSD);
 }
 
 TYPED_TEST(MatchTemplate, MatrixBatch_SAD) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     matchTemplateTest<TypeParam>(
         string(TEST_DIR "/MatchTemplate/matrix_sad_batch.test"), AF_SAD);
 }
diff --git a/test/medfilt.cpp b/test/medfilt.cpp
index 1939379974..2d874cb3ae 100644
--- a/test/medfilt.cpp
+++ b/test/medfilt.cpp
@@ -80,24 +80,28 @@ void medfiltTest(string pTestFile, dim_t w_len, dim_t w_wid,
 }
 
 TYPED_TEST(MedianFilter, ZERO_PAD_3x3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     medfiltTest<TypeParam>(
         string(TEST_DIR "/medianfilter/zero_pad_3x3_window.test"), 3, 3,
         AF_PAD_ZERO);
 }
 
 TYPED_TEST(MedianFilter, SYMMETRIC_PAD_3x3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     medfiltTest<TypeParam>(
         string(TEST_DIR "/medianfilter/symmetric_pad_3x3_window.test"), 3, 3,
         AF_PAD_SYM);
 }
 
 TYPED_TEST(MedianFilter, BATCH_ZERO_PAD_3x3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     medfiltTest<TypeParam>(
         string(TEST_DIR "/medianfilter/batch_zero_pad_3x3_window.test"), 3, 3,
         AF_PAD_ZERO);
 }
 
 TYPED_TEST(MedianFilter, BATCH_SYMMETRIC_PAD_3x3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     medfiltTest<TypeParam>(
         string(TEST_DIR "/medianfilter/batch_symmetric_pad_3x3_window.test"), 3,
         3, AF_PAD_SYM);
@@ -140,24 +144,28 @@ void medfilt1_Test(string pTestFile, dim_t w_wid, af_border_type pad) {
 }
 
 TYPED_TEST(MedianFilter1d, ZERO_PAD_3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     medfilt1_Test<TypeParam>(
         string(TEST_DIR "/medianfilter/zero_pad_3x1_window.test"), 3,
         AF_PAD_ZERO);
 }
 
 TYPED_TEST(MedianFilter1d, SYMMETRIC_PAD_3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     medfilt1_Test<TypeParam>(
         string(TEST_DIR "/medianfilter/symmetric_pad_3x1_window.test"), 3,
         AF_PAD_SYM);
 }
 
 TYPED_TEST(MedianFilter1d, BATCH_ZERO_PAD_3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     medfilt1_Test<TypeParam>(
         string(TEST_DIR "/medianfilter/batch_zero_pad_3x1_window.test"), 3,
         AF_PAD_ZERO);
 }
 
 TYPED_TEST(MedianFilter1d, BATCH_SYMMETRIC_PAD_3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     medfilt1_Test<TypeParam>(
         string(TEST_DIR "/medianfilter/batch_symmetric_pad_3x1_window.test"), 3,
         AF_PAD_SYM);
@@ -338,6 +346,7 @@ TYPED_TEST(MedianFilter1d, InvalidPadType) { medfilt1d_PadTest<TypeParam>(); }
 using af::array;
 
 TEST(MedianFilter, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const dim_t w_len = 3;
     const dim_t w_wid = 3;
 
@@ -365,6 +374,7 @@ TEST(MedianFilter, CPP) {
 }
 
 TEST(MedianFilter1d, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const dim_t w_wid = 3;
 
     vector<dim4> numDims;
@@ -391,6 +401,7 @@ TEST(MedianFilter1d, CPP) {
 }
 
 TEST(MedianFilter, Docs) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     float input[] = {1.0000,  2.0000,  3.0000,  4.0000,  5.0000,  6.0000,
                      7.0000,  8.0000,  9.0000,  10.0000, 11.0000, 12.0000,
                      13.0000, 14.0000, 15.0000, 16.0000};
@@ -431,6 +442,7 @@ using af::seq;
 using af::span;
 
 TEST(MedianFilter, GFOR) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     dim4 dims = dim4(10, 10, 3);
     array A   = iota(dims);
     array B   = constant(0, dims);
@@ -445,6 +457,7 @@ TEST(MedianFilter, GFOR) {
 }
 
 TEST(MedianFilter1d, GFOR) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     dim4 dims = dim4(10, 10, 3);
     array A   = iota(dims);
     array B   = constant(0, dims);
diff --git a/test/moments.cpp b/test/moments.cpp
index 6b02cb614a..bec90e5b5d 100644
--- a/test/moments.cpp
+++ b/test/moments.cpp
@@ -158,25 +158,30 @@ void momentsOnImageTest(string pTestFile, string pImageFile, bool isColor) {
 }
 
 TEST(IMAGE, MomentsImage) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     momentsOnImageTest(string(TEST_DIR "/moments/gray_seq_16_moments.test"),
                        string(TEST_DIR "/imageio/gray_seq_16.png"), false);
 }
 
 TEST(Image, MomentsImageBatch) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     momentsTest<float>(
         string(TEST_DIR "/moments/simple_mat_batch_moments.test"));
 }
 
 TEST(Image, MomentsBatch2D) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     momentsOnImageTest(string(TEST_DIR "/moments/color_seq_16_moments.test"),
                        string(TEST_DIR "/imageio/color_seq_16.png"), true);
 }
 
 TYPED_TEST(Image, MomentsSynthTypes) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     momentsTest<TypeParam>(string(TEST_DIR "/moments/simple_mat_moments.test"));
 }
 
 TEST(Image, Moment_Issue1957) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     array A = identity(3, 3, b8);
 
     double m00;
diff --git a/test/morph.cpp b/test/morph.cpp
index 9cc2255fb5..ad62ded8f3 100644
--- a/test/morph.cpp
+++ b/test/morph.cpp
@@ -59,16 +59,19 @@ void morphTest(string pTestFile) {
                                    maskDims.ndims(), maskDims.get(),
                                    (af_dtype)dtype_traits<inType>::af_type));
 
+    af_err af_stat;
     if (isDilation) {
-        if (isVolume)
+        if (isVolume) {
             ASSERT_SUCCESS(af_dilate3(&outArray, inArray, maskArray));
-        else
+        } else {
             ASSERT_SUCCESS(af_dilate(&outArray, inArray, maskArray));
+        }
     } else {
-        if (isVolume)
+        if (isVolume) {
             ASSERT_SUCCESS(af_erode3(&outArray, inArray, maskArray));
-        else
+        } else {
             ASSERT_SUCCESS(af_erode(&outArray, inArray, maskArray));
+        }
     }
 
     for (size_t testIter = 0; testIter < tests.size(); ++testIter) {
@@ -83,52 +86,63 @@ void morphTest(string pTestFile) {
 }
 
 TYPED_TEST(Morph, Dilate3x3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, true, false>(string(TEST_DIR "/morph/dilate3x3.test"));
 }
 
 TYPED_TEST(Morph, Erode3x3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, false, false>(string(TEST_DIR "/morph/erode3x3.test"));
 }
 
 TYPED_TEST(Morph, Dilate4x4) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, true, false>(string(TEST_DIR "/morph/dilate4x4.test"));
 }
 
 TYPED_TEST(Morph, Dilate12x12) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, true, false>(
         string(TEST_DIR "/morph/dilate12x12.test"));
 }
 
 TYPED_TEST(Morph, Erode4x4) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, false, false>(string(TEST_DIR "/morph/erode4x4.test"));
 }
 
 TYPED_TEST(Morph, Dilate3x3_Batch) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, true, false>(
         string(TEST_DIR "/morph/dilate3x3_batch.test"));
 }
 
 TYPED_TEST(Morph, Erode3x3_Batch) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, false, false>(
         string(TEST_DIR "/morph/erode3x3_batch.test"));
 }
 
 TYPED_TEST(Morph, Dilate3x3x3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, true, true>(
         string(TEST_DIR "/morph/dilate3x3x3.test"));
 }
 
 TYPED_TEST(Morph, Erode3x3x3) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, false, true>(
         string(TEST_DIR "/morph/erode3x3x3.test"));
 }
 
 TYPED_TEST(Morph, Dilate4x4x4) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, true, true>(
         string(TEST_DIR "/morph/dilate4x4x4.test"));
 }
 
 TYPED_TEST(Morph, Erode4x4x4) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphTest<TypeParam, false, true>(
         string(TEST_DIR "/morph/erode4x4x4.test"));
 }
@@ -186,10 +200,10 @@ void morphImageTest(string pTestFile, dim_t seLen) {
         ASSERT_SUCCESS(error_code);
         ASSERT_IMAGES_NEAR(goldArray, outArray, 0.018f);
 #else
-        ASSERT_EQ(error_code,
-                  (targetType != b8 && seLen > 19 ? AF_ERR_NOT_SUPPORTED
-                                                  : AF_SUCCESS));
-        if (!(targetType != b8 && seLen > 19)) {
+        if (targetType != b8 && seLen > 19) {
+            ASSERT_EQ(error_code, AF_ERR_NOT_SUPPORTED);
+        } else {
+            ASSERT_SUCCESS(error_code);
             ASSERT_IMAGES_NEAR(goldArray, outArray, 0.018f);
         }
 #endif
@@ -204,10 +218,12 @@ void morphImageTest(string pTestFile, dim_t seLen) {
 }
 
 TEST(Morph, GrayscaleDilation3x3StructuringElement) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphImageTest<float, true, false>(string(TEST_DIR "/morph/gray.test"), 3);
 }
 
 TEST(Morph, ColorImageErosion3x3StructuringElement) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     morphImageTest<float, false, true>(string(TEST_DIR "/morph/color.test"), 3);
 }
 
@@ -428,14 +444,17 @@ void cppMorphImageTest(string pTestFile) {
 }
 
 TEST(Morph, Grayscale_CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     cppMorphImageTest<float, true, false>(string(TEST_DIR "/morph/gray.test"));
 }
 
 TEST(Morph, ColorImage_CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     cppMorphImageTest<float, false, true>(string(TEST_DIR "/morph/color.test"));
 }
 
 TEST(Morph, GFOR) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     dim4 dims  = dim4(10, 10, 3);
     array A    = iota(dims);
     array B    = constant(0, dims);
@@ -451,6 +470,7 @@ TEST(Morph, GFOR) {
 }
 
 TEST(Morph, EdgeIssue1564) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     int inputData[10 * 10] = {0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -466,12 +486,13 @@ TEST(Morph, EdgeIssue1564) {
     array input(10, 10, inputData);
     int maskData[3 * 3] = {1, 1, 1, 1, 0, 1, 1, 1, 1};
     array mask(3, 3, maskData);
+    
     array dilated = dilate(input.as(b8), mask.as(b8));
 
     size_t nElems = dilated.elements();
     vector<char> outData(nElems);
     dilated.host((void*)outData.data());
-
+    
     for (size_t i = 0; i < nElems; ++i) {
         ASSERT_EQ((int)outData[i], goldData[i]);
     }
diff --git a/test/nearest_neighbour.cpp b/test/nearest_neighbour.cpp
index 01847aea65..2db885f566 100644
--- a/test/nearest_neighbour.cpp
+++ b/test/nearest_neighbour.cpp
@@ -117,24 +117,28 @@ void nearestNeighbourTest(string pTestFile, int feat_dim,
 // SSD
 /////////////////////////////////////////////////
 TYPED_TEST(NearestNeighbour, NN_SSD_100_1000_Dim0) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearestNeighbourTest<TypeParam>(
         string(TEST_DIR "/nearest_neighbour/ssd_100_1000_dim0.test"), 0,
         AF_SSD);
 }
 
 TYPED_TEST(NearestNeighbour, NN_SSD_100_1000_Dim1) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearestNeighbourTest<TypeParam>(
         string(TEST_DIR "/nearest_neighbour/ssd_100_1000_dim1.test"), 1,
         AF_SSD);
 }
 
 TYPED_TEST(NearestNeighbour, NN_SSD_500_5000_Dim0) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearestNeighbourTest<TypeParam>(
         string(TEST_DIR "/nearest_neighbour/ssd_500_5000_dim0.test"), 0,
         AF_SSD);
 }
 
 TYPED_TEST(NearestNeighbour, NN_SSD_500_5000_Dim1) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearestNeighbourTest<TypeParam>(
         string(TEST_DIR "/nearest_neighbour/ssd_500_5000_dim1.test"), 1,
         AF_SSD);
@@ -144,24 +148,28 @@ TYPED_TEST(NearestNeighbour, NN_SSD_500_5000_Dim1) {
 // SAD
 /////////////////////////////////////////////////
 TYPED_TEST(NearestNeighbour, NN_SAD_100_1000_Dim0) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearestNeighbourTest<TypeParam>(
         string(TEST_DIR "/nearest_neighbour/sad_100_1000_dim0.test"), 0,
         AF_SAD);
 }
 
 TYPED_TEST(NearestNeighbour, NN_SAD_100_1000_Dim1) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearestNeighbourTest<TypeParam>(
         string(TEST_DIR "/nearest_neighbour/sad_100_1000_dim1.test"), 1,
         AF_SAD);
 }
 
 TYPED_TEST(NearestNeighbour, NN_SAD_500_5000_Dim0) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearestNeighbourTest<TypeParam>(
         string(TEST_DIR "/nearest_neighbour/sad_500_5000_dim0.test"), 0,
         AF_SAD);
 }
 
 TYPED_TEST(NearestNeighbour, NN_SAD_500_5000_Dim1) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearestNeighbourTest<TypeParam>(
         string(TEST_DIR "/nearest_neighbour/sad_500_5000_dim1.test"), 1,
         AF_SAD);
@@ -170,6 +178,7 @@ TYPED_TEST(NearestNeighbour, NN_SAD_500_5000_Dim1) {
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(NearestNeighbourSSD, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     vector<dim4> numDims;
     vector<vector<uint>> in;
     vector<vector<uint>> tests;
@@ -206,6 +215,7 @@ TEST(NearestNeighbourSSD, CPP) {
 }
 
 TEST(NearestNeighbourSAD, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     vector<dim4> numDims;
     vector<vector<uint>> in;
     vector<vector<uint>> tests;
@@ -242,6 +252,7 @@ TEST(NearestNeighbourSAD, CPP) {
 }
 
 TEST(NearestNeighbourSSD, small) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const int ntrain            = 1;
     const int nquery            = 5;
     const int nfeat             = 2;
@@ -272,6 +283,7 @@ TEST(NearestNeighbourSSD, small) {
 }
 
 TEST(KNearestNeighbourSSD, small) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const int ntrain = 5;
     const int nquery = 3;
     const int nfeat  = 2;
@@ -435,6 +447,7 @@ INSTANTIATE_TEST_SUITE_P(KNearestNeighborsSSD, KNearestNeighborsTest,
                          testNameGenerator<KNearestNeighborsTest>);
 
 TEST_P(NearestNeighborsTest, SingleQTests) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearest_neighbors_params params = GetParam();
     array query = array(params.qdims_, params.query_.data());
     array train = array(params.tdims_, params.train_.data());
@@ -454,6 +467,7 @@ TEST_P(NearestNeighborsTest, SingleQTests) {
 }
 
 TEST_P(KNearestNeighborsTest, SingleQTests) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     nearest_neighbors_params params = GetParam();
 
     array query = array(params.qdims_, params.query_.data());
@@ -504,6 +518,7 @@ TEST(KNearestNeighbours, InvalidLargeK) {
 }
 
 TEST(NearestNeighbour, DocSnippet1) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     //! [ex_nearest_1]
     float h_pts[6] = {1.f, 2.f, 3.f, 8.f, 9.f, 10.f};
     array pts(dim4(1, 6), h_pts);
@@ -537,6 +552,7 @@ TEST(NearestNeighbour, DocSnippet1) {
 }
 
 TEST(NearestNeighbour, DocSnippet2) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     //! [ex_nearest_2]
     float h_pts[18] = {0.f, 0.f, 0.f, 1.f, 0.f, 0.f, 0.f, 1.f, 0.f,
                        8.f, 9.f, 1.f, 9.f, 8.f, 1.f, 9.f, 9.f, 1.f};
diff --git a/test/orb.cpp b/test/orb.cpp
index e519fd91dc..3ace1f4b05 100644
--- a/test/orb.cpp
+++ b/test/orb.cpp
@@ -238,14 +238,19 @@ void orbTest(string pTestFile) {
 }
 
 TYPED_TEST(ORB, Square) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     orbTest<TypeParam>(string(TEST_DIR "/orb/square.test"));
 }
 
-TYPED_TEST(ORB, Lena) { orbTest<TypeParam>(string(TEST_DIR "/orb/lena.test")); }
+TYPED_TEST(ORB, Lena) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
+    orbTest<TypeParam>(string(TEST_DIR "/orb/lena.test"));
+}
 
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(ORB, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
diff --git a/test/regions.cpp b/test/regions.cpp
index 182a22e9b5..a6f14ede81 100644
--- a/test/regions.cpp
+++ b/test/regions.cpp
@@ -71,7 +71,7 @@ void regionsTest(string pTestFile, af_connectivity connectivity,
     }
 
     ASSERT_SUCCESS(af_regions(&outArray, inArray, connectivity,
-                              (af_dtype)dtype_traits<T>::af_type));
+                               (af_dtype)dtype_traits<T>::af_type));
 
     // Get result
     T* outData = new T[idims.elements()];
@@ -97,6 +97,7 @@ void regionsTest(string pTestFile, af_connectivity connectivity,
 
 #define REGIONS_INIT(desc, file, conn, conn_type)                             \
     TYPED_TEST(Regions, desc) {                                               \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                               \
         regionsTest<TypeParam>(                                               \
             string(TEST_DIR "/regions/" #file "_" #conn ".test"), conn_type); \
     }
@@ -109,6 +110,7 @@ REGIONS_INIT(Regions3, regions_128x128, 8, AF_CONNECTIVITY_8);
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(Regions, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     vector<dim4> numDims;
     vector<vector<float>> in;
     vector<vector<float>> tests;
@@ -139,6 +141,7 @@ TEST(Regions, CPP) {
 
 ///////////////////////////////// Documentation Examples ///////////////////
 TEST(Regions, Docs_8) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     // input data
     uchar input[64] = {0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1,
                        0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0,
@@ -185,6 +188,7 @@ TEST(Regions, Docs_8) {
 }
 
 TEST(Regions, Docs_4) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     // input data
     uchar input[64] = {0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1,
                        0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0,
@@ -236,6 +240,7 @@ TEST(Regions, Docs_4) {
 }
 
 TEST(Regions, WholeImageComponent) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const int dim = 101;
     const int sz  = dim * dim;
     vector<char> input(sz, 1);
@@ -252,6 +257,7 @@ TEST(Regions, WholeImageComponent) {
 }
 
 TEST(Regions, NoComponentImage) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const int dim = 101;
     const int sz  = dim * dim;
     vector<char> input(sz, 0);
diff --git a/test/scan_by_key.cpp b/test/scan_by_key.cpp
index fe4d61d095..0ea1dd8ecb 100644
--- a/test/scan_by_key.cpp
+++ b/test/scan_by_key.cpp
@@ -127,6 +127,7 @@ void scanByKeyTest(dim4 dims, int scanDim, vector<int> nodeLengths,
 
 #define SCAN_BY_KEY_TEST(FN, X, Y, Z, W, Ti, To, INC, DIM, DSTART, DEND, EPS) \
     TEST(ScanByKey, Test_Scan_By_Key_##FN##_##Ti##_##INC##_##DIM) {           \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                               \
         dim4 dims(X, Y, Z, W);                                                \
         int scanDim = DIM;                                                    \
         int nodel[] = {37, 256};                                              \
@@ -194,6 +195,7 @@ SCAN_BY_KEY_TEST(AF_BINARY_MAX, 4 * 1024, 512, 1, 1, float, float, false, 1, -5,
                  5, 1e-3);
 
 TEST(ScanByKey, Test_Scan_By_key_Simple_0) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     dim4 dims(16, 8, 2, 1);
     int scanDim = 0;
     int nodel[] = {4, 8};
@@ -207,6 +209,7 @@ TEST(ScanByKey, Test_Scan_By_key_Simple_0) {
 }
 
 TEST(ScanByKey, Test_Scan_By_key_Simple_1) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     dim4 dims(8, 256 + 128, 1, 1);
     int scanDim = 1;
     int nodel[] = {4, 8};
@@ -220,6 +223,7 @@ TEST(ScanByKey, Test_Scan_By_key_Simple_1) {
 }
 
 TEST(ScanByKey, FixOverflowWrite) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     const int SIZE = 41000;
     vector<int> keys(SIZE, 0);
     vector<float> vals(SIZE, 1.0f);
diff --git a/test/sift.cpp b/test/sift.cpp
index 621659e259..b96325d672 100644
--- a/test/sift.cpp
+++ b/test/sift.cpp
@@ -162,9 +162,9 @@ void siftTest(string pTestFile, unsigned nLayers, float contrastThr,
             af_load_image(&inArray_f32, inFiles[testId].c_str(), false));
         ASSERT_SUCCESS(conv_image<T>(&inArray, inArray_f32));
 
-        ASSERT_SUCCESS(af_sift(&feat, &desc, inArray, nLayers, contrastThr,
-                               edgeThr, initSigma, doubleInput, 1.f / 256.f,
-                               0.05f));
+        ASSERT_SUCCESS(af_sift(&feat, &desc, inArray, nLayers,
+                                           contrastThr, edgeThr, initSigma,
+                                           doubleInput, 1.f / 256.f, 0.05f));
 
         dim_t n = 0;
         af_array x, y, score, orientation, size;
@@ -256,6 +256,7 @@ void siftTest(string pTestFile, unsigned nLayers, float contrastThr,
 #define SIFT_INIT(desc, image, nLayers, contrastThr, edgeThr, initSigma,  \
                   doubleInput)                                            \
     TYPED_TEST(SIFT, desc) {                                              \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                           \
         for (int i = 0; i < 1; i++)                                       \
             siftTest<TypeParam>(string(TEST_DIR "/sift/" #image ".test"), \
                                 nLayers, contrastThr, edgeThr, initSigma, \
@@ -272,6 +273,7 @@ SIFT_INIT(Man_NoDoubleInput, man_nodoubleinput, 3, 0.04f, 10.0f, 1.6f, false);
 ///////////////////////////////////// CPP ////////////////////////////////
 //
 TEST(SIFT, CPP) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     IMAGEIO_ENABLED_CHECK();
 
     vector<dim4> inDims;
diff --git a/test/sobel.cpp b/test/sobel.cpp
index 298d36d299..84fae1d34c 100644
--- a/test/sobel.cpp
+++ b/test/sobel.cpp
@@ -79,11 +79,13 @@ void testSobelDerivatives(string pTestFile) {
 // border type is set to cv.BORDER_REFLECT_101 in opencv
 
 TYPED_TEST(Sobel, Rectangle) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     testSobelDerivatives<TypeParam, TypeParam>(
         string(TEST_DIR "/sobel/rectangle.test"));
 }
 
 TYPED_TEST(Sobel_Integer, Rectangle) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     testSobelDerivatives<TypeParam, int>(
         string(TEST_DIR "/sobel/rectangle.test"));
 }
diff --git a/test/susan.cpp b/test/susan.cpp
index 34929c22c0..3741dd2653 100644
--- a/test/susan.cpp
+++ b/test/susan.cpp
@@ -125,6 +125,7 @@ void susanTest(string pTestFile, float t, float g) {
 
 #define SUSAN_TEST(image, tval, gval)                                         \
     TYPED_TEST(Susan, image) {                                                \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                               \
         susanTest<TypeParam>(string(TEST_DIR "/susan/" #image ".test"), tval, \
                              gval);                                           \
     }
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 84ac83839f..4e9496f601 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -92,7 +92,7 @@ typedef unsigned char uchar;
 typedef unsigned int uint;
 typedef unsigned short ushort;
 
-std::string getBackendName();
+std::string getBackendName(bool lower = false);
 std::string getTestName();
 
 std::string readNextNonEmptyLine(std::ifstream &file);
@@ -242,6 +242,15 @@ bool noHalfTests(af::dtype ty);
     if (noHalfTests((af_dtype)af::dtype_traits<type>::af_type))   \
     GTEST_SKIP() << "Device doesn't support Half"
 
+#ifdef SKIP_UNSUPPORTED_TESTS
+#define UNSUPPORTED_BACKEND(backend)                        \
+    if(backend == af::getActiveBackend())                   \
+        GTEST_SKIP() << "Skipping unsupported function on " \
+                        + getBackendName() + " backend"
+#else
+#define UNSUPPORTED_BACKEND(backend)
+#endif
+
 #define LAPACK_ENABLED_CHECK() \
     if (!af::isLAPACKAvailable()) GTEST_SKIP() << "LAPACK Not Configured."
 
diff --git a/test/threading.cpp b/test/threading.cpp
index 41c4ebb723..1b71411f0e 100644
--- a/test/threading.cpp
+++ b/test/threading.cpp
@@ -130,6 +130,7 @@ int nextTargetDeviceId() {
 
 void morphTest(const array input, const array mask, const bool isDilation,
                const array gold, int targetDevice) {
+    UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);
     setDevice(targetDevice);
 
     array out;

From 553c38d4c2cb6e6943dfa053168085b57f106346 Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Mon, 17 Feb 2025 15:48:16 -0500
Subject: [PATCH 423/473] fix fallthrough in reduce_by_key_common for u8
 (#3503)

This fixes minByKey/maxByKey for u8.
---
 src/api/c/reduce.cpp |   1 +
 test/reduce.cpp      | 163 +++++++++++++++++++++++++++----------------
 2 files changed, 103 insertions(+), 61 deletions(-)

diff --git a/src/api/c/reduce.cpp b/src/api/c/reduce.cpp
index 8e1e670506..15be8b39e8 100644
--- a/src/api/c/reduce.cpp
+++ b/src/api/c/reduce.cpp
@@ -280,6 +280,7 @@ static af_err reduce_by_key_common(af_array *keys_out, af_array *vals_out,
             case u8:
                 reduce_key<op, uchar, uchar>(keys_out, vals_out, keys, vals,
                                              dim);
+                break;
             case f16:
                 reduce_key<op, half, half>(keys_out, vals_out, keys, vals, dim);
                 break;
diff --git a/test/reduce.cpp b/test/reduce.cpp
index 0b8317a960..0a36431a54 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -36,10 +36,14 @@ using std::vector;
 template<typename T>
 class Reduce : public ::testing::Test {};
 
+template<typename T>
+class ReduceByKey : public ::testing::Test {};
+
 typedef ::testing::Types<float, double, cfloat, cdouble, uint, int, intl, uintl,
                          uchar, short, ushort>
     TestTypes;
 TYPED_TEST_SUITE(Reduce, TestTypes);
+TYPED_TEST_SUITE(ReduceByKey, TestTypes);
 
 typedef af_err (*reduceFunc)(af_array *, const af_array, const int);
 
@@ -154,6 +158,16 @@ struct promote_type<ushort, af_product> {
     typedef uint type;
 };
 
+// float16 is promoted to float32 for sum and product
+template<>
+struct promote_type<half_float::half, af_sum> {
+    typedef float type;
+};
+template<>
+struct promote_type<half_float::half, af_product> {
+    typedef float type;
+};
+
 #define REDUCE_TESTS(FN)                                                       \
     TYPED_TEST(Reduce, Test_##FN) {                                            \
         reduceTest<TypeParam, typename promote_type<TypeParam, af_##FN>::type, \
@@ -598,12 +612,16 @@ TEST_P(ReduceByKeyP, SumDim2) {
     ASSERT_ARRAYS_NEAR(valsReducedGold, valsReduced, 1e-5);
 }
 
-TEST(ReduceByKey, MultiBlockReduceSingleval) {
+TYPED_TEST(ReduceByKey, MultiBlockReduceSingleval) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
     array keys = constant(0, 1024 * 1024, s32);
-    array vals = constant(1, 1024 * 1024, f32);
+    array vals = constant(1, 1024 * 1024,
+            (af_dtype)af::dtype_traits<TypeParam>::af_type);
 
     array keyResGold      = constant(0, 1);
-    array valsReducedGold = constant(1024 * 1024, 1, f32);
+    using promoted_t = typename promote_type<TypeParam, af_sum>::type;
+    array valsReducedGold = constant(1024 * 1024, 1,
+            (af_dtype)af::dtype_traits<promoted_t>::af_type);
 
     array keyRes, valsReduced;
     sumByKey(keyRes, valsReduced, keys, vals);
@@ -701,10 +719,11 @@ TEST(ReduceByKey, MultiBlockReduceByKeyRandom500) {
     reduce_by_key_test(string(TEST_DIR "/reduce/test_random500_by_key.test"));
 }
 
-TEST(ReduceByKey, productReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
+TYPED_TEST(ReduceByKey, productReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -713,15 +732,17 @@ TEST(ReduceByKey, productReduceByKey) {
     productByKey(reduced_keys, reduced_vals, keys, vals, 0, 1);
 
     const int goldSz = 5;
-    const vector<float> gold_reduce{0, 7, 6, 30, 4};
+    using promoted_t = typename promote_type<TypeParam, af_product>::type;
+    const vector<promoted_t> gold_reduce{0, 7, 6, 30, 4};
 
     ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals);
 }
 
-TEST(ReduceByKey, minReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
+TYPED_TEST(ReduceByKey, minReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -730,14 +751,15 @@ TEST(ReduceByKey, minReduceByKey) {
     minByKey(reduced_keys, reduced_vals, keys, vals);
 
     const int goldSz = 5;
-    const vector<float> gold_reduce{0, 1, 6, 2, 4};
+    const vector<TypeParam> gold_reduce{0, 1, 6, 2, 4};
     ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals);
 }
 
-TEST(ReduceByKey, maxReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
+TYPED_TEST(ReduceByKey, maxReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -746,14 +768,15 @@ TEST(ReduceByKey, maxReduceByKey) {
     maxByKey(reduced_keys, reduced_vals, keys, vals);
 
     const int goldSz = 5;
-    const vector<float> gold_reduce{0, 7, 6, 5, 4};
+    const vector<TypeParam> gold_reduce{0, 7, 6, 5, 4};
     ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals);
 }
 
-TEST(ReduceByKey, allTrueReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 1, 1, 1, 0, 1, 1, 1};
+TYPED_TEST(ReduceByKey, allTrueReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam testVals[testSz] = {0, 1, 1, 1, 0, 1, 1, 1};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -766,10 +789,11 @@ TEST(ReduceByKey, allTrueReduceByKey) {
     ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals);
 }
 
-TEST(ReduceByKey, anyTrueReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 8, 8};
-    const float testVals[testSz] = {0, 1, 1, 1, 0, 1, 0, 0};
+TYPED_TEST(ReduceByKey, anyTrueReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 8, 8};
+    const TypeParam testVals[testSz] = {0, 1, 1, 1, 0, 1, 0, 0};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -783,10 +807,11 @@ TEST(ReduceByKey, anyTrueReduceByKey) {
     ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals);
 }
 
-TEST(ReduceByKey, countReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 5};
-    const float testVals[testSz] = {0, 1, 1, 1, 0, 1, 1, 1};
+TYPED_TEST(ReduceByKey, countReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 5};
+    const TypeParam testVals[testSz] = {0, 1, 1, 1, 0, 1, 1, 1};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -799,11 +824,18 @@ TEST(ReduceByKey, countReduceByKey) {
     ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals);
 }
 
-TEST(ReduceByKey, ReduceByKeyNans) {
+TYPED_TEST(ReduceByKey, ReduceByKeyNans) {
+    if (!IsFloatingPoint<TypeParam>::value) {
+        SUCCEED() << "Not a floating point type.";
+        return;
+    }
+
     SKIP_IF_FAST_MATH_ENABLED();
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 7, NAN, 6, 2, 5, 3, 4};
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam nan = std::numeric_limits<TypeParam>::quiet_NaN();
+    const TypeParam testVals[testSz] = {0, 7, nan, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -812,14 +844,16 @@ TEST(ReduceByKey, ReduceByKeyNans) {
     productByKey(reduced_keys, reduced_vals, keys, vals, 0, 1);
 
     const int goldSz = 5;
-    const vector<float> gold_reduce{0, 7, 6, 30, 4};
+    using promoted_t = typename promote_type<TypeParam, af_product>::type;
+    const vector<promoted_t> gold_reduce{0, 7, 6, 30, 4};
     ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals);
 }
 
-TEST(ReduceByKey, nDim0ReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
+TYPED_TEST(ReduceByKey, nDim0ReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -833,20 +867,22 @@ TEST(ReduceByKey, nDim0ReduceByKey) {
     sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval);
 
     const dim4 goldSz(5, 2, 2, 2);
-    const vector<float> gold_reduce{0, 8, 6, 10, 4, 0, 8, 6, 10, 4,
+    using promoted_t = typename promote_type<TypeParam, af_sum>::type;
+    const vector<promoted_t> gold_reduce{0, 8, 6, 10, 4, 0, 8, 6, 10, 4,
 
-                                    0, 8, 6, 10, 4, 0, 8, 6, 10, 4,
+                                         0, 8, 6, 10, 4, 0, 8, 6, 10, 4,
 
-                                    0, 8, 6, 10, 4, 0, 8, 6, 10, 4,
+                                         0, 8, 6, 10, 4, 0, 8, 6, 10, 4,
 
-                                    0, 8, 6, 10, 4, 0, 8, 6, 10, 4};
+                                         0, 8, 6, 10, 4, 0, 8, 6, 10, 4};
     ASSERT_VEC_ARRAY_EQ(gold_reduce, goldSz, reduced_vals);
 }
 
-TEST(ReduceByKey, nDim1ReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
+TYPED_TEST(ReduceByKey, nDim1ReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -861,8 +897,9 @@ TEST(ReduceByKey, nDim1ReduceByKey) {
     sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval);
 
     const int goldSz                = 5;
-    const float gold_reduce[goldSz] = {0, 8, 6, 10, 4};
-    vector<float> hreduce(reduced_vals.elements());
+    using promoted_t = typename promote_type<TypeParam, af_sum>::type;
+    const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4};
+    vector<promoted_t> hreduce(reduced_vals.elements());
     reduced_vals.host(hreduce.data());
 
     for (int i = 0; i < goldSz * ntile; i++) {
@@ -870,10 +907,11 @@ TEST(ReduceByKey, nDim1ReduceByKey) {
     }
 }
 
-TEST(ReduceByKey, nDim2ReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
+TYPED_TEST(ReduceByKey, nDim2ReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -888,8 +926,9 @@ TEST(ReduceByKey, nDim2ReduceByKey) {
     sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval);
 
     const int goldSz                = 5;
-    const float gold_reduce[goldSz] = {0, 8, 6, 10, 4};
-    vector<float> h_a(reduced_vals.elements());
+    using promoted_t = typename promote_type<TypeParam, af_sum>::type;
+    const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4};
+    vector<promoted_t> h_a(reduced_vals.elements());
     reduced_vals.host(h_a.data());
 
     for (int i = 0; i < goldSz * ntile; i++) {
@@ -897,10 +936,11 @@ TEST(ReduceByKey, nDim2ReduceByKey) {
     }
 }
 
-TEST(ReduceByKey, nDim3ReduceByKey) {
-    const static int testSz      = 8;
-    const int testKeys[testSz]   = {0, 2, 2, 9, 5, 5, 5, 8};
-    const float testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
+TYPED_TEST(ReduceByKey, nDim3ReduceByKey) {
+    SUPPORTED_TYPE_CHECK(TypeParam);
+    const static int testSz          = 8;
+    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam testVals[testSz] = {0, 7, 1, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
     array vals(testSz, testVals);
@@ -915,8 +955,9 @@ TEST(ReduceByKey, nDim3ReduceByKey) {
     sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval);
 
     const int goldSz                = 5;
-    const float gold_reduce[goldSz] = {0, 8, 6, 10, 4};
-    vector<float> h_a(reduced_vals.elements());
+    using promoted_t = typename promote_type<TypeParam, af_sum>::type;
+    const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4};
+    vector<promoted_t> h_a(reduced_vals.elements());
     reduced_vals.host(h_a.data());
 
     for (int i = 0; i < goldSz * ntile; i++) {

From ccabfe60f584bc5e3a5435559822d991c7c0cded Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Thu, 20 Feb 2025 15:02:39 -0500
Subject: [PATCH 424/473] The test cmake file needs to be updated with the
 commit hash which includes the new test data needed for the test added in
 pull requests #3585 and #3587. (#3635)

---
 test/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 3fae5d68ec..4d53f4d4db 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -99,8 +99,8 @@ if(${AF_USE_RELATIVE_TEST_DIR})
 else(${AF_USE_RELATIVE_TEST_DIR})
   af_dep_check_and_populate(${testdata_prefix}
     URI https://github.com/arrayfire/arrayfire-data.git
-    #pinv large data set update change
-    REF 0144a599f913cc67c76c9227031b4100156abc25
+    #Add test file for SSAS_LinearSteps
+    REF 05703a4897c8b89b7a0ece1dbe21ede33d226f44
   )
   set(TESTDATA_SOURCE_DIR "${${testdata_prefix}_SOURCE_DIR}")
 endif(${AF_USE_RELATIVE_TEST_DIR})

From 6cea4d361b67d5f26e90ef15af0e6a1ce686a911 Mon Sep 17 00:00:00 2001
From: Fraser Cormack <frasercrmck@gmail.com>
Date: Thu, 20 Feb 2025 21:48:23 +0000
Subject: [PATCH 425/473] Fix race condition in OpenCL kernel (#3535)

Without the barrier at the end of barrierOR, it is possible for
work-item 0 to start the next loop iteration and update predicates[0]
while other work-items are still inside barrierOR reading `predicates`,
meaning they read the next loop iteration's exit condition. This results
in a divergent loop, where not all work-items reach the same barriers.

A previous fix identified this as a problem only on NVIDIA platforms,
but strictly speaking a barrier is required in all cases to avoid a spec
violation and undefined behaviour.
---
 src/backend/opencl/kernel/flood_fill.cl  | 6 ------
 src/backend/opencl/kernel/flood_fill.hpp | 2 --
 2 files changed, 8 deletions(-)

diff --git a/src/backend/opencl/kernel/flood_fill.cl b/src/backend/opencl/kernel/flood_fill.cl
index 0a7916fd49..ba8f8e109a 100644
--- a/src/backend/opencl/kernel/flood_fill.cl
+++ b/src/backend/opencl/kernel/flood_fill.cl
@@ -42,13 +42,7 @@ int barrierOR(local int *predicates) {
         barrier(CLK_LOCAL_MEM_FENCE);
     }
     int retVal = predicates[0];
-#if AF_IS_PLATFORM_NVIDIA
-    // Without the extra barrier sync after reading the reduction result,
-    // the caller's loop is going into infinite loop occasionally which is
-    // in turn randoms hangs. This doesn't seem to be an issue on non-nvidia
-    // hardware. Hence, the check.
     barrier(CLK_LOCAL_MEM_FENCE);
-#endif
     return retVal;
 }
 
diff --git a/src/backend/opencl/kernel/flood_fill.hpp b/src/backend/opencl/kernel/flood_fill.hpp
index 793ae5adcd..8035a61fd6 100644
--- a/src/backend/opencl/kernel/flood_fill.hpp
+++ b/src/backend/opencl/kernel/flood_fill.hpp
@@ -84,8 +84,6 @@ void floodFill(Param out, const Param image, const Param seedsx,
         DefineKeyValue(LMEM_WIDTH, (THREADS_X + 2 * RADIUS)),
         DefineKeyValue(LMEM_HEIGHT, (THREADS_Y + 2 * RADIUS)),
         DefineKeyValue(GROUP_SIZE, (THREADS_Y * THREADS_X)),
-        DefineKeyValue(AF_IS_PLATFORM_NVIDIA, (int)(AFCL_PLATFORM_NVIDIA ==
-                                                    getActivePlatformVendor())),
         getTypeBuildDefinition<T>()};
 
     auto floodStep =

From d63c391b5d21c3dda30f79ad98f15809c6d4a6e8 Mon Sep 17 00:00:00 2001
From: Filip Matzner <FloopCZ@users.noreply.github.com>
Date: Fri, 21 Feb 2025 01:03:41 +0100
Subject: [PATCH 426/473] Add support for CUDA 12.7 and 12.8 (#3636)

---
 src/backend/cuda/device_manager.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 05f775a821..c445d5784b 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -101,6 +101,8 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {12080, 9, 0, 0},
+    {12070, 9, 0, 0},
     {12060, 9, 0, 0},
     {12050, 9, 0, 0},
     {12040, 9, 0, 0},
@@ -144,6 +146,8 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
+        {12080, 525.60f, 528.33f},
+        {12070, 525.60f, 528.33f},
         {12060, 525.60f, 528.33f},
         {12050, 525.60f, 528.33f},
         {12040, 525.60f, 528.33f},

From 48b7a9e4e173bdeb8c0896f1962f497a742d166f Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Wed, 26 Feb 2025 01:16:24 +0100
Subject: [PATCH 427/473] Join does not always respect the order of provided
 parameters (#3511) (#3513)

* Issue3511.  Join does not always respect order of provided parameters.

* Make test name more descriptive

---------

Co-authored-by: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
---
 src/backend/common/jit/Node.cpp |   6 ++
 src/backend/common/jit/Node.hpp |   1 +
 src/backend/cuda/jit.cpp        |  27 ++++----
 src/backend/oneapi/jit.cpp      |   4 +-
 src/backend/opencl/jit.cpp      | 118 ++++++++++++++++++--------------
 test/join.cpp                   |  46 +++++++++++++
 6 files changed, 137 insertions(+), 65 deletions(-)

diff --git a/src/backend/common/jit/Node.cpp b/src/backend/common/jit/Node.cpp
index f77d68e260..09c001a724 100644
--- a/src/backend/common/jit/Node.cpp
+++ b/src/backend/common/jit/Node.cpp
@@ -42,6 +42,7 @@ int Node::getNodesMap(Node_map_t &node_map, vector<Node *> &full_nodes,
 }
 
 std::string getFuncName(const vector<Node *> &output_nodes,
+                        const vector<int> &output_ids,
                         const vector<Node *> &full_nodes,
                         const vector<Node_ids> &full_ids, const bool is_linear,
                         const bool loop0, const bool loop1, const bool loop2,
@@ -59,6 +60,11 @@ std::string getFuncName(const vector<Node *> &output_nodes,
         funcName += node->getNameStr();
     }
 
+    for (const int id : output_ids) {
+        funcName += '-';
+        funcName += std::to_string(id);
+    }
+
     for (int i = 0; i < static_cast<int>(full_nodes.size()); i++) {
         full_nodes[i]->genKerName(funcName, full_ids[i]);
     }
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 8f2e0183b6..2cc3164fb5 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -326,6 +326,7 @@ struct Node_ids {
 };
 
 std::string getFuncName(const std::vector<Node *> &output_nodes,
+                        const std::vector<int> &output_ids,
                         const std::vector<Node *> &full_nodes,
                         const std::vector<Node_ids> &full_ids,
                         const bool is_linear, const bool loop0,
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 146cb07db2..9346491145 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -244,16 +244,18 @@ struct Param {
             node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
             // Generate the core function body, needs children ids as well
             node->genFuncs(opsStream, ids_curr);
-            for (auto outIt{begin(output_ids)}, endIt{end(output_ids)};
-                 (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) {
-                // Generate also output parameters
-                outParamStream << (oid == 0 ? "" : ",\n") << "Param<"
-                               << full_nodes[ids_curr.id]->getTypeStr()
-                               << "> out" << oid;
-                // Generate code to write the output (offset already in ptr)
-                opsStream << "out" << oid << ".ptr[idx] = val" << ids_curr.id
-                          << ";\n";
-                ++oid;
+            for (size_t output_idx{0}; output_idx < output_ids.size();
+                 ++output_idx) {
+                if (output_ids[output_idx] == ids_curr.id) {
+                    // Generate also output parameters
+                    outParamStream << (oid == 0 ? "" : ",\n") << "Param<"
+                                   << full_nodes[ids_curr.id]->getTypeStr()
+                                   << "> out" << oid;
+                    // Generate code to write the output (offset already in ptr)
+                    opsStream << "out" << output_idx << ".ptr[idx] = val"
+                              << ids_curr.id << ";\n";
+                    ++oid;
+                }
             }
         }
 
@@ -322,8 +324,9 @@ static CUfunction getKernel(const vector<Node*>& output_nodes,
                             const bool is_linear, const bool loop0,
                             const bool loop1, const bool loop2,
                             const bool loop3) {
-    const string funcName{getFuncName(output_nodes, full_nodes, full_ids,
-                                      is_linear, loop0, loop1, loop2, loop3)};
+    const string funcName{getFuncName(output_nodes, output_ids, full_nodes,
+                                      full_ids, is_linear, loop0, loop1, loop2,
+                                      loop3)};
     // A forward lookup in module cache helps avoid recompiling
     // the JIT source generated from identical JIT-trees.
     const auto entry{
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index ecd5bc04b9..a112e99436 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -478,8 +478,8 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
     full_nodes.clear();
     for (Node_ptr& node : node_clones) { full_nodes.push_back(node.get()); }
 
-    const string funcName{getFuncName(output_nodes, full_nodes, full_ids,
-                                      is_linear, false, false, false,
+    const string funcName{getFuncName(output_nodes, output_ids, full_nodes,
+                                      full_ids, is_linear, false, false, false,
                                       outputs[0].info.dims[2] > 1)};
 
     getQueue().submit([&](sycl::handler& h) {
diff --git a/src/backend/opencl/jit.cpp b/src/backend/opencl/jit.cpp
index 7ace33cd96..c0858c3cc5 100644
--- a/src/backend/opencl/jit.cpp
+++ b/src/backend/opencl/jit.cpp
@@ -188,62 +188,77 @@ __kernel void )JIT";
     thread_local stringstream outOffsetStream;
     thread_local stringstream inOffsetsStream;
     thread_local stringstream opsStream;
+    thread_local stringstream kerStream;
 
-    int oid{0};
-    for (size_t i{0}; i < full_nodes.size(); i++) {
-        const auto& node{full_nodes[i]};
-        const auto& ids_curr{full_ids[i]};
-        // Generate input parameters, only needs current id
-        node->genParams(inParamStream, ids_curr.id, is_linear);
-        // Generate input offsets, only needs current id
-        node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
-        // Generate the core function body, needs children ids as well
-        node->genFuncs(opsStream, ids_curr);
-        for (auto outIt{begin(output_ids)}, endIt{end(output_ids)};
-             (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) {
-            // Generate also output parameters
-            outParamStream << "__global "
-                           << full_nodes[ids_curr.id]->getTypeStr() << " *out"
-                           << oid << ", int offset" << oid << ",\n";
-            // Apply output offset
-            outOffsetStream << "\nout" << oid << " += offset" << oid << ';';
-            // Generate code to write the output
-            opsStream << "out" << oid << "[idx] = val" << ids_curr.id << ";\n";
-            ++oid;
+    string ret;
+    try {
+        int oid{0};
+        for (size_t i{0}; i < full_nodes.size(); i++) {
+            const auto& node{full_nodes[i]};
+            const auto& ids_curr{full_ids[i]};
+            // Generate input parameters, only needs current id
+            node->genParams(inParamStream, ids_curr.id, is_linear);
+            // Generate input offsets, only needs current id
+            node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
+            // Generate the core function body, needs children ids as well
+            node->genFuncs(opsStream, ids_curr);
+            for (size_t output_idx{0}; output_idx < output_ids.size();
+                 ++output_idx) {
+                if (output_ids[output_idx] == ids_curr.id) {
+                    outParamStream
+                        << "__global " << full_nodes[ids_curr.id]->getTypeStr()
+                        << " *out" << oid << ", int offset" << oid << ",\n";
+                    // Apply output offset
+                    outOffsetStream << "\nout" << oid << " += offset" << oid
+                                    << ';';
+                    // Generate code to write the output
+                    opsStream << "out" << output_idx << "[idx] = val"
+                              << ids_curr.id << ";\n";
+                    ++oid;
+                }
+            }
         }
-    }
 
-    thread_local stringstream kerStream;
-    kerStream << kernelVoid << funcName << "(\n"
-              << inParamStream.str() << outParamStream.str() << dimParams << ")"
-              << blockStart;
-    if (is_linear) {
-        kerStream << linearInit << inOffsetsStream.str()
-                  << outOffsetStream.str() << '\n';
-        if (loop0) kerStream << linearLoop0Start;
-        kerStream << "\n\n" << opsStream.str();
-        if (loop0) kerStream << linearLoop0End;
-        kerStream << linearEnd;
-    } else {
-        if (loop0) {
-            kerStream << stridedLoop0Init << outOffsetStream.str() << '\n'
-                      << stridedLoop0Start;
+        kerStream << kernelVoid << funcName << "(\n"
+                  << inParamStream.str() << outParamStream.str() << dimParams
+                  << ")" << blockStart;
+        if (is_linear) {
+            kerStream << linearInit << inOffsetsStream.str()
+                      << outOffsetStream.str() << '\n';
+            if (loop0) kerStream << linearLoop0Start;
+            kerStream << "\n\n" << opsStream.str();
+            if (loop0) kerStream << linearLoop0End;
+            kerStream << linearEnd;
         } else {
-            kerStream << stridedLoopNInit << outOffsetStream.str() << '\n';
-            if (loop3) kerStream << stridedLoop3Init;
-            if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
-            if (loop3) kerStream << stridedLoop3Start;
+            if (loop0) {
+                kerStream << stridedLoop0Init << outOffsetStream.str() << '\n'
+                          << stridedLoop0Start;
+            } else {
+                kerStream << stridedLoopNInit << outOffsetStream.str() << '\n';
+                if (loop3) kerStream << stridedLoop3Init;
+                if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
+                if (loop3) kerStream << stridedLoop3Start;
+            }
+            kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
+            if (loop3) kerStream << stridedLoop3End;
+            if (loop1) kerStream << stridedLoop1End;
+            if (loop0) kerStream << stridedLoop0End;
+            kerStream << stridedEnd;
         }
-        kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
-        if (loop3) kerStream << stridedLoop3End;
-        if (loop1) kerStream << stridedLoop1End;
-        if (loop0) kerStream << stridedLoop0End;
-        kerStream << stridedEnd;
+        kerStream << blockEnd;
+        ret = kerStream.str();
+    } catch (...) {
+        // Prepare for next round
+        inParamStream.str("");
+        outParamStream.str("");
+        inOffsetsStream.str("");
+        outOffsetStream.str("");
+        opsStream.str("");
+        kerStream.str("");
+        throw;
     }
-    kerStream << blockEnd;
-    const string ret{kerStream.str()};
 
-    // Prepare for next round, limit memory
+    // Prepare for next round
     inParamStream.str("");
     outParamStream.str("");
     inOffsetsStream.str("");
@@ -259,8 +274,9 @@ cl::Kernel getKernel(const vector<Node*>& output_nodes,
                      const vector<Node*>& full_nodes,
                      const vector<Node_ids>& full_ids, const bool is_linear,
                      const bool loop0, const bool loop1, const bool loop3) {
-    const string funcName{getFuncName(output_nodes, full_nodes, full_ids,
-                                      is_linear, loop0, loop1, false, loop3)};
+    const string funcName{getFuncName(output_nodes, output_ids, full_nodes,
+                                      full_ids, is_linear, loop0, loop1, false,
+                                      loop3)};
     // A forward lookup in module cache helps avoid recompiling the JIT
     // source generated from identical JIT-trees.
     const auto entry{
diff --git a/test/join.cpp b/test/join.cpp
index cf33fccb67..4d25e8a6ae 100644
--- a/test/join.cpp
+++ b/test/join.cpp
@@ -15,6 +15,7 @@
 #include <af/index.h>
 #include <af/traits.hpp>
 
+#include <array>
 #include <complex>
 #include <iostream>
 #include <numeric>
@@ -266,3 +267,48 @@ TEST(Join, ManyEmpty) {
     ASSERT_ARRAYS_EQ(gold, eace);
     ASSERT_ARRAYS_EQ(gold, acee);
 }
+
+TEST(Join, respect_parameters_order_ISSUE3511) {
+    const float column_host1[] = {1., 2., 3.};
+    const float column_host2[] = {4., 5., 6.};
+    const af::array buf1(3, 1, column_host1);
+    const af::array buf2(3, 1, column_host2);
+
+    // We need to avoid that JIT arrays are evaluated during whatever call,
+    // so we will have to work with copies for single use
+    const af::array jit1{buf1 + 1.0};
+    const af::array jit2{buf2 + 2.0};
+    const std::array<af::array, 8> cases{jit1,  -jit1,       jit1 + 1.0, jit2,
+                                         -jit2, jit1 + jit2, buf1,       buf2};
+    const std::array<char*, 8> cases_name{"JIT1", "-JIT1", "JIT1+1.0",
+                                          "JIT2", "-JIT2", "JIT1+JIT2",
+                                          "BUF1", "BUF2"};
+    assert(cases.size() == cases_name.size());
+    for (size_t cl0{0}; cl0 < cases.size(); ++cl0) {
+        for (size_t cl1{0}; cl1 < cases.size(); ++cl1) {
+            printf("Testing: af::join(1,%s,%s)\n", cases_name[cl0],
+                   cases_name[cl1]);
+            const array col0{cases[cl0]};
+            const array col1{cases[cl1]};
+            const array result{af::join(1, col0, col1)};
+            ASSERT_ARRAYS_EQ(result(af::span, 0), col0);
+            ASSERT_ARRAYS_EQ(result(af::span, 1), col1);
+        }
+    }
+    // Join of 3 arrays
+    for (size_t cl0{0}; cl0 < cases.size(); ++cl0) {
+        for (size_t cl1{0}; cl1 < cases.size(); ++cl1) {
+            for (size_t cl2{0}; cl2 < cases.size(); ++cl2) {
+                printf("Testing: af::join(1,%s,%s,%s)\n", cases_name[cl0],
+                       cases_name[cl1], cases_name[cl2]);
+                const array col0{cases[cl0]};
+                const array col1{cases[cl1]};
+                const array col2{cases[cl2]};
+                const array result{af::join(1, col0, col1, col2)};
+                ASSERT_ARRAYS_EQ(result(af::span, 0), col0);
+                ASSERT_ARRAYS_EQ(result(af::span, 1), col1);
+                ASSERT_ARRAYS_EQ(result(af::span, 2), col2);
+            }
+        }
+    }
+}

From 408b504c4259ec2cb28c625c0906ab12fd1019d7 Mon Sep 17 00:00:00 2001
From: Umar Arshad <umar@arrayfire.com>
Date: Fri, 7 Mar 2025 12:09:45 -0500
Subject: [PATCH 428/473] Add f16 support for modulus and norm (#3258)

* Add support for f16 for modulus operations

* Update float math functions to use f<op>f functions to maintain types

float math functions in CUDA have the format f<op>f

* Add additional binary tests for integer types

* Add tests for norm

* Add support for half for norm

* Added tests for norm and modulus. Made consistent modulus for ints in cpu backend according to other backends

* Added more mod tests

* Updated documentation to reflect status quo of the mod and rem expected outputs

* Update copyright

---------

Co-authored-by: Edwin <edwinsolisf12@gmail.com>
Co-authored-by: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Co-authored-by: Christophe Murphy <christophe.murphy@outlook.com>
---
 include/af/arith.h              |  14 +-
 src/api/c/norm.cpp              |  67 ++++----
 src/backend/cpu/binary.hpp      |   5 +-
 src/backend/cuda/binary.hpp     |   9 +-
 src/backend/cuda/kernel/jit.cuh |  12 +-
 src/backend/opencl/binary.hpp   |   7 +-
 test/CMakeLists.txt             |   3 +-
 test/binary.cpp                 |  17 +-
 test/math.cpp                   |  50 +++++-
 test/norm.cpp                   | 285 ++++++++++++++++++++++++++++++++
 10 files changed, 423 insertions(+), 46 deletions(-)
 create mode 100644 test/norm.cpp

diff --git a/include/af/arith.h b/include/af/arith.h
index 9b02e668b6..5e470f448b 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2025, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -104,6 +104,9 @@ namespace af
     /// @{
     /// C++ Interface to calculate the remainder.
     ///
+    /// For integers, it returns the same output as modulus (% operator)
+    /// For floating point numbers, it returns the same as std::remainder from <cmath>
+    /// 
     /// \param[in] lhs numerator; can be an array or a scalar
     /// \param[in] rhs denominator; can be an array or a scalar
     /// \return        remainder
@@ -121,6 +124,9 @@ namespace af
     /// @{
     /// C++ Interface to calculate the modulus.
     ///
+    /// For integers, it returns the same output as modulus (% operator)
+    /// For floating point numbers, it returns the same as std::fmod from <cmath>
+    ///
     /// \param[in] lhs dividend; can be an array or a scalar
     /// \param[in] rhs divisor; can be an array or a scalar
     /// \return        modulus
@@ -984,6 +990,9 @@ extern "C" {
     /**
        C Interface to calculate the remainder.
 
+       For integers, it returns the same output as modulus (% operator)
+       For floating point numbers, it returns the same as `remainder` from <math.h>
+
        \param[out] out   remainder
        \param[in]  lhs   numerator
        \param[in]  rhs   denominator
@@ -998,6 +1007,9 @@ extern "C" {
     /**
        C Interface to calculate the modulus.
 
+       For integers, it returns the same output as modulus (% operator)
+       For floating point numbers, it returns the same as `fmod` from <math.h>
+
        \param[out] out   modulus
        \param[in]  lhs   dividend
        \param[in]  rhs   divisor
diff --git a/src/api/c/norm.cpp b/src/api/c/norm.cpp
index 84444eed58..7eef41afcc 100644
--- a/src/api/c/norm.cpp
+++ b/src/api/c/norm.cpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2025, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -10,6 +10,7 @@
 #include <arith.hpp>
 #include <backend.hpp>
 #include <common/ArrayInfo.hpp>
+#include <common/cast.hpp>
 #include <common/err_common.hpp>
 #include <complex.hpp>
 #include <copy.hpp>
@@ -24,6 +25,7 @@
 #include <af/traits.hpp>
 
 using af::dim4;
+using arrayfire::common::cast;
 using detail::arithOp;
 using detail::Array;
 using detail::cdouble;
@@ -35,15 +37,21 @@ using detail::reduce;
 using detail::reduce_all;
 using detail::scalar;
 
+template<typename T>
+using normReductionResult =
+    typename std::conditional<std::is_same<T, arrayfire::common::half>::value, float,
+                              T>::type;
+
 template<typename T>
 double matrixNorm(const Array<T> &A, double p) {
+    using RT = normReductionResult<T>;
     if (p == 1) {
-        Array<T> colSum = reduce<af_add_t, T, T>(A, 0);
-        return getScalar<T>(reduce_all<af_max_t, T, T>(colSum));
+        Array<RT> colSum = reduce<af_add_t, T, normReductionResult<T>>(A, 0);
+        return getScalar<RT>(reduce_all<af_max_t, RT, RT>(colSum));
     }
     if (p == af::Inf) {
-        Array<T> rowSum = reduce<af_add_t, T, T>(A, 1);
-        return getScalar<T>(reduce_all<af_max_t, T, T>(rowSum));
+        Array<RT> rowSum = reduce<af_add_t, T, RT>(A, 1);
+        return getScalar<RT>(reduce_all<af_max_t, RT, RT>(rowSum));
     }
 
     AF_ERROR("This type of norm is not supported in ArrayFire\n",
@@ -52,41 +60,45 @@ double matrixNorm(const Array<T> &A, double p) {
 
 template<typename T>
 double vectorNorm(const Array<T> &A, double p) {
-    if (p == 1) { return getScalar<T>(reduce_all<af_add_t, T, T>(A)); }
+    using RT = normReductionResult<T>;
+    if (p == 1) { return getScalar<RT>(reduce_all<af_add_t, T, RT>(A)); }
     if (p == af::Inf) {
-        return getScalar<T>(reduce_all<af_max_t, T, T>(A));
+        return getScalar<RT>(reduce_all<af_max_t, RT, RT>(cast<RT>(A)));
     } else if (p == 2) {
         Array<T> A_sq = arithOp<T, af_mul_t>(A, A, A.dims());
-        return std::sqrt(getScalar<T>(reduce_all<af_add_t, T, T>(A_sq)));
+        return std::sqrt(getScalar<RT>(reduce_all<af_add_t, T, RT>(A_sq)));
     }
 
     Array<T> P   = createValueArray<T>(A.dims(), scalar<T>(p));
     Array<T> A_p = arithOp<T, af_pow_t>(A, P, A.dims());
-    return std::pow(getScalar<T>(reduce_all<af_add_t, T, T>(A_p)), T(1.0 / p));
+    return std::pow(getScalar<RT>(reduce_all<af_add_t, T, RT>(A_p)), (1.0 / p));
 }
 
 template<typename T>
 double LPQNorm(const Array<T> &A, double p, double q) {
-    Array<T> A_p_norm = createEmptyArray<T>(dim4());
+    using RT           = normReductionResult<T>;
+    Array<RT> A_p_norm = createEmptyArray<RT>(dim4());
 
     if (p == 1) {
-        A_p_norm = reduce<af_add_t, T, T>(A, 0);
+        A_p_norm = reduce<af_add_t, T, RT>(A, 0);
     } else {
-        Array<T> P    = createValueArray<T>(A.dims(), scalar<T>(p));
-        Array<T> invP = createValueArray<T>(A.dims(), scalar<T>(1.0 / p));
+        Array<T> P     = createValueArray<T>(A.dims(), scalar<T>(p));
+        Array<RT> invP = createValueArray<RT>(A.dims(), scalar<RT>(1.0 / p));
 
-        Array<T> A_p     = arithOp<T, af_pow_t>(A, P, A.dims());
-        Array<T> A_p_sum = reduce<af_add_t, T, T>(A_p, 0);
-        A_p_norm         = arithOp<T, af_pow_t>(A_p_sum, invP, invP.dims());
+        Array<T> A_p      = arithOp<T, af_pow_t>(A, P, A.dims());
+        Array<RT> A_p_sum = reduce<af_add_t, T, RT>(A_p, 0);
+        A_p_norm          = arithOp<RT, af_pow_t>(A_p_sum, invP, invP.dims());
     }
 
-    if (q == 1) { return getScalar<T>(reduce_all<af_add_t, T, T>(A_p_norm)); }
+    if (q == 1) {
+        return getScalar<RT>(reduce_all<af_add_t, RT, RT>(A_p_norm));
+    }
 
-    Array<T> Q          = createValueArray<T>(A_p_norm.dims(), scalar<T>(q));
-    Array<T> A_p_norm_q = arithOp<T, af_pow_t>(A_p_norm, Q, Q.dims());
+    Array<RT> Q          = createValueArray<RT>(A_p_norm.dims(), scalar<RT>(q));
+    Array<RT> A_p_norm_q = arithOp<RT, af_pow_t>(A_p_norm, Q, Q.dims());
 
-    return std::pow(getScalar<T>(reduce_all<af_add_t, T, T>(A_p_norm_q)),
-                    T(1.0 / q));
+    return std::pow(getScalar<RT>(reduce_all<af_add_t, RT, RT>(A_p_norm_q)),
+                    (1.0 / q));
 }
 
 template<typename T>
@@ -98,21 +110,13 @@ double norm(const af_array a, const af_norm_type type, const double p,
 
     switch (type) {
         case AF_NORM_EUCLID: return vectorNorm(A, 2);
-
         case AF_NORM_VECTOR_1: return vectorNorm(A, 1);
-
         case AF_NORM_VECTOR_INF: return vectorNorm(A, af::Inf);
-
         case AF_NORM_VECTOR_P: return vectorNorm(A, p);
-
         case AF_NORM_MATRIX_1: return matrixNorm(A, 1);
-
         case AF_NORM_MATRIX_INF: return matrixNorm(A, af::Inf);
-
         case AF_NORM_MATRIX_2: return matrixNorm(A, 2);
-
         case AF_NORM_MATRIX_L_PQ: return LPQNorm(A, p, q);
-
         default:
             AF_ERROR("This type of norm is not supported in ArrayFire\n",
                      AF_ERR_NOT_SUPPORTED);
@@ -123,17 +127,13 @@ af_err af_norm(double *out, const af_array in, const af_norm_type type,
                const double p, const double q) {
     try {
         const ArrayInfo &i_info = getInfo(in);
-
         if (i_info.ndims() > 2) {
             AF_ERROR("solve can not be used in batch mode", AF_ERR_BATCH);
         }
 
         af_dtype i_type = i_info.getType();
-
         ARG_ASSERT(1, i_info.isFloating());  // Only floating and complex types
-
         *out = 0;
-
         if (i_info.ndims() == 0) { return AF_SUCCESS; }
 
         switch (i_type) {
@@ -141,6 +141,7 @@ af_err af_norm(double *out, const af_array in, const af_norm_type type,
             case f64: *out = norm<double>(in, type, p, q); break;
             case c32: *out = norm<cfloat>(in, type, p, q); break;
             case c64: *out = norm<cdouble>(in, type, p, q); break;
+            case f16: *out = norm<arrayfire::common::half>(in, type, p, q); break;
             default: TYPE_ERROR(1, i_type);
         }
     }
diff --git a/src/backend/cpu/binary.hpp b/src/backend/cpu/binary.hpp
index 3d130ba520..8d28501053 100644
--- a/src/backend/cpu/binary.hpp
+++ b/src/backend/cpu/binary.hpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2021, ArrayFire
+ * Copyright (c) 2025, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -89,8 +89,7 @@ LOGIC_CPLX_FN(double, af_or_t, ||)
 
 template<typename T>
 static T __mod(T lhs, T rhs) {
-    T res = lhs % rhs;
-    return (res < 0) ? abs(rhs - res) : res;
+    return lhs % rhs; // Same as other backends
 }
 
 template<typename T>
diff --git a/src/backend/cuda/binary.hpp b/src/backend/cuda/binary.hpp
index 20f2bea9a6..ca707f30be 100644
--- a/src/backend/cuda/binary.hpp
+++ b/src/backend/cuda/binary.hpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2025, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -60,7 +60,7 @@ BINARY_TYPE_1(bitshiftr)
     };                                               \
     template<typename To>                            \
     struct BinOp<To, float, af_##fn##_t> {           \
-        const char *name() { return "f" #fn; }       \
+        const char *name() { return "f" #fn "f"; }   \
     };                                               \
     template<typename To>                            \
     struct BinOp<To, double, af_##fn##_t> {          \
@@ -80,6 +80,11 @@ BINARY_TYPE_2(max)
 BINARY_TYPE_2(rem)
 BINARY_TYPE_2(mod)
 
+template<>
+struct BinOp<common::half, common::half, af_mod_t> {
+    const char *name() { return "hmod"; }
+};
+
 template<typename To, typename Ti>
 struct BinOp<To, Ti, af_pow_t> {
     const char *name() { return "__pow"; }
diff --git a/src/backend/cuda/kernel/jit.cuh b/src/backend/cuda/kernel/jit.cuh
index 76fd344010..879d46f3c2 100644
--- a/src/backend/cuda/kernel/jit.cuh
+++ b/src/backend/cuda/kernel/jit.cuh
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2025, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -73,6 +73,7 @@ typedef cuDoubleComplex cdouble;
 
 #define __convert_char(val) (char)((val) != 0)
 #define frem(lhs, rhs) remainder((lhs), (rhs))
+#define fremf(lhs, rhs) remainderf((lhs), (rhs))
 
 // ----------------------------------------------
 // COMPLEX FLOAT OPERATIONS
@@ -214,6 +215,15 @@ __device__ __inline__ int __isinf<__half>(const __half in) {
 #endif
 }
 
+__device__ __inline__
+__half hmod(const __half lhs, const __half rhs) {
+#if __CUDA_ARCH__ >= 530
+    return __hsub(lhs, __hmul(htrunc(__hdiv(lhs, rhs)), rhs));
+#else
+    return __float2half(fmodf(__half2float(lhs), __half2float(rhs)));
+#endif
+}
+
 template<typename T>
 static __device__ __inline__ int __isnan(const T in) {
     return isnan(in);
diff --git a/src/backend/opencl/binary.hpp b/src/backend/opencl/binary.hpp
index 02291d566a..39f340942a 100644
--- a/src/backend/opencl/binary.hpp
+++ b/src/backend/opencl/binary.hpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2025, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -80,6 +80,11 @@ BINARY_TYPE_2(max)
 BINARY_TYPE_2(rem)
 BINARY_TYPE_2(mod)
 
+template<>
+struct BinOp<common::half, common::half, af_mod_t> {
+    const char *name() { return "fmod"; }
+};
+
 template<typename To, typename Ti>
 struct BinOp<To, Ti, af_pow_t> {
     const char *name() { return "__pow"; }
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 4d53f4d4db..8107f3c063 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-# Copyright (c) 2020, ArrayFire
+# Copyright (c) 2025, ArrayFire
 # All rights reserved.
 #
 # This file is distributed under 3-clause BSD license.
@@ -359,6 +359,7 @@ make_test(SRC moments.cpp)
 make_test(SRC morph.cpp)
 make_test(SRC nearest_neighbour.cpp CXX11)
 make_test(SRC nodevice.cpp CXX11)
+make_test(SRC norm.cpp CXX11)
 
 if(OpenCL_FOUND)
   make_test(SRC ocl_ext_context.cpp
diff --git a/test/binary.cpp b/test/binary.cpp
index a274c11346..ed5b2c0869 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2025, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -40,6 +40,11 @@ T mod(T a, T b) {
     return std::fmod(a, b);
 }
 
+template<typename T>
+T rem(T x, T y) {
+    return remainder(x, y);
+}
+
 af::array randgen(const int num, dtype ty) {
     af::array tmp = round(1 + 2 * af::randu(num, f32)).as(ty);
     tmp.eval();
@@ -181,6 +186,7 @@ BINARY_TESTS_NEAR(float, float, float, div, 1e-3)  // FIXME
 BINARY_TESTS_FLOAT(min)
 BINARY_TESTS_FLOAT(max)
 BINARY_TESTS_NEAR(float, float, float, mod, 1e-5)  // FIXME
+BINARY_TESTS_FLOAT(rem)
 
 BINARY_TESTS_DOUBLE(add)
 BINARY_TESTS_DOUBLE(sub)
@@ -189,6 +195,7 @@ BINARY_TESTS_DOUBLE(div)
 BINARY_TESTS_DOUBLE(min)
 BINARY_TESTS_DOUBLE(max)
 BINARY_TESTS_DOUBLE(mod)
+BINARY_TESTS_DOUBLE(rem)
 
 BINARY_TESTS_NEAR_FLOAT(atan2)
 BINARY_TESTS_NEAR_FLOAT(pow)
@@ -205,18 +212,26 @@ BINARY_TESTS_NEAR_DOUBLE(hypot)
 BINARY_TESTS_INT(add)
 BINARY_TESTS_INT(sub)
 BINARY_TESTS_INT(mul)
+BINARY_TESTS_INT(div)
+BINARY_TESTS_INT(pow)
 
 BINARY_TESTS_UINT(add)
 BINARY_TESTS_UINT(sub)
 BINARY_TESTS_UINT(mul)
+BINARY_TESTS_UINT(div)
+BINARY_TESTS_UINT(pow)
 
 BINARY_TESTS_INTL(add)
 BINARY_TESTS_INTL(sub)
 BINARY_TESTS_INTL(mul)
+BINARY_TESTS_INTL(div)
+BINARY_TESTS_INTL(pow)
 
 BINARY_TESTS_UINTL(add)
 BINARY_TESTS_UINTL(sub)
 BINARY_TESTS_UINTL(mul)
+BINARY_TESTS_UINTL(div)
+BINARY_TESTS_UINTL(pow)
 
 BINARY_TESTS_CFLOAT(add)
 BINARY_TESTS_CFLOAT(sub)
diff --git a/test/math.cpp b/test/math.cpp
index 8e2243e13c..ee42a11423 100644
--- a/test/math.cpp
+++ b/test/math.cpp
@@ -1,5 +1,5 @@
 /*******************************************************
- * Copyright (c) 2014, ArrayFire
+ * Copyright (c) 2025, ArrayFire
  * All rights reserved.
  *
  * This file is distributed under 3-clause BSD license.
@@ -46,7 +46,7 @@ T rsqrt(T in) {
 }
 
 #define MATH_TEST(T, func, err, lo, hi)                                        \
-    TEST(MathTests, Test_##func##_##T) {                                       \
+    TEST(Math, func##_##T) {                                                   \
         try {                                                                  \
             SUPPORTED_TYPE_CHECK(T);                                           \
             af_dtype ty = (af_dtype)dtype_traits<T>::af_type;                  \
@@ -135,7 +135,7 @@ MATH_TESTS_REAL(erf)
 MATH_TESTS_REAL(erfc)
 #endif
 
-TEST(MathTests, Not) {
+TEST(Math, Not) {
     array a  = randu(5, 5, b8);
     array b  = !a;
     char *ha = a.host<char>();
@@ -146,3 +146,47 @@ TEST(MathTests, Not) {
     af_free_host(ha);
     af_free_host(hb);
 }
+
+TEST(Math, Modulus) {
+    af::dim4 shape(2, 2);
+    std::vector<long long> aData{3, 3, 3, 3};
+    std::vector<long long> bData{2, 2, 2, 2};
+
+    auto a    = af::array(shape, aData.data(), afHost);
+    auto b    = af::array(shape, bData.data(), afHost);
+    auto rem  = a % b;
+    auto neg_rem = -a % b;
+
+    ASSERT_ARRAYS_EQ(af::constant(1, shape, s64), rem);
+    ASSERT_ARRAYS_EQ(af::constant(-1, shape, s64), neg_rem);
+}
+
+TEST(Math, ModulusFloat) {
+    SUPPORTED_TYPE_CHECK(half_float::half);
+    af::dim4 shape(2, 2);
+
+    auto a     = af::constant(3, shape, af::dtype::f16);
+    auto b     = af::constant(2, shape, af::dtype::f16);
+    auto a32   = af::constant(3, shape, af::dtype::f32);
+    auto b32   = af::constant(2, shape, af::dtype::f32);
+    auto a64   = af::constant(3, shape, af::dtype::f64);
+    auto b64   = af::constant(2, shape, af::dtype::f64);
+
+    auto rem   = a % b;
+    auto rem32 = a32 % b32;
+    auto rem64 = a64 % b64;
+
+    auto neg_rem = -a % b;
+    auto neg_rem32 = -a32 % b32;
+    auto neg_rem64 = -a64 % b64;
+    
+    ASSERT_ARRAYS_EQ(af::constant(1, shape, af::dtype::f16), rem);
+    ASSERT_ARRAYS_EQ(af::constant(1, shape, af::dtype::f32), rem32);
+    ASSERT_ARRAYS_EQ(af::constant(1, shape, af::dtype::f64), rem64);
+
+    ASSERT_ARRAYS_EQ(af::constant(-1, shape, af::dtype::f16), neg_rem);
+    ASSERT_ARRAYS_EQ(af::constant(-1, shape, af::dtype::f32), neg_rem32);
+    ASSERT_ARRAYS_EQ(af::constant(-1, shape, af::dtype::f64), neg_rem64);
+
+    ASSERT_ARRAYS_EQ(rem32.as(f16), rem);
+}
diff --git a/test/norm.cpp b/test/norm.cpp
new file mode 100644
index 0000000000..c795c112c3
--- /dev/null
+++ b/test/norm.cpp
@@ -0,0 +1,285 @@
+/*******************************************************
+ * Copyright (c) 2025, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <arrayfire.h>
+#include <gtest/gtest.h>
+#include <testHelpers.hpp>
+#include <sstream>
+
+using af::array;
+using af::constant;
+using af::dim4;
+using std::complex;
+using std::stringstream;
+using std::vector;
+
+std::ostream &operator<<(std::ostream &os, af::normType nt) {
+    switch (nt) {
+        case AF_NORM_VECTOR_1: os << "AF_NORM_VECTOR_1"; break;
+        case AF_NORM_VECTOR_INF: os << "AF_NORM_VECTOR_INF"; break;
+        case AF_NORM_VECTOR_2: os << "AF_NORM_VECTOR_2"; break;
+        case AF_NORM_VECTOR_P: os << "AF_NORM_VECTOR_P"; break;
+        case AF_NORM_MATRIX_1: os << "AF_NORM_MATRIX_1"; break;
+        case AF_NORM_MATRIX_INF: os << "AF_NORM_MATRIX_INF"; break;
+        case AF_NORM_MATRIX_2: os << "AF_NORM_MATRIX_2"; break;
+        case AF_NORM_MATRIX_L_PQ: os << "AF_NORM_MATRIX_L_PQ"; break;
+    }
+    return os;
+}
+
+template<typename T>
+double cpu_norm1_impl(af::dim4 &dims, std::vector<T> &value) {
+    int M = dims[0];
+    int N = dims[1];
+
+    double norm1 = std::numeric_limits<double>::lowest();
+    for (int n = 0; n < N; n++) {
+        T *columnN = value.data() + n * M;
+        double sum = 0;
+        for (int m = 0; m < M; m++) { sum += abs(columnN[m]); }
+        norm1 = std::max(norm1, sum);
+    }
+    return norm1;
+}
+
+template<typename T>
+double cpu_norm_pq_impl(af::dim4 &dims, std::vector<T> &value, double p, double q) {
+    int N = dims[0];
+    int M = dims[1];
+
+    double norm = 0;
+    for (int n = 0; n < N; n++) {
+        T *columnN = value.data() + n * M;
+        double sum = 0;
+        
+        for (int m = 0; m < M; m++) { sum += std::pow(std::abs(columnN[m]), p); }
+
+        norm += std::pow(sum, q / p);
+    }
+    norm = std::pow(norm, 1.0 / q);
+
+    return norm;
+}
+
+double cpu_norm1(af::array &value) {
+    double norm1;
+    af::dim4 dims = value.dims();
+    if (value.type() == f16) {
+        vector<half_float::half> values(value.elements());
+        value.host(values.data());
+        norm1 = cpu_norm1_impl<half_float::half>(dims, values);
+    } else if (value.type() == c32 || value.type() == c64) {
+        vector<complex<double> > values(value.elements());
+        value.as(c64).host(values.data());
+        norm1 = cpu_norm1_impl<complex<double> >(dims, values);
+    } else {
+        vector<double> values(value.elements());
+        value.as(f64).host(values.data());
+        norm1 = cpu_norm1_impl<double>(dims, values);
+    }
+    return norm1;
+}
+
+double cpu_norm_pq(af::array &value, double p, double q) {
+    double norm2;
+    af::dim4 dims = value.dims();
+    if (value.type() == f16) {
+        vector<half_float::half> values(value.elements());
+        value.host(values.data());
+        norm2 = cpu_norm_pq_impl<half_float::half>(dims, values, p, q);
+    } else if (value.type() == c32 || value.type() == c64) {
+        vector<complex<double> > values(value.elements());
+        value.as(c64).host(values.data());
+        norm2 = cpu_norm_pq_impl<complex<double> >(dims, values, p, q);
+    } else {
+        vector<double> values(value.elements());
+        value.as(f64).host(values.data());
+        norm2 = cpu_norm_pq_impl<double>(dims, values, p, q);
+    }
+    return norm2;
+}
+
+template<typename T>
+double cpu_norm_inf_impl(af::dim4 &dims, std::vector<T> &value) {
+    int M = dims[0];
+    int N = dims[1];
+
+    double norm_inf = std::numeric_limits<double>::lowest();
+    for (int m = 0; m < M; m++) {
+        T *rowM    = value.data() + m;
+        double sum = 0;
+        for (int n = 0; n < N; n++) { sum += abs(rowM[n * M]); }
+        norm_inf = std::max(norm_inf, sum);
+    }
+    return norm_inf;
+}
+
+double cpu_norm_inf(af::array &value) {
+    double norm_inf;
+    af::dim4 dims = value.dims();
+    if (value.type() == c32 || value.type() == c64) {
+        vector<complex<double> > values(value.elements());
+        value.as(c64).host(values.data());
+        norm_inf = cpu_norm_inf_impl<complex<double> >(dims, values);
+    } else {
+        vector<double> values(value.elements());
+        value.as(f64).host(values.data());
+        norm_inf = cpu_norm_inf_impl<double>(dims, values);
+    }
+    return norm_inf;
+}
+
+using norm_params = std::tuple<af::dim4, af::dtype>;
+class Norm
+    : public ::testing::TestWithParam<std::tuple<af::dim4, af::dtype> > {};
+
+INSTANTIATE_TEST_CASE_P(
+    Norm, Norm,
+    ::testing::Combine(::testing::Values(dim4(3, 3), dim4(32, 32), dim4(33, 33),
+                                         dim4(64, 64), dim4(128, 128),
+                                         dim4(129, 129), dim4(256, 256),
+                                         dim4(257, 257)),
+                       ::testing::Values(f32, f64, c32, c64, f16)),
+    [](const ::testing::TestParamInfo<Norm::ParamType> info) {
+        stringstream ss;
+        using std::get;
+        ss << "dims_" << get<0>(info.param)[0] << "_" << get<0>(info.param)[1]
+           << "_dtype_" << get<1>(info.param);
+        return ss.str();
+    });
+
+TEST_P(Norm, Identity_AF_NORM_MATRIX_1) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+
+    array identity = af::identity(get<0>(param), get<1>(param));
+    double result  = norm(identity, AF_NORM_MATRIX_1);
+    double norm1   = cpu_norm1(identity);
+
+    ASSERT_DOUBLE_EQ(norm1, result);
+}
+
+TEST_P(Norm, Random_AF_NORM_MATRIX_1) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+
+    array in      = af::randu(get<0>(param), get<1>(param)) - 0.5f;
+    double result = norm(in, AF_NORM_MATRIX_1);
+    double norm1  = cpu_norm1(in);
+
+    ASSERT_NEAR(norm1, result, 2e-4);
+}
+
+TEST_P(Norm, Random_AF_NORM_VECTOR_1) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+
+    af::dim4 dims = get<0>(param);
+    dims[1] = 1; // Test a vector
+
+    array in      = af::randu(dims, get<1>(param)) - 0.5f;
+    double result = norm(in, AF_NORM_VECTOR_1);
+    double norm1  = cpu_norm_pq(in, 1, 1);
+
+    ASSERT_NEAR(norm1, result, 2e-4);
+}
+
+TEST_P(Norm, Random_AF_NORM_VECTOR_INF) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+
+    af::dim4 dims = get<0>(param);
+    dims[1] = 1; // Test a vector
+
+    array in      = af::randu(dims, get<1>(param)) - 0.5f;
+    double result = norm(in, AF_NORM_VECTOR_INF);
+    double norm_inf  = cpu_norm_inf(in);
+
+    ASSERT_NEAR(norm_inf, result, 2e-4);
+}
+
+TEST_P(Norm, Random_AF_NORM_VECTOR_2) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+
+    af::dim4 dims = get<0>(param);
+    dims[1] = 1; // Test a vector
+
+    array in      = af::randu(dims, get<1>(param)) - 0.5f;
+    double result = norm(in, AF_NORM_VECTOR_2);
+    double norm2  = cpu_norm_pq(in, 1, 2); // vectors lie in first dims so swap p and q
+
+    ASSERT_NEAR(norm2, result, 3e-4);
+}
+
+TEST_P(Norm, Random_AF_NORM_VECTOR_P_P_EQUAL_3_POINT_5) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+
+    af::dim4 dims = get<0>(param);
+    dims[1] = 1; // Test a vector
+
+    array in      = af::randu(dims, get<1>(param)) - 0.5f;
+    double result = norm(in, AF_NORM_VECTOR_P, 3.5);
+    double normp  = cpu_norm_pq(in, 1, 3.5); // vectors lie in first dims so swap p and q
+
+    ASSERT_NEAR(normp, result, 3e-4);
+}
+
+TEST_P(Norm, Identity_AF_NORM_MATRIX_2_NOT_SUPPORTED) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+    try {
+        double result =
+            norm(af::identity(get<0>(param), get<1>(param)), AF_NORM_MATRIX_2);
+        FAIL();
+    } catch (af::exception &ex) {
+        ASSERT_EQ(AF_ERR_NOT_SUPPORTED, ex.err());
+        return;
+    }
+    FAIL();
+}
+
+TEST_P(Norm, Identity_AF_NORM_MATRIX_INF) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+    array in        = af::identity(get<0>(param), get<1>(param));
+    double result   = norm(in, AF_NORM_MATRIX_INF);
+    double norm_inf = cpu_norm_inf(in);
+
+    ASSERT_DOUBLE_EQ(norm_inf, result);
+}
+
+TEST_P(Norm, Random_AF_NORM_MATRIX_INF) {
+    using std::get;
+    norm_params param = GetParam();
+    if (get<1>(param) == f16) SUPPORTED_TYPE_CHECK(half_float::half);
+    if (get<1>(param) == f64) SUPPORTED_TYPE_CHECK(double);
+    array in        = af::randu(get<0>(param), get<1>(param));
+    double result   = norm(in, AF_NORM_MATRIX_INF);
+    double norm_inf = cpu_norm_inf(in);
+
+    ASSERT_NEAR(norm_inf, result, 2e-4);
+}

From e073df6b6a5a21b20cf7bc99b5e046c1c1be1569 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Fri, 7 Mar 2025 18:14:22 -0500
Subject: [PATCH 429/473] Use offsets for CSR/COO to dense conversion in OpenCL
 and oneAPI (#3633)

* Offset values for sparse arrays are now taken into account when converting from sparse CSR/COO to dense for the OpenCL and oneAPI back ends. Test has been updated to also confirm fix for COO sparse format.

* Separate CSR and COO sparse to dense with offset tests and give them a more descriptive name.
---
 src/backend/oneapi/kernel/sparse.hpp   | 27 ++++++++++++-------
 src/backend/opencl/kernel/coo2dense.cl |  6 ++---
 test/sparse.cpp                        | 36 +++++++++++++++++++-------
 3 files changed, 47 insertions(+), 22 deletions(-)

diff --git a/src/backend/oneapi/kernel/sparse.hpp b/src/backend/oneapi/kernel/sparse.hpp
index 8cc7f99fcc..b7bc316267 100644
--- a/src/backend/oneapi/kernel/sparse.hpp
+++ b/src/backend/oneapi/kernel/sparse.hpp
@@ -54,9 +54,9 @@ class coo2DenseCreateKernel {
                 g.get_group_id(0) * g.get_local_range(0) * REPEAT + i;
             if (id >= values_.dims[0]) return;
 
-            T v   = vPtr_[id];
-            int r = rPtr_[id];
-            int c = cPtr_[id];
+            T v   = vPtr_[id + values_.offset];
+            int r = rPtr_[id + rowIdx_.offset];
+            int c = cPtr_[id + colIdx_.offset];
 
             int offset = r + c * output_.strides[1];
 
@@ -101,12 +101,15 @@ class csr2DenseCreateKernel {
    public:
     csr2DenseCreateKernel(write_accessor<T> output, read_accessor<T> values,
                           read_accessor<int> rowidx, read_accessor<int> colidx,
-                          const int M)
+                          const int M, const int v_off, const int r_off, const int c_off)
         : output_(output)
         , values_(values)
         , rowidx_(rowidx)
         , colidx_(colidx)
-        , M_(M) {}
+        , M_(M)
+        , v_off_(v_off)
+        , r_off_(r_off)
+        , c_off_(c_off) {}
 
     void operator()(sycl::nd_item<2> it) const {
         sycl::group g = it.get_group();
@@ -114,10 +117,10 @@ class csr2DenseCreateKernel {
         int lid = it.get_local_id(0);
         for (int rowId = g.get_group_id(0); rowId < M_;
              rowId += it.get_group_range(0)) {
-            int colStart = rowidx_[rowId];
-            int colEnd   = rowidx_[rowId + 1];
+            int colStart = rowidx_[rowId + r_off_];
+            int colEnd   = rowidx_[rowId + r_off_ + 1];
             for (int colId = colStart + lid; colId < colEnd; colId += THREADS) {
-                output_[rowId + colidx_[colId] * M_] = values_[colId];
+                output_[rowId + colidx_[colId + c_off_] * M_] = values_[colId + v_off_];
             }
         }
     }
@@ -128,6 +131,9 @@ class csr2DenseCreateKernel {
     read_accessor<int> rowidx_;
     read_accessor<int> colidx_;
     const int M_;
+    const int v_off_;
+    const int r_off_;
+    const int c_off_;
 };
 
 template<typename T>
@@ -151,7 +157,10 @@ void csr2dense(Param<T> output, const Param<T> values, const Param<int> rowIdx,
                                 sycl::no_init};
         h.parallel_for(sycl::nd_range{global, local},
                        csr2DenseCreateKernel<T, threads>(
-                           d_output, d_values, d_rowIdx, d_colIdx, M));
+                           d_output, d_values, d_rowIdx, d_colIdx, M,
+                           static_cast<int>(values.info.offset),
+                           static_cast<int>(rowIdx.info.offset),
+                           static_cast<int>(colIdx.info.offset)));
     });
 
     ONEAPI_DEBUG_FINISH(getQueue());
diff --git a/src/backend/opencl/kernel/coo2dense.cl b/src/backend/opencl/kernel/coo2dense.cl
index 539c98ada1..85afbfcd4b 100644
--- a/src/backend/opencl/kernel/coo2dense.cl
+++ b/src/backend/opencl/kernel/coo2dense.cl
@@ -17,9 +17,9 @@ kernel void coo2Dense(global T *oPtr, const KParam output, global const T *vPtr,
         const int id = i + get_group_id(0) * dimSize * reps;
         if (id >= values.dims[0]) return;
 
-        T v   = vPtr[id];
-        int r = rPtr[id];
-        int c = cPtr[id];
+        T v   = vPtr[id + values.offset];
+        int r = rPtr[id + rowIdx.offset];
+        int c = cPtr[id + colIdx.offset];
 
         int offset = r + c * output.strides[1];
 
diff --git a/test/sparse.cpp b/test/sparse.cpp
index 9e3f29ae35..f1e1b67d72 100644
--- a/test/sparse.cpp
+++ b/test/sparse.cpp
@@ -110,24 +110,40 @@ TEST(Sparse, ISSUE_1745) {
                               row_idx.get(), col_idx.get(), AF_STORAGE_CSR));
 }
 
-TEST(Sparse, ISSUE_1918) {
+TEST(Sparse, offsets_work_csr_to_dense_ISSUE_1918) {
     array reference(2,2);
     reference(0, span) = 0;
     reference(1, span) = 2;
-    array output;
     float value[] = { 1, 1, 2, 2 };
-    int index[] = { -1, 1, 2 };
-    int row[] = { 0, 2, 2, 0, 0, 2 };
+    int row_csr[] = { 0, 2, 2, 0, 0, 2 };
     int col[] = { 0, 1, 0, 1 };
     array values(4, 1, value, afHost);
-    array rows(6, 1, row, afHost);
+    array rows_csr(6, 1, row_csr, afHost);
     array cols(4, 1, col, afHost);
-    array S;
+    array S_csr;
   
-    S = sparse(2, 2, values(seq(2, 3)), rows(seq(3, 5)), cols(seq(2, 3)));
-    output = dense(S);
+    S_csr = sparse(2, 2, values(seq(2, 3)), rows_csr(seq(3, 5)), cols(seq(2, 3)));
+    array output_csr = dense(S_csr);
 
-    ASSERT_ARRAYS_EQ(reference, output);
+    EXPECT_ARRAYS_EQ(reference, output_csr);
+}
+
+TEST(Sparse, offsets_work_coo_to_dense_ISSUE_1918) {
+    array reference(2,2);
+    reference(0, span) = 0;
+    reference(1, span) = 2;
+    float value[] = { 1, 1, 2, 2 };
+    int row_coo[] = { 0, 0, 1, 1 };
+    int col[] = { 0, 1, 0, 1 };
+    array values(4, 1, value, afHost);
+    array rows_coo(4, 1, row_coo, afHost);
+    array cols(4, 1, col, afHost);
+    array S_coo;
+  
+    S_coo = sparse(2, 2, values(seq(2, 3)), rows_coo(seq(2, 3)), cols(seq(2, 3)), AF_STORAGE_COO);
+    array output_coo = dense(S_coo);
+
+    EXPECT_ARRAYS_EQ(reference, output_coo);
 }
 
 TEST(Sparse, ISSUE_2134_COO) {
@@ -457,4 +473,4 @@ TEST(Sparse, CPPDenseToSparseConversions) {
     ASSERT_ARRAYS_EQ(
         non_zero_T,
         af::sparseGetValues(csr_sparse_arr));  // csr values are transposed
-}
\ No newline at end of file
+}

From cdbbc75fc4f8d89a3c93369694cc38dd396c334d Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Wed, 4 Oct 2023 04:14:35 -0400
Subject: [PATCH 430/473] signed 8-bit integer support

---
 include/af/arith.h                            | 31 ++++++------
 include/af/array.h                            | 43 +++++++++-------
 include/af/defines.h                          |  1 +
 include/af/traits.hpp                         | 10 ++++
 src/api/c/anisotropic_diffusion.cpp           |  1 +
 src/api/c/array.cpp                           | 14 ++++++
 src/api/c/assign.cpp                          |  4 ++
 src/api/c/bilateral.cpp                       |  2 +
 src/api/c/binary.cpp                          |  6 +++
 src/api/c/canny.cpp                           |  5 ++
 src/api/c/cast.cpp                            |  2 +
 src/api/c/clamp.cpp                           |  2 +
 src/api/c/convolve.cpp                        |  9 ++++
 src/api/c/corrcoef.cpp                        |  2 +
 src/api/c/covariance.cpp                      |  2 +
 src/api/c/data.cpp                            | 10 ++++
 src/api/c/deconvolution.cpp                   |  3 ++
 src/api/c/device.cpp                          |  3 ++
 src/api/c/diff.cpp                            |  3 ++
 src/api/c/dog.cpp                             |  2 +
 src/api/c/exampleFunction.cpp                 |  1 +
 src/api/c/fast.cpp                            |  5 ++
 src/api/c/fftconvolve.cpp                     |  5 ++
 src/api/c/filters.cpp                         |  5 ++
 src/api/c/flip.cpp                            |  2 +
 src/api/c/handle.cpp                          |  6 +++
 src/api/c/hist.cpp                            |  5 ++
 src/api/c/histeq.cpp                          |  2 +
 src/api/c/histogram.cpp                       |  5 ++
 src/api/c/image.cpp                           |  2 +
 src/api/c/imageio.cpp                         |  4 +-
 src/api/c/imageio2.cpp                        |  5 +-
 src/api/c/implicit.cpp                        |  3 +-
 src/api/c/index.cpp                           |  5 ++
 src/api/c/internal.cpp                        |  9 ++++
 src/api/c/join.cpp                            |  3 ++
 src/api/c/match_template.cpp                  |  5 ++
 src/api/c/mean.cpp                            |  5 ++
 src/api/c/meanshift.cpp                       |  5 ++
 src/api/c/median.cpp                          |  3 ++
 src/api/c/memory.cpp                          |  8 +++
 src/api/c/moddims.cpp                         |  3 ++
 src/api/c/morph.cpp                           |  3 ++
 src/api/c/nearest_neighbour.cpp               |  5 ++
 src/api/c/plot.cpp                            | 11 ++++
 src/api/c/print.cpp                           |  6 +++
 src/api/c/random.cpp                          |  3 ++
 src/api/c/reduce.cpp                          | 28 +++++++++++
 src/api/c/reorder.cpp                         |  2 +
 src/api/c/replace.cpp                         |  3 ++
 src/api/c/resize.cpp                          |  2 +
 src/api/c/rgb_gray.cpp                        |  4 ++
 src/api/c/rotate.cpp                          |  2 +
 src/api/c/sat.cpp                             |  2 +
 src/api/c/scan.cpp                            |  6 +++
 src/api/c/select.cpp                          |  6 +++
 src/api/c/set.cpp                             |  4 ++
 src/api/c/shift.cpp                           |  2 +
 src/api/c/sobel.cpp                           |  4 ++
 src/api/c/sort.cpp                            | 10 ++++
 src/api/c/stdev.cpp                           |  3 ++
 src/api/c/stream.cpp                          |  3 ++
 src/api/c/surface.cpp                         |  4 ++
 src/api/c/susan.cpp                           |  5 ++
 src/api/c/tile.cpp                            |  2 +
 src/api/c/transform.cpp                       |  2 +
 src/api/c/transpose.cpp                       |  3 ++
 src/api/c/type_util.cpp                       |  1 +
 src/api/c/type_util.hpp                       |  5 ++
 src/api/c/unary.cpp                           |  2 +
 src/api/c/unwrap.cpp                          |  4 ++
 src/api/c/var.cpp                             | 14 ++++++
 src/api/c/vector_field.cpp                    | 12 +++++
 src/api/c/where.cpp                           |  2 +
 src/api/c/wrap.cpp                            |  2 +
 src/api/cpp/array.cpp                         |  7 +++
 src/api/cpp/corrcoef.cpp                      |  1 +
 src/api/cpp/data.cpp                          |  1 +
 src/api/cpp/device.cpp                        |  1 +
 src/api/cpp/mean.cpp                          |  1 +
 src/api/cpp/median.cpp                        |  1 +
 src/api/cpp/reduce.cpp                        |  3 ++
 src/api/cpp/stdev.cpp                         |  1 +
 src/api/cpp/var.cpp                           |  1 +
 src/backend/common/TemplateTypename.hpp       |  1 +
 src/backend/common/cast.cpp                   |  3 ++
 src/backend/common/cast.hpp                   | 29 +++++------
 src/backend/common/graphics_common.cpp        |  1 +
 src/backend/common/half.hpp                   |  9 ++++
 src/backend/common/jit/BinaryNode.cpp         |  4 ++
 src/backend/common/jit/Node.hpp               |  2 +
 src/backend/common/moddims.cpp                |  1 +
 src/backend/common/traits.hpp                 |  3 +-
 src/backend/common/util.cpp                   |  2 +
 src/backend/cpu/Array.cpp                     |  1 +
 src/backend/cpu/assign.cpp                    |  1 +
 src/backend/cpu/bilateral.cpp                 |  1 +
 src/backend/cpu/cast.hpp                      |  1 +
 src/backend/cpu/convolve.cpp                  |  1 +
 src/backend/cpu/copy.cpp                      |  5 ++
 src/backend/cpu/diagonal.cpp                  |  1 +
 src/backend/cpu/diff.cpp                      |  1 +
 src/backend/cpu/exampleFunction.cpp           |  1 +
 src/backend/cpu/fast.cpp                      |  1 +
 src/backend/cpu/fftconvolve.cpp               |  1 +
 src/backend/cpu/hist_graphics.cpp             |  1 +
 src/backend/cpu/histogram.cpp                 |  1 +
 src/backend/cpu/identity.cpp                  |  1 +
 src/backend/cpu/image.cpp                     |  1 +
 src/backend/cpu/index.cpp                     |  1 +
 src/backend/cpu/iota.cpp                      |  1 +
 src/backend/cpu/ireduce.cpp                   |  2 +
 src/backend/cpu/join.cpp                      |  2 +
 src/backend/cpu/kernel/random_engine.hpp      |  5 ++
 .../kernel/sort_by_key/sort_by_key_impl.cpp   |  2 +-
 src/backend/cpu/kernel/sort_by_key_impl.hpp   |  1 +
 src/backend/cpu/lookup.cpp                    |  3 ++
 src/backend/cpu/match_template.cpp            |  1 +
 src/backend/cpu/mean.cpp                      |  1 +
 src/backend/cpu/meanshift.cpp                 |  1 +
 src/backend/cpu/medfilt.cpp                   |  1 +
 src/backend/cpu/memory.cpp                    |  1 +
 src/backend/cpu/moments.cpp                   |  1 +
 src/backend/cpu/morph.cpp                     |  1 +
 src/backend/cpu/nearest_neighbour.cpp         |  1 +
 src/backend/cpu/plot.cpp                      |  1 +
 src/backend/cpu/random_engine.cpp             |  1 +
 src/backend/cpu/range.cpp                     |  1 +
 src/backend/cpu/reduce.cpp                    |  8 +++
 src/backend/cpu/reorder.cpp                   |  1 +
 src/backend/cpu/reshape.cpp                   |  4 ++
 src/backend/cpu/resize.cpp                    |  1 +
 src/backend/cpu/rotate.cpp                    |  1 +
 src/backend/cpu/scan.cpp                      |  1 +
 src/backend/cpu/select.cpp                    |  1 +
 src/backend/cpu/set.cpp                       |  1 +
 src/backend/cpu/shift.cpp                     |  1 +
 src/backend/cpu/sobel.cpp                     |  1 +
 src/backend/cpu/sort.cpp                      |  1 +
 src/backend/cpu/sort_by_key.cpp               |  2 +
 src/backend/cpu/sort_index.cpp                |  1 +
 src/backend/cpu/surface.cpp                   |  1 +
 src/backend/cpu/susan.cpp                     |  1 +
 src/backend/cpu/tile.cpp                      |  1 +
 src/backend/cpu/transform.cpp                 |  1 +
 src/backend/cpu/transpose.cpp                 |  1 +
 src/backend/cpu/triangle.cpp                  |  1 +
 src/backend/cpu/types.hpp                     |  1 +
 src/backend/cpu/unwrap.cpp                    |  1 +
 src/backend/cpu/vector_field.cpp              |  1 +
 src/backend/cpu/where.cpp                     |  1 +
 src/backend/cpu/wrap.cpp                      |  1 +
 src/backend/cuda/Array.cpp                    |  1 +
 src/backend/cuda/all.cu                       |  1 +
 src/backend/cuda/any.cu                       |  1 +
 src/backend/cuda/assign.cpp                   |  1 +
 src/backend/cuda/bilateral.cpp                |  1 +
 src/backend/cuda/cast.hpp                     |  1 +
 src/backend/cuda/convolve.cpp                 |  1 +
 src/backend/cuda/copy.cpp                     |  5 ++
 src/backend/cuda/count.cu                     |  1 +
 src/backend/cuda/cudaDataType.hpp             | 16 ++++++
 src/backend/cuda/cudnn.cpp                    |  6 +++
 src/backend/cuda/diagonal.cpp                 |  1 +
 src/backend/cuda/diff.cpp                     |  1 +
 src/backend/cuda/exampleFunction.cpp          |  1 +
 src/backend/cuda/fast.cu                      |  1 +
 src/backend/cuda/fast_pyramid.cpp             |  1 +
 src/backend/cuda/fftconvolve.cpp              |  1 +
 src/backend/cuda/hist_graphics.cpp            |  1 +
 src/backend/cuda/histogram.cpp                |  1 +
 src/backend/cuda/identity.cpp                 |  1 +
 src/backend/cuda/image.cpp                    |  1 +
 src/backend/cuda/index.cpp                    |  1 +
 src/backend/cuda/iota.cpp                     |  1 +
 src/backend/cuda/ireduce.cpp                  |  2 +
 src/backend/cuda/jit.cpp                      |  3 ++
 src/backend/cuda/join.cpp                     |  2 +
 .../cuda/kernel/convolve_separable.cpp        |  1 +
 src/backend/cuda/kernel/copy.cuh              | 13 +++++
 src/backend/cuda/kernel/random_engine.hpp     | 13 +++++
 src/backend/cuda/kernel/shared.hpp            |  1 +
 .../thrust_sort_by_key_impl.cu                |  2 +-
 .../cuda/kernel/thrust_sort_by_key_impl.hpp   |  1 +
 src/backend/cuda/lookup.cpp                   |  3 ++
 src/backend/cuda/match_template.cpp           |  1 +
 src/backend/cuda/math.hpp                     |  8 +++
 src/backend/cuda/max.cu                       |  1 +
 src/backend/cuda/mean.cu                      |  1 +
 src/backend/cuda/meanshift.cpp                |  1 +
 src/backend/cuda/medfilt.cpp                  |  1 +
 src/backend/cuda/memory.cpp                   |  1 +
 src/backend/cuda/min.cu                       |  1 +
 src/backend/cuda/moments.cpp                  |  1 +
 src/backend/cuda/morph.cpp                    |  1 +
 src/backend/cuda/nearest_neighbour.cu         |  1 +
 src/backend/cuda/pad_array_borders.cpp        |  1 +
 src/backend/cuda/plot.cpp                     |  1 +
 src/backend/cuda/product.cu                   |  1 +
 src/backend/cuda/random_engine.cu             |  1 +
 src/backend/cuda/range.cpp                    |  1 +
 src/backend/cuda/reorder.cpp                  |  1 +
 src/backend/cuda/reshape.cpp                  |  3 ++
 src/backend/cuda/resize.cpp                   |  1 +
 src/backend/cuda/rotate.cpp                   |  1 +
 src/backend/cuda/scan.cpp                     |  1 +
 src/backend/cuda/select.cpp                   |  1 +
 src/backend/cuda/set.cu                       |  1 +
 src/backend/cuda/shift.cpp                    |  1 +
 src/backend/cuda/sobel.cpp                    |  1 +
 src/backend/cuda/sort.cu                      |  1 +
 src/backend/cuda/sort_by_key.cu               |  2 +
 src/backend/cuda/sort_index.cu                |  1 +
 src/backend/cuda/sum.cu                       |  2 +
 src/backend/cuda/surface.cpp                  |  1 +
 src/backend/cuda/susan.cpp                    |  1 +
 src/backend/cuda/tile.cpp                     |  1 +
 src/backend/cuda/transform.cpp                |  1 +
 src/backend/cuda/transpose.cpp                |  1 +
 src/backend/cuda/transpose_inplace.cpp        |  1 +
 src/backend/cuda/triangle.cpp                 |  1 +
 src/backend/cuda/types.hpp                    |  6 +++
 src/backend/cuda/unwrap.cpp                   |  1 +
 src/backend/cuda/vector_field.cpp             |  1 +
 src/backend/cuda/where.cpp                    |  1 +
 src/backend/cuda/wrap.cpp                     |  1 +
 src/backend/oneapi/Array.cpp                  |  1 +
 src/backend/oneapi/all.cpp                    |  1 +
 src/backend/oneapi/any.cpp                    |  1 +
 src/backend/oneapi/assign.cpp                 |  1 +
 src/backend/oneapi/bilateral.cpp              |  1 +
 src/backend/oneapi/cast.hpp                   |  1 +
 src/backend/oneapi/convolve.cpp               |  1 +
 src/backend/oneapi/convolve_separable.cpp     |  1 +
 src/backend/oneapi/copy.cpp                   |  5 ++
 src/backend/oneapi/count.cpp                  |  1 +
 src/backend/oneapi/diagonal.cpp               |  1 +
 src/backend/oneapi/diff.cpp                   |  1 +
 src/backend/oneapi/exampleFunction.cpp        |  1 +
 src/backend/oneapi/fast.cpp                   |  1 +
 src/backend/oneapi/fftconvolve.cpp            |  1 +
 src/backend/oneapi/hist_graphics.cpp          |  1 +
 src/backend/oneapi/histogram.cpp              |  1 +
 src/backend/oneapi/identity.cpp               |  1 +
 src/backend/oneapi/image.cpp                  |  1 +
 src/backend/oneapi/index.cpp                  |  1 +
 src/backend/oneapi/iota.cpp                   |  1 +
 src/backend/oneapi/ireduce.cpp                |  2 +
 src/backend/oneapi/jit.cpp                    |  3 ++
 src/backend/oneapi/join.cpp                   |  2 +
 src/backend/oneapi/kernel/convolve1.hpp       |  1 +
 src/backend/oneapi/kernel/convolve2.hpp       |  1 +
 src/backend/oneapi/kernel/convolve3.hpp       |  1 +
 .../oneapi/kernel/convolve_separable.cpp      |  1 +
 src/backend/oneapi/kernel/memcopy.hpp         | 14 ++++++
 .../oneapi/kernel/random_engine_write.hpp     | 14 ++++++
 .../kernel/sort_by_key/sort_by_key_impl.cpp   |  2 +-
 .../oneapi/kernel/sort_by_key_impl.hpp        |  1 +
 src/backend/oneapi/lookup.cpp                 |  3 ++
 src/backend/oneapi/match_template.cpp         |  1 +
 src/backend/oneapi/max.cpp                    |  1 +
 src/backend/oneapi/mean.cpp                   |  1 +
 src/backend/oneapi/meanshift.cpp              |  1 +
 src/backend/oneapi/medfilt.cpp                |  1 +
 src/backend/oneapi/memory.cpp                 |  1 +
 src/backend/oneapi/min.cpp                    |  1 +
 src/backend/oneapi/moments.cpp                |  1 +
 src/backend/oneapi/morph.cpp                  |  1 +
 src/backend/oneapi/nearest_neighbour.cpp      |  1 +
 src/backend/oneapi/plot.cpp                   |  1 +
 src/backend/oneapi/product.cpp                |  1 +
 src/backend/oneapi/random_engine.cpp          |  1 +
 src/backend/oneapi/range.cpp                  |  1 +
 src/backend/oneapi/reorder.cpp                |  1 +
 src/backend/oneapi/reshape.cpp                |  3 ++
 src/backend/oneapi/resize.cpp                 |  1 +
 src/backend/oneapi/rotate.cpp                 |  1 +
 src/backend/oneapi/scan.cpp                   |  1 +
 src/backend/oneapi/select.cpp                 |  1 +
 src/backend/oneapi/set.cpp                    |  1 +
 src/backend/oneapi/shift.cpp                  |  1 +
 src/backend/oneapi/sobel.cpp                  |  1 +
 src/backend/oneapi/sort.cpp                   |  1 +
 src/backend/oneapi/sort_by_key.cpp            |  2 +
 src/backend/oneapi/sort_index.cpp             |  1 +
 src/backend/oneapi/sum.cpp                    |  2 +
 src/backend/oneapi/surface.cpp                |  1 +
 src/backend/oneapi/susan.cpp                  |  1 +
 src/backend/oneapi/tile.cpp                   |  1 +
 src/backend/oneapi/transform.cpp              |  1 +
 src/backend/oneapi/transpose.cpp              |  1 +
 src/backend/oneapi/transpose_inplace.cpp      |  1 +
 src/backend/oneapi/triangle.cpp               |  1 +
 src/backend/oneapi/types.hpp                  | 10 ++++
 src/backend/oneapi/unwrap.cpp                 |  1 +
 src/backend/oneapi/vector_field.cpp           |  1 +
 src/backend/oneapi/where.cpp                  |  1 +
 src/backend/oneapi/wrap.cpp                   |  1 +
 src/backend/opencl/Array.cpp                  |  1 +
 src/backend/opencl/CMakeLists.txt             |  1 +
 src/backend/opencl/all.cpp                    |  1 +
 src/backend/opencl/any.cpp                    |  1 +
 src/backend/opencl/assign.cpp                 |  1 +
 src/backend/opencl/bilateral.cpp              |  1 +
 src/backend/opencl/cast.hpp                   |  5 ++
 src/backend/opencl/compile_module.cpp         |  3 ++
 src/backend/opencl/convolve.cpp               |  1 +
 src/backend/opencl/convolve_separable.cpp     |  1 +
 src/backend/opencl/copy.cpp                   |  5 ++
 src/backend/opencl/count.cpp                  |  1 +
 src/backend/opencl/diagonal.cpp               |  1 +
 src/backend/opencl/diff.cpp                   |  1 +
 src/backend/opencl/exampleFunction.cpp        |  1 +
 src/backend/opencl/fast.cpp                   |  1 +
 src/backend/opencl/fftconvolve.cpp            |  1 +
 src/backend/opencl/flood_fill.cpp             |  1 +
 src/backend/opencl/hist_graphics.cpp          |  1 +
 src/backend/opencl/histogram.cpp              |  1 +
 src/backend/opencl/identity.cpp               |  1 +
 src/backend/opencl/image.cpp                  |  1 +
 src/backend/opencl/index.cpp                  |  1 +
 src/backend/opencl/iota.cpp                   |  1 +
 src/backend/opencl/ireduce.cpp                |  2 +
 src/backend/opencl/join.cpp                   |  2 +
 src/backend/opencl/kernel/convolve/conv1.cpp  |  1 +
 .../opencl/kernel/convolve/conv2_s8.cpp       | 20 ++++++++
 src/backend/opencl/kernel/convolve/conv3.cpp  |  1 +
 .../opencl/kernel/convolve_separable.cpp      |  1 +
 .../opencl/kernel/random_engine_write.cl      | 50 +++++++++++++++++++
 .../kernel/sort_by_key/sort_by_key_impl.cpp   |  2 +-
 .../opencl/kernel/sort_by_key_impl.hpp        |  1 +
 src/backend/opencl/lookup.cpp                 |  3 ++
 src/backend/opencl/match_template.cpp         |  1 +
 src/backend/opencl/max.cpp                    |  1 +
 src/backend/opencl/mean.cpp                   |  1 +
 src/backend/opencl/meanshift.cpp              |  1 +
 src/backend/opencl/medfilt.cpp                |  1 +
 src/backend/opencl/memory.cpp                 |  1 +
 src/backend/opencl/min.cpp                    |  1 +
 src/backend/opencl/moments.cpp                |  1 +
 src/backend/opencl/morph.cpp                  |  1 +
 src/backend/opencl/nearest_neighbour.cpp      |  1 +
 src/backend/opencl/plot.cpp                   |  1 +
 src/backend/opencl/product.cpp                |  1 +
 src/backend/opencl/random_engine.cpp          |  1 +
 src/backend/opencl/range.cpp                  |  1 +
 src/backend/opencl/reorder.cpp                |  1 +
 src/backend/opencl/resize.cpp                 |  1 +
 src/backend/opencl/rotate.cpp                 |  1 +
 src/backend/opencl/scan.cpp                   |  1 +
 src/backend/opencl/select.cpp                 |  1 +
 src/backend/opencl/set.cpp                    |  1 +
 src/backend/opencl/shift.cpp                  |  1 +
 src/backend/opencl/sobel.cpp                  |  1 +
 src/backend/opencl/sort.cpp                   |  1 +
 src/backend/opencl/sort_by_key.cpp            |  2 +
 src/backend/opencl/sort_index.cpp             |  1 +
 src/backend/opencl/sum.cpp                    |  2 +
 src/backend/opencl/surface.cpp                |  1 +
 src/backend/opencl/susan.cpp                  |  1 +
 src/backend/opencl/tile.cpp                   |  1 +
 src/backend/opencl/transform.cpp              |  1 +
 src/backend/opencl/transpose.cpp              |  1 +
 src/backend/opencl/transpose_inplace.cpp      |  1 +
 src/backend/opencl/triangle.cpp               |  1 +
 src/backend/opencl/types.cpp                  |  1 +
 src/backend/opencl/types.hpp                  | 10 ++++
 src/backend/opencl/unwrap.cpp                 |  1 +
 src/backend/opencl/vector_field.cpp           |  1 +
 src/backend/opencl/where.cpp                  |  1 +
 src/backend/opencl/wrap.cpp                   |  1 +
 test/anisotropic_diffusion.cpp                |  2 +-
 test/array.cpp                                | 15 +++++-
 test/arrayfire_test.cpp                       | 23 +++++++++
 test/arrayio.cpp                              |  4 +-
 test/assign.cpp                               |  4 +-
 test/bilateral.cpp                            |  3 +-
 test/binary.cpp                               |  6 ++-
 test/canny.cpp                                |  2 +-
 test/cast.cpp                                 |  2 +
 test/clamp.cpp                                |  1 +
 test/compare.cpp                              |  4 +-
 test/constant.cpp                             |  3 +-
 test/convolve.cpp                             |  4 +-
 test/corrcoef.cpp                             |  3 +-
 test/covariance.cpp                           | 10 ++--
 test/diagonal.cpp                             |  4 +-
 test/diff1.cpp                                |  2 +-
 test/diff2.cpp                                |  2 +-
 test/dog.cpp                                  |  3 +-
 test/fast.cpp                                 |  2 +-
 test/fftconvolve.cpp                          |  4 +-
 test/gen_index.cpp                            |  5 +-
 test/half.cpp                                 |  2 +
 test/histogram.cpp                            |  2 +-
 test/index.cpp                                |  7 +--
 test/inverse_deconv.cpp                       |  2 +-
 test/iota.cpp                                 |  3 +-
 test/iterative_deconv.cpp                     |  2 +-
 test/join.cpp                                 |  4 +-
 test/match_template.cpp                       |  3 +-
 test/mean.cpp                                 | 10 ++--
 test/meanshift.cpp                            |  4 +-
 test/medfilt.cpp                              |  3 +-
 test/memory.cpp                               |  3 +-
 test/moddims.cpp                              |  4 +-
 test/morph.cpp                                |  3 +-
 test/nearest_neighbour.cpp                    |  9 +++-
 test/pad_borders.cpp                          |  4 +-
 test/random.cpp                               | 12 ++---
 test/range.cpp                                |  5 +-
 test/reduce.cpp                               | 12 ++++-
 test/reorder.cpp                              |  2 +-
 test/replace.cpp                              |  2 +-
 test/resize.cpp                               |  4 +-
 test/rotate.cpp                               |  3 +-
 test/rotate_linear.cpp                        |  3 +-
 test/sat.cpp                                  |  4 +-
 test/select.cpp                               |  2 +-
 test/shift.cpp                                |  3 +-
 test/sobel.cpp                                |  3 +-
 test/sort.cpp                                 |  4 +-
 test/sort_by_key.cpp                          |  4 +-
 test/sort_index.cpp                           |  4 +-
 test/stdev.cpp                                |  9 ++--
 test/susan.cpp                                |  3 +-
 test/testHelpers.hpp                          |  1 +
 test/tile.cpp                                 |  4 +-
 test/transform.cpp                            |  2 +-
 test/translate.cpp                            |  2 +-
 test/transpose.cpp                            |  4 +-
 test/transpose_inplace.cpp                    |  4 +-
 test/triangle.cpp                             |  3 +-
 test/unwrap.cpp                               |  3 +-
 test/var.cpp                                  |  6 +--
 test/where.cpp                                |  2 +-
 test/wrap.cpp                                 |  3 +-
 test/write.cpp                                |  2 +-
 438 files changed, 1135 insertions(+), 159 deletions(-)
 create mode 100644 src/backend/opencl/kernel/convolve/conv2_s8.cpp

diff --git a/include/af/arith.h b/include/af/arith.h
index 5e470f448b..0dd2eb2c1f 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -910,21 +910,22 @@ extern "C" {
        be performed by ArrayFire. The following table shows which casts will
        be optimized out. outer -> inner -> outer
 
-       | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | u8 | b8 | s64 | u64 | s16 | u16 | f16 |
-       |---------|-----|-----|-----|-----|-----|-----|----|----|-----|-----|-----|-----|-----|
-       | f32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-       | f64     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-       | c32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-       | c64     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-       | s32     | x   | x   | x   | x   | x   | x   |    |    | x   | x   |     |     | x   |
-       | u32     | x   | x   | x   | x   | x   | x   |    |    | x   | x   |     |     | x   |
-       | u8      | x   | x   | x   | x   | x   | x   | x  | x  | x   | x   | x   | x   | x   |
-       | b8      | x   | x   | x   | x   | x   | x   | x  | x  | x   | x   | x   | x   | x   |
-       | s64     | x   | x   | x   | x   |     |     |    |    | x   | x   |     |     | x   |
-       | u64     | x   | x   | x   | x   |     |     |    |    | x   | x   |     |     | x   |
-       | s16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
-       | u16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
-       | f16     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+       | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | s8 | u8 | b8 | s64 | u64 | s16 | u16 | f16 |
+       |---------|-----|-----|-----|-----|-----|-----|----|----|----|-----|-----|-----|-----|-----|
+       | f32     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
+       | f64     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
+       | c32     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
+       | c64     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
+       | s32     | x   | x   | x   | x   | x   | x   |    |    |    | x   | x   |     |     | x   |
+       | u32     | x   | x   | x   | x   | x   | x   |    |    |    | x   | x   |     |     | x   |
+       | s8      | x   | x   | x   | x   | x   | x   | x  | x  | x  | x   | x   | x   | x   | x   |
+       | u8      | x   | x   | x   | x   | x   | x   | x  | x  | x  | x   | x   | x   | x   | x   |
+       | b8      | x   | x   | x   | x   | x   | x   | x  | x  | x  | x   | x   | x   | x   | x   |
+       | s64     | x   | x   | x   | x   |     |     |    |    |    | x   | x   |     |     | x   |
+       | u64     | x   | x   | x   | x   |     |     |    |    |    | x   | x   |     |     | x   |
+       | s16     | x   | x   | x   | x   | x   | x   |    |    |    | x   | x   | x   | x   | x   |
+       | u16     | x   | x   | x   | x   | x   | x   |    |    |    | x   | x   | x   | x   | x   |
+       | f16     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
 
        If you want to avoid this behavior use, af_eval after the first cast
        operation. This will ensure that the cast operation is performed on the
diff --git a/include/af/array.h b/include/af/array.h
index 4186b95d08..a442147565 100644
--- a/include/af/array.h
+++ b/include/af/array.h
@@ -82,6 +82,7 @@ namespace af
             array_proxy& operator OP(const unsigned &a);            \
             array_proxy& operator OP(const bool &a);                \
             array_proxy& operator OP(const char &a);                \
+            array_proxy& operator OP(const signed char &a);         \
             array_proxy& operator OP(const unsigned char &a);       \
             array_proxy& operator OP(const long  &a);               \
             array_proxy& operator OP(const unsigned long &a);       \
@@ -762,8 +763,8 @@ namespace af
         bool isfloating() const;
 
         /**
-           \brief Returns true if the array type is \ref u8, \ref b8, \ref s32
-                  \ref u32, \ref s64, \ref u64, \ref s16, \ref u16
+           \brief Returns true if the array type is \ref s8, \ref u8, \ref b8,
+                  \ref s32, \ref u32, \ref s64, \ref u64, \ref s16, \ref u16
         */
         bool isinteger() const;
 
@@ -953,21 +954,22 @@ namespace af
         /// and then back to f64, then the cast to f32 will be skipped and that
         /// operation will *NOT* be performed by ArrayFire. The following table
         /// shows which casts will be optimized out. outer -> inner -> outer
-        /// | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | u8 | b8 | s64 | u64 | s16 | u16 | f16 |
-        /// |---------|-----|-----|-----|-----|-----|-----|----|----|-----|-----|-----|-----|-----|
-        /// | f32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-        /// | f64     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-        /// | c32     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-        /// | c64     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
-        /// | s32     | x   | x   | x   | x   | x   | x   |    |    | x   | x   |     |     | x   |
-        /// | u32     | x   | x   | x   | x   | x   | x   |    |    | x   | x   |     |     | x   |
-        /// | u8      | x   | x   | x   | x   | x   | x   | x  | x  | x   | x   | x   | x   | x   |
-        /// | b8      | x   | x   | x   | x   | x   | x   | x  | x  | x   | x   | x   | x   | x   |
-        /// | s64     | x   | x   | x   | x   |     |     |    |    | x   | x   |     |     | x   |
-        /// | u64     | x   | x   | x   | x   |     |     |    |    | x   | x   |     |     | x   |
-        /// | s16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
-        /// | u16     | x   | x   | x   | x   | x   | x   |    |    | x   | x   | x   | x   | x   |
-        /// | f16     | x   | x   | x   | x   |     |     |    |    |     |     |     |     | x   |
+        /// | inner-> | f32 | f64 | c32 | c64 | s32 | u32 | s8 | u8 | b8 | s64 | u64 | s16 | u16 | f16 |
+        /// |---------|-----|-----|-----|-----|-----|-----|----|----|----|-----|-----|-----|-----|-----|
+        /// | f32     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
+        /// | f64     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
+        /// | c32     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
+        /// | c64     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
+        /// | s32     | x   | x   | x   | x   | x   | x   |    |    |    | x   | x   |     |     | x   |
+        /// | u32     | x   | x   | x   | x   | x   | x   |    |    |    | x   | x   |     |     | x   |
+        /// | s8      | x   | x   | x   | x   | x   | x   | x  | x  | x  | x   | x   | x   | x   | x   |
+        /// | u8      | x   | x   | x   | x   | x   | x   | x  | x  | x  | x   | x   | x   | x   | x   |
+        /// | b8      | x   | x   | x   | x   | x   | x   | x  | x  | x  | x   | x   | x   | x   | x   |
+        /// | s64     | x   | x   | x   | x   |     |     |    |    |    | x   | x   |     |     | x   |
+        /// | u64     | x   | x   | x   | x   |     |     |    |    |    | x   | x   |     |     | x   |
+        /// | s16     | x   | x   | x   | x   | x   | x   |    |    |    | x   | x   | x   | x   | x   |
+        /// | u16     | x   | x   | x   | x   | x   | x   |    |    |    | x   | x   | x   | x   | x   |
+        /// | f16     | x   | x   | x   | x   |     |     |    |    |    |     |     |     |     | x   |
         /// If you want to avoid this behavior use af_eval after the first cast
         /// operation. This will ensure that the cast operation is performed on
         /// the af::array
@@ -998,6 +1000,7 @@ namespace af
         array& OP2(const unsigned &val);            /**< \copydoc OP2##(const array &) */ \
         array& OP2(const bool &val);                /**< \copydoc OP2##(const array &) */ \
         array& OP2(const char &val);                /**< \copydoc OP2##(const array &) */ \
+        array& OP2(const signed char &val);         /**< \copydoc OP2##(const array &) */ \
         array& OP2(const unsigned char &val);       /**< \copydoc OP2##(const array &) */ \
         array& OP2(const long  &val);               /**< \copydoc OP2##(const array &) */ \
         array& OP2(const unsigned long &val);       /**< \copydoc OP2##(const array &) */ \
@@ -1144,6 +1147,7 @@ namespace af
     AFAPI array OP (const int& lhs, const array& rhs);                /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const unsigned& lhs, const array& rhs);           /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const char& lhs, const array& rhs);               /**< \copydoc OP##(const array&, const array&) */ \
+    AFAPI array OP (const signed char& lhs, const array& rhs);        /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const unsigned char& lhs, const array& rhs);      /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const long& lhs, const array& rhs);               /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const unsigned long& lhs, const array& rhs);      /**< \copydoc OP##(const array&, const array&) */ \
@@ -1157,6 +1161,7 @@ namespace af
     AFAPI array OP (const array& lhs, const int& rhs);                /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const unsigned& rhs);           /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const char& rhs);               /**< \copydoc OP##(const array&, const array&) */ \
+    AFAPI array OP (const array& lhs, const signed char& rhs);        /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const unsigned char& rhs);      /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const long& rhs);               /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const unsigned long& rhs);      /**< \copydoc OP##(const array&, const array&) */ \
@@ -1394,6 +1399,7 @@ namespace af
     AFAPI array operator&(const array& lhs, const long long& rhs);
     AFAPI array operator&(const array& lhs, const long& rhs);
     AFAPI array operator&(const array& lhs, const short& rhs);
+    AFAPI array operator&(const array& lhs, const signed char& rhs);
     AFAPI array operator&(const array& lhs, const unsigned char& rhs);
     AFAPI array operator&(const array& lhs, const unsigned long long& rhs);
     AFAPI array operator&(const array& lhs, const unsigned long& rhs);
@@ -1409,6 +1415,7 @@ namespace af
     AFAPI array operator&(const long long& lhs, const array& rhs);
     AFAPI array operator&(const long& lhs, const array& rhs);
     AFAPI array operator&(const short& lhs, const array& rhs);
+    AFAPI array operator&(const signed char& lhs, const array& rhs);
     AFAPI array operator&(const unsigned char& lhs, const array& rhs);
     AFAPI array operator&(const unsigned long long& lhs, const array& rhs);
     AFAPI array operator&(const unsigned long& lhs, const array& rhs);
@@ -1437,6 +1444,7 @@ namespace af
     AFAPI array operator&&(const array& lhs, const long long& rhs);
     AFAPI array operator&&(const array& lhs, const long& rhs);
     AFAPI array operator&&(const array& lhs, const short& rhs);
+    AFAPI array operator&&(const array& lhs, const signed char& rhs);
     AFAPI array operator&&(const array& lhs, const unsigned char& rhs);
     AFAPI array operator&&(const array& lhs, const unsigned long long& rhs);
     AFAPI array operator&&(const array& lhs, const unsigned long& rhs);
@@ -1452,6 +1460,7 @@ namespace af
     AFAPI array operator&&(const long long& lhs, const array& rhs);
     AFAPI array operator&&(const long& lhs, const array& rhs);
     AFAPI array operator&&(const short& lhs, const array& rhs);
+    AFAPI array operator&&(const signed char& lhs, const array& rhs);
     AFAPI array operator&&(const unsigned char& lhs, const array& rhs);
     AFAPI array operator&&(const unsigned long long& lhs, const array& rhs);
     AFAPI array operator&&(const unsigned long& lhs, const array& rhs);
diff --git a/include/af/defines.h b/include/af/defines.h
index da6c5591de..4be88f97bd 100644
--- a/include/af/defines.h
+++ b/include/af/defines.h
@@ -227,6 +227,7 @@ typedef enum {
 #if AF_API_VERSION >= 37
     , f16    ///< 16-bit floating point value
 #endif
+    , s8     ///< 8-bit signed integral value /// TODO AF_API_VERSION
 } af_dtype;
 
 typedef enum {
diff --git a/include/af/traits.hpp b/include/af/traits.hpp
index 6c7d1bf5fa..330435a929 100644
--- a/include/af/traits.hpp
+++ b/include/af/traits.hpp
@@ -175,6 +175,16 @@ struct dtype_traits<half> {
     static const char* getName() { return "half"; }
 };
 #endif
+
+template<>
+struct dtype_traits<signed char> {
+    enum {
+        af_type = s8 ,
+        ctype = f32
+    };
+    typedef signed char base_type;
+    static const char* getName() { return "schar"; }
+};
 }
 
 #endif
diff --git a/src/api/c/anisotropic_diffusion.cpp b/src/api/c/anisotropic_diffusion.cpp
index 3c77f8644c..6268accb3b 100644
--- a/src/api/c/anisotropic_diffusion.cpp
+++ b/src/api/c/anisotropic_diffusion.cpp
@@ -90,6 +90,7 @@ af_err af_anisotropic_diffusion(af_array* out, const af_array in,
             case u32:
             case s16:
             case u16:
+            case s8:
             case u8:
                 output = diffusion<float>(input, dt, K, iterations, F, eq);
                 break;
diff --git a/src/api/c/array.cpp b/src/api/c/array.cpp
index 4e1877e364..d164faabdb 100644
--- a/src/api/c/array.cpp
+++ b/src/api/c/array.cpp
@@ -30,6 +30,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createDeviceDataArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -47,6 +48,7 @@ af_err af_get_data_ptr(void *data, const af_array arr) {
             case b8:  copyData(static_cast<char*    >(data), arr); break;
             case s32: copyData(static_cast<int*     >(data), arr); break;
             case u32: copyData(static_cast<unsigned*>(data), arr); break;
+            case s8:  copyData(static_cast<schar*   >(data), arr); break;
             case u8:  copyData(static_cast<uchar*   >(data), arr); break;
             case s64: copyData(static_cast<intl*    >(data), arr); break;
             case u64: copyData(static_cast<uintl*   >(data), arr); break;
@@ -96,6 +98,9 @@ af_err af_create_array(af_array *result, const void *const data,
             case u32:
                 out = createHandleFromData(d, static_cast<const uint *>(data));
                 break;
+            case s8:
+                out = createHandleFromData(d, static_cast<const schar *>(data));
+                break;
             case u8:
                 out = createHandleFromData(d, static_cast<const uchar *>(data));
                 break;
@@ -175,6 +180,7 @@ af_err af_copy_array(af_array *out, const af_array in) {
                 case b8: res = copyArray<char>(in); break;
                 case s32: res = copyArray<int>(in); break;
                 case u32: res = copyArray<uint>(in); break;
+                case s8: res = copyArray<schar>(in); break;
                 case u8: res = copyArray<uchar>(in); break;
                 case s64: res = copyArray<intl>(in); break;
                 case u64: res = copyArray<uintl>(in); break;
@@ -205,6 +211,7 @@ af_err af_get_data_ref_count(int *use_count, const af_array in) {
             case b8: res = getUseCount<char>(in); break;
             case s32: res = getUseCount<int>(in); break;
             case u32: res = getUseCount<uint>(in); break;
+            case s8: res = getUseCount<schar>(in); break;
             case u8: res = getUseCount<uchar>(in); break;
             case s64: res = getUseCount<intl>(in); break;
             case u64: res = getUseCount<uintl>(in); break;
@@ -242,6 +249,7 @@ af_err af_release_array(af_array arr) {
                 case b8: releaseHandle<char>(arr); break;
                 case s32: releaseHandle<int>(arr); break;
                 case u32: releaseHandle<uint>(arr); break;
+                case s8: releaseHandle<schar>(arr); break;
                 case u8: releaseHandle<uchar>(arr); break;
                 case s64: releaseHandle<intl>(arr); break;
                 case u64: releaseHandle<uintl>(arr); break;
@@ -308,6 +316,9 @@ af_err af_write_array(af_array arr, const void *data, const size_t bytes,
             case u32:
                 write_array(arr, static_cast<const uint *>(data), bytes, src);
                 break;
+            case s8:
+                write_array(arr, static_cast<const schar *>(data), bytes, src);
+                break;
             case u8:
                 write_array(arr, static_cast<const uchar *>(data), bytes, src);
                 break;
@@ -433,6 +444,9 @@ af_err af_get_scalar(void *output_value, const af_array arr) {
             case u32:
                 getScalar<uint>(reinterpret_cast<uint *>(output_value), arr);
                 break;
+            case s8:
+                getScalar<schar>(reinterpret_cast<schar *>(output_value), arr);
+                break;
             case u8:
                 getScalar<uchar>(reinterpret_cast<uchar *>(output_value), arr);
                 break;
diff --git a/src/api/c/assign.cpp b/src/api/c/assign.cpp
index 22f11255e9..bdf505048d 100644
--- a/src/api/c/assign.cpp
+++ b/src/api/c/assign.cpp
@@ -42,6 +42,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createSubArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -122,6 +123,7 @@ static if_real<T> assign(Array<T>& out, const vector<af_seq> iv,
         case u64: assign<T, uintl>(out, iv, getArray<uintl>(in)); break;
         case s16: assign<T, short>(out, iv, getArray<short>(in)); break;
         case u16: assign<T, ushort>(out, iv, getArray<ushort>(in)); break;
+        case s8: assign<T, schar>(out, iv, getArray<schar>(in)); break;
         case u8: assign<T, uchar>(out, iv, getArray<uchar>(in)); break;
         case b8: assign<T, char>(out, iv, getArray<char>(in)); break;
         case f16: assign<T, half>(out, iv, getArray<half>(in)); break;
@@ -201,6 +203,7 @@ af_err af_assign_seq(af_array* out, const af_array lhs, const unsigned ndims,
                     case u64: assign(getArray<uintl>(res), inSeqs, rhs); break;
                     case s16: assign(getArray<short>(res), inSeqs, rhs); break;
                     case u16: assign(getArray<ushort>(res), inSeqs, rhs); break;
+                    case s8: assign(getArray<schar>(res), inSeqs, rhs); break;
                     case u8: assign(getArray<uchar>(res), inSeqs, rhs); break;
                     case b8: assign(getArray<char>(res), inSeqs, rhs); break;
                     case f16: assign(getArray<half>(res), inSeqs, rhs); break;
@@ -382,6 +385,7 @@ af_err af_assign_gen(af_array* out, const af_array lhs, const dim_t ndims,
                 case s32: genAssign<int>(output, ptr, rhs); break;
                 case s16: genAssign<short>(output, ptr, rhs); break;
                 case u16: genAssign<ushort>(output, ptr, rhs); break;
+                case s8: genAssign<schar>(output, ptr, rhs); break;
                 case u8: genAssign<uchar>(output, ptr, rhs); break;
                 case b8: genAssign<char>(output, ptr, rhs); break;
                 case f16: genAssign<half>(output, ptr, rhs); break;
diff --git a/src/api/c/bilateral.cpp b/src/api/c/bilateral.cpp
index 44e15c725c..aeec279ea5 100644
--- a/src/api/c/bilateral.cpp
+++ b/src/api/c/bilateral.cpp
@@ -19,6 +19,7 @@
 
 using af::dim4;
 using detail::bilateral;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -50,6 +51,7 @@ af_err af_bilateral(af_array *out, const af_array in, const float ssigma,
             case b8: output = bilateral<char>(in, ssigma, csigma); break;
             case s32: output = bilateral<int>(in, ssigma, csigma); break;
             case u32: output = bilateral<uint>(in, ssigma, csigma); break;
+            case s8: output = bilateral<schar>(in, ssigma, csigma); break;
             case u8: output = bilateral<uchar>(in, ssigma, csigma); break;
             case s16: output = bilateral<short>(in, ssigma, csigma); break;
             case u16: output = bilateral<ushort>(in, ssigma, csigma); break;
diff --git a/src/api/c/binary.cpp b/src/api/c/binary.cpp
index ee727c264a..eebe62bdbb 100644
--- a/src/api/c/binary.cpp
+++ b/src/api/c/binary.cpp
@@ -43,6 +43,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -139,6 +140,7 @@ static af_err af_arith(af_array *out, const af_array lhs, const af_array rhs,
                 case c64: res = arithOp<cdouble, op>(lhs, rhs, odims); break;
                 case s32: res = arithOp<int, op>(lhs, rhs, odims); break;
                 case u32: res = arithOp<uint, op>(lhs, rhs, odims); break;
+                case s8: res = arithOp<schar, op>(lhs, rhs, odims); break;
                 case u8: res = arithOp<uchar, op>(lhs, rhs, odims); break;
                 case b8: res = arithOp<char, op>(lhs, rhs, odims); break;
                 case s64: res = arithOp<intl, op>(lhs, rhs, odims); break;
@@ -159,6 +161,7 @@ static af_err af_arith(af_array *out, const af_array lhs, const af_array rhs,
                 case c64: res = arithOpBroadcast<cdouble, op>(lhs, rhs); break;
                 case s32: res = arithOpBroadcast<int, op>(lhs, rhs); break;
                 case u32: res = arithOpBroadcast<uint, op>(lhs, rhs); break;
+                case s8: res = arithOpBroadcast<schar, op>(lhs, rhs); break;
                 case u8: res = arithOpBroadcast<uchar, op>(lhs, rhs); break;
                 case b8: res = arithOpBroadcast<char, op>(lhs, rhs); break;
                 case s64: res = arithOpBroadcast<intl, op>(lhs, rhs); break;
@@ -195,6 +198,7 @@ static af_err af_arith_real(af_array *out, const af_array lhs,
             case f64: res = arithOp<double, op>(lhs, rhs, odims); break;
             case s32: res = arithOp<int, op>(lhs, rhs, odims); break;
             case u32: res = arithOp<uint, op>(lhs, rhs, odims); break;
+            case s8: res = arithOp<schar, op>(lhs, rhs, odims); break;
             case u8: res = arithOp<uchar, op>(lhs, rhs, odims); break;
             case b8: res = arithOp<char, op>(lhs, rhs, odims); break;
             case s64: res = arithOp<intl, op>(lhs, rhs, odims); break;
@@ -567,6 +571,7 @@ static af_err af_logic(af_array *out, const af_array lhs, const af_array rhs,
             case c64: res = logicOp<cdouble, op>(lhs, rhs, odims); break;
             case s32: res = logicOp<int, op>(lhs, rhs, odims); break;
             case u32: res = logicOp<uint, op>(lhs, rhs, odims); break;
+            case s8: res = logicOp<schar, op>(lhs, rhs, odims); break;
             case u8: res = logicOp<uchar, op>(lhs, rhs, odims); break;
             case b8: res = logicOp<char, op>(lhs, rhs, odims); break;
             case s64: res = logicOp<intl, op>(lhs, rhs, odims); break;
@@ -650,6 +655,7 @@ static af_err af_bitwise(af_array *out, const af_array lhs, const af_array rhs,
         switch (type) {
             case s32: res = bitOp<int, op>(lhs, rhs, odims); break;
             case u32: res = bitOp<uint, op>(lhs, rhs, odims); break;
+            case s8: res = bitOp<schar, op>(lhs, rhs, odims); break;
             case u8: res = bitOp<uchar, op>(lhs, rhs, odims); break;
             case b8: res = bitOp<char, op>(lhs, rhs, odims); break;
             case s64: res = bitOp<intl, op>(lhs, rhs, odims); break;
diff --git a/src/api/c/canny.cpp b/src/api/c/canny.cpp
index ef3ad029cd..b68b8d4ed0 100644
--- a/src/api/c/canny.cpp
+++ b/src/api/c/canny.cpp
@@ -53,6 +53,7 @@ using detail::logicOp;
 using detail::reduce;
 using detail::reduce_all;
 using detail::scan;
+using detail::schar;
 using detail::sobelDerivatives;
 using detail::uchar;
 using detail::uint;
@@ -265,6 +266,10 @@ af_err af_canny(af_array* out, const af_array in, const af_canny_threshold ct,
                 output = cannyHelper<ushort>(getArray<ushort>(in), t1, ct, t2,
                                              sw, isf);
                 break;
+            case s8:
+                output = cannyHelper<schar>(getArray<schar>(in), t1, ct, t2, sw,
+                                            isf);
+                break;
             case u8:
                 output = cannyHelper<uchar>(getArray<uchar>(in), t1, ct, t2, sw,
                                             isf);
diff --git a/src/api/c/cast.cpp b/src/api/c/cast.cpp
index 328c81ca65..7b421d28bb 100644
--- a/src/api/c/cast.cpp
+++ b/src/api/c/cast.cpp
@@ -28,6 +28,7 @@ using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -54,6 +55,7 @@ static af_array cast(const af_array in, const af_dtype type) {
             case c64: return getHandle(castArray<cdouble>(in));
             case s32: return getHandle(castArray<int>(in));
             case u32: return getHandle(castArray<uint>(in));
+            case s8: return getHandle(castArray<schar>(in));
             case u8: return getHandle(castArray<uchar>(in));
             case b8: return getHandle(castArray<char>(in));
             case s64: return getHandle(castArray<intl>(in));
diff --git a/src/api/c/clamp.cpp b/src/api/c/clamp.cpp
index fb821d3bf3..8c31469e55 100644
--- a/src/api/c/clamp.cpp
+++ b/src/api/c/clamp.cpp
@@ -28,6 +28,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -64,6 +65,7 @@ af_err af_clamp(af_array* out, const af_array in, const af_array lo,
             case c64: res = clampOp<cdouble>(in, lo, hi, odims); break;
             case s32: res = clampOp<int>(in, lo, hi, odims); break;
             case u32: res = clampOp<uint>(in, lo, hi, odims); break;
+            case s8: res = clampOp<schar>(in, lo, hi, odims); break;
             case u8: res = clampOp<uchar>(in, lo, hi, odims); break;
             case b8: res = clampOp<char>(in, lo, hi, odims); break;
             case s64: res = clampOp<intl>(in, lo, hi, odims); break;
diff --git a/src/api/c/convolve.cpp b/src/api/c/convolve.cpp
index 61af7b1b16..8d37c5d285 100644
--- a/src/api/c/convolve.cpp
+++ b/src/api/c/convolve.cpp
@@ -33,6 +33,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::convolve;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -196,6 +197,10 @@ af_err convolve(af_array *out, const af_array signal, const af_array filter,
                 output = convolve<uchar, float>(signal, filter, convBT, rank,
                                                 expand);
                 break;
+            case s8:
+                output = convolve<schar, float>(signal, filter, convBT, rank,
+                                                expand);
+                break;
             case b8:
                 output =
                     convolve<char, float>(signal, filter, convBT, rank, expand);
@@ -311,6 +316,10 @@ af_err af_convolve2_sep(af_array *out, const af_array col_filter,
                 output = convolve2<uchar, float>(signal, col_filter, row_filter,
                                                  expand);
                 break;
+            case s8:
+                output = convolve2<schar, float>(signal, col_filter, row_filter,
+                                                 expand);
+                break;
             case b8:
                 output = convolve2<char, float>(signal, col_filter, row_filter,
                                                 expand);
diff --git a/src/api/c/corrcoef.cpp b/src/api/c/corrcoef.cpp
index fd767fb0ba..fde3788dac 100644
--- a/src/api/c/corrcoef.cpp
+++ b/src/api/c/corrcoef.cpp
@@ -30,6 +30,7 @@ using detail::Array;
 using detail::getScalar;
 using detail::intl;
 using detail::reduce_all;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -89,6 +90,7 @@ af_err af_corrcoef(double* realVal, double* imagVal, const af_array X,
             case u64: *realVal = corrcoef<uintl, double>(X, Y); break;
             case s16: *realVal = corrcoef<short, float>(X, Y); break;
             case u16: *realVal = corrcoef<ushort, float>(X, Y); break;
+            case s8: *realVal = corrcoef<schar, float>(X, Y); break;
             case u8: *realVal = corrcoef<uchar, float>(X, Y); break;
             case b8: *realVal = corrcoef<char, float>(X, Y); break;
             default: TYPE_ERROR(1, xType);
diff --git a/src/api/c/covariance.cpp b/src/api/c/covariance.cpp
index f364558b11..a4241a8f0a 100644
--- a/src/api/c/covariance.cpp
+++ b/src/api/c/covariance.cpp
@@ -31,6 +31,7 @@ using detail::intl;
 using detail::mean;
 using detail::reduce;
 using detail::scalar;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -97,6 +98,7 @@ af_err af_cov_v2(af_array* out, const af_array X, const af_array Y,
             case u64: output = cov<uintl, double>(X, Y, bias); break;
             case s16: output = cov<short, float>(X, Y, bias); break;
             case u16: output = cov<ushort, float>(X, Y, bias); break;
+            case s8: output = cov<schar, float>(X, Y, bias); break;
             case u8: output = cov<uchar, float>(X, Y, bias); break;
             default: TYPE_ERROR(1, xType);
         }
diff --git a/src/api/c/data.cpp b/src/api/c/data.cpp
index 60ede3d4f6..324936e76e 100644
--- a/src/api/c/data.cpp
+++ b/src/api/c/data.cpp
@@ -35,6 +35,7 @@ using detail::iota;
 using detail::padArrayBorders;
 using detail::range;
 using detail::scalar;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -58,6 +59,7 @@ af_err af_constant(af_array *result, const double value, const unsigned ndims,
             case b8: out = createHandleFromValue<char>(d, value); break;
             case s32: out = createHandleFromValue<int>(d, value); break;
             case u32: out = createHandleFromValue<uint>(d, value); break;
+            case s8: out = createHandleFromValue<schar>(d, value); break;
             case u8: out = createHandleFromValue<uchar>(d, value); break;
             case s64: out = createHandleFromValue<intl>(d, value); break;
             case u64: out = createHandleFromValue<uintl>(d, value); break;
@@ -159,6 +161,7 @@ af_err af_identity(af_array *out, const unsigned ndims, const dim_t *const dims,
             case c64: result = identity_<cdouble>(d); break;
             case s32: result = identity_<int>(d); break;
             case u32: result = identity_<uint>(d); break;
+            case s8: result = identity_<schar>(d); break;
             case u8: result = identity_<uchar>(d); break;
             case u64: result = identity_<uintl>(d); break;
             case s64: result = identity_<intl>(d); break;
@@ -202,6 +205,7 @@ af_err af_range(af_array *result, const unsigned ndims, const dim_t *const dims,
             case u64: out = range_<uintl>(d, seq_dim); break;
             case s16: out = range_<short>(d, seq_dim); break;
             case u16: out = range_<ushort>(d, seq_dim); break;
+            case s8: out = range_<schar>(d, seq_dim); break;
             case u8: out = range_<uchar>(d, seq_dim); break;
             case f16: out = range_<half>(d, seq_dim); break;
             default: TYPE_ERROR(4, type);
@@ -242,6 +246,7 @@ af_err af_iota(af_array *result, const unsigned ndims, const dim_t *const dims,
             case u64: out = iota_<uintl>(d, t); break;
             case s16: out = iota_<short>(d, t); break;
             case u16: out = iota_<ushort>(d, t); break;
+            case s8: out = iota_<schar>(d, t); break;
             case u8: out = iota_<uchar>(d, t); break;
             case f16: out = iota_<half>(d, t); break;
             default: TYPE_ERROR(4, type);
@@ -285,6 +290,7 @@ af_err af_diag_create(af_array *out, const af_array in, const int num) {
             case u64: result = diagCreate<uintl>(in, num); break;
             case s16: result = diagCreate<short>(in, num); break;
             case u16: result = diagCreate<ushort>(in, num); break;
+            case s8: result = diagCreate<schar>(in, num); break;
             case u8:
                 result = diagCreate<uchar>(in, num);
                 break;
@@ -324,6 +330,7 @@ af_err af_diag_extract(af_array *out, const af_array in, const int num) {
             case u64: result = diagExtract<uintl>(in, num); break;
             case s16: result = diagExtract<short>(in, num); break;
             case u16: result = diagExtract<ushort>(in, num); break;
+            case s8: result = diagExtract<schar>(in, num); break;
             case u8:
                 result = diagExtract<uchar>(in, num);
                 break;
@@ -366,6 +373,7 @@ af_err af_lower(af_array *out, const af_array in, bool is_unit_diag) {
             case u64: res = triangle<uintl>(in, false, is_unit_diag); break;
             case s16: res = triangle<short>(in, false, is_unit_diag); break;
             case u16: res = triangle<ushort>(in, false, is_unit_diag); break;
+            case s8: res = triangle<schar>(in, false, is_unit_diag); break;
             case u8: res = triangle<uchar>(in, false, is_unit_diag); break;
             case b8: res = triangle<char>(in, false, is_unit_diag); break;
             case f16: res = triangle<half>(in, false, is_unit_diag); break;
@@ -395,6 +403,7 @@ af_err af_upper(af_array *out, const af_array in, bool is_unit_diag) {
             case u64: res = triangle<uintl>(in, true, is_unit_diag); break;
             case s16: res = triangle<short>(in, true, is_unit_diag); break;
             case u16: res = triangle<ushort>(in, true, is_unit_diag); break;
+            case s8: res = triangle<schar>(in, true, is_unit_diag); break;
             case u8: res = triangle<uchar>(in, true, is_unit_diag); break;
             case b8: res = triangle<char>(in, true, is_unit_diag); break;
             case f16: res = triangle<half>(in, true, is_unit_diag); break;
@@ -449,6 +458,7 @@ af_err af_pad(af_array *out, const af_array in, const unsigned begin_ndims,
             case u64: res = pad<uintl>(in, lPad, uPad, pad_type); break;
             case s16: res = pad<short>(in, lPad, uPad, pad_type); break;
             case u16: res = pad<ushort>(in, lPad, uPad, pad_type); break;
+            case s8: res = pad<schar>(in, lPad, uPad, pad_type); break;
             case u8: res = pad<uchar>(in, lPad, uPad, pad_type); break;
             case b8: res = pad<char>(in, lPad, uPad, pad_type); break;
             case f16: res = pad<half>(in, lPad, uPad, pad_type); break;
diff --git a/src/api/c/deconvolution.cpp b/src/api/c/deconvolution.cpp
index f579eeadf8..19ad89e5db 100644
--- a/src/api/c/deconvolution.cpp
+++ b/src/api/c/deconvolution.cpp
@@ -43,6 +43,7 @@ using detail::createValueArray;
 using detail::logicOp;
 using detail::padArrayBorders;
 using detail::scalar;
+using detail::schar;
 using detail::select_scalar;
 using detail::shift;
 using detail::uchar;
@@ -226,6 +227,7 @@ af_err af_iterative_deconv(af_array* out, const af_array in, const af_array ker,
             case u16:
                 res = iterDeconv<ushort>(in, ker, iters, rfac, algo);
                 break;
+            case s8: res = iterDeconv<schar>(in, ker, iters, rfac, algo); break;
             case u8: res = iterDeconv<uchar>(in, ker, iters, rfac, algo); break;
             default: TYPE_ERROR(1, inputType);
         }
@@ -323,6 +325,7 @@ af_err af_inverse_deconv(af_array* out, const af_array in, const af_array psf,
             case f32: res = invDeconv<float>(in, psf, gamma, algo); break;
             case s16: res = invDeconv<short>(in, psf, gamma, algo); break;
             case u16: res = invDeconv<ushort>(in, psf, gamma, algo); break;
+            case s8: res = invDeconv<schar>(in, psf, gamma, algo); break;
             case u8: res = invDeconv<uchar>(in, psf, gamma, algo); break;
             default: TYPE_ERROR(1, inputType);
         }
diff --git a/src/api/c/device.cpp b/src/api/c/device.cpp
index ef37888523..7427a1a4e5 100644
--- a/src/api/c/device.cpp
+++ b/src/api/c/device.cpp
@@ -47,6 +47,7 @@ using detail::init;
 using detail::intl;
 using detail::isDoubleSupported;
 using detail::isHalfSupported;
+using detail::schar;
 using detail::setDevice;
 using detail::uchar;
 using detail::uint;
@@ -290,6 +291,7 @@ af_err af_eval(af_array arr) {
                 case c64: eval<cdouble>(arr); break;
                 case s32: eval<int>(arr); break;
                 case u32: eval<uint>(arr); break;
+                case s8: eval<schar>(arr); break;
                 case u8: eval<uchar>(arr); break;
                 case b8: eval<char>(arr); break;
                 case s64: eval<intl>(arr); break;
@@ -344,6 +346,7 @@ af_err af_eval_multiple(int num, af_array* arrays) {
             case c64: evalMultiple<cdouble>(num, arrays); break;
             case s32: evalMultiple<int>(num, arrays); break;
             case u32: evalMultiple<uint>(num, arrays); break;
+            case s8: evalMultiple<schar>(num, arrays); break;
             case u8: evalMultiple<uchar>(num, arrays); break;
             case b8: evalMultiple<char>(num, arrays); break;
             case s64: evalMultiple<intl>(num, arrays); break;
diff --git a/src/api/c/diff.cpp b/src/api/c/diff.cpp
index c579f0b53e..f75d5c1ab1 100644
--- a/src/api/c/diff.cpp
+++ b/src/api/c/diff.cpp
@@ -21,6 +21,7 @@ using arrayfire::getHandle;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -64,6 +65,7 @@ af_err af_diff1(af_array* out, const af_array in, const int dim) {
             case u64: output = diff1<uintl>(in, dim); break;
             case s16: output = diff1<short>(in, dim); break;
             case u16: output = diff1<ushort>(in, dim); break;
+            case s8: output = diff1<schar>(in, dim); break;
             case u8: output = diff1<uchar>(in, dim); break;
             default: TYPE_ERROR(1, type);
         }
@@ -101,6 +103,7 @@ af_err af_diff2(af_array* out, const af_array in, const int dim) {
             case u64: output = diff2<uintl>(in, dim); break;
             case s16: output = diff2<short>(in, dim); break;
             case u16: output = diff2<ushort>(in, dim); break;
+            case s8: output = diff2<schar>(in, dim); break;
             case u8: output = diff2<uchar>(in, dim); break;
             default: TYPE_ERROR(1, type);
         }
diff --git a/src/api/c/dog.cpp b/src/api/c/dog.cpp
index fbbe94d211..848262daab 100644
--- a/src/api/c/dog.cpp
+++ b/src/api/c/dog.cpp
@@ -22,6 +22,7 @@ using af::dim4;
 using detail::arithOp;
 using detail::Array;
 using detail::convolve;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -70,6 +71,7 @@ af_err af_dog(af_array* out, const af_array in, const int radius1,
             case u32: output = dog<uint, float>(in, radius1, radius2); break;
             case s16: output = dog<short, float>(in, radius1, radius2); break;
             case u16: output = dog<ushort, float>(in, radius1, radius2); break;
+            case s8: output = dog<schar, float>(in, radius1, radius2); break;
             case u8: output = dog<uchar, float>(in, radius1, radius2); break;
             default: TYPE_ERROR(1, type);
         }
diff --git a/src/api/c/exampleFunction.cpp b/src/api/c/exampleFunction.cpp
index 4a7a52f6bd..a58336f90c 100644
--- a/src/api/c/exampleFunction.cpp
+++ b/src/api/c/exampleFunction.cpp
@@ -76,6 +76,7 @@ af_err af_example_function(af_array* out, const af_array a,
             case f32: output = example<float>(a, a, param); break;
             case s32: output = example<int>(a, a, param); break;
             case u32: output = example<uint>(a, a, param); break;
+            case s8: output = example<schar>(a, a, param); break;
             case u8: output = example<uchar>(a, a, param); break;
             case b8: output = example<char>(a, a, param); break;
             case c32: output = example<cfloat>(a, a, param); break;
diff --git a/src/api/c/fast.cpp b/src/api/c/fast.cpp
index ed8822c402..08834ce4f4 100644
--- a/src/api/c/fast.cpp
+++ b/src/api/c/fast.cpp
@@ -22,6 +22,7 @@ using af::dim4;
 using detail::Array;
 using detail::createEmptyArray;
 using detail::createValueArray;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -96,6 +97,10 @@ af_err af_fast(af_features *out, const af_array in, const float thr,
                 *out = fast<ushort>(in, thr, arc_length, non_max, feature_ratio,
                                     edge);
                 break;
+            case s8:
+                *out = fast<schar>(in, thr, arc_length, non_max, feature_ratio,
+                                   edge);
+                break;
             case u8:
                 *out = fast<uchar>(in, thr, arc_length, non_max, feature_ratio,
                                    edge);
diff --git a/src/api/c/fftconvolve.cpp b/src/api/c/fftconvolve.cpp
index 5e69d5d0ce..ead2247c51 100644
--- a/src/api/c/fftconvolve.cpp
+++ b/src/api/c/fftconvolve.cpp
@@ -35,6 +35,7 @@ using detail::createSubArray;
 using detail::fftconvolve;
 using detail::intl;
 using detail::real;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -211,6 +212,10 @@ af_err fft_convolve(af_array *out, const af_array signal, const af_array filter,
                 output =
                     fftconvolve<uchar>(signal, filter, expand, convBT, baseDim);
                 break;
+            case s8:
+                output =
+                    fftconvolve<schar>(signal, filter, expand, convBT, baseDim);
+                break;
             case b8:
                 output =
                     fftconvolve<char>(signal, filter, expand, convBT, baseDim);
diff --git a/src/api/c/filters.cpp b/src/api/c/filters.cpp
index dc0067f257..4c154c16fb 100644
--- a/src/api/c/filters.cpp
+++ b/src/api/c/filters.cpp
@@ -18,6 +18,7 @@
 #include <af/signal.h>
 
 using af::dim4;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -64,6 +65,7 @@ af_err af_medfilt1(af_array *out, const af_array in, const dim_t wind_width,
             case u16:
                 output = medfilt1<ushort>(in, wind_width, edge_pad);
                 break;
+            case s8: output = medfilt1<schar>(in, wind_width, edge_pad); break;
             case u8: output = medfilt1<uchar>(in, wind_width, edge_pad); break;
             default: TYPE_ERROR(1, type);
         }
@@ -129,6 +131,9 @@ af_err af_medfilt2(af_array *out, const af_array in, const dim_t wind_length,
                 output =
                     medfilt2<ushort>(in, wind_length, wind_width, edge_pad);
                 break;
+            case s8:
+                output = medfilt2<schar>(in, wind_length, wind_width, edge_pad);
+                break;
             case u8:
                 output = medfilt2<uchar>(in, wind_length, wind_width, edge_pad);
                 break;
diff --git a/src/api/c/flip.cpp b/src/api/c/flip.cpp
index 080af47aac..4aea98ec73 100644
--- a/src/api/c/flip.cpp
+++ b/src/api/c/flip.cpp
@@ -25,6 +25,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uintl;
 using detail::ushort;
@@ -61,6 +62,7 @@ af_err af_flip(af_array *result, const af_array in, const unsigned dim) {
             case u64: out = flip<uintl>(in, dim); break;
             case s16: out = flip<short>(in, dim); break;
             case u16: out = flip<ushort>(in, dim); break;
+            case s8: out = flip<schar>(in, dim); break;
             case u8: out = flip<uchar>(in, dim); break;
             default: TYPE_ERROR(1, in_type);
         }
diff --git a/src/api/c/handle.cpp b/src/api/c/handle.cpp
index 9c980af9f0..d67f4ae9a1 100644
--- a/src/api/c/handle.cpp
+++ b/src/api/c/handle.cpp
@@ -21,6 +21,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createDeviceDataArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -46,6 +47,7 @@ af_array retain(const af_array in) {
             case f64: return retainHandle<double>(in);
             case s32: return retainHandle<int>(in);
             case u32: return retainHandle<uint>(in);
+            case s8: return retainHandle<schar>(in);
             case u8: return retainHandle<uchar>(in);
             case c32: return retainHandle<detail::cfloat>(in);
             case c64: return retainHandle<detail::cdouble>(in);
@@ -70,6 +72,7 @@ af_array createHandle(const dim4 &d, af_dtype dtype) {
         case b8:  return createHandle<char   >(d);
         case s32: return createHandle<int    >(d);
         case u32: return createHandle<uint   >(d);
+        case s8:  return createHandle<schar  >(d);
         case u8:  return createHandle<uchar  >(d);
         case s64: return createHandle<intl   >(d);
         case u64: return createHandle<uintl  >(d);
@@ -91,6 +94,7 @@ af_array createHandleFromValue(const dim4 &d, double val, af_dtype dtype) {
         case b8:  return createHandleFromValue<char   >(d, val);
         case s32: return createHandleFromValue<int    >(d, val);
         case u32: return createHandleFromValue<uint   >(d, val);
+        case s8:  return createHandleFromValue<schar  >(d, val);
         case u8:  return createHandleFromValue<uchar  >(d, val);
         case s64: return createHandleFromValue<intl   >(d, val);
         case u64: return createHandleFromValue<uintl  >(d, val);
@@ -113,6 +117,7 @@ af_array createHandleFromDeviceData(const af::dim4 &d, af_dtype dtype,
         case b8:  return getHandle(createDeviceDataArray<char   >(d, data, false));
         case s32: return getHandle(createDeviceDataArray<int    >(d, data, false));
         case u32: return getHandle(createDeviceDataArray<uint   >(d, data, false));
+        case s8:  return getHandle(createDeviceDataArray<schar  >(d, data, false));
         case u8:  return getHandle(createDeviceDataArray<uchar  >(d, data, false));
         case s64: return getHandle(createDeviceDataArray<intl   >(d, data, false));
         case u64: return getHandle(createDeviceDataArray<uintl  >(d, data, false));
@@ -182,5 +187,6 @@ INSTANTIATE(char);
 INSTANTIATE(short);
 INSTANTIATE(ushort);
 INSTANTIATE(half);
+INSTANTIATE(schar);
 
 }  // namespace arrayfire
diff --git a/src/api/c/hist.cpp b/src/api/c/hist.cpp
index 1e250b5df4..0d8f9bfe6b 100644
--- a/src/api/c/hist.cpp
+++ b/src/api/c/hist.cpp
@@ -29,6 +29,7 @@ using detail::Array;
 using detail::copy_histogram;
 using detail::forgeManager;
 using detail::getScalar;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -133,6 +134,10 @@ af_err af_draw_hist(const af_window window, const af_array X,
                 chart =
                     setup_histogram<ushort>(window, X, minval, maxval, props);
                 break;
+            case s8:
+                chart =
+                    setup_histogram<schar>(window, X, minval, maxval, props);
+                break;
             case u8:
                 chart =
                     setup_histogram<uchar>(window, X, minval, maxval, props);
diff --git a/src/api/c/histeq.cpp b/src/api/c/histeq.cpp
index da2a7579d8..faed6a238c 100644
--- a/src/api/c/histeq.cpp
+++ b/src/api/c/histeq.cpp
@@ -33,6 +33,7 @@ using detail::intl;
 using detail::lookup;
 using detail::reduce_all;
 using detail::scan;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -95,6 +96,7 @@ af_err af_hist_equal(af_array* out, const af_array in, const af_array hist) {
             case u16: output = hist_equal<ushort, uint>(in, hist); break;
             case s64: output = hist_equal<intl, uint>(in, hist); break;
             case u64: output = hist_equal<uintl, uint>(in, hist); break;
+            case s8: output = hist_equal<schar, uint>(in, hist); break;
             case u8: output = hist_equal<uchar, uint>(in, hist); break;
             default: TYPE_ERROR(1, dataType);
         }
diff --git a/src/api/c/histogram.cpp b/src/api/c/histogram.cpp
index aa2744bb6c..69c6d71de5 100644
--- a/src/api/c/histogram.cpp
+++ b/src/api/c/histogram.cpp
@@ -15,6 +15,7 @@
 #include <af/image.h>
 
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -74,6 +75,10 @@ af_err af_histogram(af_array *out, const af_array in, const unsigned nbins,
                 output = histogram<uintl>(in, nbins, minval, maxval,
                                           info.isLinear());
                 break;
+            case s8:
+                output = histogram<schar>(in, nbins, minval, maxval,
+                                          info.isLinear());
+                break;
             case u8:
                 output = histogram<uchar>(in, nbins, minval, maxval,
                                           info.isLinear());
diff --git a/src/api/c/image.cpp b/src/api/c/image.cpp
index 425530806c..4650c0ec3d 100644
--- a/src/api/c/image.cpp
+++ b/src/api/c/image.cpp
@@ -39,6 +39,7 @@ using detail::Array;
 using detail::copy_image;
 using detail::createValueArray;
 using detail::forgeManager;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -102,6 +103,7 @@ af_err af_draw_image(const af_window window, const af_array in,
             case u32: image = convert_and_copy_image<uint>(in); break;
             case s16: image = convert_and_copy_image<short>(in); break;
             case u16: image = convert_and_copy_image<ushort>(in); break;
+            case s8: image = convert_and_copy_image<schar>(in); break;
             case u8: image = convert_and_copy_image<uchar>(in); break;
             default: TYPE_ERROR(1, type);
         }
diff --git a/src/api/c/imageio.cpp b/src/api/c/imageio.cpp
index be5f528922..0f87e4df17 100644
--- a/src/api/c/imageio.cpp
+++ b/src/api/c/imageio.cpp
@@ -75,7 +75,7 @@ static af_err readImage(af_array* rImage, const uchar* pSrcLine,
             if (fo_color == 1) {
                 pDst0[indx] = static_cast<T>(*(src + (x * step)));
             } else if (fo_color >= 3) {
-                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) {
+                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) { // FIXME s8?
                     pDst0[indx] =
                         static_cast<float>(*(src + (x * step + FI_RGBA_RED)));
                     pDst1[indx] =
@@ -201,7 +201,7 @@ static af_err readImage(af_array* rImage, const uchar* pSrcLine,
             if (fo_color == 1) {
                 pDst[indx] = static_cast<T>(*(src + (x * step)));
             } else if (fo_color >= 3) {
-                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) {
+                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) { // FIXME s8?
                     r = *(src + (x * step + FI_RGBA_RED));
                     g = *(src + (x * step + FI_RGBA_GREEN));
                     b = *(src + (x * step + FI_RGBA_BLUE));
diff --git a/src/api/c/imageio2.cpp b/src/api/c/imageio2.cpp
index 7130202397..4a00212207 100644
--- a/src/api/c/imageio2.cpp
+++ b/src/api/c/imageio2.cpp
@@ -71,7 +71,7 @@ static af_err readImage_t(af_array* rImage, const uchar* pSrcLine,
             if (fi_color == 1) {
                 pDst0[indx] = *(src + (x * step));
             } else if (fi_color >= 3) {
-                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) {
+                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) { // FIXME s8?
                     pDst0[indx] = *(src + (x * step + FI_RGBA_RED));
                     pDst1[indx] = *(src + (x * step + FI_RGBA_GREEN));
                     pDst2[indx] = *(src + (x * step + FI_RGBA_BLUE));
@@ -102,6 +102,7 @@ static af_err readImage_t(af_array* rImage, const uchar* pSrcLine,
 }
 
 FREE_IMAGE_TYPE getFIT(FI_CHANNELS channels, af_dtype type) {
+    // FIXME s8?
     if (channels == AFFI_GRAY) {
         if (type == u8) { return FIT_BITMAP; }
         if (type == u16) {
@@ -364,7 +365,7 @@ static void save_t(T* pDstLine, const af_array in, const dim4& dims,
             if (channels == 1) {
                 *(pDstLine + x * step) = pSrc0[indx];  // r -> 0
             } else if (channels >= 3) {
-                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) {
+                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) { // FIXME s8?
                     *(pDstLine + x * step + FI_RGBA_RED) =
                         pSrc0[indx];  // r -> 0
                     *(pDstLine + x * step + FI_RGBA_GREEN) =
diff --git a/src/api/c/implicit.cpp b/src/api/c/implicit.cpp
index f30afda7eb..d045769cbd 100644
--- a/src/api/c/implicit.cpp
+++ b/src/api/c/implicit.cpp
@@ -14,7 +14,7 @@ Implicit type mimics C/C++ behavior.
 
 Order of precedence:
 - complex > real
-- double > float > uintl > intl > uint > int > uchar > char
+- double > float > uintl > intl > uint > int > uchar > schar > char
 */
 
 af_dtype implicit(const af_dtype lty, const af_dtype rty) {
@@ -38,6 +38,7 @@ af_dtype implicit(const af_dtype lty, const af_dtype rty) {
     if ((lty == u16) || (rty == u16)) { return u16; }
     if ((lty == s16) || (rty == s16)) { return s16; }
     if ((lty == u8) || (rty == u8)) { return u8; }
+    if ((lty == s8) || (rty == s8)) { return s8; }
     if ((lty == b8) && (rty == b8)) { return b8; }
 
     return f32;
diff --git a/src/api/c/index.cpp b/src/api/c/index.cpp
index 1c7484f2bf..a697f8457c 100644
--- a/src/api/c/index.cpp
+++ b/src/api/c/index.cpp
@@ -40,6 +40,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::index;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -115,6 +116,7 @@ af_err af_index(af_array* result, const af_array in, const unsigned ndims,
             case u16: out = indexBySeqs<ushort>(in, indices_); break;
             case s64: out = indexBySeqs<intl>(in, indices_); break;
             case u64: out = indexBySeqs<uintl>(in, indices_); break;
+            case s8: out = indexBySeqs<schar>(in, indices_); break;
             case u8: out = indexBySeqs<uchar>(in, indices_); break;
             case f16: out = indexBySeqs<half>(in, indices_); break;
             default: TYPE_ERROR(1, type);
@@ -148,6 +150,7 @@ static af_array lookup(const af_array& in, const af_array& idx,
         case u64: return lookup<uintl, idx_t>(in, idx, dim);
         case s16: return lookup<short, idx_t>(in, idx, dim);
         case u16: return lookup<ushort, idx_t>(in, idx, dim);
+        case s8: return lookup<schar, idx_t>(in, idx, dim);
         case u8: return lookup<uchar, idx_t>(in, idx, dim);
         case b8: return lookup<char, idx_t>(in, idx, dim);
         case f16: return lookup<half, idx_t>(in, idx, dim);
@@ -185,6 +188,7 @@ af_err af_lookup(af_array* out, const af_array in, const af_array indices,
             case u16: output = lookup<ushort>(in, indices, dim); break;
             case s64: output = lookup<intl>(in, indices, dim); break;
             case u64: output = lookup<uintl>(in, indices, dim); break;
+            case s8: output = lookup<schar>(in, indices, dim); break;
             case u8: output = lookup<uchar>(in, indices, dim); break;
             case f16: output = lookup<half>(in, indices, dim); break;
             default: TYPE_ERROR(1, idxType);
@@ -289,6 +293,7 @@ af_err af_index_gen(af_array* out, const af_array in, const dim_t ndims,
             case s32: output = genIndex<int>(in, ptr); break;
             case u16: output = genIndex<ushort>(in, ptr); break;
             case s16: output = genIndex<short>(in, ptr); break;
+            case s8: output = genIndex<schar>(in, ptr); break;
             case u8: output = genIndex<uchar>(in, ptr); break;
             case b8: output = genIndex<char>(in, ptr); break;
             case f16: output = genIndex<half>(in, ptr); break;
diff --git a/src/api/c/internal.cpp b/src/api/c/internal.cpp
index 38c0c96dfe..c0314981cb 100644
--- a/src/api/c/internal.cpp
+++ b/src/api/c/internal.cpp
@@ -25,6 +25,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createStridedArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -120,6 +121,11 @@ af_err af_create_strided_array(af_array *arr, const void *data,
                     dims, strides, offset, static_cast<uchar *>(in_data),
                     isdev));
                 break;
+            case s8:
+                res = getHandle(createStridedArray<schar>(
+                    dims, strides, offset, static_cast<schar *>(in_data),
+                    isdev));
+                break;
             case f16:
                 res = getHandle(createStridedArray<half>(
                     dims, strides, offset, static_cast<half *>(in_data),
@@ -175,6 +181,7 @@ af_err af_get_raw_ptr(void **ptr, const af_array arr) {
             case s16: res = getRawPtr(getArray<short>(arr)); break;
             case b8: res = getRawPtr(getArray<char>(arr)); break;
             case u8: res = getRawPtr(getArray<uchar>(arr)); break;
+            case s8: res = getRawPtr(getArray<schar>(arr)); break;
             case f16: res = getRawPtr(getArray<half>(arr)); break;
             default: TYPE_ERROR(6, ty);
         }
@@ -212,6 +219,7 @@ af_err af_is_owner(bool *result, const af_array arr) {
             case s16: res = getArray<short>(arr).isOwner(); break;
             case b8: res = getArray<char>(arr).isOwner(); break;
             case u8: res = getArray<uchar>(arr).isOwner(); break;
+            case s8: res = getArray<schar>(arr).isOwner(); break;
             case f16: res = getArray<half>(arr).isOwner(); break;
             default: TYPE_ERROR(6, ty);
         }
@@ -241,6 +249,7 @@ af_err af_get_allocated_bytes(size_t *bytes, const af_array arr) {
             case s16: res = getArray<short>(arr).getAllocatedBytes(); break;
             case b8: res = getArray<char>(arr).getAllocatedBytes(); break;
             case u8: res = getArray<uchar>(arr).getAllocatedBytes(); break;
+            case s8: res = getArray<schar>(arr).getAllocatedBytes(); break;
             case f16: res = getArray<half>(arr).getAllocatedBytes(); break;
             default: TYPE_ERROR(6, ty);
         }
diff --git a/src/api/c/join.cpp b/src/api/c/join.cpp
index 4c47fbe495..d3e9cda6b5 100644
--- a/src/api/c/join.cpp
+++ b/src/api/c/join.cpp
@@ -26,6 +26,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createEmptyArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -98,6 +99,7 @@ af_err af_join(af_array *out, const int dim, const af_array first,
             case u64: output = join<uintl>(dim, first, second); break;
             case s16: output = join<short>(dim, first, second); break;
             case u16: output = join<ushort>(dim, first, second); break;
+            case s8: output = join<schar>(dim, first, second); break;
             case u8: output = join<uchar>(dim, first, second); break;
             case f16: output = join<half>(dim, first, second); break;
             default: TYPE_ERROR(1, finfo.getType());
@@ -169,6 +171,7 @@ af_err af_join_many(af_array *out, const int dim, const unsigned n_arrays,
             case u64: output = join_many<uintl>(dim, n_arrays, inputs); break;
             case s16: output = join_many<short>(dim, n_arrays, inputs); break;
             case u16: output = join_many<ushort>(dim, n_arrays, inputs); break;
+            case s8: output = join_many<schar>(dim, n_arrays, inputs); break;
             case u8: output = join_many<uchar>(dim, n_arrays, inputs); break;
             case f16: output = join_many<half>(dim, n_arrays, inputs); break;
             default: TYPE_ERROR(1, assertType);
diff --git a/src/api/c/match_template.cpp b/src/api/c/match_template.cpp
index 6882711a7f..91d81c383c 100644
--- a/src/api/c/match_template.cpp
+++ b/src/api/c/match_template.cpp
@@ -19,6 +19,7 @@
 
 using af::dim4;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -82,6 +83,10 @@ af_err af_match_template(af_array* out, const af_array search_img,
             case b8:
                 output = match_template<char>(search_img, template_img, m_type);
                 break;
+            case s8:
+                output =
+                    match_template<schar>(search_img, template_img, m_type);
+                break;
             case u8:
                 output =
                     match_template<uchar>(search_img, template_img, m_type);
diff --git a/src/api/c/mean.cpp b/src/api/c/mean.cpp
index af9021983e..65fe057155 100644
--- a/src/api/c/mean.cpp
+++ b/src/api/c/mean.cpp
@@ -31,6 +31,7 @@ using detail::imag;
 using detail::intl;
 using detail::mean;
 using detail::real;
+using detail::schar;
 using detail::uchar;
 using detail::uintl;
 using detail::ushort;
@@ -77,6 +78,7 @@ af_err af_mean(af_array *out, const af_array in, const dim_t dim) {
             case u64: output = mean<uintl, double>(in, dim); break;
             case s16: output = mean<short, float>(in, dim); break;
             case u16: output = mean<ushort, float>(in, dim); break;
+            case s8: output = mean<schar, float>(in, dim); break;
             case u8: output = mean<uchar, float>(in, dim); break;
             case b8: output = mean<char, float>(in, dim); break;
             case c32: output = mean<cfloat, cfloat>(in, dim); break;
@@ -127,6 +129,7 @@ af_err af_mean_weighted(af_array *out, const af_array in,
             case u32:
             case s16:
             case u16:
+            case s8:
             case u8:
             case b8: output = mean<float>(in, w, dim); break;
             case f64:
@@ -158,6 +161,7 @@ af_err af_mean_all(double *realVal, double *imagVal, const af_array in) {
             case u64: *realVal = mean<uintl, double>(in); break;
             case s16: *realVal = mean<short, float>(in); break;
             case u16: *realVal = mean<ushort, float>(in); break;
+            case s8: *realVal = mean<schar, float>(in); break;
             case u8: *realVal = mean<uchar, float>(in); break;
             case b8: *realVal = mean<char, float>(in); break;
             case f16:
@@ -200,6 +204,7 @@ af_err af_mean_all_weighted(double *realVal, double *imagVal, const af_array in,
             case u32:
             case s16:
             case u16:
+            case s8:
             case u8:
             case b8:
             case f16: *realVal = mean<float>(in, weights); break;
diff --git a/src/api/c/meanshift.cpp b/src/api/c/meanshift.cpp
index 0c8322cafe..bf09bc4d2a 100644
--- a/src/api/c/meanshift.cpp
+++ b/src/api/c/meanshift.cpp
@@ -18,6 +18,7 @@
 using af::dim4;
 using detail::intl;
 using detail::meanshift;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -84,6 +85,10 @@ af_err af_mean_shift(af_array *out, const af_array in,
                 output = mean_shift<uintl>(in, spatial_sigma, chromatic_sigma,
                                            num_iterations, is_color);
                 break;
+            case s8:
+                output = mean_shift<schar>(in, spatial_sigma, chromatic_sigma,
+                                           num_iterations, is_color);
+                break;
             case u8:
                 output = mean_shift<uchar>(in, spatial_sigma, chromatic_sigma,
                                            num_iterations, is_color);
diff --git a/src/api/c/median.cpp b/src/api/c/median.cpp
index 5e22c1c36a..2fd0de18d8 100644
--- a/src/api/c/median.cpp
+++ b/src/api/c/median.cpp
@@ -23,6 +23,7 @@
 using af::dim4;
 using detail::Array;
 using detail::division;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -169,6 +170,7 @@ af_err af_median_all(double* realVal, double* imagVal,  // NOLINT
             case u32: *realVal = median<uint>(in); break;
             case s16: *realVal = median<short>(in); break;
             case u16: *realVal = median<ushort>(in); break;
+            case s8: *realVal = median<schar>(in); break;
             case u8: *realVal = median<uchar>(in); break;
             default: TYPE_ERROR(1, type);
         }
@@ -193,6 +195,7 @@ af_err af_median(af_array* out, const af_array in, const dim_t dim) {
             case u32: output = median<uint>(in, dim); break;
             case s16: output = median<short>(in, dim); break;
             case u16: output = median<ushort>(in, dim); break;
+            case s8: output = median<schar>(in, dim); break;
             case u8: output = median<uchar>(in, dim); break;
             default: TYPE_ERROR(1, type);
         }
diff --git a/src/api/c/memory.cpp b/src/api/c/memory.cpp
index fbff61720e..665a51ac9c 100644
--- a/src/api/c/memory.cpp
+++ b/src/api/c/memory.cpp
@@ -42,6 +42,7 @@ using detail::memUnlock;
 using detail::pinnedAlloc;
 using detail::pinnedFree;
 using detail::printMemInfo;
+using detail::schar;
 using detail::signalMemoryCleanup;
 using detail::uchar;
 using detail::uint;
@@ -95,6 +96,9 @@ af_err af_device_array(af_array *arr, void *data, const unsigned ndims,
             case u16:
                 res = getHandle(createDeviceDataArray<ushort>(d, data));
                 break;
+            case s8:
+                res = getHandle(createDeviceDataArray<schar>(d, data));
+                break;
             case u8:
                 res = getHandle(createDeviceDataArray<uchar>(d, data));
                 break;
@@ -130,6 +134,7 @@ af_err af_get_device_ptr(void **data, const af_array arr) {
             case u64: *data = getDevicePtr(getArray<uintl>(arr)); break;
             case s16: *data = getDevicePtr(getArray<short>(arr)); break;
             case u16: *data = getDevicePtr(getArray<ushort>(arr)); break;
+            case s8: *data = getDevicePtr(getArray<schar>(arr)); break;
             case u8: *data = getDevicePtr(getArray<uchar>(arr)); break;
             case b8: *data = getDevicePtr(getArray<char>(arr)); break;
             case f16: *data = getDevicePtr(getArray<half>(arr)); break;
@@ -164,6 +169,7 @@ af_err af_lock_array(const af_array arr) {
             case u64: lockArray<uintl>(arr); break;
             case s16: lockArray<short>(arr); break;
             case u16: lockArray<ushort>(arr); break;
+            case s8: lockArray<schar>(arr); break;
             case u8: lockArray<uchar>(arr); break;
             case b8: lockArray<char>(arr); break;
             case f16: lockArray<half>(arr); break;
@@ -196,6 +202,7 @@ af_err af_is_locked_array(bool *res, const af_array arr) {
             case u64: *res = checkUserLock<uintl>(arr); break;
             case s16: *res = checkUserLock<short>(arr); break;
             case u16: *res = checkUserLock<ushort>(arr); break;
+            case s8: *res = checkUserLock<schar>(arr); break;
             case u8: *res = checkUserLock<uchar>(arr); break;
             case b8: *res = checkUserLock<char>(arr); break;
             case f16: *res = checkUserLock<half>(arr); break;
@@ -229,6 +236,7 @@ af_err af_unlock_array(const af_array arr) {
             case u64: unlockArray<uintl>(arr); break;
             case s16: unlockArray<short>(arr); break;
             case u16: unlockArray<ushort>(arr); break;
+            case s8: unlockArray<schar>(arr); break;
             case u8: unlockArray<uchar>(arr); break;
             case b8: unlockArray<char>(arr); break;
             case f16: unlockArray<half>(arr); break;
diff --git a/src/api/c/moddims.cpp b/src/api/c/moddims.cpp
index 4f6f0f310d..f419a2fb04 100644
--- a/src/api/c/moddims.cpp
+++ b/src/api/c/moddims.cpp
@@ -22,6 +22,7 @@ using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -66,6 +67,7 @@ af_err af_moddims(af_array* out, const af_array in, const unsigned ndims,
             case b8: output = modDims<char>(in, newDims); break;
             case s32: output = modDims<int>(in, newDims); break;
             case u32: output = modDims<uint>(in, newDims); break;
+            case s8: output = modDims<schar>(in, newDims); break;
             case u8: output = modDims<uchar>(in, newDims); break;
             case s64: output = modDims<intl>(in, newDims); break;
             case u64: output = modDims<uintl>(in, newDims); break;
@@ -99,6 +101,7 @@ af_err af_flat(af_array* out, const af_array in) {
                 case b8: output = flat<char>(in); break;
                 case s32: output = flat<int>(in); break;
                 case u32: output = flat<uint>(in); break;
+                case s8: output = flat<schar>(in); break;
                 case u8: output = flat<uchar>(in); break;
                 case s64: output = flat<intl>(in); break;
                 case u64: output = flat<uintl>(in); break;
diff --git a/src/api/c/morph.cpp b/src/api/c/morph.cpp
index efaf6cc53a..418b84e8a9 100644
--- a/src/api/c/morph.cpp
+++ b/src/api/c/morph.cpp
@@ -34,6 +34,7 @@ using detail::createEmptyArray;
 using detail::createValueArray;
 using detail::logicOp;
 using detail::scalar;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::unaryOp;
@@ -137,6 +138,7 @@ af_err morph(af_array *out, const af_array &in, const af_array &mask,
             case u32: output = morph<uint>(in, mask, isDilation); break;
             case s16: output = morph<short>(in, mask, isDilation); break;
             case u16: output = morph<ushort>(in, mask, isDilation); break;
+            case s8: output = morph<schar>(in, mask, isDilation); break;
             case u8: output = morph<uchar>(in, mask, isDilation); break;
             default: TYPE_ERROR(1, type);
         }
@@ -170,6 +172,7 @@ af_err morph3d(af_array *out, const af_array &in, const af_array &mask,
             case u32: output = morph3d<uint>(in, mask, isDilation); break;
             case s16: output = morph3d<short>(in, mask, isDilation); break;
             case u16: output = morph3d<ushort>(in, mask, isDilation); break;
+            case s8: output = morph3d<schar>(in, mask, isDilation); break;
             case u8: output = morph3d<uchar>(in, mask, isDilation); break;
             default: TYPE_ERROR(1, type);
         }
diff --git a/src/api/c/nearest_neighbour.cpp b/src/api/c/nearest_neighbour.cpp
index abc2a7b65b..10543649d9 100644
--- a/src/api/c/nearest_neighbour.cpp
+++ b/src/api/c/nearest_neighbour.cpp
@@ -21,6 +21,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createEmptyArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -128,6 +129,10 @@ af_err af_nearest_neighbour(af_array* idx, af_array* dist, const af_array query,
                                                     dist_dim, n_dist,
                                                     dist_type);
                     break;
+                case s8:
+                    nearest_neighbour<schar, int>(&oIdx, &oDist, query, train,
+                                                  dist_dim, n_dist, dist_type);
+                    break;
                 case u8:
                     nearest_neighbour<uchar, uint>(&oIdx, &oDist, query, train,
                                                    dist_dim, n_dist, dist_type);
diff --git a/src/api/c/plot.cpp b/src/api/c/plot.cpp
index c2d954d481..be5aab06b1 100644
--- a/src/api/c/plot.cpp
+++ b/src/api/c/plot.cpp
@@ -35,6 +35,7 @@ using detail::Array;
 using detail::copy_plot;
 using detail::forgeManager;
 using detail::reduce;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -166,6 +167,10 @@ af_err plotWrapper(const af_window window, const af_array in,
                 chart = setup_plot<ushort>(window, in, dims[order_dim], props,
                                            ptype, marker);
                 break;
+            case s8:
+                chart = setup_plot<schar>(window, in, dims[order_dim], props,
+                                          ptype, marker);
+                break;
             case u8:
                 chart = setup_plot<uchar>(window, in, dims[order_dim], props,
                                           ptype, marker);
@@ -240,6 +245,9 @@ af_err plotWrapper(const af_window window, const af_array X, const af_array Y,
             case u16:
                 chart = setup_plot<ushort>(window, in, 3, props, ptype, marker);
                 break;
+            case s8:
+                chart = setup_plot<schar>(window, in, 3, props, ptype, marker);
+                break;
             case u8:
                 chart = setup_plot<uchar>(window, in, 3, props, ptype, marker);
                 break;
@@ -307,6 +315,9 @@ af_err plotWrapper(const af_window window, const af_array X, const af_array Y,
             case u16:
                 chart = setup_plot<ushort>(window, in, 2, props, ptype, marker);
                 break;
+            case s8:
+                chart = setup_plot<schar>(window, in, 2, props, ptype, marker);
+                break;
             case u8:
                 chart = setup_plot<uchar>(window, in, 2, props, ptype, marker);
                 break;
diff --git a/src/api/c/print.cpp b/src/api/c/print.cpp
index 48fea73b48..2f1ae15c8d 100644
--- a/src/api/c/print.cpp
+++ b/src/api/c/print.cpp
@@ -36,6 +36,7 @@ using arrayfire::common::SparseArray;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -162,6 +163,7 @@ af_err af_print_array(af_array arr) {
                 case b8: print<char>(NULL, arr, 4); break;
                 case s32: print<int>(NULL, arr, 4); break;
                 case u32: print<unsigned>(NULL, arr, 4); break;
+                case s8: print<schar>(NULL, arr, 4); break;
                 case u8: print<uchar>(NULL, arr, 4); break;
                 case s64: print<intl>(NULL, arr, 4); break;
                 case u64: print<uintl>(NULL, arr, 4); break;
@@ -201,6 +203,7 @@ af_err af_print_array_gen(const char *exp, const af_array arr,
                 case b8: print<char>(exp, arr, precision); break;
                 case s32: print<int>(exp, arr, precision); break;
                 case u32: print<unsigned>(exp, arr, precision); break;
+                case s8: print<schar>(exp, arr, precision); break;
                 case u8: print<uchar>(exp, arr, precision); break;
                 case s64: print<intl>(exp, arr, precision); break;
                 case u64: print<uintl>(exp, arr, precision); break;
@@ -259,6 +262,9 @@ af_err af_array_to_string(char **output, const char *exp, const af_array arr,
                 case u32:
                     print<unsigned>(exp, arr, precision, ss, transpose);
                     break;
+                case s8:
+                    print<schar>(exp, arr, precision, ss, transpose);
+                    break;
                 case u8:
                     print<uchar>(exp, arr, precision, ss, transpose);
                     break;
diff --git a/src/api/c/random.cpp b/src/api/c/random.cpp
index 915e733974..6508786f53 100644
--- a/src/api/c/random.cpp
+++ b/src/api/c/random.cpp
@@ -42,6 +42,7 @@ using detail::createEmptyArray;
 using detail::createHostDataArray;
 using detail::intl;
 using detail::normalDistribution;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -296,6 +297,7 @@ af_err af_random_uniform(af_array *out, const unsigned ndims,
             case u64: result = uniformDistribution_<uintl>(d, e); break;
             case s16: result = uniformDistribution_<short>(d, e); break;
             case u16: result = uniformDistribution_<ushort>(d, e); break;
+            case s8: result = uniformDistribution_<schar>(d, e); break;
             case u8: result = uniformDistribution_<uchar>(d, e); break;
             case b8: result = uniformDistribution_<char>(d, e); break;
             case f16: result = uniformDistribution_<half>(d, e); break;
@@ -362,6 +364,7 @@ af_err af_randu(af_array *out, const unsigned ndims, const dim_t *const dims,
             case u64: result = uniformDistribution_<uintl>(d, e); break;
             case s16: result = uniformDistribution_<short>(d, e); break;
             case u16: result = uniformDistribution_<ushort>(d, e); break;
+            case s8: result = uniformDistribution_<schar>(d, e); break;
             case u8: result = uniformDistribution_<uchar>(d, e); break;
             case b8: result = uniformDistribution_<char>(d, e); break;
             case f16: result = uniformDistribution_<half>(d, e); break;
diff --git a/src/api/c/reduce.cpp b/src/api/c/reduce.cpp
index 15be8b39e8..65d3f85209 100644
--- a/src/api/c/reduce.cpp
+++ b/src/api/c/reduce.cpp
@@ -30,6 +30,7 @@ using detail::getScalar;
 using detail::imag;
 using detail::intl;
 using detail::real;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -107,6 +108,7 @@ static af_err reduce_type(af_array *out, const af_array in, const int dim) {
             case s16: res = reduce<op, short, To>(in, dim); break;
             case b8: res = reduce<op, char, To>(in, dim); break;
             case u8: res = reduce<op, uchar, To>(in, dim); break;
+            case s8: res = reduce<op, schar, To>(in, dim); break;
             case f16: res = reduce<op, half, To>(in, dim); break;
             default: TYPE_ERROR(1, type);
         }
@@ -171,6 +173,9 @@ static af_err reduce_by_key_type(af_array *keys_out, af_array *vals_out,
             case u8:
                 reduce_key<op, uchar, To>(keys_out, vals_out, keys, vals, dim);
                 break;
+            case s8:
+                reduce_key<op, schar, To>(keys_out, vals_out, keys, vals, dim);
+                break;
             case f16:
                 reduce_key<op, half, To>(keys_out, vals_out, keys, vals, dim);
                 break;
@@ -210,6 +215,7 @@ static af_err reduce_common(af_array *out, const af_array in, const int dim) {
             case s16: res = reduce<op, short, short>(in, dim); break;
             case b8: res = reduce<op, char, char>(in, dim); break;
             case u8: res = reduce<op, uchar, uchar>(in, dim); break;
+            case s8: res = reduce<op, schar, schar>(in, dim); break;
             case f16: res = reduce<op, half, half>(in, dim); break;
             default: TYPE_ERROR(1, type);
         }
@@ -281,6 +287,10 @@ static af_err reduce_by_key_common(af_array *keys_out, af_array *vals_out,
                 reduce_key<op, uchar, uchar>(keys_out, vals_out, keys, vals,
                                              dim);
                 break;
+            case s8:
+                reduce_key<op, schar, schar>(keys_out, vals_out, keys, vals,
+                                             dim);
+                break;
             case f16:
                 reduce_key<op, half, half>(keys_out, vals_out, keys, vals, dim);
                 break;
@@ -343,6 +353,9 @@ static af_err reduce_promote(af_array *out, const af_array in, const int dim,
             case u8:
                 res = reduce<op, uchar, uint>(in, dim, change_nan, nanval);
                 break;
+            case s8:
+                res = reduce<op, schar, int>(in, dim, change_nan, nanval);
+                break;
             case b8: {
                 if (op == af_mul_t) {
                     res = reduce<af_and_t, char, char>(in, dim, change_nan,
@@ -425,6 +438,10 @@ static af_err reduce_promote_by_key(af_array *keys_out, af_array *vals_out,
                 reduce_key<op, uchar, uint>(keys_out, vals_out, keys, vals, dim,
                                             change_nan, nanval);
                 break;
+            case s8:
+                reduce_key<op, schar, int>(keys_out, vals_out, keys, vals, dim,
+                                           change_nan, nanval);
+                break;
             case b8:
                 reduce_key<af_notzero_t, char, uint>(
                     keys_out, vals_out, keys, vals, dim, change_nan, nanval);
@@ -575,6 +592,7 @@ static af_err reduce_all_type(double *real, double *imag, const af_array in) {
             case s16: *real = reduce_all<op, short,   To>(in); break;
             case b8:  *real = reduce_all<op, char,    To>(in); break;
             case u8:  *real = reduce_all<op, uchar,   To>(in); break;
+            case s8:  *real = reduce_all<op, schar,   To>(in); break;
             case f16: *real = reduce_all<op, half,    To>(in); break;
             // clang-format on
             default: TYPE_ERROR(1, type);
@@ -606,6 +624,7 @@ static af_err reduce_all_type_array(af_array *out, const af_array in) {
             case s16: res = reduce_all_array<op, short,   To>(in); break;
             case b8:  res = reduce_all_array<op, char,    To>(in); break;
             case u8:  res = reduce_all_array<op, uchar,   To>(in); break;
+            case s8:  res = reduce_all_array<op, schar,   To>(in); break;
             case f16: res = reduce_all_array<op, half,    To>(in); break;
             // clang-format on
             default: TYPE_ERROR(1, type);
@@ -644,6 +663,7 @@ static af_err reduce_all_common(double *real_val, double *imag_val,
             case s16: *real_val = reduce_all<op, short,  short>(in); break;
             case b8:  *real_val = reduce_all<op, char,   char>(in); break;
             case u8:  *real_val = reduce_all<op, uchar,  uchar>(in); break;
+            case s8:  *real_val = reduce_all<op, schar,  schar>(in); break;
             case f16: *real_val = reduce_all<op, half,   half>(in); break;
             // clang-format on
             case c32:
@@ -689,6 +709,7 @@ static af_err reduce_all_common_array(af_array *out, const af_array in) {
             case s16: res = reduce_all_array<op, short,  short>(in); break;
             case b8:  res = reduce_all_array<op, char,   char>(in); break;
             case u8:  res = reduce_all_array<op, uchar,  uchar>(in); break;
+            case s8:  res = reduce_all_array<op, schar,  schar>(in); break;
             case f16: res = reduce_all_array<op, half,   half>(in); break;
             // clang-format on
             case c32: res = reduce_all_array<op, cfloat, cfloat>(in); break;
@@ -728,6 +749,7 @@ static af_err reduce_all_promote(double *real_val, double *imag_val,
             case u16: *real_val = reduce_all<op, ushort,   uint>(in, change_nan, nanval); break;
             case s16: *real_val = reduce_all<op, short,     int>(in, change_nan, nanval); break;
             case u8:  *real_val = reduce_all<op, uchar,    uint>(in, change_nan, nanval); break;
+            case s8:  *real_val = reduce_all<op, schar,     int>(in, change_nan, nanval); break;
             // clang-format on
             case b8: {
                 if (op == af_mul_t) {
@@ -813,6 +835,9 @@ static af_err reduce_all_promote_array(af_array *out, const af_array in,
             case u8:
                 res = reduce_all_array<op, uchar, uint>(in, change_nan, nanval);
                 break;
+            case s8:
+                res = reduce_all_array<op, schar, int>(in, change_nan, nanval);
+                break;
             case b8: {
                 if (op == af_mul_t) {
                     res = reduce_all_array<af_and_t, char, char>(in, change_nan,
@@ -953,6 +978,7 @@ static af_err ireduce_common(af_array *val, af_array *idx, const af_array in,
             case s16: ireduce<op, short>(&res, &loc, in, dim); break;
             case b8: ireduce<op, char>(&res, &loc, in, dim); break;
             case u8: ireduce<op, uchar>(&res, &loc, in, dim); break;
+            case s8: ireduce<op, schar>(&res, &loc, in, dim); break;
             case f16: ireduce<op, half>(&res, &loc, in, dim); break;
             default: TYPE_ERROR(1, type);
         }
@@ -1028,6 +1054,7 @@ static af_err rreduce_common(af_array *val, af_array *idx, const af_array in,
                 break;
             case b8: rreduce<op, char>(&res, &loc, in, dim, ragged_len); break;
             case u8: rreduce<op, uchar>(&res, &loc, in, dim, ragged_len); break;
+            case s8: rreduce<op, schar>(&res, &loc, in, dim, ragged_len); break;
             case f16: rreduce<op, half>(&res, &loc, in, dim, ragged_len); break;
             default: TYPE_ERROR(2, type);
         }
@@ -1086,6 +1113,7 @@ static af_err ireduce_all_common(double *real_val, double *imag_val,
                 break;
             case b8: *real_val = ireduce_all<op, char, double>(loc, in); break;
             case u8: *real_val = ireduce_all<op, uchar, double>(loc, in); break;
+            case s8: *real_val = ireduce_all<op, schar, double>(loc, in); break;
 
             case c32:
                 cfval = ireduce_all<op, cfloat>(loc, in);
diff --git a/src/api/c/reorder.cpp b/src/api/c/reorder.cpp
index 556e1f0e20..e29fb621c0 100644
--- a/src/api/c/reorder.cpp
+++ b/src/api/c/reorder.cpp
@@ -25,6 +25,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -108,6 +109,7 @@ af_err af_reorder(af_array *out, const af_array in, const af::dim4 &rdims) {
             case b8: output = reorder<char>(in, rdims); break;
             case s32: output = reorder<int>(in, rdims); break;
             case u32: output = reorder<uint>(in, rdims); break;
+            case s8: output = reorder<schar>(in, rdims); break;
             case u8: output = reorder<uchar>(in, rdims); break;
             case s64: output = reorder<intl>(in, rdims); break;
             case u64: output = reorder<uintl>(in, rdims); break;
diff --git a/src/api/c/replace.cpp b/src/api/c/replace.cpp
index b8fdd75e02..7bf66cc439 100644
--- a/src/api/c/replace.cpp
+++ b/src/api/c/replace.cpp
@@ -27,6 +27,7 @@ using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::select_scalar;
 using detail::uchar;
 using detail::uint;
@@ -74,6 +75,7 @@ af_err af_replace(af_array a, const af_array cond, const af_array b) {
             case u64: replace<uintl>(a, cond, b); break;
             case s16: replace<short>(a, cond, b); break;
             case u16: replace<ushort>(a, cond, b); break;
+            case s8: replace<schar>(a, cond, b); break;
             case u8: replace<uchar>(a, cond, b); break;
             case b8: replace<char>(a, cond, b); break;
             default: TYPE_ERROR(2, ainfo.getType());
@@ -116,6 +118,7 @@ af_err replaceScalar(af_array a, const af_array cond, const ScalarType b) {
             case u64: replace_scalar<uintl>(a, cond, b); break;
             case s16: replace_scalar<short>(a, cond, b); break;
             case u16: replace_scalar<ushort>(a, cond, b); break;
+            case s8: replace_scalar<schar>(a, cond, b); break;
             case u8: replace_scalar<uchar>(a, cond, b); break;
             case b8: replace_scalar<char>(a, cond, b); break;
             default: TYPE_ERROR(2, ainfo.getType());
diff --git a/src/api/c/resize.cpp b/src/api/c/resize.cpp
index 8b6df743da..814d4df0c8 100644
--- a/src/api/c/resize.cpp
+++ b/src/api/c/resize.cpp
@@ -19,6 +19,7 @@
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -68,6 +69,7 @@ af_err af_resize(af_array* out, const af_array in, const dim_t odim0,
             case u64: output = resize<uintl>(in, odim0, odim1, method); break;
             case s16: output = resize<short>(in, odim0, odim1, method); break;
             case u16: output = resize<ushort>(in, odim0, odim1, method); break;
+            case s8: output = resize<schar>(in, odim0, odim1, method); break;
             case u8: output = resize<uchar>(in, odim0, odim1, method); break;
             case b8: output = resize<char>(in, odim0, odim1, method); break;
             default: TYPE_ERROR(1, type);
diff --git a/src/api/c/rgb_gray.cpp b/src/api/c/rgb_gray.cpp
index 3bea06e855..c7abe042bc 100644
--- a/src/api/c/rgb_gray.cpp
+++ b/src/api/c/rgb_gray.cpp
@@ -30,6 +30,7 @@ using detail::createEmptyArray;
 using detail::createValueArray;
 using detail::join;
 using detail::scalar;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -157,6 +158,9 @@ af_err convert(af_array* out, const af_array in, const float r, const float g,
             case u8:
                 output = convert<uchar, float, isRGB2GRAY>(in, r, g, b);
                 break;
+            case s8:
+                output = convert<schar, float, isRGB2GRAY>(in, r, g, b);
+                break;
             default: TYPE_ERROR(1, iType); break;
         }
         std::swap(*out, output);
diff --git a/src/api/c/rotate.cpp b/src/api/c/rotate.cpp
index 762f77d7f4..50397a310a 100644
--- a/src/api/c/rotate.cpp
+++ b/src/api/c/rotate.cpp
@@ -19,6 +19,7 @@ using af::dim4;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -76,6 +77,7 @@ af_err af_rotate(af_array *out, const af_array in, const float theta,
             case u64: output = rotate<uintl>(in, theta, odims, method); break;
             case s16: output = rotate<short>(in, theta, odims, method); break;
             case u16: output = rotate<ushort>(in, theta, odims, method); break;
+            case s8: output = rotate<schar>(in, theta, odims, method); break;
             case u8:
             case b8: output = rotate<uchar>(in, theta, odims, method); break;
             default: TYPE_ERROR(1, itype);
diff --git a/src/api/c/sat.cpp b/src/api/c/sat.cpp
index 3ff72abacc..8715f4865c 100644
--- a/src/api/c/sat.cpp
+++ b/src/api/c/sat.cpp
@@ -18,6 +18,7 @@ using arrayfire::common::integralImage;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -44,6 +45,7 @@ af_err af_sat(af_array* out, const af_array in) {
             case s32: output = sat<int, int>(in); break;
             case u32: output = sat<uint, uint>(in); break;
             case b8: output = sat<int, char>(in); break;
+            case s8: output = sat<int, schar>(in); break;
             case u8: output = sat<uint, uchar>(in); break;
             case s64: output = sat<intl, intl>(in); break;
             case u64: output = sat<uintl, uintl>(in); break;
diff --git a/src/api/c/scan.cpp b/src/api/c/scan.cpp
index d8a3a7a95d..cac89d6c01 100644
--- a/src/api/c/scan.cpp
+++ b/src/api/c/scan.cpp
@@ -21,6 +21,7 @@
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -141,6 +142,7 @@ af_err af_accum(af_array* out, const af_array in, const int dim) {
             case u16: res = scan<af_add_t, ushort, uint>(in, dim); break;
             case s16: res = scan<af_add_t, short, int>(in, dim); break;
             case u8: res = scan<af_add_t, uchar, uint>(in, dim); break;
+            case s8: res = scan<af_add_t, schar, int>(in, dim); break;
             // Make sure you are adding only "1" for every non zero value, even
             // if op == af_add_t
             case b8: res = scan<af_notzero_t, char, uint>(in, dim); break;
@@ -204,6 +206,9 @@ af_err af_scan(af_array* out, const af_array in, const int dim, af_binary_op op,
             case u8:
                 res = scan_op<uchar, uint>(in, dim, op, inclusive_scan);
                 break;
+            case s8:
+                res = scan_op<schar, int>(in, dim, op, inclusive_scan);
+                break;
             case b8:
                 res = scan_op<char, uint>(in, dim, op, inclusive_scan);
                 break;
@@ -252,6 +257,7 @@ af_err af_scan_by_key(af_array* out, const af_array key, const af_array in,
                 break;
             case s16:
             case s32:
+            case s8:
                 res = scan_op<int, int>(key, in, dim, op, inclusive_scan);
                 break;
             case u64:
diff --git a/src/api/c/select.cpp b/src/api/c/select.cpp
index dec47166e7..c161aa5e9b 100644
--- a/src/api/c/select.cpp
+++ b/src/api/c/select.cpp
@@ -26,6 +26,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createSelectNode;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -76,6 +77,7 @@ af_err af_select(af_array* out, const af_array cond, const af_array a,
             case u64: res = select<uintl>(cond, a, b, odims); break;
             case s16: res = select<short>(cond, a, b, odims); break;
             case u16: res = select<ushort>(cond, a, b, odims); break;
+            case s8: res = select<schar>(cond, a, b, odims); break;
             case u8: res = select<uchar>(cond, a, b, odims); break;
             case b8: res = select<char>(cond, a, b, odims); break;
             case f16: res = select<half>(cond, a, b, odims); break;
@@ -163,6 +165,10 @@ af_err selectScalar(af_array* out, const af_array cond, const af_array e,
                 res = select_scalar<uintl, ScalarType, IsScalarTrueOutput>(
                     cond, e, c, odims);
                 break;
+            case s8:
+                res = select_scalar<schar, ScalarType, IsScalarTrueOutput>(
+                    cond, e, c, odims);
+                break;
             case u8:
                 res = select_scalar<uchar, ScalarType, IsScalarTrueOutput>(
                     cond, e, c, odims);
diff --git a/src/api/c/set.cpp b/src/api/c/set.cpp
index bf8b66e3c8..3353d7c5ee 100644
--- a/src/api/c/set.cpp
+++ b/src/api/c/set.cpp
@@ -18,6 +18,7 @@
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -51,6 +52,7 @@ af_err af_set_unique(af_array* out, const af_array in, const bool is_sorted) {
             case s64: res = setUnique<intl>(in, is_sorted); break;
             case u64: res = setUnique<uintl>(in, is_sorted); break;
             case b8: res = setUnique<char>(in, is_sorted); break;
+            case s8: res = setUnique<schar>(in, is_sorted); break;
             case u8: res = setUnique<uchar>(in, is_sorted); break;
             default: TYPE_ERROR(1, type);
         }
@@ -98,6 +100,7 @@ af_err af_set_union(af_array* out, const af_array first, const af_array second,
             case s64: res = setUnion<intl>(first, second, is_unique); break;
             case u64: res = setUnion<uintl>(first, second, is_unique); break;
             case b8: res = setUnion<char>(first, second, is_unique); break;
+            case s8: res = setUnion<schar>(first, second, is_unique); break;
             case u8: res = setUnion<uchar>(first, second, is_unique); break;
             default: TYPE_ERROR(1, first_type);
         }
@@ -156,6 +159,7 @@ af_err af_set_intersect(af_array* out, const af_array first,
                 res = setIntersect<uintl>(first, second, is_unique);
                 break;
             case b8: res = setIntersect<char>(first, second, is_unique); break;
+            case s8: res = setIntersect<schar>(first, second, is_unique); break;
             case u8: res = setIntersect<uchar>(first, second, is_unique); break;
             default: TYPE_ERROR(1, first_type);
         }
diff --git a/src/api/c/shift.cpp b/src/api/c/shift.cpp
index 42052fbfbc..cf195d2026 100644
--- a/src/api/c/shift.cpp
+++ b/src/api/c/shift.cpp
@@ -17,6 +17,7 @@
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -49,6 +50,7 @@ af_err af_shift(af_array *out, const af_array in, const int sdims[4]) {
             case u64: output = shift<uintl>(in, sdims); break;
             case s16: output = shift<short>(in, sdims); break;
             case u16: output = shift<ushort>(in, sdims); break;
+            case s8: output = shift<schar>(in, sdims); break;
             case u8: output = shift<uchar>(in, sdims); break;
             default: TYPE_ERROR(1, type);
         }
diff --git a/src/api/c/sobel.cpp b/src/api/c/sobel.cpp
index 6184d5502a..d466db1617 100644
--- a/src/api/c/sobel.cpp
+++ b/src/api/c/sobel.cpp
@@ -21,6 +21,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -66,6 +67,9 @@ af_err af_sobel_operator(af_array *dx, af_array *dy, const af_array img,
                 output = sobelDerivatives<ushort, int>(img, ker_size);
                 break;
             case b8: output = sobelDerivatives<char, int>(img, ker_size); break;
+            case s8:
+                output = sobelDerivatives<schar, int>(img, ker_size);
+                break;
             case u8:
                 output = sobelDerivatives<uchar, int>(img, ker_size);
                 break;
diff --git a/src/api/c/sort.cpp b/src/api/c/sort.cpp
index 4ec1c0a466..b917b8b3c5 100644
--- a/src/api/c/sort.cpp
+++ b/src/api/c/sort.cpp
@@ -27,6 +27,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createEmptyArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -59,6 +60,7 @@ af_err af_sort(af_array *out, const af_array in, const unsigned dim,
             case u16: val = sort<ushort>(in, dim, isAscending); break;
             case s64: val = sort<intl>(in, dim, isAscending); break;
             case u64: val = sort<uintl>(in, dim, isAscending); break;
+            case s8: val = sort<schar>(in, dim, isAscending); break;
             case u8: val = sort<uchar>(in, dim, isAscending); break;
             case b8: val = sort<char>(in, dim, isAscending); break;
             default: TYPE_ERROR(1, type);
@@ -118,6 +120,7 @@ af_err af_sort_index(af_array *out, af_array *indices, const af_array in,
             case u64:
                 sort_index<uintl>(&val, &idx, in, dim, isAscending);
                 break;
+            case s8: sort_index<schar>(&val, &idx, in, dim, isAscending); break;
             case u8: sort_index<uchar>(&val, &idx, in, dim, isAscending); break;
             case b8: sort_index<char>(&val, &idx, in, dim, isAscending); break;
             default: TYPE_ERROR(1, type);
@@ -185,6 +188,9 @@ void sort_by_key_tmplt(af_array *okey, af_array *oval, const af_array ikey,
         case u64:
             sort_by_key<Tk, uintl>(okey, oval, ikey, ival, dim, isAscending);
             break;
+        case s8:
+            sort_by_key<Tk, schar>(okey, oval, ikey, ival, dim, isAscending);
+            break;
         case u8:
             sort_by_key<Tk, uchar>(okey, oval, ikey, ival, dim, isAscending);
             break;
@@ -249,6 +255,10 @@ af_err af_sort_by_key(af_array *out_keys, af_array *out_values,
                 sort_by_key_tmplt<uintl>(&oKey, &oVal, keys, values, dim,
                                          isAscending);
                 break;
+            case s8:
+                sort_by_key_tmplt<schar>(&oKey, &oVal, keys, values, dim,
+                                         isAscending);
+                break;
             case u8:
                 sort_by_key_tmplt<uchar>(&oKey, &oVal, keys, values, dim,
                                          isAscending);
diff --git a/src/api/c/stdev.cpp b/src/api/c/stdev.cpp
index 7f64bf3355..d5589f4d39 100644
--- a/src/api/c/stdev.cpp
+++ b/src/api/c/stdev.cpp
@@ -38,6 +38,7 @@ using detail::mean;
 using detail::reduce;
 using detail::reduce_all;
 using detail::scalar;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -111,6 +112,7 @@ af_err af_stdev_all_v2(double* realVal, double* imagVal, const af_array in,
             case u16: *realVal = stdev<ushort, float>(in, bias); break;
             case s64: *realVal = stdev<intl, double>(in, bias); break;
             case u64: *realVal = stdev<uintl, double>(in, bias); break;
+            case s8: *realVal = stdev<schar, float>(in, bias); break;
             case u8: *realVal = stdev<uchar, float>(in, bias); break;
             case b8: *realVal = stdev<char, float>(in, bias); break;
             // TODO(umar): FIXME: sqrt(complex) is not present in cuda/opencl
@@ -152,6 +154,7 @@ af_err af_stdev_v2(af_array* out, const af_array in, const af_var_bias bias,
             case u16: output = stdev<ushort, float>(in, dim, bias); break;
             case s64: output = stdev<intl, double>(in, dim, bias); break;
             case u64: output = stdev<uintl, double>(in, dim, bias); break;
+            case s8: output = stdev<schar, float>(in, dim, bias); break;
             case u8: output = stdev<uchar, float>(in, dim, bias); break;
             case b8: output = stdev<char, float>(in, dim, bias); break;
             // TODO(umar): FIXME: sqrt(complex) is not present in cuda/opencl
diff --git a/src/api/c/stream.cpp b/src/api/c/stream.cpp
index 1be207c66d..45265e69b5 100644
--- a/src/api/c/stream.cpp
+++ b/src/api/c/stream.cpp
@@ -28,6 +28,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::createHostDataArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -141,6 +142,7 @@ af_err af_save_array(int *index, const char *key, const af_array arr,
             case b8: id = save<char>(key, arr, filename, append); break;
             case s32: id = save<int>(key, arr, filename, append); break;
             case u32: id = save<unsigned>(key, arr, filename, append); break;
+            case s8: id = save<schar>(key, arr, filename, append); break;
             case u8: id = save<uchar>(key, arr, filename, append); break;
             case s64: id = save<intl>(key, arr, filename, append); break;
             case u64: id = save<uintl>(key, arr, filename, append); break;
@@ -240,6 +242,7 @@ static af_array readArrayV1(const char *filename, const unsigned index) {
         case b8: out = readDataToArray<char>(fs); break;
         case s32: out = readDataToArray<int>(fs); break;
         case u32: out = readDataToArray<uint>(fs); break;
+        case s8: out = readDataToArray<schar>(fs); break;
         case u8: out = readDataToArray<uchar>(fs); break;
         case s64: out = readDataToArray<intl>(fs); break;
         case u64: out = readDataToArray<uintl>(fs); break;
diff --git a/src/api/c/surface.cpp b/src/api/c/surface.cpp
index b2a6404a33..d748677269 100644
--- a/src/api/c/surface.cpp
+++ b/src/api/c/surface.cpp
@@ -38,6 +38,7 @@ using detail::createEmptyArray;
 using detail::forgeManager;
 using detail::getScalar;
 using detail::reduce_all;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -190,6 +191,9 @@ af_err af_draw_surface(const af_window window, const af_array xVals,
             case u16:
                 chart = setup_surface<ushort>(window, xVals, yVals, S, props);
                 break;
+            case s8:
+                chart = setup_surface<schar>(window, xVals, yVals, S, props);
+                break;
             case u8:
                 chart = setup_surface<uchar>(window, xVals, yVals, S, props);
                 break;
diff --git a/src/api/c/susan.cpp b/src/api/c/susan.cpp
index 0621f7eb16..8ea7dc8945 100644
--- a/src/api/c/susan.cpp
+++ b/src/api/c/susan.cpp
@@ -24,6 +24,7 @@ using detail::cfloat;
 using detail::createEmptyArray;
 using detail::createValueArray;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::ushort;
@@ -98,6 +99,10 @@ af_err af_susan(af_features* out, const af_array in, const unsigned radius,
                 *out = susan<ushort>(in, radius, diff_thr, geom_thr,
                                      feature_ratio, edge);
                 break;
+            case s8:
+                *out = susan<schar>(in, radius, diff_thr, geom_thr,
+                                    feature_ratio, edge);
+                break;
             case u8:
                 *out = susan<uchar>(in, radius, diff_thr, geom_thr,
                                     feature_ratio, edge);
diff --git a/src/api/c/tile.cpp b/src/api/c/tile.cpp
index ce512e9958..2a50f12c43 100644
--- a/src/api/c/tile.cpp
+++ b/src/api/c/tile.cpp
@@ -26,6 +26,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -60,6 +61,7 @@ af_err af_tile(af_array *out, const af_array in, const af::dim4 &tileDims) {
             case u64: output = tile<uintl>(in, tileDims); break;
             case s16: output = tile<short>(in, tileDims); break;
             case u16: output = tile<ushort>(in, tileDims); break;
+            case s8: output = tile<schar>(in, tileDims); break;
             case u8: output = tile<uchar>(in, tileDims); break;
             case f16: output = tile<half>(in, tileDims); break;
             default: TYPE_ERROR(1, type);
diff --git a/src/api/c/transform.cpp b/src/api/c/transform.cpp
index 9bdaceb149..259d13840e 100644
--- a/src/api/c/transform.cpp
+++ b/src/api/c/transform.cpp
@@ -19,6 +19,7 @@ using af::dim4;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -158,6 +159,7 @@ void af_transform_common(af_array *out, const af_array in, const af_array tf,
     case u64: transform<uintl  >(out, in, tf, method, inverse, perspective);  break;
     case s16: transform<short  >(out, in, tf, method, inverse, perspective);  break;
     case u16: transform<ushort >(out, in, tf, method, inverse, perspective);  break;
+    case s8:  transform<schar  >(out, in, tf, method, inverse, perspective);  break;
     case u8:  transform<uchar  >(out, in, tf, method, inverse, perspective);  break;
     case b8:  transform<char   >(out, in, tf, method, inverse, perspective);  break;
     default:  TYPE_ERROR(1, itype);
diff --git a/src/api/c/transpose.cpp b/src/api/c/transpose.cpp
index 82ae18fef2..9d2fd48cbd 100644
--- a/src/api/c/transpose.cpp
+++ b/src/api/c/transpose.cpp
@@ -24,6 +24,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -67,6 +68,7 @@ af_err af_transpose(af_array* out, af_array in, const bool conjugate) {
             case b8: output = trs<char>(in, conjugate); break;
             case s32: output = trs<int>(in, conjugate); break;
             case u32: output = trs<uint>(in, conjugate); break;
+            case s8: output = trs<schar>(in, conjugate); break;
             case u8: output = trs<uchar>(in, conjugate); break;
             case s64: output = trs<intl>(in, conjugate); break;
             case u64: output = trs<uintl>(in, conjugate); break;
@@ -107,6 +109,7 @@ af_err af_transpose_inplace(af_array in, const bool conjugate) {
             case b8: transpose_inplace<char>(in, conjugate); break;
             case s32: transpose_inplace<int>(in, conjugate); break;
             case u32: transpose_inplace<uint>(in, conjugate); break;
+            case s8: transpose_inplace<schar>(in, conjugate); break;
             case u8: transpose_inplace<uchar>(in, conjugate); break;
             case s64: transpose_inplace<intl>(in, conjugate); break;
             case u64: transpose_inplace<uintl>(in, conjugate); break;
diff --git a/src/api/c/type_util.cpp b/src/api/c/type_util.cpp
index c78b85b1da..d409c0d868 100644
--- a/src/api/c/type_util.cpp
+++ b/src/api/c/type_util.cpp
@@ -20,6 +20,7 @@ size_t size_of(af_dtype type) {
             case f64: return sizeof(double);
             case s32: return sizeof(int);
             case u32: return sizeof(unsigned);
+            case s8: return sizeof(signed char);
             case u8: return sizeof(unsigned char);
             case b8: return sizeof(unsigned char);
             case c32: return sizeof(float) * 2;
diff --git a/src/api/c/type_util.hpp b/src/api/c/type_util.hpp
index 4214882492..8e6a7ff9cf 100644
--- a/src/api/c/type_util.hpp
+++ b/src/api/c/type_util.hpp
@@ -16,6 +16,11 @@ struct ToNum {
     inline T operator()(T val) { return val; }
 };
 
+template<>
+struct ToNum<signed char> {
+    inline int operator()(signed char val) { return static_cast<int>(val); }
+};
+
 template<>
 struct ToNum<unsigned char> {
     inline int operator()(unsigned char val) { return static_cast<int>(val); }
diff --git a/src/api/c/unary.cpp b/src/api/c/unary.cpp
index 6d8b584ace..505c831e74 100644
--- a/src/api/c/unary.cpp
+++ b/src/api/c/unary.cpp
@@ -43,6 +43,7 @@ using detail::intl;
 using detail::logicOp;
 using detail::real;
 using detail::scalar;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -598,6 +599,7 @@ af_err af_bitnot(af_array *out, const af_array in) {
         switch (type) {
             case s32: res = bitOpNot<int>(in); break;
             case u32: res = bitOpNot<uint>(in); break;
+            case s8: res = bitOpNot<schar>(in); break;
             case u8: res = bitOpNot<uchar>(in); break;
             case b8: res = bitOpNot<char>(in); break;
             case s64: res = bitOpNot<intl>(in); break;
diff --git a/src/api/c/unwrap.cpp b/src/api/c/unwrap.cpp
index ee0ac2a16e..6f09a6b7eb 100644
--- a/src/api/c/unwrap.cpp
+++ b/src/api/c/unwrap.cpp
@@ -20,6 +20,7 @@ using detail::Array;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -81,6 +82,9 @@ af_err af_unwrap(af_array* out, const af_array in, const dim_t wx,
             case u16:
                 output = unwrap<ushort>(in, wx, wy, sx, sy, px, py, is_column);
                 break;
+            case s8:
+                output = unwrap<schar>(in, wx, wy, sx, sy, px, py, is_column);
+                break;
             case u8:
                 output = unwrap<uchar>(in, wx, wy, sx, sy, px, py, is_column);
                 break;
diff --git a/src/api/c/var.cpp b/src/api/c/var.cpp
index c82c1ca0cd..64a5d8f693 100644
--- a/src/api/c/var.cpp
+++ b/src/api/c/var.cpp
@@ -43,6 +43,7 @@ using detail::real;
 using detail::reduce;
 using detail::reduce_all;
 using detail::scalar;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -225,6 +226,9 @@ af_err af_var_v2(af_array* out, const af_array in, const af_var_bias bias,
             case u64:
                 output = var_<uintl, double>(in, no_weights, bias, dim);
                 break;
+            case s8:
+                output = var_<schar, float>(in, no_weights, bias, dim);
+                break;
             case u8:
                 output = var_<uchar, float>(in, no_weights, bias, dim);
                 break;
@@ -298,6 +302,10 @@ af_err af_var_weighted(af_array* out, const af_array in, const af_array weights,
                 output = var_<uintl, double>(in, weights,
                                              AF_VARIANCE_POPULATION, dim);
                 break;
+            case s8:
+                output = var_<schar, float>(in, weights, AF_VARIANCE_POPULATION,
+                                            dim);
+                break;
             case u8:
                 output = var_<uchar, float>(in, weights, AF_VARIANCE_POPULATION,
                                             dim);
@@ -347,6 +355,7 @@ af_err af_var_all_v2(double* realVal, double* imagVal, const af_array in,
             case u16: *realVal = varAll<ushort, float>(in, bias); break;
             case s64: *realVal = varAll<intl, double>(in, bias); break;
             case u64: *realVal = varAll<uintl, double>(in, bias); break;
+            case s8: *realVal = varAll<schar, float>(in, bias); break;
             case u8: *realVal = varAll<uchar, float>(in, bias); break;
             case b8: *realVal = varAll<char, float>(in, bias); break;
             case f16: *realVal = varAll<half, float>(in, bias); break;
@@ -390,6 +399,7 @@ af_err af_var_all_weighted(double* realVal, double* imagVal, const af_array in,
             case u16: *realVal = varAll<ushort, float>(in, weights); break;
             case s64: *realVal = varAll<intl, double>(in, weights); break;
             case u64: *realVal = varAll<uintl, double>(in, weights); break;
+            case s8: *realVal = varAll<schar, float>(in, weights); break;
             case u8: *realVal = varAll<uchar, float>(in, weights); break;
             case b8: *realVal = varAll<char, float>(in, weights); break;
             case f16: *realVal = varAll<half, float>(in, weights); break;
@@ -453,6 +463,10 @@ af_err af_meanvar(af_array* mean, af_array* var, const af_array in,
                 tie(*mean, *var) =
                     meanvar<uintl, double>(in, weights, bias, dim);
                 break;
+            case s8:
+                tie(*mean, *var) =
+                    meanvar<schar, float>(in, weights, bias, dim);
+                break;
             case u8:
                 tie(*mean, *var) =
                     meanvar<uchar, float>(in, weights, bias, dim);
diff --git a/src/api/c/vector_field.cpp b/src/api/c/vector_field.cpp
index 701db6fc12..9eba21811c 100644
--- a/src/api/c/vector_field.cpp
+++ b/src/api/c/vector_field.cpp
@@ -35,6 +35,7 @@ using detail::copy_vector_field;
 using detail::createEmptyArray;
 using detail::forgeManager;
 using detail::reduce;
+using detail::schar;
 using detail::transpose;
 using detail::uchar;
 using detail::uint;
@@ -183,6 +184,9 @@ af_err vectorFieldWrapper(const af_window window, const af_array points,
             case u16:
                 chart = setup_vector_field<ushort>(window, pnts, dirs, props);
                 break;
+            case s8:
+                chart = setup_vector_field<schar>(window, pnts, dirs, props);
+                break;
             case u8:
                 chart = setup_vector_field<uchar>(window, pnts, dirs, props);
                 break;
@@ -289,6 +293,10 @@ af_err vectorFieldWrapper(const af_window window, const af_array xPoints,
                 chart = setup_vector_field<ushort>(window, points, directions,
                                                    props);
                 break;
+            case s8:
+                chart = setup_vector_field<schar>(window, points, directions,
+                                                  props);
+                break;
             case u8:
                 chart = setup_vector_field<uchar>(window, points, directions,
                                                   props);
@@ -383,6 +391,10 @@ af_err vectorFieldWrapper(const af_window window, const af_array xPoints,
                 chart = setup_vector_field<ushort>(window, points, directions,
                                                    props);
                 break;
+            case s8:
+                chart = setup_vector_field<schar>(window, points, directions,
+                                                  props);
+                break;
             case u8:
                 chart = setup_vector_field<uchar>(window, points, directions,
                                                   props);
diff --git a/src/api/c/where.cpp b/src/api/c/where.cpp
index 4aeb7b60ba..6f83aed17d 100644
--- a/src/api/c/where.cpp
+++ b/src/api/c/where.cpp
@@ -18,6 +18,7 @@
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -51,6 +52,7 @@ af_err af_where(af_array* idx, const af_array in) {
             case u64: res = where<uintl>(in); break;
             case s16: res = where<short>(in); break;
             case u16: res = where<ushort>(in); break;
+            case s8: res = where<schar>(in); break;
             case u8: res = where<uchar>(in); break;
             case b8: res = where<char>(in); break;
             default: TYPE_ERROR(1, type);
diff --git a/src/api/c/wrap.cpp b/src/api/c/wrap.cpp
index f436f37350..e3c06a4642 100644
--- a/src/api/c/wrap.cpp
+++ b/src/api/c/wrap.cpp
@@ -19,6 +19,7 @@ using af::dim4;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -75,6 +76,7 @@ void af_wrap_common(af_array* out, const af_array in, const dim_t ox,
         case u64: wrap<uintl  >(out, in, wx, wy, sx, sy, px, py, is_column);  break;
         case s16: wrap<short  >(out, in, wx, wy, sx, sy, px, py, is_column);  break;
         case u16: wrap<ushort >(out, in, wx, wy, sx, sy, px, py, is_column);  break;
+        case s8:  wrap<schar  >(out, in, wx, wy, sx, sy, px, py, is_column);  break;
         case u8:  wrap<uchar  >(out, in, wx, wy, sx, sy, px, py, is_column);  break;
         case b8:  wrap<char   >(out, in, wx, wy, sx, sy, px, py, is_column);  break;
         default:  TYPE_ERROR(1, in_type);
diff --git a/src/api/cpp/array.cpp b/src/api/cpp/array.cpp
index 1d61c63c2d..418d94c52b 100644
--- a/src/api/cpp/array.cpp
+++ b/src/api/cpp/array.cpp
@@ -236,6 +236,7 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 INSTANTIATE(unsigned)
 INSTANTIATE(int)
+INSTANTIATE(signed char)
 INSTANTIATE(unsigned char)
 INSTANTIATE(char)
 INSTANTIATE(long long)
@@ -701,6 +702,7 @@ MEM_FUNC(af_array, get)
     ASSIGN_TYPE(long long, OP)          \
     ASSIGN_TYPE(unsigned long long, OP) \
     ASSIGN_TYPE(char, OP)               \
+    ASSIGN_TYPE(signed char, OP)        \
     ASSIGN_TYPE(unsigned char, OP)      \
     ASSIGN_TYPE(bool, OP)               \
     ASSIGN_TYPE(short, OP)              \
@@ -828,6 +830,7 @@ array &array::operator=(const array &other) {
     ASSIGN_TYPE(long long, OP)                                    \
     ASSIGN_TYPE(unsigned long long, OP)                           \
     ASSIGN_TYPE(char, OP)                                         \
+    ASSIGN_TYPE(signed char, OP)                                  \
     ASSIGN_TYPE(unsigned char, OP)                                \
     ASSIGN_TYPE(bool, OP)                                         \
     ASSIGN_TYPE(short, OP)                                        \
@@ -863,6 +866,7 @@ ASSIGN_OP(/=, af_div)
     ASSIGN_TYPE(long long, OP)          \
     ASSIGN_TYPE(unsigned long long, OP) \
     ASSIGN_TYPE(char, OP)               \
+    ASSIGN_TYPE(signed char, OP)        \
     ASSIGN_TYPE(unsigned char, OP)      \
     ASSIGN_TYPE(bool, OP)               \
     ASSIGN_TYPE(short, OP)              \
@@ -939,6 +943,7 @@ af::dtype implicit_dtype(af::dtype scalar_type, af::dtype array_type) {
     BINARY_TYPE(long long, OP, release_func, s64)                      \
     BINARY_TYPE(unsigned long long, OP, release_func, u64)             \
     BINARY_TYPE(char, OP, release_func, b8)                            \
+    BINARY_TYPE(signed char, OP, release_func, s8)                     \
     BINARY_TYPE(unsigned char, OP, release_func, u8)                   \
     BINARY_TYPE(bool, OP, release_func, b8)                            \
     BINARY_TYPE(short, OP, release_func, s16)                          \
@@ -1038,6 +1043,7 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 INSTANTIATE(unsigned)
 INSTANTIATE(int)
+INSTANTIATE(signed char)
 INSTANTIATE(unsigned char)
 INSTANTIATE(char)
 INSTANTIATE(long long)
@@ -1080,6 +1086,7 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 INSTANTIATE(unsigned)
 INSTANTIATE(int)
+INSTANTIATE(signed char)
 INSTANTIATE(unsigned char)
 INSTANTIATE(char)
 INSTANTIATE(long long)
diff --git a/src/api/cpp/corrcoef.cpp b/src/api/cpp/corrcoef.cpp
index f90be68b5f..dbedad5aee 100644
--- a/src/api/cpp/corrcoef.cpp
+++ b/src/api/cpp/corrcoef.cpp
@@ -26,6 +26,7 @@ INSTANTIATE_CORRCOEF(double);
 INSTANTIATE_CORRCOEF(int);
 INSTANTIATE_CORRCOEF(unsigned int);
 INSTANTIATE_CORRCOEF(char);
+INSTANTIATE_CORRCOEF(signed char);
 INSTANTIATE_CORRCOEF(unsigned char);
 INSTANTIATE_CORRCOEF(long long);
 INSTANTIATE_CORRCOEF(unsigned long long);
diff --git a/src/api/cpp/data.cpp b/src/api/cpp/data.cpp
index 3f86520bd0..f5eb8c2544 100644
--- a/src/api/cpp/data.cpp
+++ b/src/api/cpp/data.cpp
@@ -130,6 +130,7 @@ CONSTANT(float);
 CONSTANT(int);
 CONSTANT(unsigned);
 CONSTANT(char);
+CONSTANT(signed char);
 CONSTANT(unsigned char);
 CONSTANT(cfloat);
 CONSTANT(cdouble);
diff --git a/src/api/cpp/device.cpp b/src/api/cpp/device.cpp
index 89aab84754..b62589097e 100644
--- a/src/api/cpp/device.cpp
+++ b/src/api/cpp/device.cpp
@@ -192,6 +192,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(unsigned)
+INSTANTIATE(signed char)
 INSTANTIATE(unsigned char)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/api/cpp/mean.cpp b/src/api/cpp/mean.cpp
index c03a83fa51..61693ca40d 100644
--- a/src/api/cpp/mean.cpp
+++ b/src/api/cpp/mean.cpp
@@ -81,6 +81,7 @@ INSTANTIATE_MEAN(double);
 INSTANTIATE_MEAN(int);
 INSTANTIATE_MEAN(unsigned int);
 INSTANTIATE_MEAN(char);
+INSTANTIATE_MEAN(signed char);
 INSTANTIATE_MEAN(unsigned char);
 INSTANTIATE_MEAN(long long);
 INSTANTIATE_MEAN(unsigned long long);
diff --git a/src/api/cpp/median.cpp b/src/api/cpp/median.cpp
index 5f4b88fb2a..b288df74a9 100644
--- a/src/api/cpp/median.cpp
+++ b/src/api/cpp/median.cpp
@@ -27,6 +27,7 @@ INSTANTIATE_MEDIAN(double);
 INSTANTIATE_MEDIAN(int);
 INSTANTIATE_MEDIAN(unsigned int);
 INSTANTIATE_MEDIAN(char);
+INSTANTIATE_MEDIAN(signed char);
 INSTANTIATE_MEDIAN(unsigned char);
 INSTANTIATE_MEDIAN(long long);
 INSTANTIATE_MEDIAN(unsigned long long);
diff --git a/src/api/cpp/reduce.cpp b/src/api/cpp/reduce.cpp
index cfdadf85ae..8dc47fcab9 100644
--- a/src/api/cpp/reduce.cpp
+++ b/src/api/cpp/reduce.cpp
@@ -191,6 +191,7 @@ void max(array &val, array &idx, const array &in, const int dim) {
     INSTANTIATE_REAL(fnC, fnCPP, short)              \
     INSTANTIATE_REAL(fnC, fnCPP, unsigned short)     \
     INSTANTIATE_REAL(fnC, fnCPP, char)               \
+    INSTANTIATE_REAL(fnC, fnCPP, signed char)        \
     INSTANTIATE_REAL(fnC, fnCPP, unsigned char)      \
     INSTANTIATE_CPLX(fnC, fnCPP, af_cfloat, float)   \
     INSTANTIATE_CPLX(fnC, fnCPP, af_cdouble, double)
@@ -294,6 +295,7 @@ INSTANTIATE(product_nan, product)
     INSTANTIATE_COMPAT(fnCPP, fnCompat, long long)          \
     INSTANTIATE_COMPAT(fnCPP, fnCompat, unsigned long long) \
     INSTANTIATE_COMPAT(fnCPP, fnCompat, char)               \
+    INSTANTIATE_COMPAT(fnCPP, fnCompat, signed char)        \
     INSTANTIATE_COMPAT(fnCPP, fnCompat, unsigned char)      \
     INSTANTIATE_COMPAT(fnCPP, fnCompat, af_cfloat)          \
     INSTANTIATE_COMPAT(fnCPP, fnCompat, af_cdouble)         \
@@ -332,6 +334,7 @@ INSTANTIATE_COMPAT(anyTrue, anytrue, bool)
     INSTANTIATE_REAL(fn, int)              \
     INSTANTIATE_REAL(fn, unsigned)         \
     INSTANTIATE_REAL(fn, char)             \
+    INSTANTIATE_REAL(fn, signed char)      \
     INSTANTIATE_REAL(fn, unsigned char)    \
     INSTANTIATE_REAL(fn, short)            \
     INSTANTIATE_REAL(fn, unsigned short)   \
diff --git a/src/api/cpp/stdev.cpp b/src/api/cpp/stdev.cpp
index a9e22d58f6..66edaf816a 100644
--- a/src/api/cpp/stdev.cpp
+++ b/src/api/cpp/stdev.cpp
@@ -60,6 +60,7 @@ INSTANTIATE_STDEV(unsigned long long);
 INSTANTIATE_STDEV(short);
 INSTANTIATE_STDEV(unsigned short);
 INSTANTIATE_STDEV(char);
+INSTANTIATE_STDEV(signed char);
 INSTANTIATE_STDEV(unsigned char);
 
 #undef INSTANTIATE_STDEV
diff --git a/src/api/cpp/var.cpp b/src/api/cpp/var.cpp
index 80cd6a63c5..66f2d76252 100644
--- a/src/api/cpp/var.cpp
+++ b/src/api/cpp/var.cpp
@@ -112,6 +112,7 @@ INSTANTIATE_VAR(unsigned long long);
 INSTANTIATE_VAR(short);
 INSTANTIATE_VAR(unsigned short);
 INSTANTIATE_VAR(char);
+INSTANTIATE_VAR(signed char);
 INSTANTIATE_VAR(unsigned char);
 INSTANTIATE_VAR(af_half);
 INSTANTIATE_VAR(half_float::half);
diff --git a/src/backend/common/TemplateTypename.hpp b/src/backend/common/TemplateTypename.hpp
index 47286af899..96dfb3c6fe 100644
--- a/src/backend/common/TemplateTypename.hpp
+++ b/src/backend/common/TemplateTypename.hpp
@@ -33,6 +33,7 @@ struct TemplateTypename {
         operator std::string() const noexcept { return #NAME; } \
     }
 
+SPECIALIZE(signed char, detail::schar);
 SPECIALIZE(unsigned char, detail::uchar);
 SPECIALIZE(unsigned int, detail::uint);
 SPECIALIZE(unsigned short, detail::ushort);
diff --git a/src/backend/common/cast.cpp b/src/backend/common/cast.cpp
index cc98f0504f..bcb2dfb519 100644
--- a/src/backend/common/cast.cpp
+++ b/src/backend/common/cast.cpp
@@ -14,6 +14,7 @@ using arrayfire::common::half;
 using detail::cdouble;
 using detail::cfloat;
 using detail::intl;
+using detail::schar;
 using detail::uchar;
 using detail::uint;
 using detail::uintl;
@@ -38,6 +39,7 @@ detail::Array<To> castArray(const af_array &in) {
         case c64: return common::cast<To, cdouble>(getArray<cdouble>(in));
         case s32: return common::cast<To, int>(getArray<int>(in));
         case u32: return common::cast<To, uint>(getArray<uint>(in));
+        case s8: return common::cast<To, schar>(getArray<schar>(in));
         case u8: return common::cast<To, uchar>(getArray<uchar>(in));
         case b8: return common::cast<To, char>(getArray<char>(in));
         case s64: return common::cast<To, intl>(getArray<intl>(in));
@@ -56,6 +58,7 @@ template detail::Array<cfloat> castArray(const af_array &in);
 template detail::Array<cdouble> castArray(const af_array &in);
 template detail::Array<int> castArray(const af_array &in);
 template detail::Array<uint> castArray(const af_array &in);
+template detail::Array<schar> castArray(const af_array &in);
 template detail::Array<uchar> castArray(const af_array &in);
 template detail::Array<char> castArray(const af_array &in);
 template detail::Array<intl> castArray(const af_array &in);
diff --git a/src/backend/common/cast.hpp b/src/backend/common/cast.hpp
index 4186a03914..c60614a8a9 100644
--- a/src/backend/common/cast.hpp
+++ b/src/backend/common/cast.hpp
@@ -31,20 +31,21 @@ namespace common {
 ///                        outer -> inner -> outer
 ///
 ///                                inner cast
-///           f32  f64  c32  c64  s32  u32   u8   b8  s64  u64  s16  u16  f16
-///     f32    x    x    x    x                                            x
-///     f64    x    x    x    x                                            x
-///  o  c32    x    x    x    x                                            x
-///  u  c64    x    x    x    x                                            x
-///  t  s32    x    x    x    x    x    x              x    x              x
-///  e  u32    x    x    x    x    x    x              x    x              x
-///  r   u8    x    x    x    x    x    x    x    x    x    x    x    x    x
-///      b8    x    x    x    x    x    x    x    x    x    x    x    x    x
-///  c  s64    x    x    x    x                        x    x              x
-///  a  u64    x    x    x    x                        x    x              x
-///  s  s16    x    x    x    x    x    x              x    x    x    x    x
-///  t  u16    x    x    x    x    x    x              x    x    x    x    x
-///     f16    x    x    x    x                                            x
+///           f32  f64  c32  c64  s32  u32   s8   u8   b8  s64  u64  s16  u16  f16
+///     f32    x    x    x    x                                                 x
+///     f64    x    x    x    x                                                 x
+///  o  c32    x    x    x    x                                                 x
+///  u  c64    x    x    x    x                                                 x
+///  t  s32    x    x    x    x    x    x                   x    x              x
+///  e  u32    x    x    x    x    x    x                   x    x              x
+///  r   s8    x    x    x    x    x    x    x    x    x    x    x    x    x    x
+///      u8    x    x    x    x    x    x    x    x    x    x    x    x    x    x
+///  c   b8    x    x    x    x    x    x    x    x    x    x    x    x    x    x
+///  a  s64    x    x    x    x                             x    x              x
+///  s  u64    x    x    x    x                             x    x              x
+///  t  s16    x    x    x    x    x    x                   x    x    x    x    x
+///     u16    x    x    x    x    x    x                   x    x    x    x    x
+///     f16    x    x    x    x                                                 x
 ///
 /// \param[in] outer The type of the second cast and the child of the
 ///            previous cast
diff --git a/src/backend/common/graphics_common.cpp b/src/backend/common/graphics_common.cpp
index 217722eb36..01f94078d4 100644
--- a/src/backend/common/graphics_common.cpp
+++ b/src/backend/common/graphics_common.cpp
@@ -139,6 +139,7 @@ INSTANTIATE_GET_FG_TYPE(float, FG_FLOAT32);
 INSTANTIATE_GET_FG_TYPE(int, FG_INT32);
 INSTANTIATE_GET_FG_TYPE(unsigned, FG_UINT32);
 INSTANTIATE_GET_FG_TYPE(char, FG_INT8);
+INSTANTIATE_GET_FG_TYPE(signed char, FG_INT8);
 INSTANTIATE_GET_FG_TYPE(unsigned char, FG_UINT8);
 INSTANTIATE_GET_FG_TYPE(unsigned short, FG_UINT16);
 INSTANTIATE_GET_FG_TYPE(short, FG_INT16);
diff --git a/src/backend/common/half.hpp b/src/backend/common/half.hpp
index 3f966c6f81..42d18be47b 100644
--- a/src/backend/common/half.hpp
+++ b/src/backend/common/half.hpp
@@ -164,6 +164,10 @@ AF_CONSTEXPR __DH__ native_half_t int2half_impl(char value) noexcept {
     return __ull2half_rn(value);
 }
 template<>
+AF_CONSTEXPR __DH__ native_half_t int2half_impl(signed char value) noexcept {
+    return __ull2half_rn(value);
+}
+template<>
 AF_CONSTEXPR __DH__ native_half_t int2half_impl(unsigned char value) noexcept {
     return __ull2half_rn(value);
 }
@@ -861,6 +865,7 @@ AF_CONSTEXPR T half2int(native_half_t value) {
 #ifdef __CUDA_ARCH__
     AF_IF_CONSTEXPR(std::is_same<T, short>::value ||
                     std::is_same<T, char>::value ||
+                    std::is_same<T, signed char>::value ||
                     std::is_same<T, unsigned char>::value) {
         return __half2short_rn(value);
     }
@@ -1044,6 +1049,10 @@ class alignas(2) half {
         return half2int<std::round_indeterminate, true, char>(data_);
     }
 
+    AF_CONSTEXPR __DH__ explicit operator signed char() const noexcept {
+        return half2int<std::round_indeterminate, true, signed char>(data_);
+    }
+
     AF_CONSTEXPR __DH__ explicit operator unsigned char() const noexcept {
         return half2int<std::round_indeterminate, true, unsigned char>(data_);
     }
diff --git a/src/backend/common/jit/BinaryNode.cpp b/src/backend/common/jit/BinaryNode.cpp
index 84c5597e31..b017394876 100644
--- a/src/backend/common/jit/BinaryNode.cpp
+++ b/src/backend/common/jit/BinaryNode.cpp
@@ -69,6 +69,7 @@ INSTANTIATE(cdouble, double, af_cplx2_t);
     INSTANTIATE(unsigned short, unsigned short, op);         \
     INSTANTIATE(unsigned long long, unsigned long long, op); \
     INSTANTIATE(long long, long long, op);                   \
+    INSTANTIATE(signed char, signed char, op);               \
     INSTANTIATE(unsigned char, unsigned char, op);           \
     INSTANTIATE(char, char, op);                             \
     INSTANTIATE(common::half, common::half, op);             \
@@ -91,6 +92,7 @@ INSTANTIATE_ARITH(af_max_t);
     INSTANTIATE(unsigned short, unsigned short, op);         \
     INSTANTIATE(unsigned long long, unsigned long long, op); \
     INSTANTIATE(long long, long long, op);                   \
+    INSTANTIATE(signed char, signed char, op);               \
     INSTANTIATE(unsigned char, unsigned char, op);           \
     INSTANTIATE(char, char, op);                             \
     INSTANTIATE(common::half, common::half, op);             \
@@ -114,6 +116,7 @@ INSTANTIATE_FLOATOPS(af_atan2_t);
     INSTANTIATE(unsigned short, unsigned short, op);         \
     INSTANTIATE(unsigned long long, unsigned long long, op); \
     INSTANTIATE(long long, long long, op);                   \
+    INSTANTIATE(signed char, signed char, op);               \
     INSTANTIATE(unsigned char, unsigned char, op);           \
     INSTANTIATE(char, char, op);                             \
     INSTANTIATE(int, int, op)
@@ -136,6 +139,7 @@ INSTANTIATE_BITOP(af_bitxor_t);
     INSTANTIATE(char, unsigned short, op);     \
     INSTANTIATE(char, unsigned long long, op); \
     INSTANTIATE(char, long long, op);          \
+    INSTANTIATE(char, signed char, op);        \
     INSTANTIATE(char, unsigned char, op);      \
     INSTANTIATE(char, char, op);               \
     INSTANTIATE(char, int, op)
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 2cc3164fb5..4641ff182c 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -98,6 +98,7 @@ static const char *getFullName(af::dtype type) {
         case u16: return detail::getFullName<unsigned short>();
         case s16: return detail::getFullName<short>();
         case b8: return detail::getFullName<char>();
+        case s8: return detail::getFullName<signed char>();
         case u8: return detail::getFullName<unsigned char>();
         case f16: return "half";
     }
@@ -117,6 +118,7 @@ static const char *getShortName(af::dtype type) {
         case u16: return detail::shortname<unsigned short>();
         case s16: return detail::shortname<short>();
         case b8: return detail::shortname<char>();
+        case s8: return detail::shortname<signed char>();
         case u8: return detail::shortname<unsigned char>();
         case f16: return "h";
     }
diff --git a/src/backend/common/moddims.cpp b/src/backend/common/moddims.cpp
index 6fbd99650e..cf9d8d6bb9 100644
--- a/src/backend/common/moddims.cpp
+++ b/src/backend/common/moddims.cpp
@@ -94,6 +94,7 @@ INSTANTIATE(double);
 INSTANTIATE(detail::cfloat);
 INSTANTIATE(detail::cdouble);
 INSTANTIATE(arrayfire::common::half);
+INSTANTIATE(signed char);
 INSTANTIATE(unsigned char);
 INSTANTIATE(char);
 INSTANTIATE(unsigned short);
diff --git a/src/backend/common/traits.hpp b/src/backend/common/traits.hpp
index 3036d91dd0..51a4b53899 100644
--- a/src/backend/common/traits.hpp
+++ b/src/backend/common/traits.hpp
@@ -24,6 +24,7 @@ namespace {
 
 inline size_t dtypeSize(af::dtype type) {
     switch (type) {
+        case s8:
         case u8:
         case b8: return 1;
         case s16:
@@ -59,7 +60,7 @@ constexpr bool isRealFloating(af::dtype type) {
 
 constexpr bool isInteger(af::dtype type) {
     return (type == s32 || type == u32 || type == s64 || type == u64 ||
-            type == s16 || type == u16 || type == u8);
+            type == s16 || type == u16 || type == s8 || type == u8);
 }
 
 constexpr bool isBool(af::dtype type) { return (type == b8); }
diff --git a/src/backend/common/util.cpp b/src/backend/common/util.cpp
index f0b24bba65..87be74fa83 100644
--- a/src/backend/common/util.cpp
+++ b/src/backend/common/util.cpp
@@ -103,6 +103,7 @@ const char* getName(af_dtype type) {
         case u64: return "unsigned long long";
         case s64: return "long long";
         case u8: return "unsigned char";
+        case s8: return "signed char";
         case b8: return "bool";
         default: return "unknown type";
     }
@@ -275,6 +276,7 @@ template string toString<int>(int);
 template string toString<unsigned short>(unsigned short);
 template string toString<short>(short);
 template string toString<unsigned char>(unsigned char);
+template string toString<signed char>(signed char);
 template string toString<char>(char);
 template string toString<long>(long);
 template string toString<long long>(long long);
diff --git a/src/backend/cpu/Array.cpp b/src/backend/cpu/Array.cpp
index dc0b5d5dad..276ea952b4 100644
--- a/src/backend/cpu/Array.cpp
+++ b/src/backend/cpu/Array.cpp
@@ -367,6 +367,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
diff --git a/src/backend/cpu/assign.cpp b/src/backend/cpu/assign.cpp
index cfeb5e168e..32af00e487 100644
--- a/src/backend/cpu/assign.cpp
+++ b/src/backend/cpu/assign.cpp
@@ -66,6 +66,7 @@ INSTANTIATE(uintl)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(int)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/bilateral.cpp b/src/backend/cpu/bilateral.cpp
index 027afb2c3b..19af80f3cb 100644
--- a/src/backend/cpu/bilateral.cpp
+++ b/src/backend/cpu/bilateral.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(char, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uint, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/cpu/cast.hpp b/src/backend/cpu/cast.hpp
index dd756eb2b3..d51b7838b8 100644
--- a/src/backend/cpu/cast.hpp
+++ b/src/backend/cpu/cast.hpp
@@ -150,6 +150,7 @@ struct UnOp<std::complex<double>, std::complex<float>, af_cast_t> {
 CAST_B8(float)
 CAST_B8(double)
 CAST_B8(int)
+CAST_B8(schar)
 CAST_B8(uchar)
 CAST_B8(char)
 
diff --git a/src/backend/cpu/convolve.cpp b/src/backend/cpu/convolve.cpp
index 20138fd9e5..2fd0e3bce3 100644
--- a/src/backend/cpu/convolve.cpp
+++ b/src/backend/cpu/convolve.cpp
@@ -111,6 +111,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/cpu/copy.cpp b/src/backend/cpu/copy.cpp
index b1d0985680..ea98c0f613 100644
--- a/src/backend/cpu/copy.cpp
+++ b/src/backend/cpu/copy.cpp
@@ -72,6 +72,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
@@ -101,6 +102,8 @@ INSTANTIATE(half)
                                           Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, ushort>(Array<ushort> & dst,       \
                                            Array<SRC_T> const &src);  \
+    template void copyArray<SRC_T, schar>(Array<schar> & dst,         \
+                                          Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, uchar>(Array<uchar> & dst,         \
                                           Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, char>(Array<char> & dst,           \
@@ -114,6 +117,7 @@ INSTANTIATE_COPY_ARRAY(int)
 INSTANTIATE_COPY_ARRAY(uint)
 INSTANTIATE_COPY_ARRAY(intl)
 INSTANTIATE_COPY_ARRAY(uintl)
+INSTANTIATE_COPY_ARRAY(schar)
 INSTANTIATE_COPY_ARRAY(uchar)
 INSTANTIATE_COPY_ARRAY(char)
 INSTANTIATE_COPY_ARRAY(ushort)
@@ -144,6 +148,7 @@ INSTANTIATE_GETSCALAR(cfloat)
 INSTANTIATE_GETSCALAR(cdouble)
 INSTANTIATE_GETSCALAR(int)
 INSTANTIATE_GETSCALAR(uint)
+INSTANTIATE_GETSCALAR(schar)
 INSTANTIATE_GETSCALAR(uchar)
 INSTANTIATE_GETSCALAR(char)
 INSTANTIATE_GETSCALAR(intl)
diff --git a/src/backend/cpu/diagonal.cpp b/src/backend/cpu/diagonal.cpp
index eddd8c0a49..1767096ed0 100644
--- a/src/backend/cpu/diagonal.cpp
+++ b/src/backend/cpu/diagonal.cpp
@@ -62,6 +62,7 @@ INSTANTIATE_DIAGONAL(uint)
 INSTANTIATE_DIAGONAL(intl)
 INSTANTIATE_DIAGONAL(uintl)
 INSTANTIATE_DIAGONAL(char)
+INSTANTIATE_DIAGONAL(schar)
 INSTANTIATE_DIAGONAL(uchar)
 INSTANTIATE_DIAGONAL(short)
 INSTANTIATE_DIAGONAL(ushort)
diff --git a/src/backend/cpu/diff.cpp b/src/backend/cpu/diff.cpp
index 8e9c67cae1..f9ced50f52 100644
--- a/src/backend/cpu/diff.cpp
+++ b/src/backend/cpu/diff.cpp
@@ -56,6 +56,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/exampleFunction.cpp b/src/backend/cpu/exampleFunction.cpp
index ee7b847524..3f677bc24b 100644
--- a/src/backend/cpu/exampleFunction.cpp
+++ b/src/backend/cpu/exampleFunction.cpp
@@ -56,6 +56,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(cfloat)
diff --git a/src/backend/cpu/fast.cpp b/src/backend/cpu/fast.cpp
index b8ac38eeaf..ac93345797 100644
--- a/src/backend/cpu/fast.cpp
+++ b/src/backend/cpu/fast.cpp
@@ -120,6 +120,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/fftconvolve.cpp b/src/backend/cpu/fftconvolve.cpp
index 728238c1ef..ff2e5b68c4 100644
--- a/src/backend/cpu/fftconvolve.cpp
+++ b/src/backend/cpu/fftconvolve.cpp
@@ -207,6 +207,7 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 INSTANTIATE(uint)
 INSTANTIATE(int)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(uintl)
diff --git a/src/backend/cpu/hist_graphics.cpp b/src/backend/cpu/hist_graphics.cpp
index 7635004c91..a77e9fe77e 100644
--- a/src/backend/cpu/hist_graphics.cpp
+++ b/src/backend/cpu/hist_graphics.cpp
@@ -43,6 +43,7 @@ void copy_histogram(const Array<T> &data, fg_histogram hist) {
 INSTANTIATE(float)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/histogram.cpp b/src/backend/cpu/histogram.cpp
index e2f8e15433..9d9c6ba8fa 100644
--- a/src/backend/cpu/histogram.cpp
+++ b/src/backend/cpu/histogram.cpp
@@ -48,6 +48,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/identity.cpp b/src/backend/cpu/identity.cpp
index 05695d7629..ce7f35bdb0 100644
--- a/src/backend/cpu/identity.cpp
+++ b/src/backend/cpu/identity.cpp
@@ -42,6 +42,7 @@ INSTANTIATE_IDENTITY(uint)
 INSTANTIATE_IDENTITY(intl)
 INSTANTIATE_IDENTITY(uintl)
 INSTANTIATE_IDENTITY(char)
+INSTANTIATE_IDENTITY(schar)
 INSTANTIATE_IDENTITY(uchar)
 INSTANTIATE_IDENTITY(short)
 INSTANTIATE_IDENTITY(ushort)
diff --git a/src/backend/cpu/image.cpp b/src/backend/cpu/image.cpp
index f11a2db4ca..2e24dec9be 100644
--- a/src/backend/cpu/image.cpp
+++ b/src/backend/cpu/image.cpp
@@ -49,6 +49,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/index.cpp b/src/backend/cpu/index.cpp
index 850239acfe..84cff747bd 100644
--- a/src/backend/cpu/index.cpp
+++ b/src/backend/cpu/index.cpp
@@ -81,6 +81,7 @@ INSTANTIATE(uintl)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(int)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/iota.cpp b/src/backend/cpu/iota.cpp
index 1e7155bcd9..fe50919783 100644
--- a/src/backend/cpu/iota.cpp
+++ b/src/backend/cpu/iota.cpp
@@ -41,6 +41,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/ireduce.cpp b/src/backend/cpu/ireduce.cpp
index 435d6ea44d..a20df27c1a 100644
--- a/src/backend/cpu/ireduce.cpp
+++ b/src/backend/cpu/ireduce.cpp
@@ -105,6 +105,7 @@ INSTANTIATE(af_min_t, uint)
 INSTANTIATE(af_min_t, intl)
 INSTANTIATE(af_min_t, uintl)
 INSTANTIATE(af_min_t, char)
+INSTANTIATE(af_min_t, schar)
 INSTANTIATE(af_min_t, uchar)
 INSTANTIATE(af_min_t, short)
 INSTANTIATE(af_min_t, ushort)
@@ -120,6 +121,7 @@ INSTANTIATE(af_max_t, uint)
 INSTANTIATE(af_max_t, intl)
 INSTANTIATE(af_max_t, uintl)
 INSTANTIATE(af_max_t, char)
+INSTANTIATE(af_max_t, schar)
 INSTANTIATE(af_max_t, uchar)
 INSTANTIATE(af_max_t, short)
 INSTANTIATE(af_max_t, ushort)
diff --git a/src/backend/cpu/join.cpp b/src/backend/cpu/join.cpp
index e9fed65df1..602f2db7f9 100644
--- a/src/backend/cpu/join.cpp
+++ b/src/backend/cpu/join.cpp
@@ -70,6 +70,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
@@ -90,6 +91,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/kernel/random_engine.hpp b/src/backend/cpu/kernel/random_engine.hpp
index 09c2bff20c..0ab49f8a80 100644
--- a/src/backend/cpu/kernel/random_engine.hpp
+++ b/src/backend/cpu/kernel/random_engine.hpp
@@ -115,6 +115,11 @@ uchar transform<uchar>(uint *val, uint index) {
     return v;
 }
 
+template<>
+schar transform<schar>(uint *val, uint index) {
+    return transform<uchar>(val, index);
+}
+
 template<>
 ushort transform<ushort>(uint *val, uint index) {
     ushort v = val[index >> 1U] >> (16U * (index & 1U)) & 0x0000ffff;
diff --git a/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp
index 6ac6875f3e..5873e93117 100644
--- a/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp
+++ b/src/backend/cpu/kernel/sort_by_key/sort_by_key_impl.cpp
@@ -9,7 +9,7 @@
 
 #include <kernel/sort_by_key_impl.hpp>
 
-// SBK_TYPES:float double int uint intl uintl short ushort char uchar
+// SBK_TYPES:float double int uint intl uintl short ushort char schar uchar
 
 namespace arrayfire {
 namespace cpu {
diff --git a/src/backend/cpu/kernel/sort_by_key_impl.hpp b/src/backend/cpu/kernel/sort_by_key_impl.hpp
index acd7524a9b..e77e868d78 100644
--- a/src/backend/cpu/kernel/sort_by_key_impl.hpp
+++ b/src/backend/cpu/kernel/sort_by_key_impl.hpp
@@ -169,6 +169,7 @@ void sort0ByKey(Param<Tk> okey, Param<Tv> oval, bool isAscending) {
     INSTANTIATE(Tk, short)   \
     INSTANTIATE(Tk, ushort)  \
     INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, schar)   \
     INSTANTIATE(Tk, uchar)   \
     INSTANTIATE(Tk, intl)    \
     INSTANTIATE(Tk, uintl)
diff --git a/src/backend/cpu/lookup.cpp b/src/backend/cpu/lookup.cpp
index 8a5c40d55c..b8c56e297c 100644
--- a/src/backend/cpu/lookup.cpp
+++ b/src/backend/cpu/lookup.cpp
@@ -51,6 +51,8 @@ Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                                       const unsigned);                         \
     template Array<T> lookup<T, uintl>(const Array<T> &, const Array<uintl> &, \
                                        const unsigned);                        \
+    template Array<T> lookup<T, schar>(const Array<T> &, const Array<schar> &, \
+                                       const unsigned);                        \
     template Array<T> lookup<T, uchar>(const Array<T> &, const Array<uchar> &, \
                                        const unsigned);                        \
     template Array<T> lookup<T, half>(const Array<T> &, const Array<half> &,   \
@@ -64,6 +66,7 @@ INSTANTIATE(int);
 INSTANTIATE(unsigned);
 INSTANTIATE(intl);
 INSTANTIATE(uintl);
+INSTANTIATE(schar);
 INSTANTIATE(uchar);
 INSTANTIATE(char);
 INSTANTIATE(ushort);
diff --git a/src/backend/cpu/match_template.cpp b/src/backend/cpu/match_template.cpp
index d3cfb26b4a..6b4d0f1b91 100644
--- a/src/backend/cpu/match_template.cpp
+++ b/src/backend/cpu/match_template.cpp
@@ -51,6 +51,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(char, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uint, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/cpu/mean.cpp b/src/backend/cpu/mean.cpp
index 6a256113f7..2323442110 100644
--- a/src/backend/cpu/mean.cpp
+++ b/src/backend/cpu/mean.cpp
@@ -141,6 +141,7 @@ INSTANTIATE(intl, double, double);
 INSTANTIATE(uintl, double, double);
 INSTANTIATE(short, float, float);
 INSTANTIATE(ushort, float, float);
+INSTANTIATE(schar, float, float);
 INSTANTIATE(uchar, float, float);
 INSTANTIATE(char, float, float);
 INSTANTIATE(cfloat, float, cfloat);
diff --git a/src/backend/cpu/meanshift.cpp b/src/backend/cpu/meanshift.cpp
index d52b56a99e..878aa4cacb 100644
--- a/src/backend/cpu/meanshift.cpp
+++ b/src/backend/cpu/meanshift.cpp
@@ -50,6 +50,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/medfilt.cpp b/src/backend/cpu/medfilt.cpp
index 53497be8c9..4c952fc762 100644
--- a/src/backend/cpu/medfilt.cpp
+++ b/src/backend/cpu/medfilt.cpp
@@ -63,6 +63,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(ushort)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/memory.cpp b/src/backend/cpu/memory.cpp
index 9bbb41d458..0a32186f2e 100644
--- a/src/backend/cpu/memory.cpp
+++ b/src/backend/cpu/memory.cpp
@@ -106,6 +106,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/cpu/moments.cpp b/src/backend/cpu/moments.cpp
index bd5c520eac..09db606bd4 100644
--- a/src/backend/cpu/moments.cpp
+++ b/src/backend/cpu/moments.cpp
@@ -49,6 +49,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/morph.cpp b/src/backend/cpu/morph.cpp
index add13de416..e526e7c066 100644
--- a/src/backend/cpu/morph.cpp
+++ b/src/backend/cpu/morph.cpp
@@ -67,6 +67,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(ushort)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/nearest_neighbour.cpp b/src/backend/cpu/nearest_neighbour.cpp
index 2979090dd9..0581e97ab6 100644
--- a/src/backend/cpu/nearest_neighbour.cpp
+++ b/src/backend/cpu/nearest_neighbour.cpp
@@ -67,6 +67,7 @@ INSTANTIATE(int, int)
 INSTANTIATE(uint, uint)
 INSTANTIATE(intl, intl)
 INSTANTIATE(uintl, uintl)
+INSTANTIATE(schar, int)
 INSTANTIATE(uchar, uint)
 INSTANTIATE(ushort, uint)
 INSTANTIATE(short, int)
diff --git a/src/backend/cpu/plot.cpp b/src/backend/cpu/plot.cpp
index abf1a7b397..1ca6ae7882 100644
--- a/src/backend/cpu/plot.cpp
+++ b/src/backend/cpu/plot.cpp
@@ -46,6 +46,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/random_engine.cpp b/src/backend/cpu/random_engine.cpp
index 3e1c8745c8..d42a7bdae1 100644
--- a/src/backend/cpu/random_engine.cpp
+++ b/src/backend/cpu/random_engine.cpp
@@ -149,6 +149,7 @@ INSTANTIATE_UNIFORM(uint)
 INSTANTIATE_UNIFORM(intl)
 INSTANTIATE_UNIFORM(uintl)
 INSTANTIATE_UNIFORM(char)
+INSTANTIATE_UNIFORM(schar)
 INSTANTIATE_UNIFORM(uchar)
 INSTANTIATE_UNIFORM(short)
 INSTANTIATE_UNIFORM(ushort)
diff --git a/src/backend/cpu/range.cpp b/src/backend/cpu/range.cpp
index 3b782837e0..ad100da4d4 100644
--- a/src/backend/cpu/range.cpp
+++ b/src/backend/cpu/range.cpp
@@ -54,6 +54,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(ushort)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/reduce.cpp b/src/backend/cpu/reduce.cpp
index 6ce141b316..5b13d6f96f 100644
--- a/src/backend/cpu/reduce.cpp
+++ b/src/backend/cpu/reduce.cpp
@@ -145,6 +145,7 @@ INSTANTIATE(af_min_t, uint, uint)
 INSTANTIATE(af_min_t, intl, intl)
 INSTANTIATE(af_min_t, uintl, uintl)
 INSTANTIATE(af_min_t, char, char)
+INSTANTIATE(af_min_t, schar, schar)
 INSTANTIATE(af_min_t, uchar, uchar)
 INSTANTIATE(af_min_t, short, short)
 INSTANTIATE(af_min_t, ushort, ushort)
@@ -160,6 +161,7 @@ INSTANTIATE(af_max_t, uint, uint)
 INSTANTIATE(af_max_t, intl, intl)
 INSTANTIATE(af_max_t, uintl, uintl)
 INSTANTIATE(af_max_t, char, char)
+INSTANTIATE(af_max_t, schar, schar)
 INSTANTIATE(af_max_t, uchar, uchar)
 INSTANTIATE(af_max_t, short, short)
 INSTANTIATE(af_max_t, ushort, ushort)
@@ -180,6 +182,8 @@ INSTANTIATE(af_add_t, uintl, uintl)
 INSTANTIATE(af_add_t, uintl, double)
 INSTANTIATE(af_add_t, char, int)
 INSTANTIATE(af_add_t, char, float)
+INSTANTIATE(af_add_t, schar, int)
+INSTANTIATE(af_add_t, schar, float)
 INSTANTIATE(af_add_t, uchar, uint)
 INSTANTIATE(af_add_t, uchar, float)
 INSTANTIATE(af_add_t, short, int)
@@ -199,6 +203,7 @@ INSTANTIATE(af_mul_t, uint, uint)
 INSTANTIATE(af_mul_t, intl, intl)
 INSTANTIATE(af_mul_t, uintl, uintl)
 INSTANTIATE(af_mul_t, char, int)
+INSTANTIATE(af_mul_t, schar, int)
 INSTANTIATE(af_mul_t, uchar, uint)
 INSTANTIATE(af_mul_t, short, int)
 INSTANTIATE(af_mul_t, ushort, uint)
@@ -214,6 +219,7 @@ INSTANTIATE(af_notzero_t, uint, uint)
 INSTANTIATE(af_notzero_t, intl, uint)
 INSTANTIATE(af_notzero_t, uintl, uint)
 INSTANTIATE(af_notzero_t, char, uint)
+INSTANTIATE(af_notzero_t, schar, uint)
 INSTANTIATE(af_notzero_t, uchar, uint)
 INSTANTIATE(af_notzero_t, short, uint)
 INSTANTIATE(af_notzero_t, ushort, uint)
@@ -229,6 +235,7 @@ INSTANTIATE(af_or_t, uint, char)
 INSTANTIATE(af_or_t, intl, char)
 INSTANTIATE(af_or_t, uintl, char)
 INSTANTIATE(af_or_t, char, char)
+INSTANTIATE(af_or_t, schar, char)
 INSTANTIATE(af_or_t, uchar, char)
 INSTANTIATE(af_or_t, short, char)
 INSTANTIATE(af_or_t, ushort, char)
@@ -244,6 +251,7 @@ INSTANTIATE(af_and_t, uint, char)
 INSTANTIATE(af_and_t, intl, char)
 INSTANTIATE(af_and_t, uintl, char)
 INSTANTIATE(af_and_t, char, char)
+INSTANTIATE(af_and_t, schar, char)
 INSTANTIATE(af_and_t, uchar, char)
 INSTANTIATE(af_and_t, short, char)
 INSTANTIATE(af_and_t, ushort, char)
diff --git a/src/backend/cpu/reorder.cpp b/src/backend/cpu/reorder.cpp
index 67233542bd..dd0a43ccac 100644
--- a/src/backend/cpu/reorder.cpp
+++ b/src/backend/cpu/reorder.cpp
@@ -39,6 +39,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
diff --git a/src/backend/cpu/reshape.cpp b/src/backend/cpu/reshape.cpp
index b2d46eb066..31a0053684 100644
--- a/src/backend/cpu/reshape.cpp
+++ b/src/backend/cpu/reshape.cpp
@@ -40,6 +40,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
@@ -68,6 +69,8 @@ INSTANTIATE(ushort)
                                                 const dim4 &, short, double); \
     template Array<ushort> reshape<SRC_T, ushort>(                            \
         const Array<SRC_T> &, const dim4 &, ushort, double);                  \
+    template Array<schar> reshape<SRC_T, schar>(const Array<SRC_T> &,         \
+                                                const dim4 &, schar, double); \
     template Array<uchar> reshape<SRC_T, uchar>(const Array<SRC_T> &,         \
                                                 const dim4 &, uchar, double); \
     template Array<char> reshape<SRC_T, char>(const Array<SRC_T> &,           \
@@ -79,6 +82,7 @@ INSTANTIATE_PAD_ARRAY(int)
 INSTANTIATE_PAD_ARRAY(uint)
 INSTANTIATE_PAD_ARRAY(intl)
 INSTANTIATE_PAD_ARRAY(uintl)
+INSTANTIATE_PAD_ARRAY(schar)
 INSTANTIATE_PAD_ARRAY(uchar)
 INSTANTIATE_PAD_ARRAY(char)
 INSTANTIATE_PAD_ARRAY(ushort)
diff --git a/src/backend/cpu/resize.cpp b/src/backend/cpu/resize.cpp
index 4f899d89d8..ffc473fd4e 100644
--- a/src/backend/cpu/resize.cpp
+++ b/src/backend/cpu/resize.cpp
@@ -53,6 +53,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/rotate.cpp b/src/backend/cpu/rotate.cpp
index 0e9806a2af..bed34b7bf3 100644
--- a/src/backend/cpu/rotate.cpp
+++ b/src/backend/cpu/rotate.cpp
@@ -53,6 +53,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/scan.cpp b/src/backend/cpu/scan.cpp
index af5c4d9efe..7f6843f99a 100644
--- a/src/backend/cpu/scan.cpp
+++ b/src/backend/cpu/scan.cpp
@@ -84,6 +84,7 @@ Array<To> scan(const Array<Ti>& in, const int dim, bool inclusive_scan) {
     INSTANTIATE_SCAN(ROp, uintl, uintl)     \
     INSTANTIATE_SCAN(ROp, char, int)        \
     INSTANTIATE_SCAN(ROp, char, uint)       \
+    INSTANTIATE_SCAN(ROp, schar, int)       \
     INSTANTIATE_SCAN(ROp, uchar, uint)      \
     INSTANTIATE_SCAN(ROp, short, int)       \
     INSTANTIATE_SCAN(ROp, ushort, uint)
diff --git a/src/backend/cpu/select.cpp b/src/backend/cpu/select.cpp
index 96849cecd1..8258cae47a 100644
--- a/src/backend/cpu/select.cpp
+++ b/src/backend/cpu/select.cpp
@@ -51,6 +51,7 @@ INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/set.cpp b/src/backend/cpu/set.cpp
index 838ad7675e..6db13c8760 100644
--- a/src/backend/cpu/set.cpp
+++ b/src/backend/cpu/set.cpp
@@ -120,6 +120,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/shift.cpp b/src/backend/cpu/shift.cpp
index f8942f641f..d812cbde89 100644
--- a/src/backend/cpu/shift.cpp
+++ b/src/backend/cpu/shift.cpp
@@ -37,6 +37,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/sobel.cpp b/src/backend/cpu/sobel.cpp
index 68bddee784..5708348295 100644
--- a/src/backend/cpu/sobel.cpp
+++ b/src/backend/cpu/sobel.cpp
@@ -44,6 +44,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(int, int)
 INSTANTIATE(uint, int)
 INSTANTIATE(char, int)
+INSTANTIATE(schar, int)
 INSTANTIATE(uchar, int)
 INSTANTIATE(short, int)
 INSTANTIATE(ushort, int)
diff --git a/src/backend/cpu/sort.cpp b/src/backend/cpu/sort.cpp
index e5067a8dba..41c6b75147 100644
--- a/src/backend/cpu/sort.cpp
+++ b/src/backend/cpu/sort.cpp
@@ -98,6 +98,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/sort_by_key.cpp b/src/backend/cpu/sort_by_key.cpp
index 169b598558..efe8eba2f1 100644
--- a/src/backend/cpu/sort_by_key.cpp
+++ b/src/backend/cpu/sort_by_key.cpp
@@ -71,6 +71,7 @@ void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
     INSTANTIATE(Tk, int)     \
     INSTANTIATE(Tk, uint)    \
     INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, schar)   \
     INSTANTIATE(Tk, uchar)   \
     INSTANTIATE(Tk, short)   \
     INSTANTIATE(Tk, ushort)  \
@@ -82,6 +83,7 @@ INSTANTIATE1(double)
 INSTANTIATE1(int)
 INSTANTIATE1(uint)
 INSTANTIATE1(char)
+INSTANTIATE1(schar)
 INSTANTIATE1(uchar)
 INSTANTIATE1(short)
 INSTANTIATE1(ushort)
diff --git a/src/backend/cpu/sort_index.cpp b/src/backend/cpu/sort_index.cpp
index cec724c85d..8b1f4a1319 100644
--- a/src/backend/cpu/sort_index.cpp
+++ b/src/backend/cpu/sort_index.cpp
@@ -75,6 +75,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/surface.cpp b/src/backend/cpu/surface.cpp
index e861dbeac7..d86bd6f469 100644
--- a/src/backend/cpu/surface.cpp
+++ b/src/backend/cpu/surface.cpp
@@ -47,6 +47,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/susan.cpp b/src/backend/cpu/susan.cpp
index 6ab2bfba78..c5321deb16 100644
--- a/src/backend/cpu/susan.cpp
+++ b/src/backend/cpu/susan.cpp
@@ -73,6 +73,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/tile.cpp b/src/backend/cpu/tile.cpp
index d2a8d3ab7c..884bfed40d 100644
--- a/src/backend/cpu/tile.cpp
+++ b/src/backend/cpu/tile.cpp
@@ -47,6 +47,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/transform.cpp b/src/backend/cpu/transform.cpp
index 9a57424250..bbcf689f25 100644
--- a/src/backend/cpu/transform.cpp
+++ b/src/backend/cpu/transform.cpp
@@ -58,6 +58,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/transpose.cpp b/src/backend/cpu/transpose.cpp
index 7cd713afd6..a9f6f9d3d5 100644
--- a/src/backend/cpu/transpose.cpp
+++ b/src/backend/cpu/transpose.cpp
@@ -51,6 +51,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/cpu/triangle.cpp b/src/backend/cpu/triangle.cpp
index 8e3b0569b2..6c276ca4bd 100644
--- a/src/backend/cpu/triangle.cpp
+++ b/src/backend/cpu/triangle.cpp
@@ -58,6 +58,7 @@ INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/types.hpp b/src/backend/cpu/types.hpp
index 27a678af82..f1f58e7006 100644
--- a/src/backend/cpu/types.hpp
+++ b/src/backend/cpu/types.hpp
@@ -31,6 +31,7 @@ using cdouble = std::complex<double>;
 using cfloat  = std::complex<float>;
 using intl    = long long;
 using uint    = unsigned int;
+using schar   = signed char;
 using uchar   = unsigned char;
 using uintl   = unsigned long long;
 using ushort  = unsigned short;
diff --git a/src/backend/cpu/unwrap.cpp b/src/backend/cpu/unwrap.cpp
index 49086fad49..dca2433ff8 100644
--- a/src/backend/cpu/unwrap.cpp
+++ b/src/backend/cpu/unwrap.cpp
@@ -55,6 +55,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cpu/vector_field.cpp b/src/backend/cpu/vector_field.cpp
index 2a7549de81..efe207be09 100644
--- a/src/backend/cpu/vector_field.cpp
+++ b/src/backend/cpu/vector_field.cpp
@@ -58,6 +58,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/where.cpp b/src/backend/cpu/where.cpp
index 3eb65015f0..30f70efcb0 100644
--- a/src/backend/cpu/where.cpp
+++ b/src/backend/cpu/where.cpp
@@ -73,6 +73,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cpu/wrap.cpp b/src/backend/cpu/wrap.cpp
index d502bc85ad..0c0d397e3f 100644
--- a/src/backend/cpu/wrap.cpp
+++ b/src/backend/cpu/wrap.cpp
@@ -49,6 +49,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/Array.cpp b/src/backend/cuda/Array.cpp
index 9193f329de..e0d5f73f5a 100644
--- a/src/backend/cuda/Array.cpp
+++ b/src/backend/cuda/Array.cpp
@@ -493,6 +493,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
diff --git a/src/backend/cuda/all.cu b/src/backend/cuda/all.cu
index 3ff42ad599..fa0681dbaf 100644
--- a/src/backend/cuda/all.cu
+++ b/src/backend/cuda/all.cu
@@ -24,6 +24,7 @@ INSTANTIATE(af_and_t, uint, char)
 INSTANTIATE(af_and_t, intl, char)
 INSTANTIATE(af_and_t, uintl, char)
 INSTANTIATE(af_and_t, char, char)
+INSTANTIATE(af_and_t, schar, char)
 INSTANTIATE(af_and_t, uchar, char)
 INSTANTIATE(af_and_t, short, char)
 INSTANTIATE(af_and_t, ushort, char)
diff --git a/src/backend/cuda/any.cu b/src/backend/cuda/any.cu
index 34092c94d3..801dcb6c10 100644
--- a/src/backend/cuda/any.cu
+++ b/src/backend/cuda/any.cu
@@ -24,6 +24,7 @@ INSTANTIATE(af_or_t, uint, char)
 INSTANTIATE(af_or_t, intl, char)
 INSTANTIATE(af_or_t, uintl, char)
 INSTANTIATE(af_or_t, char, char)
+INSTANTIATE(af_or_t, schar, char)
 INSTANTIATE(af_or_t, uchar, char)
 INSTANTIATE(af_or_t, short, char)
 INSTANTIATE(af_or_t, ushort, char)
diff --git a/src/backend/cuda/assign.cpp b/src/backend/cuda/assign.cpp
index 67bcbd1291..b65265dc8b 100644
--- a/src/backend/cuda/assign.cpp
+++ b/src/backend/cuda/assign.cpp
@@ -73,6 +73,7 @@ INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/bilateral.cpp b/src/backend/cuda/bilateral.cpp
index f9f828018d..6d56640fa8 100644
--- a/src/backend/cuda/bilateral.cpp
+++ b/src/backend/cuda/bilateral.cpp
@@ -34,6 +34,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(char, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uint, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/cuda/cast.hpp b/src/backend/cuda/cast.hpp
index 9328dd5052..214d24845a 100644
--- a/src/backend/cuda/cast.hpp
+++ b/src/backend/cuda/cast.hpp
@@ -34,6 +34,7 @@ struct CastOp {
 CAST_FN(int)
 CAST_FN(unsigned int)
 CAST_FN(unsigned char)
+CAST_FN(signed char)
 CAST_FN(unsigned short)
 CAST_FN(short)
 CAST_FN(float)
diff --git a/src/backend/cuda/convolve.cpp b/src/backend/cuda/convolve.cpp
index 3a33c6f64f..043bfdcc9e 100644
--- a/src/backend/cuda/convolve.cpp
+++ b/src/backend/cuda/convolve.cpp
@@ -95,6 +95,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/cuda/copy.cpp b/src/backend/cuda/copy.cpp
index f8472a7dfb..5d1701d965 100644
--- a/src/backend/cuda/copy.cpp
+++ b/src/backend/cuda/copy.cpp
@@ -113,6 +113,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
@@ -142,6 +143,8 @@ INSTANTIATE(half)
                                           Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, ushort>(Array<ushort> & dst,       \
                                            Array<SRC_T> const &src);  \
+    template void copyArray<SRC_T, schar>(Array<schar> & dst,         \
+                                          Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, uchar>(Array<uchar> & dst,         \
                                           Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, char>(Array<char> & dst,           \
@@ -157,6 +160,7 @@ INSTANTIATE_COPY_ARRAY(intl)
 INSTANTIATE_COPY_ARRAY(uintl)
 INSTANTIATE_COPY_ARRAY(short)
 INSTANTIATE_COPY_ARRAY(ushort)
+INSTANTIATE_COPY_ARRAY(schar)
 INSTANTIATE_COPY_ARRAY(uchar)
 INSTANTIATE_COPY_ARRAY(char)
 INSTANTIATE_COPY_ARRAY(half)
@@ -187,6 +191,7 @@ INSTANTIATE_GETSCALAR(cfloat)
 INSTANTIATE_GETSCALAR(cdouble)
 INSTANTIATE_GETSCALAR(int)
 INSTANTIATE_GETSCALAR(uint)
+INSTANTIATE_GETSCALAR(schar)
 INSTANTIATE_GETSCALAR(uchar)
 INSTANTIATE_GETSCALAR(char)
 INSTANTIATE_GETSCALAR(intl)
diff --git a/src/backend/cuda/count.cu b/src/backend/cuda/count.cu
index 373def999c..3cb5806a88 100644
--- a/src/backend/cuda/count.cu
+++ b/src/backend/cuda/count.cu
@@ -26,6 +26,7 @@ INSTANTIATE(af_notzero_t, uintl, uint)
 INSTANTIATE(af_notzero_t, short, uint)
 INSTANTIATE(af_notzero_t, ushort, uint)
 INSTANTIATE(af_notzero_t, char, uint)
+INSTANTIATE(af_notzero_t, schar, uint)
 INSTANTIATE(af_notzero_t, uchar, uint)
 INSTANTIATE(af_notzero_t, half, uint)
 }  // namespace cuda
diff --git a/src/backend/cuda/cudaDataType.hpp b/src/backend/cuda/cudaDataType.hpp
index 1da3429e60..3746d0b4b9 100644
--- a/src/backend/cuda/cudaDataType.hpp
+++ b/src/backend/cuda/cudaDataType.hpp
@@ -44,6 +44,22 @@ inline cudaDataType_t getType<common::half>() {
     return CUDA_R_16F;
 }
 
+template<>
+inline cudaDataType_t getType<uchar>() {
+    return CUDA_R_8I;
+}
+
+template<>
+inline cudaDataType_t getType<schar>() {
+    return CUDA_R_8I;
+}
+
+/* only supports LStride/RStride % 4 == 0 */
+template<>
+inline cudaDataType_t getType<int>() {
+    return CUDA_R_32I;
+}
+
 template<typename T>
 inline cudaDataType_t getComputeType() {
     return getType<T>();
diff --git a/src/backend/cuda/cudnn.cpp b/src/backend/cuda/cudnn.cpp
index 39ee3305e6..5b8a500d00 100644
--- a/src/backend/cuda/cudnn.cpp
+++ b/src/backend/cuda/cudnn.cpp
@@ -64,6 +64,12 @@ cudnnDataType_t getCudnnDataType<int>() {
 }
 
 #if CUDNN_VERSION >= 7100
+/// TODONT COMMIT
+template<>
+cudnnDataType_t getCudnnDataType<signed char>() {
+    return CUDNN_DATA_INT8;
+}
+
 template<>
 cudnnDataType_t getCudnnDataType<unsigned char>() {
     return CUDNN_DATA_UINT8;
diff --git a/src/backend/cuda/diagonal.cpp b/src/backend/cuda/diagonal.cpp
index cbf3180a70..b5dd2b5c0b 100644
--- a/src/backend/cuda/diagonal.cpp
+++ b/src/backend/cuda/diagonal.cpp
@@ -54,6 +54,7 @@ INSTANTIATE_DIAGONAL(uint)
 INSTANTIATE_DIAGONAL(intl)
 INSTANTIATE_DIAGONAL(uintl)
 INSTANTIATE_DIAGONAL(char)
+INSTANTIATE_DIAGONAL(schar)
 INSTANTIATE_DIAGONAL(uchar)
 INSTANTIATE_DIAGONAL(short)
 INSTANTIATE_DIAGONAL(ushort)
diff --git a/src/backend/cuda/diff.cpp b/src/backend/cuda/diff.cpp
index 55bb68ece0..b21ab36b72 100644
--- a/src/backend/cuda/diff.cpp
+++ b/src/backend/cuda/diff.cpp
@@ -55,6 +55,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/exampleFunction.cpp b/src/backend/cuda/exampleFunction.cpp
index b94f9f8e54..12bf635785 100644
--- a/src/backend/cuda/exampleFunction.cpp
+++ b/src/backend/cuda/exampleFunction.cpp
@@ -60,6 +60,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(cfloat)
diff --git a/src/backend/cuda/fast.cu b/src/backend/cuda/fast.cu
index 7744d4b6d6..63e9a57cb4 100644
--- a/src/backend/cuda/fast.cu
+++ b/src/backend/cuda/fast.cu
@@ -62,6 +62,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/fast_pyramid.cpp b/src/backend/cuda/fast_pyramid.cpp
index 97228af248..ba0b6dfbf4 100644
--- a/src/backend/cuda/fast_pyramid.cpp
+++ b/src/backend/cuda/fast_pyramid.cpp
@@ -120,6 +120,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/fftconvolve.cpp b/src/backend/cuda/fftconvolve.cpp
index ed22d0ea85..cb8359423e 100644
--- a/src/backend/cuda/fftconvolve.cpp
+++ b/src/backend/cuda/fftconvolve.cpp
@@ -112,6 +112,7 @@ INSTANTIATE(float)
 INSTANTIATE(uint)
 INSTANTIATE(int)
 INSTANTIATE(uchar)
+INSTANTIATE(schar)
 INSTANTIATE(char)
 INSTANTIATE(uintl)
 INSTANTIATE(intl)
diff --git a/src/backend/cuda/hist_graphics.cpp b/src/backend/cuda/hist_graphics.cpp
index 6678281db6..cabadeb1ad 100644
--- a/src/backend/cuda/hist_graphics.cpp
+++ b/src/backend/cuda/hist_graphics.cpp
@@ -69,6 +69,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
diff --git a/src/backend/cuda/histogram.cpp b/src/backend/cuda/histogram.cpp
index ca7e6ced86..f012d6e64b 100644
--- a/src/backend/cuda/histogram.cpp
+++ b/src/backend/cuda/histogram.cpp
@@ -41,6 +41,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/identity.cpp b/src/backend/cuda/identity.cpp
index 995b09a9d9..ee62dcf549 100644
--- a/src/backend/cuda/identity.cpp
+++ b/src/backend/cuda/identity.cpp
@@ -37,6 +37,7 @@ INSTANTIATE_IDENTITY(uint)
 INSTANTIATE_IDENTITY(intl)
 INSTANTIATE_IDENTITY(uintl)
 INSTANTIATE_IDENTITY(char)
+INSTANTIATE_IDENTITY(schar)
 INSTANTIATE_IDENTITY(uchar)
 INSTANTIATE_IDENTITY(short)
 INSTANTIATE_IDENTITY(ushort)
diff --git a/src/backend/cuda/image.cpp b/src/backend/cuda/image.cpp
index 810d36d968..23bccf616e 100644
--- a/src/backend/cuda/image.cpp
+++ b/src/backend/cuda/image.cpp
@@ -70,6 +70,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/index.cpp b/src/backend/cuda/index.cpp
index d8acf90c12..dbb7d1ad60 100644
--- a/src/backend/cuda/index.cpp
+++ b/src/backend/cuda/index.cpp
@@ -90,6 +90,7 @@ INSTANTIATE(int)
 INSTANTIATE(uintl)
 INSTANTIATE(intl)
 INSTANTIATE(uchar)
+INSTANTIATE(schar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/iota.cpp b/src/backend/cuda/iota.cpp
index d9afef41c5..0ac6dbee74 100644
--- a/src/backend/cuda/iota.cpp
+++ b/src/backend/cuda/iota.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/ireduce.cpp b/src/backend/cuda/ireduce.cpp
index 94cd340a66..a2236230d4 100644
--- a/src/backend/cuda/ireduce.cpp
+++ b/src/backend/cuda/ireduce.cpp
@@ -62,6 +62,7 @@ INSTANTIATE(af_min_t, uintl)
 INSTANTIATE(af_min_t, short)
 INSTANTIATE(af_min_t, ushort)
 INSTANTIATE(af_min_t, char)
+INSTANTIATE(af_min_t, schar)
 INSTANTIATE(af_min_t, uchar)
 INSTANTIATE(af_min_t, half)
 
@@ -77,6 +78,7 @@ INSTANTIATE(af_max_t, uintl)
 INSTANTIATE(af_max_t, short)
 INSTANTIATE(af_max_t, ushort)
 INSTANTIATE(af_max_t, char)
+INSTANTIATE(af_max_t, schar)
 INSTANTIATE(af_max_t, uchar)
 INSTANTIATE(af_max_t, half)
 }  // namespace cuda
diff --git a/src/backend/cuda/jit.cpp b/src/backend/cuda/jit.cpp
index 9346491145..171ec66f61 100644
--- a/src/backend/cuda/jit.cpp
+++ b/src/backend/cuda/jit.cpp
@@ -552,6 +552,7 @@ template void evalNodes<cdouble>(Param<cdouble> out, Node* node);
 template void evalNodes<int>(Param<int> out, Node* node);
 template void evalNodes<uint>(Param<uint> out, Node* node);
 template void evalNodes<char>(Param<char> out, Node* node);
+template void evalNodes<schar>(Param<schar> out, Node* node);
 template void evalNodes<uchar>(Param<uchar> out, Node* node);
 template void evalNodes<intl>(Param<intl> out, Node* node);
 template void evalNodes<uintl>(Param<uintl> out, Node* node);
@@ -573,6 +574,8 @@ template void evalNodes<uint>(vector<Param<uint>>& out,
                               const vector<Node*>& node);
 template void evalNodes<char>(vector<Param<char>>& out,
                               const vector<Node*>& node);
+template void evalNodes<schar>(vector<Param<schar>>& out,
+                               const vector<Node*>& node);
 template void evalNodes<uchar>(vector<Param<uchar>>& out,
                                const vector<Node*>& node);
 template void evalNodes<intl>(vector<Param<intl>>& out,
diff --git a/src/backend/cuda/join.cpp b/src/backend/cuda/join.cpp
index 3eed6f7fb5..5065412342 100644
--- a/src/backend/cuda/join.cpp
+++ b/src/backend/cuda/join.cpp
@@ -209,6 +209,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(half)
@@ -229,6 +230,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(half)
diff --git a/src/backend/cuda/kernel/convolve_separable.cpp b/src/backend/cuda/kernel/convolve_separable.cpp
index 3c18a02240..14a62d1f1e 100644
--- a/src/backend/cuda/kernel/convolve_separable.cpp
+++ b/src/backend/cuda/kernel/convolve_separable.cpp
@@ -22,6 +22,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uchar, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
 INSTANTIATE(short, float)
diff --git a/src/backend/cuda/kernel/copy.cuh b/src/backend/cuda/kernel/copy.cuh
index 9e771e8c52..20f6bfa021 100644
--- a/src/backend/cuda/kernel/copy.cuh
+++ b/src/backend/cuda/kernel/copy.cuh
@@ -49,6 +49,18 @@ convertType<char, compute_t<common::half>>(char value) {
     return compute_t<common::half>(value);
 }
 
+template<>
+__inline__ __device__ schar
+convertType<compute_t<common::half>, schar>(compute_t<common::half> value) {
+    return (schar)((short)value);
+}
+
+template<>
+__inline__ __device__ compute_t<common::half>
+convertType<schar, compute_t<common::half>>(schar value) {
+    return compute_t<common::half>(value);
+}
+
 template<>
 __inline__ __device__ uchar
 convertType<compute_t<common::half>, uchar>(compute_t<common::half> value) {
@@ -90,6 +102,7 @@ OTHER_SPECIALIZATIONS(intl)
 OTHER_SPECIALIZATIONS(uintl)
 OTHER_SPECIALIZATIONS(short)
 OTHER_SPECIALIZATIONS(ushort)
+OTHER_SPECIALIZATIONS(schar)
 OTHER_SPECIALIZATIONS(uchar)
 OTHER_SPECIALIZATIONS(char)
 OTHER_SPECIALIZATIONS(common::half)
diff --git a/src/backend/cuda/kernel/random_engine.hpp b/src/backend/cuda/kernel/random_engine.hpp
index 07ba4163a2..a5e2305885 100644
--- a/src/backend/cuda/kernel/random_engine.hpp
+++ b/src/backend/cuda/kernel/random_engine.hpp
@@ -312,6 +312,12 @@ __device__ static void writeOut128Bytes(uchar *out, const uint &index,
     out[index + 15 * blockDim.x] = r4 >> 24;
 }
 
+__device__ static void writeOut128Bytes(schar *out, const uint &index,
+                                        const uint &r1, const uint &r2,
+                                        const uint &r3, const uint &r4) {
+    writeOut128Bytes((uchar *)(out), index, r1, r2, r3, r4);
+}
+
 __device__ static void writeOut128Bytes(char *out, const uint &index,
                                         const uint &r1, const uint &r2,
                                         const uint &r3, const uint &r4) {
@@ -535,6 +541,13 @@ __device__ static void partialWriteOut128Bytes(uchar *out, const uint &index,
     }
 }
 
+__device__ static void partialWriteOut128Bytes(schar *out, const uint &index,
+                                               const uint &r1, const uint &r2,
+                                               const uint &r3, const uint &r4,
+                                               const uint &elements) {
+    partialWriteOut128Bytes((uchar *)(out), index, r1, r2, r3, r4, elements);
+}
+
 __device__ static void partialWriteOut128Bytes(char *out, const uint &index,
                                                const uint &r1, const uint &r2,
                                                const uint &r3, const uint &r4,
diff --git a/src/backend/cuda/kernel/shared.hpp b/src/backend/cuda/kernel/shared.hpp
index 55d9f70a64..d1f15653c3 100644
--- a/src/backend/cuda/kernel/shared.hpp
+++ b/src/backend/cuda/kernel/shared.hpp
@@ -53,6 +53,7 @@ SPECIALIZE(int)
 SPECIALIZE(uint)
 SPECIALIZE(short)
 SPECIALIZE(ushort)
+SPECIALIZE(schar)
 SPECIALIZE(uchar)
 SPECIALIZE(intl)
 SPECIALIZE(uintl)
diff --git a/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu b/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu
index 19b291356c..7a7e3616c9 100644
--- a/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu
+++ b/src/backend/cuda/kernel/thrust_sort_by_key/thrust_sort_by_key_impl.cu
@@ -11,7 +11,7 @@
 
 // This file instantiates sort_by_key as separate object files from CMake
 // The 3 lines below are read by CMake to determenine the instantiations
-// SBK_TYPES:float double int uint intl uintl short ushort char uchar
+// SBK_TYPES:float double int uint intl uintl short ushort char schar uchar
 // SBK_INSTS:0 1
 
 namespace arrayfire {
diff --git a/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp b/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp
index e4695ac48e..e909a786de 100644
--- a/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp
+++ b/src/backend/cuda/kernel/thrust_sort_by_key_impl.hpp
@@ -39,6 +39,7 @@ void thrustSortByKey(Tk *keyPtr, Tv *valPtr, int elements, bool isAscending) {
     INSTANTIATE(Tk, cfloat)  \
     INSTANTIATE(Tk, cdouble) \
     INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, schar)   \
     INSTANTIATE(Tk, uchar)
 
 #define INSTANTIATE1(Tk)    \
diff --git a/src/backend/cuda/lookup.cpp b/src/backend/cuda/lookup.cpp
index 133db5ba26..ca5b8f79ed 100644
--- a/src/backend/cuda/lookup.cpp
+++ b/src/backend/cuda/lookup.cpp
@@ -54,6 +54,8 @@ Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                                       const unsigned);                         \
     template Array<T> lookup<T, uintl>(const Array<T> &, const Array<uintl> &, \
                                        const unsigned);                        \
+    template Array<T> lookup<T, schar>(const Array<T> &, const Array<schar> &, \
+                                       const unsigned);                        \
     template Array<T> lookup<T, uchar>(const Array<T> &, const Array<uchar> &, \
                                        const unsigned);                        \
     template Array<T> lookup<T, half>(const Array<T> &, const Array<half> &,   \
@@ -67,6 +69,7 @@ INSTANTIATE(int);
 INSTANTIATE(unsigned);
 INSTANTIATE(intl);
 INSTANTIATE(uintl);
+INSTANTIATE(schar);
 INSTANTIATE(uchar);
 INSTANTIATE(char);
 INSTANTIATE(short);
diff --git a/src/backend/cuda/match_template.cpp b/src/backend/cuda/match_template.cpp
index d82137bb5c..63b50435b7 100644
--- a/src/backend/cuda/match_template.cpp
+++ b/src/backend/cuda/match_template.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(char, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uint, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/cuda/math.hpp b/src/backend/cuda/math.hpp
index 6986bcb445..28574ac7e2 100644
--- a/src/backend/cuda/math.hpp
+++ b/src/backend/cuda/math.hpp
@@ -192,6 +192,14 @@ inline __device__ uintl maxval<uintl>() {
     return 1ULL << (8 * sizeof(uintl) - 1);
 }
 template<>
+inline __device__ schar maxval<schar>() {
+    return 0x7f;
+}
+template<>
+inline __device__ schar minval<schar>() {
+    return 0x80;
+}
+template<>
 inline __device__ char maxval<char>() {
     return 0x7f;
 }
diff --git a/src/backend/cuda/max.cu b/src/backend/cuda/max.cu
index 03f712b303..9fe7b92409 100644
--- a/src/backend/cuda/max.cu
+++ b/src/backend/cuda/max.cu
@@ -24,6 +24,7 @@ INSTANTIATE(af_max_t, uint, uint)
 INSTANTIATE(af_max_t, intl, intl)
 INSTANTIATE(af_max_t, uintl, uintl)
 INSTANTIATE(af_max_t, char, char)
+INSTANTIATE(af_max_t, schar, schar)
 INSTANTIATE(af_max_t, uchar, uchar)
 INSTANTIATE(af_max_t, short, short)
 INSTANTIATE(af_max_t, ushort, ushort)
diff --git a/src/backend/cuda/mean.cu b/src/backend/cuda/mean.cu
index 9b1eea74e9..b4dab3b866 100644
--- a/src/backend/cuda/mean.cu
+++ b/src/backend/cuda/mean.cu
@@ -63,6 +63,7 @@ INSTANTIATE(uintl, double, double);
 INSTANTIATE(short, float, float);
 INSTANTIATE(ushort, float, float);
 INSTANTIATE(uchar, float, float);
+INSTANTIATE(schar, float, float);
 INSTANTIATE(char, float, float);
 INSTANTIATE(cfloat, float, cfloat);
 INSTANTIATE(cdouble, double, cdouble);
diff --git a/src/backend/cuda/meanshift.cpp b/src/backend/cuda/meanshift.cpp
index d72d1aa041..83d12cb3ef 100644
--- a/src/backend/cuda/meanshift.cpp
+++ b/src/backend/cuda/meanshift.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/medfilt.cpp b/src/backend/cuda/medfilt.cpp
index c80c95c21f..cca97dd644 100644
--- a/src/backend/cuda/medfilt.cpp
+++ b/src/backend/cuda/medfilt.cpp
@@ -58,6 +58,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/memory.cpp b/src/backend/cuda/memory.cpp
index dafbef1ce8..616547d6af 100644
--- a/src/backend/cuda/memory.cpp
+++ b/src/backend/cuda/memory.cpp
@@ -117,6 +117,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/cuda/min.cu b/src/backend/cuda/min.cu
index 72a3f1beef..b0fad5733c 100644
--- a/src/backend/cuda/min.cu
+++ b/src/backend/cuda/min.cu
@@ -24,6 +24,7 @@ INSTANTIATE(af_min_t, uint, uint)
 INSTANTIATE(af_min_t, intl, intl)
 INSTANTIATE(af_min_t, uintl, uintl)
 INSTANTIATE(af_min_t, char, char)
+INSTANTIATE(af_min_t, schar, schar)
 INSTANTIATE(af_min_t, uchar, uchar)
 INSTANTIATE(af_min_t, short, short)
 INSTANTIATE(af_min_t, ushort, ushort)
diff --git a/src/backend/cuda/moments.cpp b/src/backend/cuda/moments.cpp
index 34c8cf753f..fa37b033e1 100644
--- a/src/backend/cuda/moments.cpp
+++ b/src/backend/cuda/moments.cpp
@@ -51,6 +51,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/morph.cpp b/src/backend/cuda/morph.cpp
index a49fd5a40e..f09f20bded 100644
--- a/src/backend/cuda/morph.cpp
+++ b/src/backend/cuda/morph.cpp
@@ -53,6 +53,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/nearest_neighbour.cu b/src/backend/cuda/nearest_neighbour.cu
index ca6a11a1c6..dc10695f8a 100644
--- a/src/backend/cuda/nearest_neighbour.cu
+++ b/src/backend/cuda/nearest_neighbour.cu
@@ -67,6 +67,7 @@ INSTANTIATE(int, int)
 INSTANTIATE(uint, uint)
 INSTANTIATE(intl, intl)
 INSTANTIATE(uintl, uintl)
+INSTANTIATE(schar, int)
 INSTANTIATE(uchar, uint)
 INSTANTIATE(short, int)
 INSTANTIATE(ushort, uint)
diff --git a/src/backend/cuda/pad_array_borders.cpp b/src/backend/cuda/pad_array_borders.cpp
index bf41b5f2e7..af563733d2 100644
--- a/src/backend/cuda/pad_array_borders.cpp
+++ b/src/backend/cuda/pad_array_borders.cpp
@@ -48,6 +48,7 @@ INSTANTIATE_PAD_ARRAY_BORDERS(int)
 INSTANTIATE_PAD_ARRAY_BORDERS(uint)
 INSTANTIATE_PAD_ARRAY_BORDERS(intl)
 INSTANTIATE_PAD_ARRAY_BORDERS(uintl)
+INSTANTIATE_PAD_ARRAY_BORDERS(schar)
 INSTANTIATE_PAD_ARRAY_BORDERS(uchar)
 INSTANTIATE_PAD_ARRAY_BORDERS(char)
 INSTANTIATE_PAD_ARRAY_BORDERS(ushort)
diff --git a/src/backend/cuda/plot.cpp b/src/backend/cuda/plot.cpp
index e012377305..e69b149790 100644
--- a/src/backend/cuda/plot.cpp
+++ b/src/backend/cuda/plot.cpp
@@ -70,6 +70,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
diff --git a/src/backend/cuda/product.cu b/src/backend/cuda/product.cu
index c4fff43b93..fb26c95562 100644
--- a/src/backend/cuda/product.cu
+++ b/src/backend/cuda/product.cu
@@ -24,6 +24,7 @@ INSTANTIATE(af_mul_t, uint, uint)
 INSTANTIATE(af_mul_t, intl, intl)
 INSTANTIATE(af_mul_t, uintl, uintl)
 INSTANTIATE(af_mul_t, char, int)
+INSTANTIATE(af_mul_t, schar, int)
 INSTANTIATE(af_mul_t, uchar, uint)
 INSTANTIATE(af_mul_t, short, int)
 INSTANTIATE(af_mul_t, ushort, uint)
diff --git a/src/backend/cuda/random_engine.cu b/src/backend/cuda/random_engine.cu
index a63ead0bf8..26cdbdc23b 100644
--- a/src/backend/cuda/random_engine.cu
+++ b/src/backend/cuda/random_engine.cu
@@ -143,6 +143,7 @@ INSTANTIATE_UNIFORM(uint)
 INSTANTIATE_UNIFORM(intl)
 INSTANTIATE_UNIFORM(uintl)
 INSTANTIATE_UNIFORM(char)
+INSTANTIATE_UNIFORM(schar)
 INSTANTIATE_UNIFORM(uchar)
 INSTANTIATE_UNIFORM(short)
 INSTANTIATE_UNIFORM(ushort)
diff --git a/src/backend/cuda/range.cpp b/src/backend/cuda/range.cpp
index 55a2553649..f821f283f7 100644
--- a/src/backend/cuda/range.cpp
+++ b/src/backend/cuda/range.cpp
@@ -48,6 +48,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/reorder.cpp b/src/backend/cuda/reorder.cpp
index c81fd02f6a..286dcde6ad 100644
--- a/src/backend/cuda/reorder.cpp
+++ b/src/backend/cuda/reorder.cpp
@@ -43,6 +43,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
diff --git a/src/backend/cuda/reshape.cpp b/src/backend/cuda/reshape.cpp
index 9d6e57549f..329b7883cb 100644
--- a/src/backend/cuda/reshape.cpp
+++ b/src/backend/cuda/reshape.cpp
@@ -49,6 +49,8 @@ Array<outType> reshape(const Array<inType> &in, const dim4 &outDims,
                                                 dim4 const &, short, double); \
     template Array<ushort> reshape<SRC_T, ushort>(                            \
         Array<SRC_T> const &, dim4 const &, ushort, double);                  \
+    template Array<schar> reshape<SRC_T, schar>(Array<SRC_T> const &,         \
+                                                dim4 const &, schar, double); \
     template Array<uchar> reshape<SRC_T, uchar>(Array<SRC_T> const &,         \
                                                 dim4 const &, uchar, double); \
     template Array<char> reshape<SRC_T, char>(Array<SRC_T> const &,           \
@@ -64,6 +66,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(half)
diff --git a/src/backend/cuda/resize.cpp b/src/backend/cuda/resize.cpp
index 97dc8a7da8..dec6f09d26 100644
--- a/src/backend/cuda/resize.cpp
+++ b/src/backend/cuda/resize.cpp
@@ -41,6 +41,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/rotate.cpp b/src/backend/cuda/rotate.cpp
index 2f46894aef..7edb0de7a6 100644
--- a/src/backend/cuda/rotate.cpp
+++ b/src/backend/cuda/rotate.cpp
@@ -36,6 +36,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/scan.cpp b/src/backend/cuda/scan.cpp
index 10002cbbad..cf3f2a0b70 100644
--- a/src/backend/cuda/scan.cpp
+++ b/src/backend/cuda/scan.cpp
@@ -47,6 +47,7 @@ Array<To> scan(const Array<Ti>& in, const int dim, bool inclusive_scan) {
     INSTANTIATE_SCAN(ROp, uintl, uintl)     \
     INSTANTIATE_SCAN(ROp, char, int)        \
     INSTANTIATE_SCAN(ROp, char, uint)       \
+    INSTANTIATE_SCAN(ROp, schar, int)       \
     INSTANTIATE_SCAN(ROp, uchar, uint)      \
     INSTANTIATE_SCAN(ROp, short, int)       \
     INSTANTIATE_SCAN(ROp, ushort, uint)
diff --git a/src/backend/cuda/select.cpp b/src/backend/cuda/select.cpp
index b13df55bfe..0b78263efd 100644
--- a/src/backend/cuda/select.cpp
+++ b/src/backend/cuda/select.cpp
@@ -127,6 +127,7 @@ INSTANTIATE(uint);
 INSTANTIATE(intl);
 INSTANTIATE(uintl);
 INSTANTIATE(char);
+INSTANTIATE(schar);
 INSTANTIATE(uchar);
 INSTANTIATE(short);
 INSTANTIATE(ushort);
diff --git a/src/backend/cuda/set.cu b/src/backend/cuda/set.cu
index fbbbc28c0a..d558d6e938 100644
--- a/src/backend/cuda/set.cu
+++ b/src/backend/cuda/set.cu
@@ -122,6 +122,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/shift.cpp b/src/backend/cuda/shift.cpp
index 6f88a38472..f073d3c844 100644
--- a/src/backend/cuda/shift.cpp
+++ b/src/backend/cuda/shift.cpp
@@ -68,6 +68,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/sobel.cpp b/src/backend/cuda/sobel.cpp
index 5200f69a45..1861d0c76c 100644
--- a/src/backend/cuda/sobel.cpp
+++ b/src/backend/cuda/sobel.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(int, int)
 INSTANTIATE(uint, int)
 INSTANTIATE(char, int)
+INSTANTIATE(schar, int)
 INSTANTIATE(uchar, int)
 INSTANTIATE(short, int)
 INSTANTIATE(ushort, int)
diff --git a/src/backend/cuda/sort.cu b/src/backend/cuda/sort.cu
index 9970ddd8b2..d56899a87d 100644
--- a/src/backend/cuda/sort.cu
+++ b/src/backend/cuda/sort.cu
@@ -54,6 +54,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/sort_by_key.cu b/src/backend/cuda/sort_by_key.cu
index bd19d16240..21d9efc5b2 100644
--- a/src/backend/cuda/sort_by_key.cu
+++ b/src/backend/cuda/sort_by_key.cu
@@ -67,6 +67,7 @@ void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
     INSTANTIATE(Tk, short)   \
     INSTANTIATE(Tk, ushort)  \
     INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, schar)   \
     INSTANTIATE(Tk, uchar)   \
     INSTANTIATE(Tk, intl)    \
     INSTANTIATE(Tk, uintl)
@@ -78,6 +79,7 @@ INSTANTIATE1(uint)
 INSTANTIATE1(short)
 INSTANTIATE1(ushort)
 INSTANTIATE1(char)
+INSTANTIATE1(schar)
 INSTANTIATE1(uchar)
 INSTANTIATE1(intl)
 INSTANTIATE1(uintl)
diff --git a/src/backend/cuda/sort_index.cu b/src/backend/cuda/sort_index.cu
index 039e77a147..d923f7c6e9 100644
--- a/src/backend/cuda/sort_index.cu
+++ b/src/backend/cuda/sort_index.cu
@@ -63,6 +63,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/sum.cu b/src/backend/cuda/sum.cu
index 44cfec9449..6a52c2c369 100644
--- a/src/backend/cuda/sum.cu
+++ b/src/backend/cuda/sum.cu
@@ -29,6 +29,8 @@ INSTANTIATE(af_add_t, uintl, uintl)
 INSTANTIATE(af_add_t, uintl, double)
 INSTANTIATE(af_add_t, char, int)
 INSTANTIATE(af_add_t, char, float)
+INSTANTIATE(af_add_t, schar, int)
+INSTANTIATE(af_add_t, schar, float)
 INSTANTIATE(af_add_t, uchar, uint)
 INSTANTIATE(af_add_t, uchar, float)
 INSTANTIATE(af_add_t, short, int)
diff --git a/src/backend/cuda/surface.cpp b/src/backend/cuda/surface.cpp
index bef751239b..61f3457036 100644
--- a/src/backend/cuda/surface.cpp
+++ b/src/backend/cuda/surface.cpp
@@ -71,6 +71,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
diff --git a/src/backend/cuda/susan.cpp b/src/backend/cuda/susan.cpp
index 4d0fcc078c..5f1d07d913 100644
--- a/src/backend/cuda/susan.cpp
+++ b/src/backend/cuda/susan.cpp
@@ -74,6 +74,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/tile.cpp b/src/backend/cuda/tile.cpp
index f93982eb43..edd2a7b686 100644
--- a/src/backend/cuda/tile.cpp
+++ b/src/backend/cuda/tile.cpp
@@ -48,6 +48,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/transform.cpp b/src/backend/cuda/transform.cpp
index baba9b1a04..e0d0509c8d 100644
--- a/src/backend/cuda/transform.cpp
+++ b/src/backend/cuda/transform.cpp
@@ -37,6 +37,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/transpose.cpp b/src/backend/cuda/transpose.cpp
index faa4659b68..03d6f3b91d 100644
--- a/src/backend/cuda/transpose.cpp
+++ b/src/backend/cuda/transpose.cpp
@@ -45,6 +45,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/cuda/transpose_inplace.cpp b/src/backend/cuda/transpose_inplace.cpp
index ff89730d47..dcc8c5664b 100644
--- a/src/backend/cuda/transpose_inplace.cpp
+++ b/src/backend/cuda/transpose_inplace.cpp
@@ -37,6 +37,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/cuda/triangle.cpp b/src/backend/cuda/triangle.cpp
index 4ec0a04e6f..c32e984626 100644
--- a/src/backend/cuda/triangle.cpp
+++ b/src/backend/cuda/triangle.cpp
@@ -48,6 +48,7 @@ INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/types.hpp b/src/backend/cuda/types.hpp
index 34815cba66..2230948f3a 100644
--- a/src/backend/cuda/types.hpp
+++ b/src/backend/cuda/types.hpp
@@ -35,6 +35,7 @@ namespace cuda {
 using cdouble = cuDoubleComplex;
 using cfloat  = cuFloatComplex;
 using intl    = long long;
+using schar   = signed char;
 using uchar   = unsigned char;
 using uint    = unsigned int;
 using uintl   = unsigned long long;
@@ -82,6 +83,10 @@ inline const char *shortname<char>(bool caps) {
     return caps ? "J" : "j";
 }
 template<>
+inline const char *shortname<schar>(bool caps) {
+    return caps ? "A" : "a"; // TODO
+}
+template<>
 inline const char *shortname<uchar>(bool caps) {
     return caps ? "V" : "v";
 }
@@ -120,6 +125,7 @@ SPECIALIZE(double)
 SPECIALIZE(cfloat)
 SPECIALIZE(cdouble)
 SPECIALIZE(char)
+SPECIALIZE(signed char)
 SPECIALIZE(unsigned char)
 SPECIALIZE(short)
 SPECIALIZE(unsigned short)
diff --git a/src/backend/cuda/unwrap.cpp b/src/backend/cuda/unwrap.cpp
index 6eae7d428b..9d96aec1d9 100644
--- a/src/backend/cuda/unwrap.cpp
+++ b/src/backend/cuda/unwrap.cpp
@@ -55,6 +55,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/cuda/vector_field.cpp b/src/backend/cuda/vector_field.cpp
index 2868979772..a0528cddb1 100644
--- a/src/backend/cuda/vector_field.cpp
+++ b/src/backend/cuda/vector_field.cpp
@@ -105,6 +105,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace cuda
diff --git a/src/backend/cuda/where.cpp b/src/backend/cuda/where.cpp
index efd488d26e..862b25fa24 100644
--- a/src/backend/cuda/where.cpp
+++ b/src/backend/cuda/where.cpp
@@ -36,6 +36,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/cuda/wrap.cpp b/src/backend/cuda/wrap.cpp
index d8963cacd9..dd7901cc0e 100644
--- a/src/backend/cuda/wrap.cpp
+++ b/src/backend/cuda/wrap.cpp
@@ -44,6 +44,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/Array.cpp b/src/backend/oneapi/Array.cpp
index 8165e6fb08..57c8f111ee 100644
--- a/src/backend/oneapi/Array.cpp
+++ b/src/backend/oneapi/Array.cpp
@@ -582,6 +582,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
diff --git a/src/backend/oneapi/all.cpp b/src/backend/oneapi/all.cpp
index ad09e4aff1..e4e86232d2 100644
--- a/src/backend/oneapi/all.cpp
+++ b/src/backend/oneapi/all.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_and_t, uint, char)
 INSTANTIATE(af_and_t, intl, char)
 INSTANTIATE(af_and_t, uintl, char)
 INSTANTIATE(af_and_t, char, char)
+INSTANTIATE(af_and_t, schar, char)
 INSTANTIATE(af_and_t, uchar, char)
 INSTANTIATE(af_and_t, short, char)
 INSTANTIATE(af_and_t, ushort, char)
diff --git a/src/backend/oneapi/any.cpp b/src/backend/oneapi/any.cpp
index bdf600e9a9..82e242a989 100644
--- a/src/backend/oneapi/any.cpp
+++ b/src/backend/oneapi/any.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_or_t, uint, char)
 INSTANTIATE(af_or_t, intl, char)
 INSTANTIATE(af_or_t, uintl, char)
 INSTANTIATE(af_or_t, char, char)
+INSTANTIATE(af_or_t, schar, char)
 INSTANTIATE(af_or_t, uchar, char)
 INSTANTIATE(af_or_t, short, char)
 INSTANTIATE(af_or_t, ushort, char)
diff --git a/src/backend/oneapi/assign.cpp b/src/backend/oneapi/assign.cpp
index def9378d2d..de436495db 100644
--- a/src/backend/oneapi/assign.cpp
+++ b/src/backend/oneapi/assign.cpp
@@ -80,6 +80,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/bilateral.cpp b/src/backend/oneapi/bilateral.cpp
index d7d5dd33b9..6520cf9ffa 100644
--- a/src/backend/oneapi/bilateral.cpp
+++ b/src/backend/oneapi/bilateral.cpp
@@ -35,6 +35,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(char, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uint, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/oneapi/cast.hpp b/src/backend/oneapi/cast.hpp
index c9b015c4f2..7d4e2be76f 100644
--- a/src/backend/oneapi/cast.hpp
+++ b/src/backend/oneapi/cast.hpp
@@ -34,6 +34,7 @@ struct CastOp {
 
 CAST_FN(int)
 CAST_FN(uint)
+CAST_FN(schar)
 CAST_FN(uchar)
 CAST_FN(float)
 CAST_FN(double)
diff --git a/src/backend/oneapi/convolve.cpp b/src/backend/oneapi/convolve.cpp
index d2cc41c588..0e443d7b77 100644
--- a/src/backend/oneapi/convolve.cpp
+++ b/src/backend/oneapi/convolve.cpp
@@ -98,6 +98,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/oneapi/convolve_separable.cpp b/src/backend/oneapi/convolve_separable.cpp
index fdf9fc952f..ddf5c27a7e 100644
--- a/src/backend/oneapi/convolve_separable.cpp
+++ b/src/backend/oneapi/convolve_separable.cpp
@@ -65,6 +65,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(short, float)
diff --git a/src/backend/oneapi/copy.cpp b/src/backend/oneapi/copy.cpp
index 506206b11e..a89023261e 100644
--- a/src/backend/oneapi/copy.cpp
+++ b/src/backend/oneapi/copy.cpp
@@ -155,6 +155,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
@@ -184,6 +185,8 @@ INSTANTIATE(half)
                                           Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, ushort>(Array<ushort> & dst,       \
                                            Array<SRC_T> const &src);  \
+    template void copyArray<SRC_T, schar>(Array<schar> & dst,         \
+                                          Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, uchar>(Array<uchar> & dst,         \
                                           Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, char>(Array<char> & dst,           \
@@ -197,6 +200,7 @@ INSTANTIATE_COPY_ARRAY(int)
 INSTANTIATE_COPY_ARRAY(uint)
 INSTANTIATE_COPY_ARRAY(intl)
 INSTANTIATE_COPY_ARRAY(uintl)
+INSTANTIATE_COPY_ARRAY(schar)
 INSTANTIATE_COPY_ARRAY(uchar)
 INSTANTIATE_COPY_ARRAY(char)
 INSTANTIATE_COPY_ARRAY(short)
@@ -238,6 +242,7 @@ INSTANTIATE_GETSCALAR(cfloat)
 INSTANTIATE_GETSCALAR(cdouble)
 INSTANTIATE_GETSCALAR(int)
 INSTANTIATE_GETSCALAR(uint)
+INSTANTIATE_GETSCALAR(schar)
 INSTANTIATE_GETSCALAR(uchar)
 INSTANTIATE_GETSCALAR(char)
 INSTANTIATE_GETSCALAR(intl)
diff --git a/src/backend/oneapi/count.cpp b/src/backend/oneapi/count.cpp
index f8ef354169..4ed59eb3b9 100644
--- a/src/backend/oneapi/count.cpp
+++ b/src/backend/oneapi/count.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_notzero_t, uint, uint)
 INSTANTIATE(af_notzero_t, intl, uint)
 INSTANTIATE(af_notzero_t, uintl, uint)
 INSTANTIATE(af_notzero_t, char, uint)
+INSTANTIATE(af_notzero_t, schar, uint)
 INSTANTIATE(af_notzero_t, uchar, uint)
 INSTANTIATE(af_notzero_t, short, uint)
 INSTANTIATE(af_notzero_t, ushort, uint)
diff --git a/src/backend/oneapi/diagonal.cpp b/src/backend/oneapi/diagonal.cpp
index a18d024585..900f53ba3c 100644
--- a/src/backend/oneapi/diagonal.cpp
+++ b/src/backend/oneapi/diagonal.cpp
@@ -54,6 +54,7 @@ INSTANTIATE_DIAGONAL(uint)
 INSTANTIATE_DIAGONAL(intl)
 INSTANTIATE_DIAGONAL(uintl)
 INSTANTIATE_DIAGONAL(char)
+INSTANTIATE_DIAGONAL(schar)
 INSTANTIATE_DIAGONAL(uchar)
 INSTANTIATE_DIAGONAL(short)
 INSTANTIATE_DIAGONAL(ushort)
diff --git a/src/backend/oneapi/diff.cpp b/src/backend/oneapi/diff.cpp
index a3c37f6a4a..01cd18e37e 100644
--- a/src/backend/oneapi/diff.cpp
+++ b/src/backend/oneapi/diff.cpp
@@ -50,6 +50,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/oneapi/exampleFunction.cpp b/src/backend/oneapi/exampleFunction.cpp
index 6159d9d1d4..9a006febff 100644
--- a/src/backend/oneapi/exampleFunction.cpp
+++ b/src/backend/oneapi/exampleFunction.cpp
@@ -59,6 +59,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(cfloat)
diff --git a/src/backend/oneapi/fast.cpp b/src/backend/oneapi/fast.cpp
index cb9ae28d4c..a5b0934f97 100644
--- a/src/backend/oneapi/fast.cpp
+++ b/src/backend/oneapi/fast.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/fftconvolve.cpp b/src/backend/oneapi/fftconvolve.cpp
index de96d94c99..85718f4f4f 100644
--- a/src/backend/oneapi/fftconvolve.cpp
+++ b/src/backend/oneapi/fftconvolve.cpp
@@ -148,6 +148,7 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 INSTANTIATE(uint)
 INSTANTIATE(int)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(uintl)
diff --git a/src/backend/oneapi/hist_graphics.cpp b/src/backend/oneapi/hist_graphics.cpp
index 3b280592b1..e016337a54 100644
--- a/src/backend/oneapi/hist_graphics.cpp
+++ b/src/backend/oneapi/hist_graphics.cpp
@@ -28,6 +28,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
diff --git a/src/backend/oneapi/histogram.cpp b/src/backend/oneapi/histogram.cpp
index 4dfece0640..872431f14c 100644
--- a/src/backend/oneapi/histogram.cpp
+++ b/src/backend/oneapi/histogram.cpp
@@ -41,6 +41,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/identity.cpp b/src/backend/oneapi/identity.cpp
index 5a838a4cf0..68a592ab88 100644
--- a/src/backend/oneapi/identity.cpp
+++ b/src/backend/oneapi/identity.cpp
@@ -37,6 +37,7 @@ INSTANTIATE_IDENTITY(uint)
 INSTANTIATE_IDENTITY(intl)
 INSTANTIATE_IDENTITY(uintl)
 INSTANTIATE_IDENTITY(char)
+INSTANTIATE_IDENTITY(schar)
 INSTANTIATE_IDENTITY(uchar)
 INSTANTIATE_IDENTITY(short)
 INSTANTIATE_IDENTITY(ushort)
diff --git a/src/backend/oneapi/image.cpp b/src/backend/oneapi/image.cpp
index 723c29fb8b..7aa8b4b667 100644
--- a/src/backend/oneapi/image.cpp
+++ b/src/backend/oneapi/image.cpp
@@ -29,6 +29,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/index.cpp b/src/backend/oneapi/index.cpp
index 2548df2011..af204b0820 100644
--- a/src/backend/oneapi/index.cpp
+++ b/src/backend/oneapi/index.cpp
@@ -83,6 +83,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/iota.cpp b/src/backend/oneapi/iota.cpp
index 6d511df23f..e775f0dde6 100644
--- a/src/backend/oneapi/iota.cpp
+++ b/src/backend/oneapi/iota.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/ireduce.cpp b/src/backend/oneapi/ireduce.cpp
index c7b4d263ab..c4bfc7604f 100644
--- a/src/backend/oneapi/ireduce.cpp
+++ b/src/backend/oneapi/ireduce.cpp
@@ -58,6 +58,7 @@ INSTANTIATE(af_min_t, uint)
 INSTANTIATE(af_min_t, intl)
 INSTANTIATE(af_min_t, uintl)
 INSTANTIATE(af_min_t, char)
+INSTANTIATE(af_min_t, schar)
 INSTANTIATE(af_min_t, uchar)
 INSTANTIATE(af_min_t, short)
 INSTANTIATE(af_min_t, ushort)
@@ -73,6 +74,7 @@ INSTANTIATE(af_max_t, uint)
 INSTANTIATE(af_max_t, intl)
 INSTANTIATE(af_max_t, uintl)
 INSTANTIATE(af_max_t, char)
+INSTANTIATE(af_max_t, schar)
 INSTANTIATE(af_max_t, uchar)
 INSTANTIATE(af_max_t, short)
 INSTANTIATE(af_max_t, ushort)
diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index a112e99436..2bd34a5dc4 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -627,6 +627,7 @@ template void evalNodes<cdouble>(Param<cdouble>& out, Node* node);
 template void evalNodes<int>(Param<int>& out, Node* node);
 template void evalNodes<uint>(Param<uint>& out, Node* node);
 template void evalNodes<char>(Param<char>& out, Node* node);
+template void evalNodes<schar>(Param<schar>& out, Node* node);
 template void evalNodes<uchar>(Param<uchar>& out, Node* node);
 template void evalNodes<intl>(Param<intl>& out, Node* node);
 template void evalNodes<uintl>(Param<uintl>& out, Node* node);
@@ -648,6 +649,8 @@ template void evalNodes<uint>(vector<Param<uint>>& out,
                               const vector<Node*>& node);
 template void evalNodes<char>(vector<Param<char>>& out,
                               const vector<Node*>& node);
+template void evalNodes<schar>(vector<Param<schar>>& out,
+                               const vector<Node*>& node);
 template void evalNodes<uchar>(vector<Param<uchar>>& out,
                                const vector<Node*>& node);
 template void evalNodes<intl>(vector<Param<intl>>& out,
diff --git a/src/backend/oneapi/join.cpp b/src/backend/oneapi/join.cpp
index e95b63c392..a64e6edb9d 100644
--- a/src/backend/oneapi/join.cpp
+++ b/src/backend/oneapi/join.cpp
@@ -272,6 +272,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(half)
@@ -292,6 +293,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(half)
diff --git a/src/backend/oneapi/kernel/convolve1.hpp b/src/backend/oneapi/kernel/convolve1.hpp
index e156308b34..41c6facae6 100644
--- a/src/backend/oneapi/kernel/convolve1.hpp
+++ b/src/backend/oneapi/kernel/convolve1.hpp
@@ -174,6 +174,7 @@ INSTANTIATE_CONV1(double, double)
 INSTANTIATE_CONV1(float, float)
 INSTANTIATE_CONV1(uint, float)
 INSTANTIATE_CONV1(int, float)
+INSTANTIATE_CONV1(schar, float)
 INSTANTIATE_CONV1(uchar, float)
 INSTANTIATE_CONV1(char, float)
 INSTANTIATE_CONV1(ushort, float)
diff --git a/src/backend/oneapi/kernel/convolve2.hpp b/src/backend/oneapi/kernel/convolve2.hpp
index b216e50917..45bfa6c108 100644
--- a/src/backend/oneapi/kernel/convolve2.hpp
+++ b/src/backend/oneapi/kernel/convolve2.hpp
@@ -195,4 +195,5 @@ INSTANTIATE_CONV2(intl, float)
 INSTANTIATE_CONV2(ushort, float)
 INSTANTIATE_CONV2(uint, float)
 INSTANTIATE_CONV2(uintl, float)
+INSTANTIATE_CONV2(schar, float)
 INSTANTIATE_CONV2(uchar, float)
diff --git a/src/backend/oneapi/kernel/convolve3.hpp b/src/backend/oneapi/kernel/convolve3.hpp
index 3ac4a50aa2..bdfcc4eb24 100644
--- a/src/backend/oneapi/kernel/convolve3.hpp
+++ b/src/backend/oneapi/kernel/convolve3.hpp
@@ -193,6 +193,7 @@ INSTANTIATE_CONV3(double, double)
 INSTANTIATE_CONV3(float, float)
 INSTANTIATE_CONV3(uint, float)
 INSTANTIATE_CONV3(int, float)
+INSTANTIATE_CONV3(schar, float)
 INSTANTIATE_CONV3(uchar, float)
 INSTANTIATE_CONV3(char, float)
 INSTANTIATE_CONV3(ushort, float)
diff --git a/src/backend/oneapi/kernel/convolve_separable.cpp b/src/backend/oneapi/kernel/convolve_separable.cpp
index 45a86efb7a..0f3dfacb30 100644
--- a/src/backend/oneapi/kernel/convolve_separable.cpp
+++ b/src/backend/oneapi/kernel/convolve_separable.cpp
@@ -200,6 +200,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/oneapi/kernel/memcopy.hpp b/src/backend/oneapi/kernel/memcopy.hpp
index b400d04673..64bd26ba1e 100644
--- a/src/backend/oneapi/kernel/memcopy.hpp
+++ b/src/backend/oneapi/kernel/memcopy.hpp
@@ -164,6 +164,19 @@ convertType<char, compute_t<arrayfire::common::half>>(char value) {
     return compute_t<arrayfire::common::half>(value);
 }
 
+template<>
+signed char inline convertType<compute_t<arrayfire::common::half>, signed char>(
+    compute_t<arrayfire::common::half> value) {
+    return (signed char)((short)value);
+}
+
+template<>
+inline compute_t<arrayfire::common::half>
+convertType<signed char, compute_t<arrayfire::common::half>>(
+    signed char value) {
+    return compute_t<arrayfire::common::half>(value);
+}
+
 template<>
 unsigned char inline convertType<compute_t<arrayfire::common::half>,
                                  unsigned char>(
@@ -197,6 +210,7 @@ OTHER_SPECIALIZATIONS(intl)
 OTHER_SPECIALIZATIONS(uintl)
 OTHER_SPECIALIZATIONS(short)
 OTHER_SPECIALIZATIONS(ushort)
+OTHER_SPECIALIZATIONS(schar)
 OTHER_SPECIALIZATIONS(uchar)
 OTHER_SPECIALIZATIONS(char)
 OTHER_SPECIALIZATIONS(arrayfire::common::half)
diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index dcd20dec13..3ebf0a113e 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -303,6 +303,12 @@ static void writeOut128Bytes(uchar *out, const uint &index, const uint groupSz,
     out[index + 15 * groupSz] = r4 >> 24;
 }
 
+static void writeOut128Bytes(schar *out, const uint &index, const uint groupSz,
+                             const uint &r1, const uint &r2, const uint &r3,
+                             const uint &r4) {
+    writeOut128Bytes((uchar *)(out), index, groupSz, r1, r2, r3, r4);
+}
+
 static void writeOut128Bytes(char *out, const uint &index, const uint groupSz,
                              const uint &r1, const uint &r2, const uint &r3,
                              const uint &r4) {
@@ -505,6 +511,14 @@ static void partialWriteOut128Bytes(uchar *out, const uint &index,
     }
 }
 
+static void partialWriteOut128Bytes(schar *out, const uint &index,
+                                    const uint groupSz, const uint &r1,
+                                    const uint &r2, const uint &r3,
+                                    const uint &r4, const uint &elements) {
+    partialWriteOut128Bytes((uchar *)(out), index, groupSz, r1, r2, r3, r4,
+                            elements);
+}
+
 static void partialWriteOut128Bytes(char *out, const uint &index,
                                     const uint groupSz, const uint &r1,
                                     const uint &r2, const uint &r3,
diff --git a/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp
index 9b04402904..0b0a8fb13f 100644
--- a/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp
+++ b/src/backend/oneapi/kernel/sort_by_key/sort_by_key_impl.cpp
@@ -9,7 +9,7 @@
 
 #include <kernel/sort_by_key_impl.hpp>
 
-// SBK_TYPES:float double int uint intl uintl short ushort char uchar half
+// SBK_TYPES:float double int uint intl uintl short ushort char schar uchar half
 
 namespace arrayfire {
 namespace oneapi {
diff --git a/src/backend/oneapi/kernel/sort_by_key_impl.hpp b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
index 6e3a0bd655..2e462db4b6 100644
--- a/src/backend/oneapi/kernel/sort_by_key_impl.hpp
+++ b/src/backend/oneapi/kernel/sort_by_key_impl.hpp
@@ -209,6 +209,7 @@ void sort0ByKey(Param<Tk> pKey, Param<Tv> pVal, bool isAscending) {
     INSTANTIATE(Tk, short)   \
     INSTANTIATE(Tk, ushort)  \
     INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, schar)   \
     INSTANTIATE(Tk, uchar)   \
     INSTANTIATE(Tk, intl)    \
     INSTANTIATE(Tk, uintl)
diff --git a/src/backend/oneapi/lookup.cpp b/src/backend/oneapi/lookup.cpp
index 9c87003375..de0a017c55 100644
--- a/src/backend/oneapi/lookup.cpp
+++ b/src/backend/oneapi/lookup.cpp
@@ -53,6 +53,8 @@ Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                                       const unsigned);                         \
     template Array<T> lookup<T, uintl>(const Array<T> &, const Array<uintl> &, \
                                        const unsigned);                        \
+    template Array<T> lookup<T, schar>(const Array<T> &, const Array<schar> &, \
+                                       const unsigned);                        \
     template Array<T> lookup<T, uchar>(const Array<T> &, const Array<uchar> &, \
                                        const unsigned);                        \
     template Array<T> lookup<T, half>(const Array<T> &, const Array<half> &,   \
@@ -66,6 +68,7 @@ INSTANTIATE(int);
 INSTANTIATE(unsigned);
 INSTANTIATE(intl);
 INSTANTIATE(uintl);
+INSTANTIATE(schar);
 INSTANTIATE(uchar);
 INSTANTIATE(char);
 INSTANTIATE(ushort);
diff --git a/src/backend/oneapi/match_template.cpp b/src/backend/oneapi/match_template.cpp
index 28794ff2eb..10b84757ac 100644
--- a/src/backend/oneapi/match_template.cpp
+++ b/src/backend/oneapi/match_template.cpp
@@ -32,6 +32,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(char, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uint, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/oneapi/max.cpp b/src/backend/oneapi/max.cpp
index 8b6ef71a10..fa21d78c1c 100644
--- a/src/backend/oneapi/max.cpp
+++ b/src/backend/oneapi/max.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_max_t, uint, uint)
 INSTANTIATE(af_max_t, intl, intl)
 INSTANTIATE(af_max_t, uintl, uintl)
 INSTANTIATE(af_max_t, char, char)
+INSTANTIATE(af_max_t, schar, schar)
 INSTANTIATE(af_max_t, uchar, uchar)
 INSTANTIATE(af_max_t, short, short)
 INSTANTIATE(af_max_t, ushort, ushort)
diff --git a/src/backend/oneapi/mean.cpp b/src/backend/oneapi/mean.cpp
index 09763bb739..2f94101f56 100644
--- a/src/backend/oneapi/mean.cpp
+++ b/src/backend/oneapi/mean.cpp
@@ -60,6 +60,7 @@ INSTANTIATE(intl, double, double);
 INSTANTIATE(uintl, double, double);
 INSTANTIATE(short, float, float);
 INSTANTIATE(ushort, float, float);
+INSTANTIATE(schar, float, float);
 INSTANTIATE(uchar, float, float);
 INSTANTIATE(char, float, float);
 INSTANTIATE(cfloat, float, cfloat);
diff --git a/src/backend/oneapi/meanshift.cpp b/src/backend/oneapi/meanshift.cpp
index 1017b9074b..825b26eb88 100644
--- a/src/backend/oneapi/meanshift.cpp
+++ b/src/backend/oneapi/meanshift.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/medfilt.cpp b/src/backend/oneapi/medfilt.cpp
index 3b1ff319c5..50c2cc3dd8 100644
--- a/src/backend/oneapi/medfilt.cpp
+++ b/src/backend/oneapi/medfilt.cpp
@@ -59,6 +59,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/memory.cpp b/src/backend/oneapi/memory.cpp
index f94b6df5a4..3482742b73 100644
--- a/src/backend/oneapi/memory.cpp
+++ b/src/backend/oneapi/memory.cpp
@@ -152,6 +152,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/oneapi/min.cpp b/src/backend/oneapi/min.cpp
index ea9900543c..fe1a5a3fa4 100644
--- a/src/backend/oneapi/min.cpp
+++ b/src/backend/oneapi/min.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_min_t, uint, uint)
 INSTANTIATE(af_min_t, intl, intl)
 INSTANTIATE(af_min_t, uintl, uintl)
 INSTANTIATE(af_min_t, char, char)
+INSTANTIATE(af_min_t, schar, schar)
 INSTANTIATE(af_min_t, uchar, uchar)
 INSTANTIATE(af_min_t, short, short)
 INSTANTIATE(af_min_t, ushort, ushort)
diff --git a/src/backend/oneapi/moments.cpp b/src/backend/oneapi/moments.cpp
index 50efe4ccd5..76e385990b 100644
--- a/src/backend/oneapi/moments.cpp
+++ b/src/backend/oneapi/moments.cpp
@@ -49,6 +49,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/morph.cpp b/src/backend/oneapi/morph.cpp
index 44fe6a6529..11f3d3df7a 100644
--- a/src/backend/oneapi/morph.cpp
+++ b/src/backend/oneapi/morph.cpp
@@ -62,6 +62,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/nearest_neighbour.cpp b/src/backend/oneapi/nearest_neighbour.cpp
index 7a34ba0fba..bec80b5cce 100644
--- a/src/backend/oneapi/nearest_neighbour.cpp
+++ b/src/backend/oneapi/nearest_neighbour.cpp
@@ -82,6 +82,7 @@ INSTANTIATE(intl, intl)
 INSTANTIATE(uintl, uintl)
 INSTANTIATE(short, int)
 INSTANTIATE(ushort, uint)
+INSTANTIATE(schar, int)
 INSTANTIATE(uchar, uint)
 
 INSTANTIATE(uintl, uint)  // For Hamming
diff --git a/src/backend/oneapi/plot.cpp b/src/backend/oneapi/plot.cpp
index d2fa041291..3bd287fbd6 100644
--- a/src/backend/oneapi/plot.cpp
+++ b/src/backend/oneapi/plot.cpp
@@ -78,6 +78,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
diff --git a/src/backend/oneapi/product.cpp b/src/backend/oneapi/product.cpp
index bc3f9421ae..4aa9cb61dd 100644
--- a/src/backend/oneapi/product.cpp
+++ b/src/backend/oneapi/product.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_mul_t, uint, uint)
 INSTANTIATE(af_mul_t, intl, intl)
 INSTANTIATE(af_mul_t, uintl, uintl)
 INSTANTIATE(af_mul_t, char, int)
+INSTANTIATE(af_mul_t, schar, int)
 INSTANTIATE(af_mul_t, uchar, uint)
 INSTANTIATE(af_mul_t, short, int)
 INSTANTIATE(af_mul_t, ushort, uint)
diff --git a/src/backend/oneapi/random_engine.cpp b/src/backend/oneapi/random_engine.cpp
index 7045dcc8cc..e3eac5da0b 100644
--- a/src/backend/oneapi/random_engine.cpp
+++ b/src/backend/oneapi/random_engine.cpp
@@ -92,6 +92,7 @@ INSTANTIATE_UNIFORM(uint)
 INSTANTIATE_UNIFORM(intl)
 INSTANTIATE_UNIFORM(uintl)
 INSTANTIATE_UNIFORM(char)
+INSTANTIATE_UNIFORM(schar)
 INSTANTIATE_UNIFORM(uchar)
 INSTANTIATE_UNIFORM(short)
 INSTANTIATE_UNIFORM(ushort)
diff --git a/src/backend/oneapi/range.cpp b/src/backend/oneapi/range.cpp
index caa8ed48bc..c08a7bea91 100644
--- a/src/backend/oneapi/range.cpp
+++ b/src/backend/oneapi/range.cpp
@@ -48,6 +48,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/reorder.cpp b/src/backend/oneapi/reorder.cpp
index d62db984e9..d9e264f70c 100644
--- a/src/backend/oneapi/reorder.cpp
+++ b/src/backend/oneapi/reorder.cpp
@@ -40,6 +40,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
diff --git a/src/backend/oneapi/reshape.cpp b/src/backend/oneapi/reshape.cpp
index 8f1b6f0ecb..2b15f686e9 100644
--- a/src/backend/oneapi/reshape.cpp
+++ b/src/backend/oneapi/reshape.cpp
@@ -50,6 +50,8 @@ Array<outType> reshape(const Array<inType> &in, const dim4 &outDims,
                                                 dim4 const &, short, double); \
     template Array<ushort> reshape<SRC_T, ushort>(                            \
         Array<SRC_T> const &, dim4 const &, ushort, double);                  \
+    template Array<schar> reshape<SRC_T, schar>(Array<SRC_T> const &,         \
+                                                dim4 const &, schar, double); \
     template Array<uchar> reshape<SRC_T, uchar>(Array<SRC_T> const &,         \
                                                 dim4 const &, uchar, double); \
     template Array<char> reshape<SRC_T, char>(Array<SRC_T> const &,           \
@@ -65,6 +67,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(half)
diff --git a/src/backend/oneapi/resize.cpp b/src/backend/oneapi/resize.cpp
index 005faf6b2b..b73f42eabb 100644
--- a/src/backend/oneapi/resize.cpp
+++ b/src/backend/oneapi/resize.cpp
@@ -40,6 +40,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/rotate.cpp b/src/backend/oneapi/rotate.cpp
index 10f1f93480..bcd7b5810a 100644
--- a/src/backend/oneapi/rotate.cpp
+++ b/src/backend/oneapi/rotate.cpp
@@ -50,6 +50,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/scan.cpp b/src/backend/oneapi/scan.cpp
index f7151ce076..9aaae59b49 100644
--- a/src/backend/oneapi/scan.cpp
+++ b/src/backend/oneapi/scan.cpp
@@ -45,6 +45,7 @@ Array<To> scan(const Array<Ti>& in, const int dim, bool inclusiveScan) {
     INSTANTIATE_SCAN(ROp, intl, intl)       \
     INSTANTIATE_SCAN(ROp, uintl, uintl)     \
     INSTANTIATE_SCAN(ROp, char, uint)       \
+    INSTANTIATE_SCAN(ROp, schar, int)       \
     INSTANTIATE_SCAN(ROp, uchar, uint)      \
     INSTANTIATE_SCAN(ROp, short, int)       \
     INSTANTIATE_SCAN(ROp, ushort, uint)
diff --git a/src/backend/oneapi/select.cpp b/src/backend/oneapi/select.cpp
index 8cb80c919d..b24b1fa340 100644
--- a/src/backend/oneapi/select.cpp
+++ b/src/backend/oneapi/select.cpp
@@ -128,6 +128,7 @@ INSTANTIATE(uint);
 INSTANTIATE(intl);
 INSTANTIATE(uintl);
 INSTANTIATE(char);
+INSTANTIATE(schar);
 INSTANTIATE(uchar);
 INSTANTIATE(short);
 INSTANTIATE(ushort);
diff --git a/src/backend/oneapi/set.cpp b/src/backend/oneapi/set.cpp
index 416efb4040..4c4b68e4b0 100644
--- a/src/backend/oneapi/set.cpp
+++ b/src/backend/oneapi/set.cpp
@@ -127,6 +127,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/shift.cpp b/src/backend/oneapi/shift.cpp
index 8a12eb81a8..7e5e31bf37 100644
--- a/src/backend/oneapi/shift.cpp
+++ b/src/backend/oneapi/shift.cpp
@@ -64,6 +64,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/sobel.cpp b/src/backend/oneapi/sobel.cpp
index 54ba117be7..e919a37b77 100644
--- a/src/backend/oneapi/sobel.cpp
+++ b/src/backend/oneapi/sobel.cpp
@@ -42,6 +42,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(int, int)
 INSTANTIATE(uint, int)
 INSTANTIATE(char, int)
+INSTANTIATE(schar, int)
 INSTANTIATE(uchar, int)
 INSTANTIATE(short, int)
 INSTANTIATE(ushort, int)
diff --git a/src/backend/oneapi/sort.cpp b/src/backend/oneapi/sort.cpp
index 4dc65a621c..9bfbeb9094 100644
--- a/src/backend/oneapi/sort.cpp
+++ b/src/backend/oneapi/sort.cpp
@@ -63,6 +63,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/sort_by_key.cpp b/src/backend/oneapi/sort_by_key.cpp
index 9ec60130cd..ba24249955 100644
--- a/src/backend/oneapi/sort_by_key.cpp
+++ b/src/backend/oneapi/sort_by_key.cpp
@@ -67,6 +67,7 @@ void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
     INSTANTIATE(Tk, short)   \
     INSTANTIATE(Tk, ushort)  \
     INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, schar)   \
     INSTANTIATE(Tk, uchar)   \
     INSTANTIATE(Tk, intl)    \
     INSTANTIATE(Tk, uintl)
@@ -78,6 +79,7 @@ INSTANTIATE1(uint)
 INSTANTIATE1(short)
 INSTANTIATE1(ushort)
 INSTANTIATE1(char)
+INSTANTIATE1(schar)
 INSTANTIATE1(uchar)
 INSTANTIATE1(intl)
 INSTANTIATE1(uintl)
diff --git a/src/backend/oneapi/sort_index.cpp b/src/backend/oneapi/sort_index.cpp
index 17de33fbad..a8c547f8a1 100644
--- a/src/backend/oneapi/sort_index.cpp
+++ b/src/backend/oneapi/sort_index.cpp
@@ -68,6 +68,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/sum.cpp b/src/backend/oneapi/sum.cpp
index fb20ce6121..990979ba25 100644
--- a/src/backend/oneapi/sum.cpp
+++ b/src/backend/oneapi/sum.cpp
@@ -29,6 +29,8 @@ INSTANTIATE(af_add_t, uintl, uintl)
 INSTANTIATE(af_add_t, uintl, double)
 INSTANTIATE(af_add_t, char, int)
 INSTANTIATE(af_add_t, char, float)
+INSTANTIATE(af_add_t, schar, int)
+INSTANTIATE(af_add_t, schar, float)
 INSTANTIATE(af_add_t, uchar, uint)
 INSTANTIATE(af_add_t, uchar, float)
 INSTANTIATE(af_add_t, short, int)
diff --git a/src/backend/oneapi/surface.cpp b/src/backend/oneapi/surface.cpp
index 2a8d604772..ac50627938 100644
--- a/src/backend/oneapi/surface.cpp
+++ b/src/backend/oneapi/surface.cpp
@@ -80,6 +80,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
diff --git a/src/backend/oneapi/susan.cpp b/src/backend/oneapi/susan.cpp
index 437259681c..b51acf13df 100644
--- a/src/backend/oneapi/susan.cpp
+++ b/src/backend/oneapi/susan.cpp
@@ -70,6 +70,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/tile.cpp b/src/backend/oneapi/tile.cpp
index aca96e4ec6..928d0e2b19 100644
--- a/src/backend/oneapi/tile.cpp
+++ b/src/backend/oneapi/tile.cpp
@@ -42,6 +42,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/transform.cpp b/src/backend/oneapi/transform.cpp
index 54b328f7fd..a277df9661 100644
--- a/src/backend/oneapi/transform.cpp
+++ b/src/backend/oneapi/transform.cpp
@@ -50,6 +50,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/transpose.cpp b/src/backend/oneapi/transpose.cpp
index 580573125f..1f41e96cde 100644
--- a/src/backend/oneapi/transpose.cpp
+++ b/src/backend/oneapi/transpose.cpp
@@ -43,6 +43,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/oneapi/transpose_inplace.cpp b/src/backend/oneapi/transpose_inplace.cpp
index ddbb14e419..013027f780 100644
--- a/src/backend/oneapi/transpose_inplace.cpp
+++ b/src/backend/oneapi/transpose_inplace.cpp
@@ -40,6 +40,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/oneapi/triangle.cpp b/src/backend/oneapi/triangle.cpp
index e418c15b93..c8ab5e2b16 100644
--- a/src/backend/oneapi/triangle.cpp
+++ b/src/backend/oneapi/triangle.cpp
@@ -49,6 +49,7 @@ INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/types.hpp b/src/backend/oneapi/types.hpp
index 4537f27987..395687396c 100644
--- a/src/backend/oneapi/types.hpp
+++ b/src/backend/oneapi/types.hpp
@@ -43,6 +43,7 @@ namespace oneapi {
 using cdouble = std::complex<double>;
 using cfloat  = std::complex<float>;
 using intl    = long long;
+using schar   = signed char;
 using uchar   = unsigned char;
 using uint    = unsigned int;
 using uintl   = unsigned long long;
@@ -95,6 +96,10 @@ inline const char *shortname<char>(bool caps) {
     return caps ? "J" : "j";
 }
 template<>
+inline const char *shortname<schar>(bool caps) {
+    return caps ? "A" : "a"; // TODO
+}
+template<>
 inline const char *shortname<uchar>(bool caps) {
     return caps ? "V" : "v";
 }
@@ -120,6 +125,11 @@ inline const char *getFullName() {
     return af::dtype_traits<T>::getName();
 }
 
+template<>
+inline const char *getFullName<schar>() {
+    return "signed char";
+}
+
 template<>
 inline const char *getFullName<cfloat>() {
     return "float2";
diff --git a/src/backend/oneapi/unwrap.cpp b/src/backend/oneapi/unwrap.cpp
index 15d60afe5d..bfc95e0f18 100644
--- a/src/backend/oneapi/unwrap.cpp
+++ b/src/backend/oneapi/unwrap.cpp
@@ -53,6 +53,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/oneapi/vector_field.cpp b/src/backend/oneapi/vector_field.cpp
index 92f310698a..d67fa73c51 100644
--- a/src/backend/oneapi/vector_field.cpp
+++ b/src/backend/oneapi/vector_field.cpp
@@ -31,6 +31,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace oneapi
diff --git a/src/backend/oneapi/where.cpp b/src/backend/oneapi/where.cpp
index bc9e45a515..fd08b975b8 100644
--- a/src/backend/oneapi/where.cpp
+++ b/src/backend/oneapi/where.cpp
@@ -36,6 +36,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/oneapi/wrap.cpp b/src/backend/oneapi/wrap.cpp
index 19e8c0260e..21c47ac007 100644
--- a/src/backend/oneapi/wrap.cpp
+++ b/src/backend/oneapi/wrap.cpp
@@ -44,6 +44,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/Array.cpp b/src/backend/opencl/Array.cpp
index b4b6bcd5a9..38fbfc4d84 100644
--- a/src/backend/opencl/Array.cpp
+++ b/src/backend/opencl/Array.cpp
@@ -585,6 +585,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index 5c920f44f8..a02ae6781d 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -454,6 +454,7 @@ target_sources(afopencl
     kernel/convolve/conv2_f32.cpp
     kernel/convolve/conv2_f64.cpp
     kernel/convolve/conv2_impl.hpp
+    kernel/convolve/conv2_s8.cpp
     kernel/convolve/conv2_s16.cpp
     kernel/convolve/conv2_s32.cpp
     kernel/convolve/conv2_s64.cpp
diff --git a/src/backend/opencl/all.cpp b/src/backend/opencl/all.cpp
index 2d2a1d4717..d81d9def34 100644
--- a/src/backend/opencl/all.cpp
+++ b/src/backend/opencl/all.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_and_t, uint, char)
 INSTANTIATE(af_and_t, intl, char)
 INSTANTIATE(af_and_t, uintl, char)
 INSTANTIATE(af_and_t, char, char)
+INSTANTIATE(af_and_t, schar, char)
 INSTANTIATE(af_and_t, uchar, char)
 INSTANTIATE(af_and_t, short, char)
 INSTANTIATE(af_and_t, ushort, char)
diff --git a/src/backend/opencl/any.cpp b/src/backend/opencl/any.cpp
index ce36f8ed90..ee2d16ab63 100644
--- a/src/backend/opencl/any.cpp
+++ b/src/backend/opencl/any.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_or_t, uint, char)
 INSTANTIATE(af_or_t, intl, char)
 INSTANTIATE(af_or_t, uintl, char)
 INSTANTIATE(af_or_t, char, char)
+INSTANTIATE(af_or_t, schar, char)
 INSTANTIATE(af_or_t, uchar, char)
 INSTANTIATE(af_or_t, short, char)
 INSTANTIATE(af_or_t, ushort, char)
diff --git a/src/backend/opencl/assign.cpp b/src/backend/opencl/assign.cpp
index 57ceeaab2d..fbe0370dde 100644
--- a/src/backend/opencl/assign.cpp
+++ b/src/backend/opencl/assign.cpp
@@ -104,6 +104,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/bilateral.cpp b/src/backend/opencl/bilateral.cpp
index 21ec82e2b6..6475377e75 100644
--- a/src/backend/opencl/bilateral.cpp
+++ b/src/backend/opencl/bilateral.cpp
@@ -34,6 +34,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(char, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uint, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/opencl/cast.hpp b/src/backend/opencl/cast.hpp
index 999d6188d9..cef1d76c0e 100644
--- a/src/backend/opencl/cast.hpp
+++ b/src/backend/opencl/cast.hpp
@@ -38,6 +38,11 @@ CAST_FN(uchar)
 CAST_FN(float)
 CAST_FN(double)
 
+template<typename Ti>
+struct CastOp<schar, Ti> {
+    const char *name() { return "convert_char"; }
+};
+
 #define CAST_CFN(TYPE)                                    \
     template<typename Ti>                                 \
     struct CastOp<TYPE, Ti> {                             \
diff --git a/src/backend/opencl/compile_module.cpp b/src/backend/opencl/compile_module.cpp
index 89d382c9c0..f0244b3b0d 100644
--- a/src/backend/opencl/compile_module.cpp
+++ b/src/backend/opencl/compile_module.cpp
@@ -81,6 +81,9 @@ const static string DEFAULT_MACROS_STR(
                                            #else\n                     \
                                            #define half short\n          \
                                            #endif\n                      \
+                                           #ifndef schar\n              \
+                                           #define schar char\n         \
+                                           #endif\n                     \
                                            #ifndef M_PI\n               \
                                            #define M_PI 3.1415926535897932384626433832795028841971693993751058209749445923078164\n \
                                            #endif\n                     \
diff --git a/src/backend/opencl/convolve.cpp b/src/backend/opencl/convolve.cpp
index f826102caf..34aa93b642 100644
--- a/src/backend/opencl/convolve.cpp
+++ b/src/backend/opencl/convolve.cpp
@@ -98,6 +98,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/opencl/convolve_separable.cpp b/src/backend/opencl/convolve_separable.cpp
index 03da468ac4..41b88b6ba8 100644
--- a/src/backend/opencl/convolve_separable.cpp
+++ b/src/backend/opencl/convolve_separable.cpp
@@ -65,6 +65,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(short, float)
diff --git a/src/backend/opencl/copy.cpp b/src/backend/opencl/copy.cpp
index 970deae518..97d54d432c 100644
--- a/src/backend/opencl/copy.cpp
+++ b/src/backend/opencl/copy.cpp
@@ -128,6 +128,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
@@ -157,6 +158,8 @@ INSTANTIATE(half)
                                           Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, ushort>(Array<ushort> & dst,       \
                                            Array<SRC_T> const &src);  \
+    template void copyArray<SRC_T, schar>(Array<schar> & dst,         \
+                                          Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, uchar>(Array<uchar> & dst,         \
                                           Array<SRC_T> const &src);   \
     template void copyArray<SRC_T, char>(Array<char> & dst,           \
@@ -170,6 +173,7 @@ INSTANTIATE_COPY_ARRAY(int)
 INSTANTIATE_COPY_ARRAY(uint)
 INSTANTIATE_COPY_ARRAY(intl)
 INSTANTIATE_COPY_ARRAY(uintl)
+INSTANTIATE_COPY_ARRAY(schar)
 INSTANTIATE_COPY_ARRAY(uchar)
 INSTANTIATE_COPY_ARRAY(char)
 INSTANTIATE_COPY_ARRAY(short)
@@ -201,6 +205,7 @@ INSTANTIATE_GETSCALAR(cfloat)
 INSTANTIATE_GETSCALAR(cdouble)
 INSTANTIATE_GETSCALAR(int)
 INSTANTIATE_GETSCALAR(uint)
+INSTANTIATE_GETSCALAR(schar)
 INSTANTIATE_GETSCALAR(uchar)
 INSTANTIATE_GETSCALAR(char)
 INSTANTIATE_GETSCALAR(intl)
diff --git a/src/backend/opencl/count.cpp b/src/backend/opencl/count.cpp
index 80f12e68cd..fe1b588f89 100644
--- a/src/backend/opencl/count.cpp
+++ b/src/backend/opencl/count.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_notzero_t, uint, uint)
 INSTANTIATE(af_notzero_t, intl, uint)
 INSTANTIATE(af_notzero_t, uintl, uint)
 INSTANTIATE(af_notzero_t, char, uint)
+INSTANTIATE(af_notzero_t, schar, uint)
 INSTANTIATE(af_notzero_t, uchar, uint)
 INSTANTIATE(af_notzero_t, short, uint)
 INSTANTIATE(af_notzero_t, ushort, uint)
diff --git a/src/backend/opencl/diagonal.cpp b/src/backend/opencl/diagonal.cpp
index 094906a77a..2d21b5f461 100644
--- a/src/backend/opencl/diagonal.cpp
+++ b/src/backend/opencl/diagonal.cpp
@@ -54,6 +54,7 @@ INSTANTIATE_DIAGONAL(uint)
 INSTANTIATE_DIAGONAL(intl)
 INSTANTIATE_DIAGONAL(uintl)
 INSTANTIATE_DIAGONAL(char)
+INSTANTIATE_DIAGONAL(schar)
 INSTANTIATE_DIAGONAL(uchar)
 INSTANTIATE_DIAGONAL(short)
 INSTANTIATE_DIAGONAL(ushort)
diff --git a/src/backend/opencl/diff.cpp b/src/backend/opencl/diff.cpp
index 020365d24c..e152301f0d 100644
--- a/src/backend/opencl/diff.cpp
+++ b/src/backend/opencl/diff.cpp
@@ -50,6 +50,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/opencl/exampleFunction.cpp b/src/backend/opencl/exampleFunction.cpp
index 10af977382..87306e329c 100644
--- a/src/backend/opencl/exampleFunction.cpp
+++ b/src/backend/opencl/exampleFunction.cpp
@@ -57,6 +57,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(cfloat)
diff --git a/src/backend/opencl/fast.cpp b/src/backend/opencl/fast.cpp
index bfe6c84177..4198cf82ba 100644
--- a/src/backend/opencl/fast.cpp
+++ b/src/backend/opencl/fast.cpp
@@ -53,6 +53,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/fftconvolve.cpp b/src/backend/opencl/fftconvolve.cpp
index f6b243baac..f5a875f41c 100644
--- a/src/backend/opencl/fftconvolve.cpp
+++ b/src/backend/opencl/fftconvolve.cpp
@@ -137,6 +137,7 @@ INSTANTIATE(double)
 INSTANTIATE(float)
 INSTANTIATE(uint)
 INSTANTIATE(int)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(uintl)
diff --git a/src/backend/opencl/flood_fill.cpp b/src/backend/opencl/flood_fill.cpp
index b57de824bd..4a759e095d 100644
--- a/src/backend/opencl/flood_fill.cpp
+++ b/src/backend/opencl/flood_fill.cpp
@@ -34,6 +34,7 @@ Array<T> floodFill(const Array<T>& image, const Array<uint>& seedsX,
 INSTANTIATE(float)
 INSTANTIATE(uint)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
diff --git a/src/backend/opencl/hist_graphics.cpp b/src/backend/opencl/hist_graphics.cpp
index 6c2a06e0b1..a20daeb700 100644
--- a/src/backend/opencl/hist_graphics.cpp
+++ b/src/backend/opencl/hist_graphics.cpp
@@ -74,6 +74,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
diff --git a/src/backend/opencl/histogram.cpp b/src/backend/opencl/histogram.cpp
index 7c3d432228..bbf7e9082e 100644
--- a/src/backend/opencl/histogram.cpp
+++ b/src/backend/opencl/histogram.cpp
@@ -41,6 +41,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/identity.cpp b/src/backend/opencl/identity.cpp
index 9d9ae55718..9aa72fc433 100644
--- a/src/backend/opencl/identity.cpp
+++ b/src/backend/opencl/identity.cpp
@@ -37,6 +37,7 @@ INSTANTIATE_IDENTITY(uint)
 INSTANTIATE_IDENTITY(intl)
 INSTANTIATE_IDENTITY(uintl)
 INSTANTIATE_IDENTITY(char)
+INSTANTIATE_IDENTITY(schar)
 INSTANTIATE_IDENTITY(uchar)
 INSTANTIATE_IDENTITY(short)
 INSTANTIATE_IDENTITY(ushort)
diff --git a/src/backend/opencl/image.cpp b/src/backend/opencl/image.cpp
index cffc2b8194..663fc63c24 100644
--- a/src/backend/opencl/image.cpp
+++ b/src/backend/opencl/image.cpp
@@ -78,6 +78,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/index.cpp b/src/backend/opencl/index.cpp
index d2864e6a81..b1cb238968 100644
--- a/src/backend/opencl/index.cpp
+++ b/src/backend/opencl/index.cpp
@@ -91,6 +91,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/iota.cpp b/src/backend/opencl/iota.cpp
index de69ca6595..87c840b419 100644
--- a/src/backend/opencl/iota.cpp
+++ b/src/backend/opencl/iota.cpp
@@ -39,6 +39,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/ireduce.cpp b/src/backend/opencl/ireduce.cpp
index ca4c916f63..d4b080389c 100644
--- a/src/backend/opencl/ireduce.cpp
+++ b/src/backend/opencl/ireduce.cpp
@@ -58,6 +58,7 @@ INSTANTIATE(af_min_t, uint)
 INSTANTIATE(af_min_t, intl)
 INSTANTIATE(af_min_t, uintl)
 INSTANTIATE(af_min_t, char)
+INSTANTIATE(af_min_t, schar)
 INSTANTIATE(af_min_t, uchar)
 INSTANTIATE(af_min_t, short)
 INSTANTIATE(af_min_t, ushort)
@@ -73,6 +74,7 @@ INSTANTIATE(af_max_t, uint)
 INSTANTIATE(af_max_t, intl)
 INSTANTIATE(af_max_t, uintl)
 INSTANTIATE(af_max_t, char)
+INSTANTIATE(af_max_t, schar)
 INSTANTIATE(af_max_t, uchar)
 INSTANTIATE(af_max_t, short)
 INSTANTIATE(af_max_t, ushort)
diff --git a/src/backend/opencl/join.cpp b/src/backend/opencl/join.cpp
index 22875d0e61..7975ecfb5a 100644
--- a/src/backend/opencl/join.cpp
+++ b/src/backend/opencl/join.cpp
@@ -227,6 +227,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(half)
@@ -247,6 +248,7 @@ INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(half)
diff --git a/src/backend/opencl/kernel/convolve/conv1.cpp b/src/backend/opencl/kernel/convolve/conv1.cpp
index 10ae600888..5bfa9668d6 100644
--- a/src/backend/opencl/kernel/convolve/conv1.cpp
+++ b/src/backend/opencl/kernel/convolve/conv1.cpp
@@ -58,6 +58,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/opencl/kernel/convolve/conv2_s8.cpp b/src/backend/opencl/kernel/convolve/conv2_s8.cpp
new file mode 100644
index 0000000000..b4b39b3f28
--- /dev/null
+++ b/src/backend/opencl/kernel/convolve/conv2_s8.cpp
@@ -0,0 +1,20 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+#include <kernel/convolve/conv2_impl.hpp>
+
+namespace arrayfire {
+namespace opencl {
+namespace kernel {
+
+INSTANTIATE(schar, float)
+
+}  // namespace kernel
+}  // namespace opencl
+}  // namespace arrayfire
diff --git a/src/backend/opencl/kernel/convolve/conv3.cpp b/src/backend/opencl/kernel/convolve/conv3.cpp
index 9a1baf9c6b..1383e8f443 100644
--- a/src/backend/opencl/kernel/convolve/conv3.cpp
+++ b/src/backend/opencl/kernel/convolve/conv3.cpp
@@ -45,6 +45,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/opencl/kernel/convolve_separable.cpp b/src/backend/opencl/kernel/convolve_separable.cpp
index 41bfa55dde..83a9116d72 100644
--- a/src/backend/opencl/kernel/convolve_separable.cpp
+++ b/src/backend/opencl/kernel/convolve_separable.cpp
@@ -95,6 +95,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(float, float)
 INSTANTIATE(uint, float)
 INSTANTIATE(int, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(char, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/opencl/kernel/random_engine_write.cl b/src/backend/opencl/kernel/random_engine_write.cl
index 8711987e44..c36c5f1d6d 100644
--- a/src/backend/opencl/kernel/random_engine_write.cl
+++ b/src/backend/opencl/kernel/random_engine_write.cl
@@ -27,6 +27,26 @@ float getFloatNegative11(uint num) {
 
 // Writes without boundary checking
 
+void writeOut128Bytes_schar(global char *out, uint index, uint r1, uint r2,
+                            uint r3, uint r4) {
+    out[index]                = r1;
+    out[index + THREADS]      = r1 >> 8;
+    out[index + 2 * THREADS]  = r1 >> 16;
+    out[index + 3 * THREADS]  = r1 >> 24;
+    out[index + 4 * THREADS]  = r2;
+    out[index + 5 * THREADS]  = r2 >> 8;
+    out[index + 6 * THREADS]  = r2 >> 16;
+    out[index + 7 * THREADS]  = r2 >> 24;
+    out[index + 8 * THREADS]  = r3;
+    out[index + 9 * THREADS]  = r3 >> 8;
+    out[index + 10 * THREADS] = r3 >> 16;
+    out[index + 11 * THREADS] = r3 >> 24;
+    out[index + 12 * THREADS] = r4;
+    out[index + 13 * THREADS] = r4 >> 8;
+    out[index + 14 * THREADS] = r4 >> 16;
+    out[index + 15 * THREADS] = r4 >> 24;
+}
+
 void writeOut128Bytes_uchar(global uchar *out, uint index, uint r1, uint r2,
                             uint r3, uint r4) {
     out[index]                = r1;
@@ -154,6 +174,36 @@ void boxMullerTransform(T *const out1, T *const out2, T r1, T r2) {
 
 // Writes with boundary checking
 
+void partialWriteOut128Bytes_schar(global char *out, uint index, uint r1,
+                                   uint r2, uint r3, uint r4, uint elements) {
+    if (index < elements) { out[index] = r1; }
+    if (index + THREADS < elements) { out[index + THREADS] = r1 >> 8; }
+    if (index + 2 * THREADS < elements) { out[index + 2 * THREADS] = r1 >> 16; }
+    if (index + 3 * THREADS < elements) { out[index + 3 * THREADS] = r1 >> 24; }
+    if (index + 4 * THREADS < elements) { out[index + 4 * THREADS] = r2; }
+    if (index + 5 * THREADS < elements) { out[index + 5 * THREADS] = r2 >> 8; }
+    if (index + 6 * THREADS < elements) { out[index + 6 * THREADS] = r2 >> 16; }
+    if (index + 7 * THREADS < elements) { out[index + 7 * THREADS] = r2 >> 24; }
+    if (index + 8 * THREADS < elements) { out[index + 8 * THREADS] = r3; }
+    if (index + 9 * THREADS < elements) { out[index + 9 * THREADS] = r3 >> 8; }
+    if (index + 10 * THREADS < elements) {
+        out[index + 10 * THREADS] = r3 >> 16;
+    }
+    if (index + 11 * THREADS < elements) {
+        out[index + 11 * THREADS] = r3 >> 24;
+    }
+    if (index + 12 * THREADS < elements) { out[index + 12 * THREADS] = r4; }
+    if (index + 13 * THREADS < elements) {
+        out[index + 13 * THREADS] = r4 >> 8;
+    }
+    if (index + 14 * THREADS < elements) {
+        out[index + 14 * THREADS] = r4 >> 16;
+    }
+    if (index + 15 * THREADS < elements) {
+        out[index + 15 * THREADS] = r4 >> 24;
+    }
+}
+
 void partialWriteOut128Bytes_uchar(global uchar *out, uint index, uint r1,
                                    uint r2, uint r3, uint r4, uint elements) {
     if (index < elements) { out[index] = r1; }
diff --git a/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp b/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp
index dd74cccc7e..dd14eee6c5 100644
--- a/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp
+++ b/src/backend/opencl/kernel/sort_by_key/sort_by_key_impl.cpp
@@ -9,7 +9,7 @@
 
 #include <kernel/sort_by_key_impl.hpp>
 
-// SBK_TYPES:float double int uint intl uintl short ushort char uchar half
+// SBK_TYPES:float double int uint intl uintl short ushort char schar uchar half
 
 namespace arrayfire {
 namespace opencl {
diff --git a/src/backend/opencl/kernel/sort_by_key_impl.hpp b/src/backend/opencl/kernel/sort_by_key_impl.hpp
index a070a60c67..f03721d01e 100644
--- a/src/backend/opencl/kernel/sort_by_key_impl.hpp
+++ b/src/backend/opencl/kernel/sort_by_key_impl.hpp
@@ -248,6 +248,7 @@ void sort0ByKey(Param pKey, Param pVal, bool isAscending) {
     INSTANTIATE(Tk, short)   \
     INSTANTIATE(Tk, ushort)  \
     INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, schar)   \
     INSTANTIATE(Tk, uchar)   \
     INSTANTIATE(Tk, intl)    \
     INSTANTIATE(Tk, uintl)   \
diff --git a/src/backend/opencl/lookup.cpp b/src/backend/opencl/lookup.cpp
index 2fee6f6ae0..36b5929f1f 100644
--- a/src/backend/opencl/lookup.cpp
+++ b/src/backend/opencl/lookup.cpp
@@ -53,6 +53,8 @@ Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
                                       const unsigned);                         \
     template Array<T> lookup<T, uintl>(const Array<T> &, const Array<uintl> &, \
                                        const unsigned);                        \
+    template Array<T> lookup<T, schar>(const Array<T> &, const Array<schar> &, \
+                                       const unsigned);                        \
     template Array<T> lookup<T, uchar>(const Array<T> &, const Array<uchar> &, \
                                        const unsigned);                        \
     template Array<T> lookup<T, half>(const Array<T> &, const Array<half> &,   \
@@ -66,6 +68,7 @@ INSTANTIATE(int);
 INSTANTIATE(unsigned);
 INSTANTIATE(intl);
 INSTANTIATE(uintl);
+INSTANTIATE(schar);
 INSTANTIATE(uchar);
 INSTANTIATE(char);
 INSTANTIATE(ushort);
diff --git a/src/backend/opencl/match_template.cpp b/src/backend/opencl/match_template.cpp
index f97bc6d353..7f02d886b3 100644
--- a/src/backend/opencl/match_template.cpp
+++ b/src/backend/opencl/match_template.cpp
@@ -37,6 +37,7 @@ INSTANTIATE(float, float)
 INSTANTIATE(char, float)
 INSTANTIATE(int, float)
 INSTANTIATE(uint, float)
+INSTANTIATE(schar, float)
 INSTANTIATE(uchar, float)
 INSTANTIATE(short, float)
 INSTANTIATE(ushort, float)
diff --git a/src/backend/opencl/max.cpp b/src/backend/opencl/max.cpp
index b2a2cdfdf0..695415517d 100644
--- a/src/backend/opencl/max.cpp
+++ b/src/backend/opencl/max.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_max_t, uint, uint)
 INSTANTIATE(af_max_t, intl, intl)
 INSTANTIATE(af_max_t, uintl, uintl)
 INSTANTIATE(af_max_t, char, char)
+INSTANTIATE(af_max_t, schar, schar)
 INSTANTIATE(af_max_t, uchar, uchar)
 INSTANTIATE(af_max_t, short, short)
 INSTANTIATE(af_max_t, ushort, ushort)
diff --git a/src/backend/opencl/mean.cpp b/src/backend/opencl/mean.cpp
index 7bd586e587..428c2812c3 100644
--- a/src/backend/opencl/mean.cpp
+++ b/src/backend/opencl/mean.cpp
@@ -59,6 +59,7 @@ INSTANTIATE(intl, double, double);
 INSTANTIATE(uintl, double, double);
 INSTANTIATE(short, float, float);
 INSTANTIATE(ushort, float, float);
+INSTANTIATE(schar, float, float);
 INSTANTIATE(uchar, float, float);
 INSTANTIATE(char, float, float);
 INSTANTIATE(cfloat, float, cfloat);
diff --git a/src/backend/opencl/meanshift.cpp b/src/backend/opencl/meanshift.cpp
index 3c6f140c98..9eaec9db9d 100644
--- a/src/backend/opencl/meanshift.cpp
+++ b/src/backend/opencl/meanshift.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/medfilt.cpp b/src/backend/opencl/medfilt.cpp
index 66a4c6969e..d3025a50b9 100644
--- a/src/backend/opencl/medfilt.cpp
+++ b/src/backend/opencl/medfilt.cpp
@@ -55,6 +55,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/memory.cpp b/src/backend/opencl/memory.cpp
index d2e0190431..7c69b33e24 100644
--- a/src/backend/opencl/memory.cpp
+++ b/src/backend/opencl/memory.cpp
@@ -162,6 +162,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/opencl/min.cpp b/src/backend/opencl/min.cpp
index 9cc6a09272..75c117caa8 100644
--- a/src/backend/opencl/min.cpp
+++ b/src/backend/opencl/min.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_min_t, uint, uint)
 INSTANTIATE(af_min_t, intl, intl)
 INSTANTIATE(af_min_t, uintl, uintl)
 INSTANTIATE(af_min_t, char, char)
+INSTANTIATE(af_min_t, schar, schar)
 INSTANTIATE(af_min_t, uchar, uchar)
 INSTANTIATE(af_min_t, short, short)
 INSTANTIATE(af_min_t, ushort, ushort)
diff --git a/src/backend/opencl/moments.cpp b/src/backend/opencl/moments.cpp
index 0b03d203c9..80afc2ece1 100644
--- a/src/backend/opencl/moments.cpp
+++ b/src/backend/opencl/moments.cpp
@@ -47,6 +47,7 @@ INSTANTIATE(float)
 INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/morph.cpp b/src/backend/opencl/morph.cpp
index e77b7a063c..a1cb86aa03 100644
--- a/src/backend/opencl/morph.cpp
+++ b/src/backend/opencl/morph.cpp
@@ -57,6 +57,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/nearest_neighbour.cpp b/src/backend/opencl/nearest_neighbour.cpp
index 535be4083f..615165a8e5 100644
--- a/src/backend/opencl/nearest_neighbour.cpp
+++ b/src/backend/opencl/nearest_neighbour.cpp
@@ -80,6 +80,7 @@ INSTANTIATE(intl, intl)
 INSTANTIATE(uintl, uintl)
 INSTANTIATE(short, int)
 INSTANTIATE(ushort, uint)
+INSTANTIATE(schar, int)
 INSTANTIATE(uchar, uint)
 
 INSTANTIATE(uintl, uint)  // For Hamming
diff --git a/src/backend/opencl/plot.cpp b/src/backend/opencl/plot.cpp
index cc7f93262e..5b7dfa69cb 100644
--- a/src/backend/opencl/plot.cpp
+++ b/src/backend/opencl/plot.cpp
@@ -75,6 +75,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
diff --git a/src/backend/opencl/product.cpp b/src/backend/opencl/product.cpp
index f13a9b9ae3..a949f87345 100644
--- a/src/backend/opencl/product.cpp
+++ b/src/backend/opencl/product.cpp
@@ -24,6 +24,7 @@ INSTANTIATE(af_mul_t, uint, uint)
 INSTANTIATE(af_mul_t, intl, intl)
 INSTANTIATE(af_mul_t, uintl, uintl)
 INSTANTIATE(af_mul_t, char, int)
+INSTANTIATE(af_mul_t, schar, int)
 INSTANTIATE(af_mul_t, uchar, uint)
 INSTANTIATE(af_mul_t, short, int)
 INSTANTIATE(af_mul_t, ushort, uint)
diff --git a/src/backend/opencl/random_engine.cpp b/src/backend/opencl/random_engine.cpp
index f2110c8be0..d307e54c2b 100644
--- a/src/backend/opencl/random_engine.cpp
+++ b/src/backend/opencl/random_engine.cpp
@@ -138,6 +138,7 @@ INSTANTIATE_UNIFORM(uint)
 INSTANTIATE_UNIFORM(intl)
 INSTANTIATE_UNIFORM(uintl)
 INSTANTIATE_UNIFORM(char)
+INSTANTIATE_UNIFORM(schar)
 INSTANTIATE_UNIFORM(uchar)
 INSTANTIATE_UNIFORM(short)
 INSTANTIATE_UNIFORM(ushort)
diff --git a/src/backend/opencl/range.cpp b/src/backend/opencl/range.cpp
index 92340d34eb..a49ba931c8 100644
--- a/src/backend/opencl/range.cpp
+++ b/src/backend/opencl/range.cpp
@@ -47,6 +47,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/reorder.cpp b/src/backend/opencl/reorder.cpp
index da485911e6..ecacccd677 100644
--- a/src/backend/opencl/reorder.cpp
+++ b/src/backend/opencl/reorder.cpp
@@ -40,6 +40,7 @@ INSTANTIATE(cfloat)
 INSTANTIATE(cdouble)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(intl)
diff --git a/src/backend/opencl/resize.cpp b/src/backend/opencl/resize.cpp
index ee7776b82f..bf3a8497b2 100644
--- a/src/backend/opencl/resize.cpp
+++ b/src/backend/opencl/resize.cpp
@@ -38,6 +38,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/rotate.cpp b/src/backend/opencl/rotate.cpp
index 46caa65c88..eab0c1da26 100644
--- a/src/backend/opencl/rotate.cpp
+++ b/src/backend/opencl/rotate.cpp
@@ -49,6 +49,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/scan.cpp b/src/backend/opencl/scan.cpp
index 0fc36366ef..649789ef91 100644
--- a/src/backend/opencl/scan.cpp
+++ b/src/backend/opencl/scan.cpp
@@ -43,6 +43,7 @@ Array<To> scan(const Array<Ti>& in, const int dim, bool inclusiveScan) {
     INSTANTIATE_SCAN(ROp, intl, intl)       \
     INSTANTIATE_SCAN(ROp, uintl, uintl)     \
     INSTANTIATE_SCAN(ROp, char, uint)       \
+    INSTANTIATE_SCAN(ROp, schar, int)       \
     INSTANTIATE_SCAN(ROp, uchar, uint)      \
     INSTANTIATE_SCAN(ROp, short, int)       \
     INSTANTIATE_SCAN(ROp, ushort, uint)
diff --git a/src/backend/opencl/select.cpp b/src/backend/opencl/select.cpp
index bbafbe989c..20c900007a 100644
--- a/src/backend/opencl/select.cpp
+++ b/src/backend/opencl/select.cpp
@@ -127,6 +127,7 @@ INSTANTIATE(uint);
 INSTANTIATE(intl);
 INSTANTIATE(uintl);
 INSTANTIATE(char);
+INSTANTIATE(schar);
 INSTANTIATE(uchar);
 INSTANTIATE(short);
 INSTANTIATE(ushort);
diff --git a/src/backend/opencl/set.cpp b/src/backend/opencl/set.cpp
index 195cf23047..1c1b74396c 100644
--- a/src/backend/opencl/set.cpp
+++ b/src/backend/opencl/set.cpp
@@ -147,6 +147,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/shift.cpp b/src/backend/opencl/shift.cpp
index 8b257f2c97..19e37286d3 100644
--- a/src/backend/opencl/shift.cpp
+++ b/src/backend/opencl/shift.cpp
@@ -64,6 +64,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/sobel.cpp b/src/backend/opencl/sobel.cpp
index e718021b42..a7651de07d 100644
--- a/src/backend/opencl/sobel.cpp
+++ b/src/backend/opencl/sobel.cpp
@@ -40,6 +40,7 @@ INSTANTIATE(double, double)
 INSTANTIATE(int, int)
 INSTANTIATE(uint, int)
 INSTANTIATE(char, int)
+INSTANTIATE(schar, int)
 INSTANTIATE(uchar, int)
 INSTANTIATE(short, int)
 INSTANTIATE(ushort, int)
diff --git a/src/backend/opencl/sort.cpp b/src/backend/opencl/sort.cpp
index 8b977316f1..e2bfcaa057 100644
--- a/src/backend/opencl/sort.cpp
+++ b/src/backend/opencl/sort.cpp
@@ -56,6 +56,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/sort_by_key.cpp b/src/backend/opencl/sort_by_key.cpp
index 2e4b2dd616..f1a89aef4d 100644
--- a/src/backend/opencl/sort_by_key.cpp
+++ b/src/backend/opencl/sort_by_key.cpp
@@ -69,6 +69,7 @@ void sort_by_key(Array<Tk> &okey, Array<Tv> &oval, const Array<Tk> &ikey,
     INSTANTIATE(Tk, short)   \
     INSTANTIATE(Tk, ushort)  \
     INSTANTIATE(Tk, char)    \
+    INSTANTIATE(Tk, schar)   \
     INSTANTIATE(Tk, uchar)   \
     INSTANTIATE(Tk, intl)    \
     INSTANTIATE(Tk, uintl)
@@ -80,6 +81,7 @@ INSTANTIATE1(uint)
 INSTANTIATE1(short)
 INSTANTIATE1(ushort)
 INSTANTIATE1(char)
+INSTANTIATE1(schar)
 INSTANTIATE1(uchar)
 INSTANTIATE1(intl)
 INSTANTIATE1(uintl)
diff --git a/src/backend/opencl/sort_index.cpp b/src/backend/opencl/sort_index.cpp
index 9c92f8406c..4840c24277 100644
--- a/src/backend/opencl/sort_index.cpp
+++ b/src/backend/opencl/sort_index.cpp
@@ -70,6 +70,7 @@ INSTANTIATE(double)
 INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/sum.cpp b/src/backend/opencl/sum.cpp
index 890280ba92..1ef26bdb89 100644
--- a/src/backend/opencl/sum.cpp
+++ b/src/backend/opencl/sum.cpp
@@ -29,6 +29,8 @@ INSTANTIATE(af_add_t, uintl, uintl)
 INSTANTIATE(af_add_t, uintl, double)
 INSTANTIATE(af_add_t, char, int)
 INSTANTIATE(af_add_t, char, float)
+INSTANTIATE(af_add_t, schar, int)
+INSTANTIATE(af_add_t, schar, float)
 INSTANTIATE(af_add_t, uchar, uint)
 INSTANTIATE(af_add_t, uchar, float)
 INSTANTIATE(af_add_t, short, int)
diff --git a/src/backend/opencl/surface.cpp b/src/backend/opencl/surface.cpp
index a0de95fb19..7a2e15276b 100644
--- a/src/backend/opencl/surface.cpp
+++ b/src/backend/opencl/surface.cpp
@@ -78,6 +78,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
diff --git a/src/backend/opencl/susan.cpp b/src/backend/opencl/susan.cpp
index 6bd78e2540..91b011120b 100644
--- a/src/backend/opencl/susan.cpp
+++ b/src/backend/opencl/susan.cpp
@@ -66,6 +66,7 @@ INSTANTIATE(double)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/tile.cpp b/src/backend/opencl/tile.cpp
index 14e2d5beac..98c7eb2bfb 100644
--- a/src/backend/opencl/tile.cpp
+++ b/src/backend/opencl/tile.cpp
@@ -41,6 +41,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/transform.cpp b/src/backend/opencl/transform.cpp
index 14ee03c962..78428ed3a7 100644
--- a/src/backend/opencl/transform.cpp
+++ b/src/backend/opencl/transform.cpp
@@ -49,6 +49,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/transpose.cpp b/src/backend/opencl/transpose.cpp
index a25fa9be28..248de43017 100644
--- a/src/backend/opencl/transpose.cpp
+++ b/src/backend/opencl/transpose.cpp
@@ -43,6 +43,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/opencl/transpose_inplace.cpp b/src/backend/opencl/transpose_inplace.cpp
index dc23873814..d6b783e5b2 100644
--- a/src/backend/opencl/transpose_inplace.cpp
+++ b/src/backend/opencl/transpose_inplace.cpp
@@ -39,6 +39,7 @@ INSTANTIATE(cdouble)
 INSTANTIATE(char)
 INSTANTIATE(int)
 INSTANTIATE(uint)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
diff --git a/src/backend/opencl/triangle.cpp b/src/backend/opencl/triangle.cpp
index cb781eeef4..346f8d1af7 100644
--- a/src/backend/opencl/triangle.cpp
+++ b/src/backend/opencl/triangle.cpp
@@ -47,6 +47,7 @@ INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
 INSTANTIATE(char)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/types.cpp b/src/backend/opencl/types.cpp
index 35c2b5745a..90393de3f9 100644
--- a/src/backend/opencl/types.cpp
+++ b/src/backend/opencl/types.cpp
@@ -95,6 +95,7 @@ INSTANTIATE(int);
 INSTANTIATE(uint);
 INSTANTIATE(intl);
 INSTANTIATE(uintl);
+INSTANTIATE(schar);
 INSTANTIATE(uchar);
 INSTANTIATE(char);
 INSTANTIATE(half);
diff --git a/src/backend/opencl/types.hpp b/src/backend/opencl/types.hpp
index 620ab74ca9..48985ab837 100644
--- a/src/backend/opencl/types.hpp
+++ b/src/backend/opencl/types.hpp
@@ -40,6 +40,7 @@ namespace opencl {
 using cdouble = cl_double2;
 using cfloat  = cl_float2;
 using intl    = long long;
+using schar   = cl_char;
 using uchar   = cl_uchar;
 using uint    = cl_uint;
 using uintl   = unsigned long long;
@@ -93,6 +94,10 @@ inline const char *shortname<char>(bool caps) {
     return caps ? "J" : "j";
 }
 template<>
+inline const char *shortname<schar>(bool caps) {
+    return caps ? "A" : "a"; // TODO
+}
+template<>
 inline const char *shortname<uchar>(bool caps) {
     return caps ? "V" : "v";
 }
@@ -118,6 +123,11 @@ inline const char *getFullName() {
     return af::dtype_traits<T>::getName();
 }
 
+template<>
+inline const char *getFullName<schar>() {
+    return "char";
+}
+
 template<>
 inline const char *getFullName<cfloat>() {
     return "float2";
diff --git a/src/backend/opencl/unwrap.cpp b/src/backend/opencl/unwrap.cpp
index c6c7a12d4f..3fb0d9a14c 100644
--- a/src/backend/opencl/unwrap.cpp
+++ b/src/backend/opencl/unwrap.cpp
@@ -53,6 +53,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/src/backend/opencl/vector_field.cpp b/src/backend/opencl/vector_field.cpp
index e470f73c9a..4d85032602 100644
--- a/src/backend/opencl/vector_field.cpp
+++ b/src/backend/opencl/vector_field.cpp
@@ -101,6 +101,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 
 }  // namespace opencl
diff --git a/src/backend/opencl/where.cpp b/src/backend/opencl/where.cpp
index c3ac797454..ae86cd8521 100644
--- a/src/backend/opencl/where.cpp
+++ b/src/backend/opencl/where.cpp
@@ -35,6 +35,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(short)
 INSTANTIATE(ushort)
diff --git a/src/backend/opencl/wrap.cpp b/src/backend/opencl/wrap.cpp
index 42d684857a..418dc9bc1f 100644
--- a/src/backend/opencl/wrap.cpp
+++ b/src/backend/opencl/wrap.cpp
@@ -42,6 +42,7 @@ INSTANTIATE(int)
 INSTANTIATE(uint)
 INSTANTIATE(intl)
 INSTANTIATE(uintl)
+INSTANTIATE(schar)
 INSTANTIATE(uchar)
 INSTANTIATE(char)
 INSTANTIATE(short)
diff --git a/test/anisotropic_diffusion.cpp b/test/anisotropic_diffusion.cpp
index 60e3c75324..a498d4cdd8 100644
--- a/test/anisotropic_diffusion.cpp
+++ b/test/anisotropic_diffusion.cpp
@@ -29,7 +29,7 @@ using std::vector;
 template<typename T>
 class AnisotropicDiffusion : public ::testing::Test {};
 
-typedef ::testing::Types<float, double, int, uint, uchar, short, ushort>
+typedef ::testing::Types<float, double, int, uint, schar, uchar, short, ushort>
     TestTypes;
 
 TYPED_TEST_SUITE(AnisotropicDiffusion, TestTypes);
diff --git a/test/array.cpp b/test/array.cpp
index b68f06820a..c5befe1fdb 100644
--- a/test/array.cpp
+++ b/test/array.cpp
@@ -21,8 +21,8 @@ using std::vector;
 template<typename T>
 class Array : public ::testing::Test {};
 
-typedef ::testing::Types<float, double, cfloat, cdouble, char, unsigned char,
-                         int, uint, intl, uintl, short, ushort,
+typedef ::testing::Types<float, double, cfloat, cdouble, char, signed char,
+                         unsigned char, int, uint, intl, uintl, short, ushort,
                          half_float::half>
     TestTypes;
 
@@ -302,6 +302,17 @@ TYPED_TEST(Array, TypeAttributes) {
             EXPECT_FALSE(one.isbool());
             EXPECT_FALSE(one.ishalf());
             break;
+        case s8:
+            EXPECT_FALSE(one.isfloating());
+            EXPECT_FALSE(one.isdouble());
+            EXPECT_FALSE(one.issingle());
+            EXPECT_FALSE(one.isrealfloating());
+            EXPECT_TRUE(one.isinteger());
+            EXPECT_TRUE(one.isreal());
+            EXPECT_FALSE(one.iscomplex());
+            EXPECT_FALSE(one.isbool());
+            EXPECT_FALSE(one.ishalf());
+            break;
         case u8:
             EXPECT_FALSE(one.isfloating());
             EXPECT_FALSE(one.isdouble());
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index db1f67a341..5b41f505d7 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -77,6 +77,7 @@ std::ostream &operator<<(std::ostream &os, af::dtype type) {
         case b8: name = "b8"; break;
         case s32: name = "s32"; break;
         case u32: name = "u32"; break;
+        case s8: name = "s8"; break;
         case u8: name = "u8"; break;
         case s64: name = "s64"; break;
         case u64: name = "u64"; break;
@@ -168,6 +169,9 @@ ::testing::AssertionResult assertArrayEq(std::string aName, std::string bName,
         case u32:
             return elemWiseEq<uint>(aName, bName, a, b, maxAbsDiff);
             break;
+        case s8:
+            return elemWiseEq<schar>(aName, bName, a, b, maxAbsDiff);
+            break;
         case u8:
             return elemWiseEq<uchar>(aName, bName, a, b, maxAbsDiff);
             break;
@@ -264,6 +268,7 @@ ::testing::AssertionResult assertImageEq(std::string aName, std::string bName,
                << "Expected: " << aName << "([" << a.dims() << "])";
 
     switch (arrDtype) {
+        case s8: return imageEq<signed char>(aName, bName, a, b, maxAbsDiff);
         case u8: return imageEq<unsigned char>(aName, bName, a, b, maxAbsDiff);
         case b8: return imageEq<char>(aName, bName, a, b, maxAbsDiff);
         case s32: return imageEq<int>(aName, bName, a, b, maxAbsDiff);
@@ -350,6 +355,7 @@ INSTANTIATE(double, float, int);
 INSTANTIATE(int, float, int);
 INSTANTIATE(unsigned int, float, int);
 INSTANTIATE(char, float, int);
+INSTANTIATE(signed char, float, int);
 INSTANTIATE(unsigned char, float, int);
 INSTANTIATE(short, float, int);
 INSTANTIATE(unsigned short, float, int);
@@ -364,6 +370,7 @@ INSTANTIATE(unsigned int, unsigned int, unsigned int);
 INSTANTIATE(long long, long long, int);
 INSTANTIATE(unsigned long long, unsigned long long, int);
 INSTANTIATE(char, char, int);
+INSTANTIATE(signed char, signed char, int);
 INSTANTIATE(unsigned char, unsigned char, int);
 INSTANTIATE(short, short, int);
 INSTANTIATE(unsigned short, unsigned short, int);
@@ -372,12 +379,19 @@ INSTANTIATE(af_half, af_half, int);
 INSTANTIATE(float, int, int);
 INSTANTIATE(unsigned int, int, int);
 INSTANTIATE(char, int, int);
+INSTANTIATE(signed char, int, int);
 INSTANTIATE(unsigned char, int, int);
 INSTANTIATE(short, int, int);
 INSTANTIATE(unsigned short, int, int);
 
+INSTANTIATE(signed char, unsigned short, int);
+INSTANTIATE(signed char, short, int);
+INSTANTIATE(signed char, unsigned char, int);
+INSTANTIATE(signed char, double, int);
+
 INSTANTIATE(unsigned char, unsigned short, int);
 INSTANTIATE(unsigned char, short, int);
+INSTANTIATE(unsigned char, signed char, int);
 INSTANTIATE(unsigned char, double, int);
 
 INSTANTIATE(long long, unsigned int, unsigned int);
@@ -386,6 +400,7 @@ INSTANTIATE(int, unsigned int, unsigned int);
 INSTANTIATE(short, unsigned int, unsigned int);
 INSTANTIATE(unsigned short, unsigned int, unsigned int);
 INSTANTIATE(char, unsigned int, unsigned int);
+INSTANTIATE(signed char, unsigned int, unsigned int);
 INSTANTIATE(unsigned char, unsigned int, unsigned int);
 INSTANTIATE(float, unsigned int, unsigned int);
 INSTANTIATE(double, unsigned int, unsigned int);
@@ -396,12 +411,14 @@ INSTANTIATE(int, unsigned int, int);
 INSTANTIATE(long long, unsigned int, int);
 INSTANTIATE(unsigned long long, unsigned int, int);
 INSTANTIATE(char, unsigned int, int);
+INSTANTIATE(signed char, unsigned int, int);
 INSTANTIATE(unsigned char, unsigned int, int);
 INSTANTIATE(short, unsigned int, int);
 INSTANTIATE(unsigned short, unsigned int, int);
 
 INSTANTIATE(float, char, int);
 INSTANTIATE(double, char, int);
+INSTANTIATE(signed char, char, int);
 INSTANTIATE(unsigned char, char, int);
 INSTANTIATE(short, char, int);
 INSTANTIATE(unsigned short, char, int);
@@ -412,6 +429,7 @@ INSTANTIATE(char, float, float);
 INSTANTIATE(int, float, float);
 INSTANTIATE(unsigned int, float, float);
 INSTANTIATE(short, float, float);
+INSTANTIATE(signed char, float, float);
 INSTANTIATE(unsigned char, float, float);
 INSTANTIATE(unsigned short, float, float);
 INSTANTIATE(double, float, float);
@@ -432,6 +450,7 @@ INSTANTIATE(unsigned int, unsigned int, float);
 INSTANTIATE(long long, long long, float);
 INSTANTIATE(unsigned long long, unsigned long long, float);
 INSTANTIATE(char, char, float);
+INSTANTIATE(signed char, signed char, float);
 INSTANTIATE(unsigned char, unsigned char, float);
 INSTANTIATE(short, short, float);
 INSTANTIATE(unsigned short, unsigned short, float);
@@ -448,6 +467,7 @@ INSTANTIATE(unsigned int, float, double);
 INSTANTIATE(short, float, double);
 INSTANTIATE(unsigned short, float, double);
 INSTANTIATE(char, float, double);
+INSTANTIATE(signed char, float, double);
 INSTANTIATE(unsigned char, float, double);
 INSTANTIATE(long long, double, double);
 INSTANTIATE(unsigned long long, double, double);
@@ -1356,6 +1376,7 @@ af_err conv_image(af_array *out, af_array in) {
 
 INSTANTIATE(float);
 INSTANTIATE(double);
+INSTANTIATE(signed char);
 INSTANTIATE(unsigned char);
 INSTANTIATE(half_float::half);
 INSTANTIATE(unsigned int);
@@ -1393,6 +1414,7 @@ af::array cpu_randu(const af::dim4 dims) {
 #define INSTANTIATE(To) template af::array cpu_randu<To>(const af::dim4 dims)
 INSTANTIATE(float);
 INSTANTIATE(double);
+INSTANTIATE(signed char);
 INSTANTIATE(unsigned char);
 INSTANTIATE(half_float::half);
 INSTANTIATE(unsigned int);
@@ -2001,6 +2023,7 @@ ::testing::AssertionResult assertRefEq(std::string hA_name,
 
 INSTANTIATE(float);
 INSTANTIATE(double);
+INSTANTIATE(signed char);
 INSTANTIATE(unsigned char);
 INSTANTIATE(half_float::half);
 INSTANTIATE(unsigned int);
diff --git a/test/arrayio.cpp b/test/arrayio.cpp
index 00d907a568..ea15165ac4 100644
--- a/test/arrayio.cpp
+++ b/test/arrayio.cpp
@@ -51,7 +51,8 @@ INSTANTIATE_TEST_SUITE_P(
                       type_params("s32", s32, 11), type_params("u32", u32, 12),
                       type_params("u8", u8, 13), type_params("b8", b8, 1),
                       type_params("s64", s64, 15), type_params("u64", u64, 16),
-                      type_params("s16", s16, 17), type_params("u16", u16, 18)),
+                      type_params("s16", s16, 17), type_params("u16", u16, 18),
+                      type_params("s8", s8, 19)),
     getTypeName);
 
 TEST_P(ArrayIOType, ReadType) {
@@ -103,6 +104,7 @@ TEST_P(ArrayIOType, ReadContent) {
         case c64: checkVals<af::cdouble>(arr, p.real, p.imag, p.type); break;
         case s32: checkVals<int>(arr, p.real, p.imag, p.type); break;
         case u32: checkVals<unsigned>(arr, p.real, p.imag, p.type); break;
+        case s8: checkVals<signed char>(arr, p.real, p.imag, p.type); break;
         case u8: checkVals<unsigned char>(arr, p.real, p.imag, p.type); break;
         case b8: checkVals<char>(arr, p.real, p.imag, p.type); break;
         case s64: checkVals<long long>(arr, p.real, p.imag, p.type); break;
diff --git a/test/assign.cpp b/test/assign.cpp
index cbfe6359b1..7b94bfa608 100644
--- a/test/assign.cpp
+++ b/test/assign.cpp
@@ -94,8 +94,8 @@ class ArrayAssign : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, cdouble, cfloat, double, int, uint, char, uchar,
-                         intl, uintl, short, ushort, half_float::half>
+typedef ::testing::Types<float, cdouble, cfloat, double, int, uint, char, schar,
+                         uchar, intl, uintl, short, ushort, half_float::half>
     TestTypes;
 
 // register the type list
diff --git a/test/bilateral.cpp b/test/bilateral.cpp
index f4ff949b55..12b27fc33f 100644
--- a/test/bilateral.cpp
+++ b/test/bilateral.cpp
@@ -73,7 +73,8 @@ TEST(BilateralOnImage, Color) {
 template<typename T>
 class BilateralOnData : public ::testing::Test {};
 
-typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
+typedef ::testing::Types<float, double, int, uint, char, schar, uchar, short,
+                         ushort>
     DataTestTypes;
 
 // register the type list
diff --git a/test/binary.cpp b/test/binary.cpp
index ed5b2c0869..3dbfa44bb9 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -496,6 +496,7 @@ INSTANTIATE_TEST_SUITE_P(
                       result_type_param(b8),
                       result_type_param(s32),
                       result_type_param(u32),
+                      result_type_param(s8),
                       result_type_param(u8),
                       result_type_param(s64),
                       result_type_param(u64),
@@ -515,6 +516,7 @@ INSTANTIATE_TEST_SUITE_P(
                       result_type_param(f32, b8, f32),
                       result_type_param(f32, s32, f32),
                       result_type_param(f32, u32, f32),
+                      result_type_param(f32, s8, f32),
                       result_type_param(f32, u8, f32),
                       result_type_param(f32, s64, f32),
                       result_type_param(f32, u64, f32),
@@ -535,6 +537,7 @@ INSTANTIATE_TEST_SUITE_P(
                       result_type_param(f64, b8,  f64),
                       result_type_param(f64, s32, f64),
                       result_type_param(f64, u32, f64),
+                      result_type_param(f64, s8,  f64),
                       result_type_param(f64, u8,  f64),
                       result_type_param(f64, s64, f64),
                       result_type_param(f64, u64, f64),
@@ -567,7 +570,8 @@ class ResultTypeScalar : public ::testing::Test {
 };
 
 typedef ::testing::Types<float, double, unsigned int, int, short,
-                         unsigned short, char, unsigned char, half_float::half>
+                         unsigned short, char, signed char, unsigned char,
+                         half_float::half>
     TestTypes;
 TYPED_TEST_SUITE(ResultTypeScalar, TestTypes);
 
diff --git a/test/canny.cpp b/test/canny.cpp
index a12ac73965..0a0fdbc08c 100644
--- a/test/canny.cpp
+++ b/test/canny.cpp
@@ -28,7 +28,7 @@ class CannyEdgeDetector : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, int, uint, short, ushort, uchar, double>
+typedef ::testing::Types<float, int, uint, short, ushort, schar, uchar, double>
     TestTypes;
 
 // register the type list
diff --git a/test/cast.cpp b/test/cast.cpp
index cb1f4e3f42..d2b4f95250 100644
--- a/test/cast.cpp
+++ b/test/cast.cpp
@@ -52,6 +52,7 @@ void cast_test() {
     REAL_TO_TESTS(Ti, char);     \
     REAL_TO_TESTS(Ti, int);      \
     REAL_TO_TESTS(Ti, unsigned); \
+    REAL_TO_TESTS(Ti, schar);    \
     REAL_TO_TESTS(Ti, uchar);    \
     REAL_TO_TESTS(Ti, intl);     \
     REAL_TO_TESTS(Ti, uintl);    \
@@ -67,6 +68,7 @@ REAL_TEST_INVOKE(double)
 REAL_TEST_INVOKE(char)
 REAL_TEST_INVOKE(int)
 REAL_TEST_INVOKE(unsigned)
+REAL_TEST_INVOKE(schar)
 REAL_TEST_INVOKE(uchar)
 REAL_TEST_INVOKE(intl)
 REAL_TEST_INVOKE(uintl)
diff --git a/test/clamp.cpp b/test/clamp.cpp
index 1e0b04b7c2..c830b06b2b 100644
--- a/test/clamp.cpp
+++ b/test/clamp.cpp
@@ -125,6 +125,7 @@ INSTANTIATE_TEST_SUITE_P(
                       clamp_params(dim4(10), f16, f16, f16, f16),
                       clamp_params(dim4(10), s32, f32, f32, f32),
                       clamp_params(dim4(10), u32, f32, f32, f32),
+                      clamp_params(dim4(10), s8,  f32, f32, f32),
                       clamp_params(dim4(10), u8,  f32, f32, f32),
                       clamp_params(dim4(10), b8,  f32, f32, f32),
                       clamp_params(dim4(10), s64, f32, f32, f32),
diff --git a/test/compare.cpp b/test/compare.cpp
index 66d9778039..877c08275f 100644
--- a/test/compare.cpp
+++ b/test/compare.cpp
@@ -23,8 +23,8 @@ using std::vector;
 template<typename T>
 class Compare : public ::testing::Test {};
 
-typedef ::testing::Types<float, double, uint, int, intl, uintl, uchar, short,
-                         ushort, half_float::half>
+typedef ::testing::Types<float, double, uint, int, intl, uintl, schar, uchar,
+                         short, ushort, half_float::half>
     TestTypes;
 TYPED_TEST_SUITE(Compare, TestTypes);
 
diff --git a/test/constant.cpp b/test/constant.cpp
index 0a75e3d974..b1d3e0a5af 100644
--- a/test/constant.cpp
+++ b/test/constant.cpp
@@ -31,7 +31,8 @@ template<typename T>
 class Constant : public ::testing::Test {};
 
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, char,
-                         uchar, uintl, intl, short, ushort, half_float::half>
+                         schar, uchar, uintl, intl, short, ushort,
+                         half_float::half>
     TestTypes;
 TYPED_TEST_SUITE(Constant, TestTypes);
 
diff --git a/test/convolve.cpp b/test/convolve.cpp
index ac731ef31c..5df8961e1b 100644
--- a/test/convolve.cpp
+++ b/test/convolve.cpp
@@ -33,8 +33,8 @@ class Convolve : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<cdouble, cfloat, float, double, int, uint, char, uchar,
-                         short, ushort, intl, uintl>
+typedef ::testing::Types<cdouble, cfloat, float, double, int, uint, char, schar,
+                         uchar, short, ushort, intl, uintl>
     TestTypes;
 
 // register the type list
diff --git a/test/corrcoef.cpp b/test/corrcoef.cpp
index 213a8de092..ffcecacd61 100644
--- a/test/corrcoef.cpp
+++ b/test/corrcoef.cpp
@@ -31,7 +31,8 @@ class CorrelationCoefficient : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, int, uint, intl, uintl, char, uchar>
+typedef ::testing::Types<float, double, int, uint, intl, uintl, char, schar,
+                         uchar>
     TestTypes;
 
 // register the type list
diff --git a/test/covariance.cpp b/test/covariance.cpp
index 4d4e4877f1..f149fbd095 100644
--- a/test/covariance.cpp
+++ b/test/covariance.cpp
@@ -34,8 +34,8 @@ class Covariance : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, int, uint, intl, uintl, uchar, short,
-                         ushort>
+typedef ::testing::Types<float, double, int, uint, intl, uintl, schar, uchar,
+                         short, ushort>
     TestTypes;
 
 // register the type list
@@ -65,9 +65,9 @@ template<typename T>
 struct covOutType {
     typedef typename cond_type<
         is_same_type<T, float>::value || is_same_type<T, int>::value ||
-            is_same_type<T, uint>::value || is_same_type<T, uchar>::value ||
-            is_same_type<T, short>::value || is_same_type<T, ushort>::value ||
-            is_same_type<T, char>::value,
+            is_same_type<T, uint>::value || is_same_type<T, schar>::value ||
+            is_same_type<T, uchar>::value || is_same_type<T, short>::value ||
+            is_same_type<T, ushort>::value || is_same_type<T, char>::value,
         float, typename elseType<T>::type>::type type;
 };
 
diff --git a/test/diagonal.cpp b/test/diagonal.cpp
index 1eecb883ae..e3031f731c 100644
--- a/test/diagonal.cpp
+++ b/test/diagonal.cpp
@@ -31,8 +31,8 @@ using std::vector;
 template<typename T>
 class Diagonal : public ::testing::Test {};
 
-typedef ::testing::Types<float, double, int, uint, char, unsigned char,
-                         half_float::half>
+typedef ::testing::Types<float, double, int, uint, char, signed char,
+                         unsigned char, half_float::half>
     TestTypes;
 TYPED_TEST_SUITE(Diagonal, TestTypes);
 
diff --git a/test/diff1.cpp b/test/diff1.cpp
index a7456fd0a2..9fdf11a91a 100644
--- a/test/diff1.cpp
+++ b/test/diff1.cpp
@@ -46,7 +46,7 @@ class Diff1 : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, intl,
-                         uintl, char, unsigned char, short, ushort>
+                         uintl, char, signed char, unsigned char, short, ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/diff2.cpp b/test/diff2.cpp
index c7c17f333f..cdc2b9909e 100644
--- a/test/diff2.cpp
+++ b/test/diff2.cpp
@@ -51,7 +51,7 @@ class Diff2 : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, intl,
-                         uintl, char, unsigned char, short, ushort>
+                         uintl, char, signed char, unsigned char, short, ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/dog.cpp b/test/dog.cpp
index 0b764f2c06..af76c23f59 100644
--- a/test/dog.cpp
+++ b/test/dog.cpp
@@ -33,7 +33,8 @@ class DOG : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
+typedef ::testing::Types<float, double, int, uint, char, schar, uchar, short,
+                         ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/fast.cpp b/test/fast.cpp
index 693c80db67..c5e3225d0e 100644
--- a/test/fast.cpp
+++ b/test/fast.cpp
@@ -61,7 +61,7 @@ class FixedFAST : public ::testing::Test {
 };
 
 typedef ::testing::Types<float, double> FloatTestTypes;
-typedef ::testing::Types<int, unsigned, short, ushort> FixedTestTypes;
+typedef ::testing::Types<int, unsigned, short, ushort, schar> FixedTestTypes;
 
 TYPED_TEST_SUITE(FloatFAST, FloatTestTypes);
 TYPED_TEST_SUITE(FixedFAST, FixedTestTypes);
diff --git a/test/fftconvolve.cpp b/test/fftconvolve.cpp
index 57d9398a04..a8f63e2f45 100644
--- a/test/fftconvolve.cpp
+++ b/test/fftconvolve.cpp
@@ -39,8 +39,8 @@ class FFTConvolveLarge : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<cfloat, cdouble, float, double, int, uint, char, uchar,
-                         intl, uintl>
+typedef ::testing::Types<cfloat, cdouble, float, double, int, uint, char, schar,
+                         uchar, intl, uintl>
     TestTypes;
 typedef ::testing::Types<float, double> TestTypesLarge;
 
diff --git a/test/gen_index.cpp b/test/gen_index.cpp
index 0716751fa0..fe684ebd27 100644
--- a/test/gen_index.cpp
+++ b/test/gen_index.cpp
@@ -108,8 +108,9 @@ INSTANTIATE_TEST_SUITE_P(
     ::testing::Combine(
         ::testing::Values(index_test(
             string(TEST_DIR "/gen_index/s0_3s0_1s1_2a.test"), dim4(4, 2, 2))),
-        ::testing::Values(f32, f64, c32, c64, u64, s64, u16, s16, u8, b8, f16),
-        ::testing::Values(f32, f64, u64, s64, u16, s16, u8, f16)),
+        ::testing::Values(f32, f64, c32, c64, u64, s64, u16, s16, s8, u8, b8,
+                          f16),
+        ::testing::Values(f32, f64, u64, s64, u16, s16, s8, u8, f16)),
     testNameGenerator);
 
 TEST_P(IndexGeneralizedLegacy, SSSA) {
diff --git a/test/half.cpp b/test/half.cpp
index 7f85950170..8afb6d5f4d 100644
--- a/test/half.cpp
+++ b/test/half.cpp
@@ -41,6 +41,7 @@ INSTANTIATE_TEST_SUITE_P(ToF16, HalfConvert,
                                            convert_params(f64, f16, 10),
                                            convert_params(s32, f16, 10),
                                            convert_params(u32, f16, 10),
+                                           convert_params(s8, f16, 10),
                                            convert_params(u8, f16, 10),
                                            convert_params(s64, f16, 10),
                                            convert_params(u64, f16, 10),
@@ -53,6 +54,7 @@ INSTANTIATE_TEST_SUITE_P(FromF16, HalfConvert,
                                            convert_params(f16, f64, 10),
                                            convert_params(f16, s32, 10),
                                            convert_params(f16, u32, 10),
+                                           convert_params(f16, s8, 10),
                                            convert_params(f16, u8, 10),
                                            convert_params(f16, s64, 10),
                                            convert_params(f16, u64, 10),
diff --git a/test/histogram.cpp b/test/histogram.cpp
index ca3df72f74..ea9431485c 100644
--- a/test/histogram.cpp
+++ b/test/histogram.cpp
@@ -33,7 +33,7 @@ class Histogram : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<half_float::half, float, double, int, uint, char,
-                         uchar, short, ushort, intl, uintl>
+                         schar, uchar, short, ushort, intl, uintl>
     TestTypes;
 
 // register the type list
diff --git a/test/index.cpp b/test/index.cpp
index c8e1a7ffb9..39491453e7 100644
--- a/test/index.cpp
+++ b/test/index.cpp
@@ -138,7 +138,7 @@ class Indexing1D : public ::testing::Test {
 };
 
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned,
-                         unsigned char, intl, uintl, short, ushort,
+                         signed char, unsigned char, intl, uintl, short, ushort,
                          half_float::half>
     AllTypes;
 TYPED_TEST_SUITE(Indexing1D, AllTypes);
@@ -710,8 +710,9 @@ class lookup : public ::testing::Test {
     virtual void SetUp() {}
 };
 
-typedef ::testing::Types<float, double, int, unsigned, unsigned char, short,
-                         ushort, intl, uintl, half_float::half>
+typedef ::testing::Types<float, double, int, unsigned, signed char,
+                         unsigned char, short, ushort, intl, uintl,
+                         half_float::half>
     ArrIdxTestTypes;
 TYPED_TEST_SUITE(lookup, ArrIdxTestTypes);
 
diff --git a/test/inverse_deconv.cpp b/test/inverse_deconv.cpp
index b6db793f4b..86ac2869ab 100644
--- a/test/inverse_deconv.cpp
+++ b/test/inverse_deconv.cpp
@@ -25,7 +25,7 @@ template<typename T>
 class InverseDeconvolution : public ::testing::Test {};
 
 // create a list of types to be tested
-typedef ::testing::Types<float, uchar, short, ushort> TestTypes;
+typedef ::testing::Types<float, schar, uchar, short, ushort> TestTypes;
 
 // register the type list
 TYPED_TEST_SUITE(InverseDeconvolution, TestTypes);
diff --git a/test/iota.cpp b/test/iota.cpp
index c776d7628e..33ff36e3ba 100644
--- a/test/iota.cpp
+++ b/test/iota.cpp
@@ -39,7 +39,8 @@ class Iota : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, int, unsigned int, intl, uintl,
-                         unsigned char, short, ushort, half_float::half>
+                         signed char, unsigned char, short, ushort,
+                         half_float::half>
     TestTypes;
 
 // register the type list
diff --git a/test/iterative_deconv.cpp b/test/iterative_deconv.cpp
index e59440b977..432c9ff533 100644
--- a/test/iterative_deconv.cpp
+++ b/test/iterative_deconv.cpp
@@ -25,7 +25,7 @@ template<typename T>
 class IterativeDeconvolution : public ::testing::Test {};
 
 // create a list of types to be tested
-typedef ::testing::Types<float, uchar, short, ushort> TestTypes;
+typedef ::testing::Types<float, schar, uchar, short, ushort> TestTypes;
 
 // register the type list
 TYPED_TEST_SUITE(IterativeDeconvolution, TestTypes);
diff --git a/test/join.cpp b/test/join.cpp
index 4d25e8a6ae..aef578bcf2 100644
--- a/test/join.cpp
+++ b/test/join.cpp
@@ -48,8 +48,8 @@ class Join : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
-                         intl, uintl, char, unsigned char, short, ushort,
-                         half_float::half>
+                         intl, uintl, char, signed char, unsigned char, short,
+                         ushort, half_float::half>
     TestTypes;
 
 // register the type list
diff --git a/test/match_template.cpp b/test/match_template.cpp
index 4ee8fc7e2d..f5f6eb4fc7 100644
--- a/test/match_template.cpp
+++ b/test/match_template.cpp
@@ -31,7 +31,8 @@ class MatchTemplate : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
+typedef ::testing::Types<float, double, int, uint, char, schar, uchar, short,
+                         ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/mean.cpp b/test/mean.cpp
index c9c6eb567b..79dd76db2d 100644
--- a/test/mean.cpp
+++ b/test/mean.cpp
@@ -40,7 +40,7 @@ class Mean : public ::testing::Test {
 // This list does not allow to cleanly add the af_half/half_float type : at the
 // moment half tested in some special unittests
 typedef ::testing::Types<cdouble, cfloat, float, double, int, uint, intl, uintl,
-                         char, uchar, short, ushort, half_float::half>
+                         char, schar, uchar, short, ushort, half_float::half>
     TestTypes;
 
 // register the type list
@@ -70,9 +70,9 @@ template<typename T>
 struct meanOutType {
     typedef typename cond_type<
         is_same_type<T, float>::value || is_same_type<T, int>::value ||
-            is_same_type<T, uint>::value || is_same_type<T, uchar>::value ||
-            is_same_type<T, short>::value || is_same_type<T, ushort>::value ||
-            is_same_type<T, char>::value,
+            is_same_type<T, uint>::value || is_same_type<T, schar>::value ||
+            is_same_type<T, uchar>::value || is_same_type<T, short>::value ||
+            is_same_type<T, ushort>::value || is_same_type<T, char>::value,
         float, typename elseType<T>::type>::type type;
 };
 
@@ -228,7 +228,7 @@ TEST(MeanAll, s32) { meanAllTest<int>(2, dim4(5, 5, 2, 2)); }
 
 TEST(MeanAll, u32) { meanAllTest<unsigned>(2, dim4(100, 1, 1, 1)); }
 
-TEST(MeanAll, s8) { meanAllTest<char>(2, dim4(5, 5, 2, 2)); }
+TEST(MeanAll, s8) { meanAllTest<schar>(2, dim4(5, 5, 2, 2)); }
 
 TEST(MeanAll, u8) { meanAllTest<uchar>(2, dim4(100, 1, 1, 1)); }
 
diff --git a/test/meanshift.cpp b/test/meanshift.cpp
index 1f0aa697b3..d91648ae52 100644
--- a/test/meanshift.cpp
+++ b/test/meanshift.cpp
@@ -28,8 +28,8 @@ class Meanshift : public ::testing::Test {
     virtual void SetUp() {}
 };
 
-typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort,
-                         intl, uintl>
+typedef ::testing::Types<float, double, int, uint, char, schar, uchar, short,
+                         ushort, intl, uintl>
     TestTypes;
 
 TYPED_TEST_SUITE(Meanshift, TestTypes);
diff --git a/test/medfilt.cpp b/test/medfilt.cpp
index 2d874cb3ae..5ef951d5b1 100644
--- a/test/medfilt.cpp
+++ b/test/medfilt.cpp
@@ -35,7 +35,8 @@ class MedianFilter1d : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
+typedef ::testing::Types<float, double, int, uint, char, schar, uchar, short,
+                         ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/memory.cpp b/test/memory.cpp
index 991756ca0b..9214ab472c 100644
--- a/test/memory.cpp
+++ b/test/memory.cpp
@@ -74,7 +74,8 @@ class MemAlloc : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
-                         intl, uintl, char, unsigned char, short, ushort>
+                         intl, uintl, char, signed char, unsigned char, short,
+                         ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/moddims.cpp b/test/moddims.cpp
index a7dea52a00..c8b98f05d1 100644
--- a/test/moddims.cpp
+++ b/test/moddims.cpp
@@ -36,8 +36,8 @@ class Moddims : public ::testing::Test {
 
 // create a list of types to be tested
 // TODO: complex types tests have to be added
-typedef ::testing::Types<float, double, int, unsigned, char, unsigned char,
-                         short, ushort, half_float::half>
+typedef ::testing::Types<float, double, int, unsigned, char, signed char,
+                         unsigned char, short, ushort, half_float::half>
     TestTypes;
 
 // register the type list
diff --git a/test/morph.cpp b/test/morph.cpp
index ad62ded8f3..b68d95076f 100644
--- a/test/morph.cpp
+++ b/test/morph.cpp
@@ -30,7 +30,8 @@ class Morph : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
+typedef ::testing::Types<float, double, int, uint, char, schar, uchar, short,
+                         ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/nearest_neighbour.cpp b/test/nearest_neighbour.cpp
index 2db885f566..82551bc31b 100644
--- a/test/nearest_neighbour.cpp
+++ b/test/nearest_neighbour.cpp
@@ -34,8 +34,8 @@ class NearestNeighbour : public ::testing::Test {
 };
 
 // create lists of types to be tested
-typedef ::testing::Types<float, double, int, uint, intl, uintl, uchar, short,
-                         ushort>
+typedef ::testing::Types<float, double, int, uint, intl, uintl, schar, uchar,
+                         short, ushort>
     TestTypes;
 
 template<typename T>
@@ -53,6 +53,11 @@ struct otype_t<ushort> {
     typedef uint otype;
 };
 
+template<>
+struct otype_t<schar> {
+    typedef int otype;
+};
+
 template<>
 struct otype_t<uchar> {
     typedef uint otype;
diff --git a/test/pad_borders.cpp b/test/pad_borders.cpp
index 028c946719..2642ed83ca 100644
--- a/test/pad_borders.cpp
+++ b/test/pad_borders.cpp
@@ -24,8 +24,8 @@ using std::vector;
 template<typename T>
 class PadBorders : public ::testing::Test {};
 
-typedef ::testing::Types<float, double, cfloat, cdouble, char, unsigned char,
-                         int, uint, intl, uintl, short,
+typedef ::testing::Types<float, double, cfloat, cdouble, char, signed char,
+                         unsigned char, int, uint, intl, uintl, short,
                          ushort /*, half_float::half*/>
     TestTypes;
 
diff --git a/test/random.cpp b/test/random.cpp
index d0860b70f2..f6fd0dd45f 100644
--- a/test/random.cpp
+++ b/test/random.cpp
@@ -36,7 +36,7 @@ class Random : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, intl,
-                         uintl, unsigned char, char, af_half>
+                         uintl, signed char, unsigned char, char, af_half>
     TestTypes;
 
 // register the type list
@@ -258,15 +258,15 @@ void testSetSeed(const uintl seed0, const uintl seed1) {
         ASSERT_EQ(h_in0[i], h_in2[i]) << "at : " << i;
 
         // Verify different arrays created with different seeds differ
-        // b8 and u9 can clash because they generate a small set of values
-        if (ty != b8 && ty != u8) {
+        // b8, s8 and u8 can clash because they generate a small set of values
+        if (ty != b8 && ty != s8 && ty != u8) {
             ASSERT_NE(h_in0[i], h_in1[i]) << "at : " << i;
         }
 
         // Verify different arrays created one after the other with same seed
-        // differ b8 and u9 can clash because they generate a small set of
+        // differ b8, s8 and u8 can clash because they generate a small set of
         // values
-        if (ty != b8 && ty != u8) {
+        if (ty != b8 && ty != s8 && ty != u8) {
             ASSERT_NE(h_in2[i], h_in3[i]) << "at : " << i;
         }
     }
@@ -394,7 +394,7 @@ void testRandomEngineSeed(randomEngineType type) {
 
     for (int i = 0; i < elem; i++) {
         ASSERT_EQ(h1[i], h3[i]) << "at : " << i;
-        if (ty != b8 && ty != u8) {
+        if (ty != b8 && ty != s8 && ty != u8) {
             ASSERT_NE(h1[i], h2[i]) << "at : " << i;
             ASSERT_NE(h3[i], h4[i]) << "at : " << i;
         }
diff --git a/test/range.cpp b/test/range.cpp
index 35708bde09..0e708160c2 100644
--- a/test/range.cpp
+++ b/test/range.cpp
@@ -46,12 +46,13 @@ class RangeMax : public Range<T> {};
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, int, unsigned int, intl, uintl,
-                         unsigned char, short, ushort, half_float::half>
+                         signed char, unsigned char, short, ushort,
+                         half_float::half>
     AllTypes;
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, int, unsigned int, intl, uintl,
-                         unsigned char, short, ushort>
+                         signed char, unsigned char, short, ushort>
     RegularTypes;
 
 // register the type list
diff --git a/test/reduce.cpp b/test/reduce.cpp
index 0a36431a54..0d4ab59225 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -40,7 +40,7 @@ template<typename T>
 class ReduceByKey : public ::testing::Test {};
 
 typedef ::testing::Types<float, double, cfloat, cdouble, uint, int, intl, uintl,
-                         uchar, short, ushort>
+                         schar, uchar, short, ushort>
     TestTypes;
 TYPED_TEST_SUITE(Reduce, TestTypes);
 TYPED_TEST_SUITE(ReduceByKey, TestTypes);
@@ -126,6 +126,10 @@ struct promote_type {
 
 // char and uchar are promoted to int for sum and product
 template<>
+struct promote_type<schar, af_sum> {
+    typedef int type;
+};
+template<>
 struct promote_type<uchar, af_sum> {
     typedef uint type;
 };
@@ -142,6 +146,10 @@ struct promote_type<ushort, af_sum> {
     typedef uint type;
 };
 template<>
+struct promote_type<schar, af_product> {
+    typedef int type;
+};
+template<>
 struct promote_type<uchar, af_product> {
     typedef uint type;
 };
@@ -389,6 +397,7 @@ array ptrToArray(size_t size, void *ptr, af_dtype type) {
         case u16: res = array(size, (unsigned short *)ptr); break;
         case s16: res = array(size, (short *)ptr); break;
         case b8: res = array(size, (char *)ptr); break;
+        case s8: res = array(size, (signed char *)ptr); break;
         case u8: res = array(size, (unsigned char *)ptr); break;
         case f16: res = array(size, (half_float::half *)ptr); break;
     }
@@ -409,6 +418,7 @@ array ptrToArray(af::dim4 size, void *ptr, af_dtype type) {
         case u16: res = array(size, (unsigned short *)ptr); break;
         case s16: res = array(size, (short *)ptr); break;
         case b8: res = array(size, (char *)ptr); break;
+        case s8: res = array(size, (signed char *)ptr); break;
         case u8: res = array(size, (unsigned char *)ptr); break;
         case f16: res = array(size, (half_float::half *)ptr); break;
     }
diff --git a/test/reorder.cpp b/test/reorder.cpp
index b06f72cdda..3109839786 100644
--- a/test/reorder.cpp
+++ b/test/reorder.cpp
@@ -44,7 +44,7 @@ class Reorder : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
-                         char, unsigned char, short, ushort>
+                         char, signed char, unsigned char, short, ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/replace.cpp b/test/replace.cpp
index 6d72cf7fc9..1156731732 100644
--- a/test/replace.cpp
+++ b/test/replace.cpp
@@ -35,7 +35,7 @@ template<typename T>
 class Replace : public ::testing::Test {};
 
 typedef ::testing::Types<half_float::half, float, double, cfloat, cdouble, uint,
-                         int, intl, uintl, uchar, char, short, ushort>
+                         int, intl, uintl, schar, uchar, char, short, ushort>
     TestTypes;
 
 TYPED_TEST_SUITE(Replace, TestTypes);
diff --git a/test/resize.cpp b/test/resize.cpp
index 423bb55416..50c46730f9 100644
--- a/test/resize.cpp
+++ b/test/resize.cpp
@@ -55,8 +55,8 @@ class ResizeI : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble> TestTypesF;
-typedef ::testing::Types<int, unsigned, intl, uintl, unsigned char, char, short,
-                         ushort>
+typedef ::testing::Types<int, unsigned, intl, uintl, signed char, unsigned char,
+                         char, short, ushort>
     TestTypesI;
 
 // register the type list
diff --git a/test/rotate.cpp b/test/rotate.cpp
index 01675fa1d7..986398f88f 100644
--- a/test/rotate.cpp
+++ b/test/rotate.cpp
@@ -34,7 +34,8 @@ class Rotate : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, cfloat, cdouble, int, intl, char, short>
+typedef ::testing::Types<float, double, cfloat, cdouble, int, intl, char, schar,
+                         short>
     TestTypes;
 
 // register the type list
diff --git a/test/rotate_linear.cpp b/test/rotate_linear.cpp
index ea19f217e7..84276a3755 100644
--- a/test/rotate_linear.cpp
+++ b/test/rotate_linear.cpp
@@ -39,7 +39,8 @@ class RotateLinear : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, cfloat, cdouble, int, intl, char, short>
+typedef ::testing::Types<float, double, cfloat, cdouble, int, intl, schar, char,
+                         short>
     TestTypes;
 
 // register the type list
diff --git a/test/sat.cpp b/test/sat.cpp
index 892e2f8f4e..f87b356b85 100644
--- a/test/sat.cpp
+++ b/test/sat.cpp
@@ -31,8 +31,8 @@ class SAT : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, int, uint, char, uchar, uintl, intl,
-                         short, ushort>
+typedef ::testing::Types<float, double, int, uint, char, schar, uchar, uintl,
+                         intl, short, ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/select.cpp b/test/select.cpp
index 0b6724d8fa..4b4c96dd21 100644
--- a/test/select.cpp
+++ b/test/select.cpp
@@ -42,7 +42,7 @@ template<typename T>
 class Select : public ::testing::Test {};
 
 typedef ::testing::Types<float, double, cfloat, cdouble, uint, int, intl, uintl,
-                         uchar, char, short, ushort, half_float::half>
+                         schar, uchar, char, short, ushort, half_float::half>
     TestTypes;
 TYPED_TEST_SUITE(Select, TestTypes);
 
diff --git a/test/shift.cpp b/test/shift.cpp
index 2de341b3bc..c86c43c8e3 100644
--- a/test/shift.cpp
+++ b/test/shift.cpp
@@ -42,7 +42,8 @@ class Shift : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
-                         intl, uintl, char, unsigned char, short, ushort>
+                         intl, uintl, char, signed char, unsigned char, short,
+                         ushort>
     TestTypes;
 // register the type list
 TYPED_TEST_SUITE(Shift, TestTypes);
diff --git a/test/sobel.cpp b/test/sobel.cpp
index 84fae1d34c..72a70ddde3 100644
--- a/test/sobel.cpp
+++ b/test/sobel.cpp
@@ -35,7 +35,8 @@ class Sobel_Integer : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double> TestTypes;
-typedef ::testing::Types<int, unsigned, char, unsigned char, short, ushort>
+typedef ::testing::Types<int, unsigned, char, signed char, unsigned char, short,
+                         ushort>
     TestTypesInt;
 
 // register the type list
diff --git a/test/sort.cpp b/test/sort.cpp
index c9da609f93..bd60edb5b5 100644
--- a/test/sort.cpp
+++ b/test/sort.cpp
@@ -40,8 +40,8 @@ class Sort : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, uint, int, uchar, short, ushort, intl,
-                         uintl>
+typedef ::testing::Types<float, double, uint, int, schar, uchar, short, ushort,
+                         intl, uintl>
     TestTypes;
 
 // register the type list
diff --git a/test/sort_by_key.cpp b/test/sort_by_key.cpp
index afd7908660..265ee570b7 100644
--- a/test/sort_by_key.cpp
+++ b/test/sort_by_key.cpp
@@ -40,8 +40,8 @@ class SortByKey : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, uint, int, uchar, short, ushort, intl,
-                         uintl>
+typedef ::testing::Types<float, double, uint, int, schar, uchar, short, ushort,
+                         intl, uintl>
     TestTypes;
 
 // register the type list
diff --git a/test/sort_index.cpp b/test/sort_index.cpp
index f3a10b9084..5e1b88a97d 100644
--- a/test/sort_index.cpp
+++ b/test/sort_index.cpp
@@ -40,8 +40,8 @@ class SortIndex : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, uint, int, uchar, short, ushort, intl,
-                         uintl>
+typedef ::testing::Types<float, double, uint, int, schar, uchar, short, ushort,
+                         intl, uintl>
     TestTypes;
 
 // register the type list
diff --git a/test/stdev.cpp b/test/stdev.cpp
index 4b93f5b220..bf95801fed 100644
--- a/test/stdev.cpp
+++ b/test/stdev.cpp
@@ -37,7 +37,8 @@ class StandardDev : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, double, int, uint, intl, uintl, char, uchar>
+typedef ::testing::Types<float, double, int, uint, intl, uintl, char, schar,
+                         uchar>
     TestTypes;
 
 // register the type list
@@ -67,9 +68,9 @@ template<typename T>
 struct sdOutType {
     typedef typename cond_type<
         is_same_type<T, float>::value || is_same_type<T, int>::value ||
-            is_same_type<T, uint>::value || is_same_type<T, uchar>::value ||
-            is_same_type<T, short>::value || is_same_type<T, ushort>::value ||
-            is_same_type<T, char>::value,
+            is_same_type<T, uint>::value || is_same_type<T, schar>::value ||
+            is_same_type<T, uchar>::value || is_same_type<T, short>::value ||
+            is_same_type<T, ushort>::value || is_same_type<T, char>::value,
         float, typename elseType<T>::type>::type type;
 };
 
diff --git a/test/susan.cpp b/test/susan.cpp
index 3741dd2653..c488bda775 100644
--- a/test/susan.cpp
+++ b/test/susan.cpp
@@ -59,7 +59,8 @@ class Susan : public ::testing::Test {
     virtual void SetUp() {}
 };
 
-typedef ::testing::Types<float, double, int, uint, char, uchar, short, ushort>
+typedef ::testing::Types<float, double, int, uint, char, schar, uchar, short,
+                         ushort>
     TestTypes;
 
 TYPED_TEST_SUITE(Susan, TestTypes);
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 4e9496f601..5f6b02b5a4 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -88,6 +88,7 @@ struct dtype_traits<half_float::half> {
 
 }  // namespace af
 
+typedef signed char schar;
 typedef unsigned char uchar;
 typedef unsigned int uint;
 typedef unsigned short ushort;
diff --git a/test/tile.cpp b/test/tile.cpp
index bc0cdddba7..3a608fa987 100644
--- a/test/tile.cpp
+++ b/test/tile.cpp
@@ -47,8 +47,8 @@ class Tile : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
-                         intl, uintl, char, unsigned char, short, ushort,
-                         half_float::half>
+                         intl, uintl, char, signed char, unsigned char, short,
+                         ushort, half_float::half>
     TestTypes;
 
 // register the type list
diff --git a/test/transform.cpp b/test/transform.cpp
index e3e0efe640..ef3b0dd4f9 100644
--- a/test/transform.cpp
+++ b/test/transform.cpp
@@ -38,7 +38,7 @@ class TransformInt : public ::testing::Test {
 };
 
 typedef ::testing::Types<float, double> TestTypes;
-typedef ::testing::Types<int, intl, uint, uintl, short, ushort, uchar>
+typedef ::testing::Types<int, intl, uint, uintl, short, ushort, schar, uchar>
     TestTypesInt;
 
 TYPED_TEST_SUITE(Transform, TestTypes);
diff --git a/test/translate.cpp b/test/translate.cpp
index 55fd570ffb..edbab15a2c 100644
--- a/test/translate.cpp
+++ b/test/translate.cpp
@@ -39,7 +39,7 @@ class TranslateInt : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble> TestTypes;
-typedef ::testing::Types<int, intl, char, short> TestTypesInt;
+typedef ::testing::Types<int, intl, char, schar, short> TestTypesInt;
 
 // register the type list
 TYPED_TEST_SUITE(Translate, TestTypes);
diff --git a/test/transpose.cpp b/test/transpose.cpp
index 72a32194fa..420f6d88e3 100644
--- a/test/transpose.cpp
+++ b/test/transpose.cpp
@@ -44,8 +44,8 @@ class Transpose : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, cfloat, double, cdouble, int, uint, char, uchar,
-                         short, ushort, half_float::half>
+typedef ::testing::Types<float, cfloat, double, cdouble, int, uint, char, schar,
+                         uchar, short, ushort, half_float::half>
     TestTypes;
 
 // register the type list
diff --git a/test/transpose_inplace.cpp b/test/transpose_inplace.cpp
index 82b071488a..7e542fd34f 100644
--- a/test/transpose_inplace.cpp
+++ b/test/transpose_inplace.cpp
@@ -30,8 +30,8 @@ class Transpose : public ::testing::Test {
 };
 
 // create a list of types to be tested
-typedef ::testing::Types<float, cfloat, double, cdouble, int, uint, char, uchar,
-                         short, ushort>
+typedef ::testing::Types<float, cfloat, double, cdouble, int, uint, char, schar,
+                         uchar, short, ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/triangle.cpp b/test/triangle.cpp
index 90b50bb6dc..a7d47832e5 100644
--- a/test/triangle.cpp
+++ b/test/triangle.cpp
@@ -35,7 +35,8 @@ template<typename T>
 class Triangle : public ::testing::Test {};
 
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, char,
-                         uchar, uintl, intl, short, ushort, half_float::half>
+                         schar, uchar, uintl, intl, short, ushort,
+                         half_float::half>
     TestTypes;
 TYPED_TEST_SUITE(Triangle, TestTypes);
 
diff --git a/test/unwrap.cpp b/test/unwrap.cpp
index f43b73e7f4..9b97059dac 100644
--- a/test/unwrap.cpp
+++ b/test/unwrap.cpp
@@ -37,7 +37,8 @@ class Unwrap : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
-                         intl, uintl, char, unsigned char, short, ushort>
+                         intl, uintl, char, signed char, unsigned char, short,
+                         ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/var.cpp b/test/var.cpp
index db846f5d57..b889413646 100644
--- a/test/var.cpp
+++ b/test/var.cpp
@@ -26,7 +26,7 @@ template<typename T>
 class Var : public ::testing::Test {};
 
 typedef ::testing::Types<float, double, cfloat, cdouble, uint, int, uintl, intl,
-                         char, uchar, short, ushort, half_float::half>
+                         char, schar, uchar, short, ushort, half_float::half>
     TestTypes;
 TYPED_TEST_SUITE(Var, TestTypes);
 
@@ -42,8 +42,8 @@ struct varOutType {
     typedef typename cond_type<
         is_same_type<T, float>::value || is_same_type<T, int>::value ||
             is_same_type<T, uint>::value || is_same_type<T, short>::value ||
-            is_same_type<T, ushort>::value || is_same_type<T, uchar>::value ||
-            is_same_type<T, char>::value,
+            is_same_type<T, ushort>::value || is_same_type<T, schar>::value ||
+            is_same_type<T, uchar>::value || is_same_type<T, char>::value,
         float, typename elseType<T>::type>::type type;
 };
 
diff --git a/test/where.cpp b/test/where.cpp
index bb5375822c..265c0d4d7b 100644
--- a/test/where.cpp
+++ b/test/where.cpp
@@ -34,7 +34,7 @@ template<typename T>
 class Where : public ::testing::Test {};
 
 typedef ::testing::Types<float, double, cfloat, cdouble, int, uint, intl, uintl,
-                         char, uchar, short, ushort>
+                         char, schar, uchar, short, ushort>
     TestTypes;
 TYPED_TEST_SUITE(Where, TestTypes);
 
diff --git a/test/wrap.cpp b/test/wrap.cpp
index baff77c5b1..4f53d9fd34 100644
--- a/test/wrap.cpp
+++ b/test/wrap.cpp
@@ -42,7 +42,8 @@ class Wrap : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, double, cfloat, cdouble, int, unsigned int,
-                         intl, uintl, char, unsigned char, short, ushort>
+                         intl, uintl, char, signed char, unsigned char, short,
+                         ushort>
     TestTypes;
 
 // register the type list
diff --git a/test/write.cpp b/test/write.cpp
index 8f18f6e954..db751939ab 100644
--- a/test/write.cpp
+++ b/test/write.cpp
@@ -34,7 +34,7 @@ class Write : public ::testing::Test {
 
 // create a list of types to be tested
 typedef ::testing::Types<float, cfloat, double, cdouble, int, unsigned, char,
-                         unsigned char, short, ushort>
+                         signed char, unsigned char, short, ushort>
     TestTypes;
 
 // register the type list

From 1636b5ae1d559dff995bda7448b11782ae3df36b Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Fri, 6 Oct 2023 10:14:22 -0400
Subject: [PATCH 431/473] fix image loading for s8 tests

---
 src/api/c/imageio.cpp   |  4 ++--
 src/api/c/imageio2.cpp  |  5 ++---
 test/arrayfire_test.cpp | 13 ++++++++++---
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/api/c/imageio.cpp b/src/api/c/imageio.cpp
index 0f87e4df17..be5f528922 100644
--- a/src/api/c/imageio.cpp
+++ b/src/api/c/imageio.cpp
@@ -75,7 +75,7 @@ static af_err readImage(af_array* rImage, const uchar* pSrcLine,
             if (fo_color == 1) {
                 pDst0[indx] = static_cast<T>(*(src + (x * step)));
             } else if (fo_color >= 3) {
-                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) { // FIXME s8?
+                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) {
                     pDst0[indx] =
                         static_cast<float>(*(src + (x * step + FI_RGBA_RED)));
                     pDst1[indx] =
@@ -201,7 +201,7 @@ static af_err readImage(af_array* rImage, const uchar* pSrcLine,
             if (fo_color == 1) {
                 pDst[indx] = static_cast<T>(*(src + (x * step)));
             } else if (fo_color >= 3) {
-                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) { // FIXME s8?
+                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) {
                     r = *(src + (x * step + FI_RGBA_RED));
                     g = *(src + (x * step + FI_RGBA_GREEN));
                     b = *(src + (x * step + FI_RGBA_BLUE));
diff --git a/src/api/c/imageio2.cpp b/src/api/c/imageio2.cpp
index 4a00212207..7130202397 100644
--- a/src/api/c/imageio2.cpp
+++ b/src/api/c/imageio2.cpp
@@ -71,7 +71,7 @@ static af_err readImage_t(af_array* rImage, const uchar* pSrcLine,
             if (fi_color == 1) {
                 pDst0[indx] = *(src + (x * step));
             } else if (fi_color >= 3) {
-                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) { // FIXME s8?
+                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) {
                     pDst0[indx] = *(src + (x * step + FI_RGBA_RED));
                     pDst1[indx] = *(src + (x * step + FI_RGBA_GREEN));
                     pDst2[indx] = *(src + (x * step + FI_RGBA_BLUE));
@@ -102,7 +102,6 @@ static af_err readImage_t(af_array* rImage, const uchar* pSrcLine,
 }
 
 FREE_IMAGE_TYPE getFIT(FI_CHANNELS channels, af_dtype type) {
-    // FIXME s8?
     if (channels == AFFI_GRAY) {
         if (type == u8) { return FIT_BITMAP; }
         if (type == u16) {
@@ -365,7 +364,7 @@ static void save_t(T* pDstLine, const af_array in, const dim4& dims,
             if (channels == 1) {
                 *(pDstLine + x * step) = pSrc0[indx];  // r -> 0
             } else if (channels >= 3) {
-                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) { // FIXME s8?
+                if (static_cast<af_dtype>(af::dtype_traits<T>::af_type) == u8) {
                     *(pDstLine + x * step + FI_RGBA_RED) =
                         pSrc0[indx];  // r -> 0
                     *(pDstLine + x * step + FI_RGBA_GREEN) =
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index 5b41f505d7..b1b82813de 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -1358,10 +1358,17 @@ af_err conv_image(af_array *out, af_array in) {
 
     T *out_data = new T[nElems];
 
-    for (int i = 0; i < (int)nElems; i++) out_data[i] = (T)in_data[i];
+    af_dtype out_type = (af_dtype)af::dtype_traits<T>::af_type
+    for (int i = 0; i < (int)nElems; i++) {
+        if (out_type == s8) {
+            // shift to avoid overflow
+            out_data[i] = (T)(std::trunc(in_data[i]) - 128.f);
+        } else {
+            out_data[i] = (T)in_data[i];
+        }
+    }
 
-    af_create_array(&outArray, out_data, idims.ndims(), idims.get(),
-                    (af_dtype)af::dtype_traits<T>::af_type);
+    af_create_array(&outArray, out_data, idims.ndims(), idims.get(), out_type);
 
     std::swap(*out, outArray);
 

From 1a0c305e1adb33d1e0120fb95cd7ece6c1fd2441 Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Sat, 14 Oct 2023 16:02:04 -0400
Subject: [PATCH 432/473] skip Richardson-Lucy test for s8

Image loading with a shift causes the test data to contain negative values.
This test passes when the data is limited to 0-127 instead.
---
 test/iterative_deconv.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/iterative_deconv.cpp b/test/iterative_deconv.cpp
index 432c9ff533..290b81f0d6 100644
--- a/test/iterative_deconv.cpp
+++ b/test/iterative_deconv.cpp
@@ -40,6 +40,11 @@ void iterDeconvImageTest(string pTestFile, const unsigned iters, const float rf,
     SUPPORTED_TYPE_CHECK(T);
     IMAGEIO_ENABLED_CHECK();
 
+    if (is_same_type<T, schar>::value &&
+        algo == AF_ITERATIVE_DECONV_RICHARDSONLUCY) {
+        GTEST_SKIP() << "Incompatible with signed values";
+    }
+
     using af::dim4;
 
     vector<dim4> inDims;

From 3eadfdc9aeb3be6ed66613162335393092420232 Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Sat, 14 Oct 2023 16:03:48 -0400
Subject: [PATCH 433/473] skip rotate_linear tests for s8

This test data cannot be trivially shifted since rotate sets out-of-bounds
values to 0, not -128. Limiting the range to 0-127 mostly works, but introduces
rounding errors between output and gold.
---
 test/rotate_linear.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/test/rotate_linear.cpp b/test/rotate_linear.cpp
index 84276a3755..1324a59a77 100644
--- a/test/rotate_linear.cpp
+++ b/test/rotate_linear.cpp
@@ -54,6 +54,10 @@ void rotateTest(string pTestFile, const unsigned resultIdx, const float angle,
                 const vector<af_seq>* seqv = NULL) {
     SUPPORTED_TYPE_CHECK(T);
 
+    if (is_same_type<T, schar>::value && (int)angle % 90 != 0) {
+        GTEST_SKIP() << "Incompatible test data for s8";
+    }
+
     vector<dim4> numDims;
     vector<vector<T>> in;
     vector<vector<T>> tests;

From 42dfafaa28f084bb4936270a38f49ec1deeff87d Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Sat, 14 Oct 2023 16:13:10 -0400
Subject: [PATCH 434/473] define s8 interface for AF_API_VERSION 310

This also slightly extends the interface macros.
---
 include/af/array.h    | 78 ++++++++++++++++++++++++++++++-------------
 include/af/defines.h  |  4 ++-
 include/af/traits.hpp |  2 ++
 3 files changed, 59 insertions(+), 25 deletions(-)

diff --git a/include/af/array.h b/include/af/array.h
index a442147565..672c2716eb 100644
--- a/include/af/array.h
+++ b/include/af/array.h
@@ -71,7 +71,7 @@ namespace af
             operator array() const;
             operator array();
 
-#define ASSIGN(OP)                                                  \
+#define ASSIGN_(OP)                                                 \
             array_proxy& operator OP(const array_proxy &a);         \
             array_proxy& operator OP(const array &a);               \
             array_proxy& operator OP(const double &a);              \
@@ -82,24 +82,31 @@ namespace af
             array_proxy& operator OP(const unsigned &a);            \
             array_proxy& operator OP(const bool &a);                \
             array_proxy& operator OP(const char &a);                \
-            array_proxy& operator OP(const signed char &a);         \
             array_proxy& operator OP(const unsigned char &a);       \
             array_proxy& operator OP(const long  &a);               \
             array_proxy& operator OP(const unsigned long &a);       \
             array_proxy& operator OP(const long long  &a);          \
             array_proxy& operator OP(const unsigned long long &a);
 
-            ASSIGN(=)
-            ASSIGN(+=)
-            ASSIGN(-=)
-            ASSIGN(*=)
-            ASSIGN(/=)
-#undef ASSIGN
-
 #if AF_API_VERSION >= 32
-#define ASSIGN(OP)                                                  \
+#define ASSIGN_32(OP)                                               \
             array_proxy& operator OP(const short &a);               \
             array_proxy& operator OP(const unsigned short &a);
+#else
+#define ASSIGN_32(OP)
+#endif
+
+#if AF_API_VERSION >= 310
+#define ASSIGN_310(OP)                                              \
+            array_proxy& operator OP(const signed char &a);
+#else
+#define ASSIGN_310(OP)
+#endif
+
+#define ASSIGN(OP)              \
+            ASSIGN_(OP)         \
+            ASSIGN_32(OP)       \
+            ASSIGN_310(OP)
 
             ASSIGN(=)
             ASSIGN(+=)
@@ -107,7 +114,9 @@ namespace af
             ASSIGN(*=)
             ASSIGN(/=)
 #undef ASSIGN
-#endif
+#undef ASSIGN_
+#undef ASSIGN_32
+#undef ASSIGN_310
 
             // af::array member functions. same behavior as those below
             af_array get();
@@ -948,7 +957,7 @@ namespace af
 
         /// \brief Casts the array into another data type
         ///
-        /// \note Consecitive casting operations may be may be optimized out if
+        /// \note Consecutive casting operations may be optimized out if
         /// the original type of the af::array is the same as the final type.
         /// For example if the original type is f64 which is then cast to f32
         /// and then back to f64, then the cast to f32 will be skipped and that
@@ -1000,24 +1009,31 @@ namespace af
         array& OP2(const unsigned &val);            /**< \copydoc OP2##(const array &) */ \
         array& OP2(const bool &val);                /**< \copydoc OP2##(const array &) */ \
         array& OP2(const char &val);                /**< \copydoc OP2##(const array &) */ \
-        array& OP2(const signed char &val);         /**< \copydoc OP2##(const array &) */ \
         array& OP2(const unsigned char &val);       /**< \copydoc OP2##(const array &) */ \
         array& OP2(const long  &val);               /**< \copydoc OP2##(const array &) */ \
         array& OP2(const unsigned long &val);       /**< \copydoc OP2##(const array &) */ \
         array& OP2(const long long  &val);          /**< \copydoc OP2##(const array &) */ \
         array& OP2(const unsigned long long &val);
 
-
 #if AF_API_VERSION >= 32
-#define ASSIGN(OP)                                                                        \
-        ASSIGN_(OP)                                                                       \
-          array& OP(const short  &val);              /**< \copydoc OP##(const array &) */ \
-          array& OP(const unsigned short &val);
+#define ASSIGN_32(OP)                                                                    \
+        array& OP(const short  &val);               /**< \copydoc OP##(const array &) */ \
+        array& OP(const unsigned short &val);
+#else
+#define ASSIGN_32(OP)
+#endif
 
+#if AF_API_VERSION >= 310
+#define ASSIGN_310(OP)                                                                   \
+        array& OP(const signed char &val);          /**< \copydoc OP##(const array &) */
 #else
-#define ASSIGN(OP) ASSIGN_(OP)
+#define ASSIGN_310(OP)
 #endif
 
+#define ASSIGN(OP)          \
+        ASSIGN_(OP)         \
+        ASSIGN_32(OP)       \
+        ASSIGN_310(OP)
 
         /// \ingroup array_mem_operator_eq
         /// @{
@@ -1083,6 +1099,8 @@ namespace af
 
 #undef ASSIGN
 #undef ASSIGN_
+#undef ASSIGN_32
+#undef ASSIGN_310
 
         ///
         /// \brief Negates the values of the array
@@ -1147,7 +1165,6 @@ namespace af
     AFAPI array OP (const int& lhs, const array& rhs);                /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const unsigned& lhs, const array& rhs);           /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const char& lhs, const array& rhs);               /**< \copydoc OP##(const array&, const array&) */ \
-    AFAPI array OP (const signed char& lhs, const array& rhs);        /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const unsigned char& lhs, const array& rhs);      /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const long& lhs, const array& rhs);               /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const unsigned long& lhs, const array& rhs);      /**< \copydoc OP##(const array&, const array&) */ \
@@ -1161,7 +1178,6 @@ namespace af
     AFAPI array OP (const array& lhs, const int& rhs);                /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const unsigned& rhs);           /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const char& rhs);               /**< \copydoc OP##(const array&, const array&) */ \
-    AFAPI array OP (const array& lhs, const signed char& rhs);        /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const unsigned char& rhs);      /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const long& rhs);               /**< \copydoc OP##(const array&, const array&) */ \
     AFAPI array OP (const array& lhs, const unsigned long& rhs);      /**< \copydoc OP##(const array&, const array&) */ \
@@ -1173,17 +1189,29 @@ namespace af
     AFAPI array OP (const array& lhs, const cdouble& rhs);
 
 #if AF_API_VERSION >= 32
-#define BIN_OP(OP)                                                                                                        \
-        BIN_OP_(OP)                                                                                                       \
+#define BIN_OP_32(OP)                                                                                                    \
         AFAPI array OP (const short& lhs, const array& rhs);           /**< \copydoc OP##(const array&, const array&) */ \
         AFAPI array OP (const unsigned short& lhs, const array& rhs);  /**< \copydoc OP##(const array&, const array&) */ \
         AFAPI array OP (const array& lhs, const short& rhs);           /**< \copydoc OP##(const array&, const array&) */ \
         AFAPI array OP (const array& lhs, const unsigned short& rhs);
 
 #else
-#define BIN_OP(OP) BIN_OP_(OP)
+#define BIN_OP_32(OP)
+#endif
+
+#if AF_API_VERSION >= 310
+#define BIN_OP_310(OP)                                                                                                  \
+    AFAPI array OP (const signed char& lhs, const array& rhs);        /**< \copydoc OP##(const array&, const array&) */ \
+    AFAPI array OP (const array& lhs, const signed char& rhs);        /**< \copydoc OP##(const array&, const array&) */
+#else
+#define BIN_OP_310(OP)
 #endif
 
+#define BIN_OP(OP)          \
+        BIN_OP_(OP)         \
+        BIN_OP_32(OP)       \
+        BIN_OP_310(OP)
+
     /// \ingroup arith_func_add
     /// @{
     /// \brief Adds two arrays or an array and a value.
@@ -1377,6 +1405,8 @@ namespace af
 
 #undef BIN_OP
 #undef BIN_OP_
+#undef BIN_OP_32
+#undef BIN_OP_310
 
     /// \ingroup arith_func_bitand
     /// @{
diff --git a/include/af/defines.h b/include/af/defines.h
index 4be88f97bd..42f71024fa 100644
--- a/include/af/defines.h
+++ b/include/af/defines.h
@@ -227,7 +227,9 @@ typedef enum {
 #if AF_API_VERSION >= 37
     , f16    ///< 16-bit floating point value
 #endif
-    , s8     ///< 8-bit signed integral value /// TODO AF_API_VERSION
+#if AF_API_VERSION >= 310
+    , s8     ///< 8-bit signed integral values
+#endif
 } af_dtype;
 
 typedef enum {
diff --git a/include/af/traits.hpp b/include/af/traits.hpp
index 330435a929..4216c3f046 100644
--- a/include/af/traits.hpp
+++ b/include/af/traits.hpp
@@ -176,6 +176,7 @@ struct dtype_traits<half> {
 };
 #endif
 
+#if AF_API_VERSION >= 310
 template<>
 struct dtype_traits<signed char> {
     enum {
@@ -185,6 +186,7 @@ struct dtype_traits<signed char> {
     typedef signed char base_type;
     static const char* getName() { return "schar"; }
 };
+#endif
 }
 
 #endif

From 0e259ccb5c36707616bcd53908909bb83de7b696 Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Mon, 16 Oct 2023 14:30:49 -0400
Subject: [PATCH 435/473] add s8 to documentation

---
 docs/details/algorithm.dox    | 10 +++++-----
 docs/details/image.dox        |  2 ++
 docs/pages/README.md          |  2 +-
 docs/pages/getting_started.md |  3 ++-
 4 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/docs/details/algorithm.dox b/docs/details/algorithm.dox
index 055750098c..69633524e2 100644
--- a/docs/details/algorithm.dox
+++ b/docs/details/algorithm.dox
@@ -22,7 +22,7 @@ Input Type          | Output Type
 --------------------|---------------------
 f32, f64, c32, c64  | same as input
 s32, s64, u32, u64  | same as input
-s16                 | s32
+s16, s8             | s32
 u16, u8, b8         | u32
 
 \copydoc batch_detail_algo
@@ -54,7 +54,7 @@ Input Type          | Output Type
 --------------------|---------------------
 f32, f64, c32, c64  | same as input
 s32, s64, u32, u64  | same as input
-s16                 | s32
+s16, s8             | s32
 u16, u8, b8         | u32
 f16                 | f32
 
@@ -76,7 +76,7 @@ Input Type          | Output Type
 --------------------|---------------------
 f32, f64, c32, c64  | same as input
 s32, u32, s64, u64  | same as input
-s16                 | s32
+s16, s8             | s32
 u16, u8, b8         | u32
 
 \copydoc batch_detail_algo
@@ -108,7 +108,7 @@ Input Type          | Output Type
 --------------------|---------------------
 f32, f64, c32, c64  | same as input
 s32, u32, s64, u64  | same as input
-s16                 | s32
+s16, s8             | s32
 u16, u8, b8         | u32
 f16                 | f32
 
@@ -340,7 +340,7 @@ Input Type          | Output Type
 --------------------|---------------------
 f32, f64, c32, c64  | same as input
 s32, s64, u32, u64  | same as input
-s16                 | s32
+s16, s8             | s32
 u16, u8, b8         | u32
 
 \copydoc batch_detail_algo
diff --git a/docs/details/image.dox b/docs/details/image.dox
index a93f1ebaed..312b88c880 100644
--- a/docs/details/image.dox
+++ b/docs/details/image.dox
@@ -1007,6 +1007,7 @@ Iterative deconvolution function excepts \ref af::array of the following types o
     - \ref f32
     - \ref s16
     - \ref u16
+    - \ref s8
     - \ref u8
 
 \note The type of output \ref af::array from deconvolution will be double if
@@ -1044,6 +1045,7 @@ Inverse deconvolution function excepts \ref af::array of the following types onl
     - \ref f32
     - \ref s16
     - \ref u16
+    - \ref s8
     - \ref u8
 
 \note The type of output \ref af::array from deconvolution will be double
diff --git a/docs/pages/README.md b/docs/pages/README.md
index 7c22adf87c..6ecb68ce4e 100644
--- a/docs/pages/README.md
+++ b/docs/pages/README.md
@@ -52,7 +52,7 @@ vectors, matrices, volumes, and
 
 It supports common [data types](\ref gettingstarted_datatypes), including
 single and double precision floating point values, complex numbers, booleans,
-and 32-bit signed and unsigned integers.
+and 8/16/32-bit signed and unsigned integers.
 
 #### Extending ArrayFire
 
diff --git a/docs/pages/getting_started.md b/docs/pages/getting_started.md
index 19660f8cc8..2bd3b4d1f6 100644
--- a/docs/pages/getting_started.md
+++ b/docs/pages/getting_started.md
@@ -28,7 +28,8 @@ can represent one of many different [basic data types](\ref af_dtype):
 * [b8](\ref b8) 8-bit boolean values (`bool`)
 * [s32](\ref s32) 32-bit signed integer (`int`)
 * [u32](\ref u32) 32-bit unsigned integer (`unsigned`)
-* [u8](\ref u8) 8-bit unsigned values (`unsigned char`)
+* [s8](\ref s8) 8-bit signed integer (`signed char`)
+* [u8](\ref u8) 8-bit unsigned integer (`unsigned char`)
 * [s64](\ref s64) 64-bit signed integer (`intl`)
 * [u64](\ref u64) 64-bit unsigned integer (`uintl`)
 * [s16](\ref s16) 16-bit signed integer (`short`)

From b0be72de52961dd14781ed7069f0204bf611ca11 Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Mon, 16 Oct 2023 16:36:29 -0400
Subject: [PATCH 436/473] add missing s8 tests

---
 test/arrayfire_test.cpp | 2 +-
 test/binary.cpp         | 6 ++++++
 test/corrcoef.cpp       | 6 +++---
 test/ireduce.cpp        | 2 ++
 test/meanvar.cpp        | 5 +++--
 test/median.cpp         | 2 ++
 test/scan.cpp           | 1 +
 test/set.cpp            | 2 ++
 8 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index b1b82813de..dedaedbf75 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -1358,7 +1358,7 @@ af_err conv_image(af_array *out, af_array in) {
 
     T *out_data = new T[nElems];
 
-    af_dtype out_type = (af_dtype)af::dtype_traits<T>::af_type
+    af_dtype out_type = (af_dtype)af::dtype_traits<T>::af_type;
     for (int i = 0; i < (int)nElems; i++) {
         if (out_type == s8) {
             // shift to avoid overflow
diff --git a/test/binary.cpp b/test/binary.cpp
index 3dbfa44bb9..7fd47bcfbd 100644
--- a/test/binary.cpp
+++ b/test/binary.cpp
@@ -324,6 +324,7 @@ UBITOP(bitnot, int)
 UBITOP(bitnot, uint)
 UBITOP(bitnot, intl)
 UBITOP(bitnot, uintl)
+UBITOP(bitnot, schar)
 UBITOP(bitnot, uchar)
 UBITOP(bitnot, short)
 UBITOP(bitnot, ushort)
@@ -414,6 +415,7 @@ DEF_TEST(Int, int)
 DEF_TEST(UShort, unsigned short)
 DEF_TEST(Short, short)
 DEF_TEST(UChar, unsigned char)
+DEF_TEST(SChar, signed char)
 
 #undef DEF_TEST
 
@@ -431,6 +433,8 @@ INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestShort,
                          testing::Range<short>(1, 180, 50));
 INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestUChar,
                          testing::Range<unsigned char>(1, 12, 5));
+INSTANTIATE_TEST_SUITE_P(PositiveValues, PowPrecisionTestSChar,
+                         testing::Range<signed char>(1, 9, 3));
 
 INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestLong,
                          testing::Range<long long>(-1e7, 0, 1e6));
@@ -438,6 +442,8 @@ INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestInt,
                          testing::Range<int>(-46340, 0, 10e3));
 INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestShort,
                          testing::Range<short>(-180, 0, 50));
+INSTANTIATE_TEST_SUITE_P(NegativeValues, PowPrecisionTestSChar,
+                         testing::Range<signed char>(-9, 0, 3));
 
 struct result_type_param {
     af_dtype result_;
diff --git a/test/corrcoef.cpp b/test/corrcoef.cpp
index ffcecacd61..e9bc5a5616 100644
--- a/test/corrcoef.cpp
+++ b/test/corrcoef.cpp
@@ -62,9 +62,9 @@ template<typename T>
 struct ccOutType {
     typedef typename cond_type<
         is_same_type<T, float>::value || is_same_type<T, int>::value ||
-            is_same_type<T, uint>::value || is_same_type<T, uchar>::value ||
-            is_same_type<T, short>::value || is_same_type<T, ushort>::value ||
-            is_same_type<T, char>::value,
+            is_same_type<T, uint>::value || is_same_type<T, schar>::value ||
+            is_same_type<T, uchar>::value || is_same_type<T, short>::value ||
+            is_same_type<T, ushort>::value || is_same_type<T, char>::value,
         float, typename elseType<T>::type>::type type;
 };
 
diff --git a/test/ireduce.cpp b/test/ireduce.cpp
index 2ebd951d46..e93a8267b4 100644
--- a/test/ireduce.cpp
+++ b/test/ireduce.cpp
@@ -103,6 +103,7 @@ MINMAXOP(min, double)
 MINMAXOP(min, int)
 MINMAXOP(min, uint)
 MINMAXOP(min, char)
+MINMAXOP(min, schar)
 MINMAXOP(min, uchar)
 
 MINMAXOP(max, float)
@@ -110,6 +111,7 @@ MINMAXOP(max, double)
 MINMAXOP(max, int)
 MINMAXOP(max, uint)
 MINMAXOP(max, char)
+MINMAXOP(max, schar)
 MINMAXOP(max, uchar)
 
 TEST(IndexedReduce, MaxIndexedSmall) {
diff --git a/test/meanvar.cpp b/test/meanvar.cpp
index 08e4702481..c7eba339a8 100644
--- a/test/meanvar.cpp
+++ b/test/meanvar.cpp
@@ -40,8 +40,8 @@ struct varOutType {
     typedef typename cond_type<
         is_same_type<T, float>::value || is_same_type<T, int>::value ||
             is_same_type<T, uint>::value || is_same_type<T, short>::value ||
-            is_same_type<T, ushort>::value || is_same_type<T, uchar>::value ||
-            is_same_type<T, char>::value,
+            is_same_type<T, ushort>::value || is_same_type<T, schar>::value ||
+            is_same_type<T, uchar>::value || is_same_type<T, char>::value,
         float, typename elseType<T>::type>::type type;
 };
 
@@ -377,5 +377,6 @@ TEST_P(MeanVarHalf, TestingCPP) {
     }
 
 // Only test small sizes because the range of the large arrays go out of bounds
+MEANVAR_TEST(SignedChar, signed char)
 MEANVAR_TEST(UnsignedChar, unsigned char)
 // MEANVAR_TEST(Bool, unsigned char) // TODO(umar): test this type
diff --git a/test/median.cpp b/test/median.cpp
index c55251e66c..4f64631c6f 100644
--- a/test/median.cpp
+++ b/test/median.cpp
@@ -119,6 +119,7 @@ void median_test(int nx, int ny = 1, int nz = 1, int nw = 1) {
 MEDIAN_FLAT(float, float)
 MEDIAN_FLAT(float, int)
 MEDIAN_FLAT(float, uint)
+MEDIAN_FLAT(float, schar)
 MEDIAN_FLAT(float, uchar)
 MEDIAN_FLAT(float, short)
 MEDIAN_FLAT(float, ushort)
@@ -151,6 +152,7 @@ MEDIAN_FLAT(double, double)
 MEDIAN(float, float)
 MEDIAN(float, int)
 MEDIAN(float, uint)
+MEDIAN(float, schar)
 MEDIAN(float, uchar)
 MEDIAN(float, short)
 MEDIAN(float, ushort)
diff --git a/test/scan.cpp b/test/scan.cpp
index a29c6e0e52..8bfbe0dd20 100644
--- a/test/scan.cpp
+++ b/test/scan.cpp
@@ -113,6 +113,7 @@ SCAN_TESTS(accum, cdouble, cdouble, cdouble);
 SCAN_TESTS(accum, unsigned, unsigned, unsigned);
 SCAN_TESTS(accum, intl, intl, intl);
 SCAN_TESTS(accum, uintl, uintl, uintl);
+SCAN_TESTS(accum, schar, schar, int);
 SCAN_TESTS(accum, uchar, uchar, unsigned);
 SCAN_TESTS(accum, short, short, int);
 SCAN_TESTS(accum, ushort, ushort, uint);
diff --git a/test/set.cpp b/test/set.cpp
index 97e05d484b..0e1ececadc 100644
--- a/test/set.cpp
+++ b/test/set.cpp
@@ -77,6 +77,7 @@ UNIQUE_TESTS(float)
 UNIQUE_TESTS(double)
 UNIQUE_TESTS(int)
 UNIQUE_TESTS(uint)
+UNIQUE_TESTS(schar)
 UNIQUE_TESTS(uchar)
 UNIQUE_TESTS(short)
 UNIQUE_TESTS(ushort)
@@ -149,6 +150,7 @@ SET_TESTS(float)
 SET_TESTS(double)
 SET_TESTS(int)
 SET_TESTS(uint)
+SET_TESTS(schar)
 SET_TESTS(uchar)
 SET_TESTS(short)
 SET_TESTS(ushort)

From 7b82364c8f49bdcb2819924b13b489f1009bfe95 Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Mon, 16 Oct 2023 16:38:02 -0400
Subject: [PATCH 437/473] bump version to 3.10

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8e0c37c19f..b299e6d72f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,7 +15,7 @@ include(CheckLanguage)
 include(CMakeModules/AF_vcpkg_options.cmake)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
-project(ArrayFire VERSION 3.9.0 LANGUAGES C CXX)
+project(ArrayFire VERSION 3.10.0 LANGUAGES C CXX)
 
 include(AFconfigure_deps_vars)
 include(AFBuildConfigurations)

From b568f87b2df71d920af211f8ea11029cf5e37923 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Fri, 14 Mar 2025 15:06:05 -0400
Subject: [PATCH 438/473] Fixes for compiling with oneAPI 2025 on Linux (#3643)

Some bugfixes and updated flags to get the code to compile successfully on Linux with oneAPI 2025
---
 CMakeLists.txt                       |  2 +-
 src/backend/oneapi/CMakeLists.txt    |  2 +-
 src/backend/oneapi/kernel/mean.hpp   | 14 +++++++-------
 src/backend/oneapi/kernel/sparse.hpp | 20 ++++++++------------
 4 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b299e6d72f..ca942f301a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -145,7 +145,7 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
   set(SYCL_COMPILER ON)
   set(MKL_THREADING "tbb_thread")
   set(MKL_INTERFACE "ilp64")
-  find_package(MKL 2024.1)
+  find_package(MKL)
 endif()
 
 af_multiple_option(NAME        AF_COMPUTE_LIBRARY
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 054681d812..702abd3125 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -355,7 +355,7 @@ target_link_libraries(afoneapi
     OpenCL::OpenCL
     OpenCL::cl2hpp
     -fno-sycl-id-queries-fit-in-int
-    $<$<PLATFORM_ID:Linux>:-fsycl-link-huge-device-code>
+    $<$<PLATFORM_ID:Linux>:-flink-huge-device-code>
     $<$<PLATFORM_ID:Linux>:-fvisibility-inlines-hidden>
     $<$<PLATFORM_ID:Linux>:-fno-sycl-rdc>
     -fsycl-max-parallel-link-jobs=${NumberOfThreads}
diff --git a/src/backend/oneapi/kernel/mean.hpp b/src/backend/oneapi/kernel/mean.hpp
index d6f33209a9..4c8533b1ec 100644
--- a/src/backend/oneapi/kernel/mean.hpp
+++ b/src/backend/oneapi/kernel/mean.hpp
@@ -614,9 +614,9 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         getQueue()
             .submit([&](sycl::handler &h) {
                 auto acc_in =
-                    tmpOut_get->template get_host_access(h, sycl::read_only);
+                    tmpOut_get->get_host_access(h, sycl::read_only);
                 auto acc_wt =
-                    tmpWt_get->template get_host_access(h, sycl::read_only);
+                    tmpWt_get->get_host_access(h, sycl::read_only);
 
                 h.host_task([acc_in, acc_wt, tmp_elements, &val] {
                     val = static_cast<compute_t<T>>(acc_in[0]);
@@ -635,9 +635,9 @@ T mean_all_weighted(Param<T> in, Param<Tw> iwt) {
         compute_t<T> val;
         getQueue()
             .submit([&](sycl::handler &h) {
-                auto acc_in = in.data->template get_host_access(
+                auto acc_in = in.data->get_host_access(
                     h, sycl::range{in_elements}, sycl::read_only);
-                auto acc_wt = iwt.data->template get_host_access(
+                auto acc_wt = iwt.data->get_host_access(
                     h, sycl::range{in_elements}, sycl::read_only);
 
                 h.host_task([acc_in, acc_wt, in_elements, &val]() {
@@ -700,9 +700,9 @@ To mean_all(Param<Ti> in) {
         getQueue()
             .submit([&](sycl::handler &h) {
                 auto out =
-                    tmpOut_get->template get_host_access(h, sycl::read_only);
+                    tmpOut_get->get_host_access(h, sycl::read_only);
                 auto ct =
-                    tmpCt_get->template get_host_access(h, sycl::read_only);
+                    tmpCt_get->get_host_access(h, sycl::read_only);
 
                 h.host_task([out, ct, tmp_elements, &val] {
                     val                  = static_cast<compute_t<To>>(out[0]);
@@ -721,7 +721,7 @@ To mean_all(Param<Ti> in) {
         getQueue()
             .submit([&](sycl::handler &h) {
                 auto acc_in =
-                    in.data->template get_host_access(h, sycl::read_only);
+                    in.data->get_host_access(h, sycl::read_only);
                 h.host_task([acc_in, in_elements, &val]() {
                     common::Transform<Ti, compute_t<To>, af_add_t> transform;
                     compute_t<Tw> count = static_cast<compute_t<Tw>>(1);
diff --git a/src/backend/oneapi/kernel/sparse.hpp b/src/backend/oneapi/kernel/sparse.hpp
index b7bc316267..24458ed77d 100644
--- a/src/backend/oneapi/kernel/sparse.hpp
+++ b/src/backend/oneapi/kernel/sparse.hpp
@@ -191,17 +191,13 @@ class dense2csrCreateKernel {
         if (gidy >= (unsigned)valinfo_.dims[1]) return;
 
         int rowoff       = rowptr_[gidx];
-        T *svalptr_ptr   = svalptr_.get_pointer();
-        int *scolptr_ptr = scolptr_.get_pointer();
-        svalptr_ptr += rowoff;
-        scolptr_ptr += rowoff;
+        auto svalptr_ptr   = svalptr_.get_pointer();
+        auto scolptr_ptr = scolptr_.get_pointer();
 
-        T *dvalptr_ptr   = dvalptr_.get_pointer();
-        int *dcolptr_ptr = dcolptr_.get_pointer();
-        dvalptr_ptr += valinfo_.offset;
-        dcolptr_ptr += colinfo_.offset;
+        auto dvalptr_ptr   = dvalptr_.get_pointer();
+        auto dcolptr_ptr = dcolptr_.get_pointer();
 
-        T val = dvalptr_ptr[gidx + gidy * (unsigned)valinfo_.strides[1]];
+        T val = dvalptr_ptr[gidx + gidy * (unsigned)valinfo_.strides[1] + valinfo_.offset];
 
         if constexpr (std::is_same_v<decltype(val), std::complex<float>> ||
                       std::is_same_v<decltype(val), std::complex<double>>) {
@@ -210,9 +206,9 @@ class dense2csrCreateKernel {
             if (val == 0) return;
         }
 
-        int oloc              = dcolptr_ptr[gidx + gidy * colinfo_.strides[1]];
-        svalptr_ptr[oloc - 1] = val;
-        scolptr_ptr[oloc - 1] = gidy;
+        int oloc              = dcolptr_ptr[gidx + gidy * colinfo_.strides[1] + colinfo_.offset];
+        svalptr_ptr[oloc + rowoff - 1] = val;
+        scolptr_ptr[oloc + rowoff - 1] = gidy;
     }
 
    private:

From 360fefb3551a7c9f91250b0ec894aad76ec6a022 Mon Sep 17 00:00:00 2001
From: John Melonakos <john.melonakos@intel.com>
Date: Fri, 14 Mar 2025 15:11:55 -0400
Subject: [PATCH 439/473] updated 3.9 install and getting started instructions
 (#3496)

* updated 3.9 install and getting started instructions

* more tweaks to the 3.9 using arrayfire instructions

* fixed release notes

* Put the sentence about OpenCL CPU drivers back in.
---
 docs/pages/install.md          |  90 ++++++++++++-------------
 docs/pages/release_notes.md    |  22 ++++---
 docs/pages/using_on_linux.md   |  54 +++++++--------
 docs/pages/using_on_windows.md | 116 ++++++++++++++++-----------------
 include/af/arith.h             |  24 ++++---
 5 files changed, 149 insertions(+), 157 deletions(-)

diff --git a/docs/pages/install.md b/docs/pages/install.md
index 555e702a1b..01b268af34 100644
--- a/docs/pages/install.md
+++ b/docs/pages/install.md
@@ -6,10 +6,9 @@ target architecture and operating system. Although ArrayFire can be [built
 from source](https://github.com/arrayfire/arrayfire), the installers
 conveniently package necessary dependencies.
 
-Install the latest device drivers before using ArrayFire. If you are going to
-target the CPU using ArrayFire’s OpenCL backend, install the OpenCL
-runtime. Drivers and runtimes should be downloaded and installed from the
-device vendor’s website.
+Install the latest device drivers before using ArrayFire. If you target the
+CPU using ArrayFire’s OpenCL backend, install the OpenCL runtime. Drivers and
+runtimes should be downloaded and installed from each device vendor's website.
 
 # Install Instructions {#InstallInstructions}
 
@@ -19,15 +18,11 @@ device vendor’s website.
 
 ## Windows {#Windows}
 
-Prior to installing ArrayFire on Windows,
-[download](https://www.microsoft.com/download/details.aspx?id=48145)
-install the Visual Studio 2015 (x64) runtime libraries.
+Once the ArrayFire has been downloaded, run the installer.
 
-Once the ArrayFire installer has been downloaded, run the installer. If you
-choose not to modify the path during the installation procedure, you'll need
-to manually add ArrayFire to the path for all users. Simply append
-`%%AF_PATH%/lib` to the PATH variable so that the loader can find ArrayFire
-DLLs.
+The installer offers the option to automatically add ArrayFire to the path for
+all users. If the installer did not do this, simply append `%%AF_PATH%/lib` to
+the PATH variable so that the loader can find ArrayFire DLLs.
 
 For more information on using ArrayFire on Windows, visit the following
 [page](http://arrayfire.org/docs/using_on_windows.htm).
@@ -36,42 +31,42 @@ For more information on using ArrayFire on Windows, visit the following
 
 There are two ways to install ArrayFire on Linux.
 1. Package Manager
-2. Using ArrayFire Linux Installer
+2. Using the ArrayFire Linux Installer
 
 As of today, approach (1) is only supported for Ubuntu 18.04 and 20.04. Please
-go through [our GitHub wiki
-page](https://github.com/arrayfire/arrayfire/wiki/Install-ArrayFire-From-Linux-Package-Managers)
-for the detailed instructions.
+go through [the GitHub
+wiki[page](https://github.com/arrayfire/arrayfire/wiki/Install-ArrayFire-From-Linux-Package-Managers)
+for detailed instructions.
 
-For approach (2), once you have downloaded the ArrayFire installer, execute
-the installer from the terminal as shown below. Set the `--prefix` argument to
-the directory you would like to install ArrayFire to - we recommend `/opt`.
+For approach (2), once the ArrayFire installer is downloaded, execute the
+installer from the terminal as shown below. Set the `--prefix` argument to the
+target install directory; we recommend `/opt`.
 
-    ./Arrayfire_*_Linux_x86_64.sh --include-subdir --prefix=/opt
+    ./ArrayFire_*_Linux_x86_64.sh --include-subdir --prefix=/opt
 
-Given sudo permissions, you can add the ArrayFire libraries via `ldconfig` like
-so:
+Given sudo permissions, the ArrayFire libraries can be added to the path via
+`ldconfig` like so:
 
     echo /opt/arrayfire/lib64 > /etc/ld.so.conf.d/arrayfire.conf
     sudo ldconfig
 
-Otherwise, you will need to set the `LD_LIBRARY_PATH` environment variable in
-order to let your shared library loader find the ArrayFire libraries.
+Otherwise, the `LD_LIBRARY_PATH` environment variable can be set so that the
+shared library loader can find the ArrayFire libraries.
 
 For more information on using ArrayFire on Linux, visit the following
 [page](http://arrayfire.org/docs/using_on_linux.htm).
 
 ### Graphics support
 
-ArrayFire allows you to do high performance visualizations via our
+ArrayFire enables high-performance visualizations via the
 [Forge](https://github.com/arrayfire/forge) library. On Linux, there are a few
-dependencies you will need to install to enable graphics support:
+dependencies to install to enable graphics support:
 
-FreeImage
-Fontconfig
-GLU (OpenGL Utility Library)
+* FreeImage
+* Fontconfig
+* GLU (OpenGL Utility Library)
 
-We show how to install these dependencies on common Linux distributions:
+To install these dependencies on common Linux distributions:
 
 __Debian, Ubuntu (14.04 and above), and other Debian derivatives__
 
@@ -84,9 +79,9 @@ __Fedora, Redhat, CentOS__
 
 ## macOS {#macOS}
 
-Once you have downloaded the ArrayFire installer, execute the installer by
-either double clicking on the ArrayFire `pkg` file or running the following
-command from your terminal:
+Once the ArrayFire installer has been downloaded, execute the installer by
+either double-clicking on the ArrayFire `pkg` file or running the following
+command:
 
     sudo installer -pkg Arrayfire-*_OSX.pkg -target /
 
@@ -95,11 +90,10 @@ For more information on using ArrayFire on macOS, visit the following
 
 ## NVIDIA Tegra devices
 
-ArrayFire is capable of running on TX1 and TX2 devices. The TK1 is no longer
-supported.
+ArrayFire is capable of running TX2 devices.
 
-Prior to installing ArrayFire, make sure you have the latest version of JetPack
-(v2.3 and above) or L4T (v24.2 and above) on your device.
+Before installing ArrayFire, make sure the latest version of JetPack (v2.3 and
+above) or L4T (v24.2 and above) is installed.
 
 ### Tegra prerequisites
 
@@ -109,27 +103,25 @@ The following dependencies are required for Tegra devices:
 
 ## Testing installation
 
-After ArrayFire is finished installing, we recommend building and running a few
-of the provided examples to verify things are working as expected.
+After ArrayFire is finished installing, we recommend building and running a
+few of the provided examples to verify things are working as expected.
 
-On Unix-like systems:
+On Windows, open the CMakeLists.txt file from CMake-GUI. Once the project is
+configured and generated, build and run the examples from Visual Studio.
+
+On Linux, run the following commands:
 
     cp -r /opt/arrayfire/share/ArrayFire/examples /tmp/examples
     cd /tmp/examples
     mkdir build
     cd build
-    cmake -DASSETS_DIR:PATH=/tmp ..
+    cmake ..
     make
-    ./helloworld/helloworld_{cpu,cuda,opencl}
-
-On Windows, open the CMakeLists.txt file from CMake-GUI and set `ASSETS_DIR`
-variable to the parent folder of examples folder. Once the project is
-configured and generated, you can build and run the examples from Visual
-Studio.
+    ./helloworld/helloworld_{cpu,cuda,oneapi,opencl}
 
 ## <a name="GettingHelp"></a> Getting help
 
 * Google Groups: https://groups.google.com/forum/#!forum/arrayfire-users
-* ArrayFire Services:  [Consulting](https://arrayfire.com/consulting/)  |  [Support](https://arrayfire.com/support/)   |  [Training](https://arrayfire.com/training/)
+* ArrayFire Services:  [Consulting](https://arrayfire.com/consulting/)  |  [Training](https://arrayfire.com/training/)
 * ArrayFire Blogs: http://arrayfire.com/blog/
-* Email: <mailto:technical@arrayfire.com>
+* Email: <mailto:support@arrayfire.com>
diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 464eba664d..1b55fea448 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -10,13 +10,15 @@ v3.9.0
 - Add broadcast support \PR{2871}
 - Improve OpenCL CPU JIT performance \PR{3257} \PR{3392}
 - Optimize thread/block calculations of several kernels \PR{3144}
-- Add support for fast math compiliation when building ArrayFire \PR{3334 \PR{3337}
+- Add support for fast math compiliation when building ArrayFire \PR{3334}
+  \PR{3337}
 - Optimize performance of fftconvolve when using floats \PR{3338}
 - Add support for CUDA 12.1 and 12.2
 - Better handling of empty arrays \PR{3398}
 - Better handling of memory in linear algebra functions in OpenCL \PR{3423}
 - Better logging with JIT kernels \PR{3468}
-- Optimize memory manager/JIT interactions for small number of buffers \PR{3468}
+- Optimize memory manager/JIT interactions for small number of buffers
+  \PR{3468}
 - Documentation improvements \PR{3485}
 - Optimize reorder function \PR{3488}
 
@@ -24,21 +26,24 @@ v3.9.0
 - Improve Errors when creating OpenCL contexts from devices \PR{3257}
 - Improvements to vcpkg builds \PR{3376 \PR{3476}
 - Fix reduce by key when nan's are present \PR{3261}
-- Fix error in convolve where the ndims parameter was forced to be equal to 2  \PR{3277}
-- Make constructors that accept dim_t to be explicit to avoid invalid conversions  \PR{3259} 
-- Fix error in randu when compiling against clang 14 \PR{3333} 
+- Fix error in convolve where the ndims parameter was forced to be equal to 2
+  \PR{3277}
+- Make constructors that accept dim_t to be explicit to avoid invalid
+  conversions \PR{3259}
+- Fix error in randu when compiling against clang 14 \PR{3333}
 - Fix bug in OpenCL linear algebra functions  \PR{3398}
-- Fix bug with thread local variables when device was changed \PR{3420} \PR{3421} 
-- Fix bug in qr related to uninitialized memory \PR{3422} 
+- Fix bug with thread local variables when device was changed \PR{3420}
+  \PR{3421}
+- Fix bug in qr related to uninitialized memory \PR{3422}
 - Fix bug in shift where the array had an empty middle dimension \PR{3488}
 
-
 ## Contributions
 
 Special thanks to our contributors:
 [Willy Born](https://github.com/willyborn)
 [Mike Mullen](https://github.com/mfzmullen)
 
+
 v3.8.3
 ======
 
@@ -101,6 +106,7 @@ Special thanks to our contributors:
 [Jacob Kahn](https://github.com/jacobkahn)
 [Willy Born](https://github.com/willyborn)
 
+
 v3.8.1
 ======
 
diff --git a/docs/pages/using_on_linux.md b/docs/pages/using_on_linux.md
index 7dbff74d2a..91035426c5 100644
--- a/docs/pages/using_on_linux.md
+++ b/docs/pages/using_on_linux.md
@@ -4,9 +4,9 @@ Using ArrayFire on Linux {#using_on_linux}
 Once you have [installed](\ref installing) ArrayFire on your system, the next
 thing to do is set up your build system. On Linux, you can create ArrayFire
 projects using almost any editor, compiler, or build system. The only
-requirements are that you include the ArrayFire header directories and link with
-the ArrayFire library you intend to use i.e. CUDA, OpenCL, CPU, or Unified
-backends.
+requirements are that you include the ArrayFire header directories and link
+with the ArrayFire library you intend to use i.e. CUDA, OpenCL, oneAPI, CPU,
+or Unified backends.
 
 ## The big picture  {#big-picture-linux}
 
@@ -15,17 +15,18 @@ installer will populate files in the following sub-directories:
 
     include/arrayfire.h         - Primary ArrayFire include file
     include/af/*.h              - Additional include files
-    lib/libaf*                  - CPU, CUDA, oneAPI and OpenCL libraries (.a, .so)
+    lib/libaf*                  - CPU, CUDA, oneAPI, and OpenCL libraries (.a, .so)
     lib/libforge*               - Visualization library
     lib/libcu*                  - CUDA backend dependencies
     lib/libOpenCL.so            - OpenCL ICD Loader library
     share/ArrayFire/cmake/*     - CMake config (find) scripts
     share/ArrayFire/examples/*  - All ArrayFire examples
 
-Because ArrayFire follows standard installation practices, you can use basically
-any build system to create and compile projects that use ArrayFire. Among the
-many possible build systems on Linux we suggest using ArrayFire with either
-CMake or Makefiles with CMake being our preferred build system.
+Because ArrayFire follows standard installation practices, you can use
+basically any build system to create and compile projects that use
+ArrayFire. Among the many possible build systems on Linux we suggest using
+ArrayFire with either CMake or Makefiles with CMake being our preferred build
+system.
 
 ## Prerequisite software
 
@@ -57,8 +58,8 @@ apt install build-essential cmake cmake-curses-gui
 ## CMake
 
 We recommend that the CMake build system be used to create ArrayFire projects.
-As [discussed above](#big-picture-linux), ArrayFire ships with a series of CMake
-scripts to make finding and using our library easy.
+As [discussed above](#big-picture-linux), ArrayFire ships with a series of
+CMake scripts to make finding and using our library easy.
 
 First create a file called `CMakeLists.txt` in your project directory:
 
@@ -74,19 +75,19 @@ and populate it with the following code:
     # Unified backend lets you choose the backend at runtime
     target_link_libraries(<my_executable> ArrayFire::af)
 
-where `my_executable` is the name of the executable you wish to create. See the
-[CMake documentation](https://cmake.org/documentation/) for more information on
-how to use CMake. To link with a specific backend directly, replace the
-`ArrayFire::af` with the following for their respective backends.
+where `my_executable` is the name of the executable you wish to create. See
+the [CMake documentation](https://cmake.org/documentation/) for more
+information on how to use CMake. To link with a specific backend directly,
+replace the `ArrayFire::af` with the following for their respective backends.
 
 * `ArrayFire::afcpu` for CPU backend.
 * `ArrayFire::afcuda` for CUDA backend.
 * `ArrayFire::afoneapi` for oneAPI backend.
 * `ArrayFire::afopencl` for OpenCL backend.
 
-Next we need to instruct CMake to create build instructions and then compile. We
-suggest using CMake's out-of-source build functionality to keep your build and
-source files cleanly separated. To do this open the CMake GUI.
+Next we need to instruct CMake to create build instructions and then
+compile. We suggest using CMake's out-of-source build functionality to keep
+your build and source files cleanly separated. To do this open the CMake GUI.
 
     cd your-project-directory
     mkdir build
@@ -98,8 +99,9 @@ source files cleanly separated. To do this open the CMake GUI.
 still help you out. When you execute CMake specify the path to ArrayFire
 installation root as `ArrayFire_DIR` variable.
 
-For example, if ArrayFire were installed locally to `/home/user/ArrayFire` then
-you would modify the `cmake` command above to contain the following definition:
+For example, if ArrayFire were installed locally to `/home/user/ArrayFire`
+then you would modify the `cmake` command above to contain the following
+definition:
 
     cmake -DArrayFire_DIR=/home/user/ArrayFire ..
 
@@ -107,18 +109,18 @@ You can also specify this information in the `ccmake` command-line interface.
 
 ## Makefiles
 
-Building ArrayFire projects with Makefiles is fairly similar to CMake except you
-must specify all paths and libraries manually.
+Building ArrayFire projects with Makefiles is fairly similar to CMake except
+you must specify all paths and libraries manually.
 
 As with any `make` project, you need to specify the include path to the
 directory containing `arrayfire.h` file. This should be `-I
 /opt/arrayfire/include` if you followed our installation instructions.
 
-Similarly, you will need to specify the path to the ArrayFire library using the
-`-L` option (e.g. `-L/opt/arrayfire/lib`) followed by the specific ArrayFire
-library you wish to use using the `-l` option (for example `-lafcpu`,
-`-lafopencl`, `-lafoneapi`, `-lafcuda`, or `-laf` for the CPU, OpenCL, oneAPI
-and CUDA, and unified backends, respectively.
+Similarly, you will need to specify the path to the ArrayFire library using
+the `-L` option (e.g. `-L/opt/arrayfire/lib`) followed by the specific
+ArrayFire library you wish to use using the `-l` option (for example
+`-lafcpu`, `-lafopencl`, `-lafoneapi`, `-lafcuda`, or `-laf` for the CPU,
+OpenCL, oneAPI, and CUDA, and unified backends, respectively.
 
 Here is a minimal example Makefile which uses ArrayFire's CPU backend:
 
diff --git a/docs/pages/using_on_windows.md b/docs/pages/using_on_windows.md
index 072445a4ae..b9084723d1 100644
--- a/docs/pages/using_on_windows.md
+++ b/docs/pages/using_on_windows.md
@@ -2,7 +2,8 @@ Using ArrayFire with Microsoft Windows and Visual Studio {#using_on_windows}
 ============================================================================
 
 If you have not already done so, please make sure you have installed,
-configured, and tested ArrayFire following the [installation instructions](#installing).
+configured, and tested ArrayFire following the [installation
+instructions](#installing).
 
 # The big picture {#big-picture-windows}
 
@@ -10,70 +11,60 @@ The ArrayFire Windows installer creates the following:
 1. **AF_PATH** environment variable to point to the installation location. The
    default install location is `C:\Program Files\ArrayFire\v3`
 2. **AF_PATH/include** : Header files for ArrayFire (include directory)
-3. **AF_PATH/lib** : All ArrayFire backends libraries, dlls and dependency dlls
-   (library directory)
-4. **AF_PATH/examples** : Examples to get started.
+3. **AF_PATH/lib** : All ArrayFire backend libraries, dlls, and dependency
+   dlls (library directory)
+4. **AF_PATH/examples** : Examples to get started
 5. **AF_PATH/cmake** : CMake config files
 6. **AF_PATH/uninstall.exe** : Uninstaller
 
-The installer will prompt the user for following three options.
-* Do not add **%%AF_PATH%/lib** to PATH
-* Add **%%AF_PATH%/lib** to PATH environment variable of current user
-* Add **%%AF_PATH%/lib** to PATH environment variable for all users
-
-If you chose not to modify PATH during installation please make sure to do so
-manually so that all applications using ArrayFire libraries will be able to find
-the required DLLs.
-
 # Build and Run Helloworld {#section1}
 
 This can be done in two ways either by using CMake build tool or using Visual
 Studio directly.
 
 ##  Using CMake {#section1part1}
-1. Download and install [CMake](https://cmake.org/download/), preferrably the
+1. Download and install [CMake](https://cmake.org/download/), preferably the
    latest version.
 2. Open CMake-GUI and set the field __Where is the source code__ to the root
    directory of examples.
 3. Set the field __Where to build the binaries__ to
-   **path_to_examples_root_dir/build** and click the `Configure` button towards
-   the lower left bottom.
-4. CMake will prompt you asking if it has to create the `build` directory if
-   it's not already present. Click yes to create the build directory.
-5. Before the configuration begins, CMake will show you a list(drop-down menu)
-   of available Visual Studio versions on your system to chose from. Select one
-   and check the radio button that says **Use default native compilers** and
-   click finish button in the bottom right corner.
-6. CMake will show you errors in red text if any once configuration is finished.
-   Ideally, you wouldn't need to do anything and CMake should be able to find
-   ArrayFire automatically. Please let us know if it didn't on your machine.
+   **path_to_examples_root_dir/build** and click the `Configure` button.
+4. CMake will prompt you to create the `build` directory if not already
+   present. Click "yes" to create the build directory.
+5. Before the configuration begins, CMake will show you a list (drop-down
+   menu) of available Visual Studio versions. Select one and check the radio
+   button that says **Use default native compilers** and click finish.
+6. CMake will show you errors in red text, if any, once configuration is
+   finished. Sometimes a second configuration is necessary.
 7. Click **Generate** button to generate the Visual Studio solution files for
    the examples.
 8. Click **Open Project** button that is right next to **Generate** button to
    open the solution file.
-9. You will see a bunch of examples segregated into three sets named after the
-   compute backends of ArrayFire: cpu, cuda & opencl if you have installed all
-   backends. Select the helloworld project from any of the installed backends
-   and mark it as startup project and hit `F5`.
+9. You will see the examples segregated into four sets named after the compute
+   backends of ArrayFire: cpu, cuda, oneapi, & opencl, if you installed all
+   backends. Select the helloworld project from any of the installed backends,
+   mark it as startup project, and hit `F5`.
 10. Once the helloworld example builds, you will see a console window with the
     output from helloworld program.
 
 ## Using Visual Studio {#section1part2}
 
-1. Open Visual Studio of your choice and create an empty C++ project.
-2. Right click the project and add an existing source file
+1. Open Visual Studio and create an empty C++ project.
+2. Right-click the project and add an existing source file
    `examples/helloworld/helloworld.cpp` to this project.
 3. Add `"$(AF_PATH)/include;"` to _Project Properties -> C/C++ -> General ->
    Additional Include Directories_.
 4. Add `"$(AF_PATH)/lib;"` to _Project Properties -> Linker -> General ->
    Additional Library Directories_.
-5. Add `afcpu.lib` or `afcuda.lib` or `afopencl.lib` to _Project Properties ->
-   Linker -> Input -> Additional Dependencies_. based on your preferred backend.
-6. (Optional) You may choose to define `NOMINMAX`, `AF_<CPU/CUDA/OPENCL>` and/or
-   `AF_<DEBUG/RELEASE>` in your projects. This can be added to _Project
-   Properties -> C/C++ -> General -> Preprocessor-> Preprocessory definitions_.
-7. Build and run the project. You will see a console window with the output from
-   helloworld program.
+5. Add `afcpu.lib`, `afcuda.lib`, `afoneapi.lib`, or `afopencl.lib` to
+   _Project Properties -> Linker -> Input -> Additional Dependencies_. based
+   on your preferred backend.
+6. (Optional) You may choose to define `NOMINMAX`,
+   `AF_<CPU/CUDA/ONEAPI/OPENCL>`, or `AF_<DEBUG/RELEASE>` in your
+   projects. This can be added to _Project Properties -> C/C++ -> General ->
+   Preprocessor-> Preprocessory definitions_.
+7. Build and run the project. You will see a console window with the output
+   from helloworld program.
 
 # Using ArrayFire within Existing Visual Studio Projects {#section2}
 This is divided into three parts:
@@ -83,10 +74,10 @@ This is divided into three parts:
 
 ## Part A: Adding ArrayFire to an existing solution (Single Backend) {#section2partA}
 
-Note: If you plan on using Native CUDA code in the project, use the steps under
-[Part B](#section2partB).
+Note: If you plan on using Native CUDA code in the project, use the steps
+under [Part B](#section2partB).
 
-Adding a single backend to an existing project is quite simple.
+Adding a single backend to an existing project is quite simple:
 
 1. Add `"$(AF_PATH)/include;"` to _Project Properties -> C/C++ -> General ->
    Additional Include Directories_.
@@ -97,8 +88,9 @@ Adding a single backend to an existing project is quite simple.
    preferred backend.
 
 ## Part B: Adding ArrayFire CUDA to a new/existing CUDA project {#section2partB}
-Lastly, if your project contains custom CUDA code, the instructions are slightly
-different as it requires using a CUDA NVCC Project:
+
+Lastly, if your project contains custom CUDA code, the instructions are
+slightly different as it requires using a CUDA NVCC Project:
 
 1. Create a custom "CUDA NVCC project" in Visual Studio
 2. Add `"$(AF_PATH)/include;"` to _Project Properties -> CUDA C/C++ -> General
@@ -108,7 +100,8 @@ different as it requires using a CUDA NVCC Project:
 4. Add `afcpu.lib`, `afcuda.lib`, `afopencl.lib`, or `af.lib` to _Project Properties ->
    Linker -> Input -> Additional Dependencies_. based on your preferred backend.
 
-### Part C: Project with all ArrayFire backends {#section2partC}
+## Part C: Project with all ArrayFire backends {#section2partC}
+
 If you wish to create a project that allows you to use all the ArrayFire
 backends with ease, you should use `af.lib` in step 3 from [Part
 A](#section2partA).
@@ -116,11 +109,12 @@ A](#section2partA).
 You can alternately download the template project from [ArrayFire Template
 Projects](https://github.com/arrayfire/arrayfire-project-templates)
 
-# <a name="section3" />Using ArrayFire with CMake
-ArrayFire ships with a series of CMake scripts to make finding and using our
+# Using ArrayFire with CMake
+
+ArrayFire ships with a series of CMake scripts to make finding and using the
 library easy.
 
-First create a file called `CMakeLists.txt` in your project directory:
+First, create a file called `CMakeLists.txt` in your project directory:
 
     cd your-project-directory
     touch CMakeLists.txt
@@ -130,13 +124,13 @@ and populate it with the following code:
     find_package(ArrayFire)
     add_executable(<my_executable> [list your source files here])
 
-    # To use Unified backend, do the following.
-    # Unified backend lets you choose the backend at runtime
+    # The Unified backend lets you choose the backend at runtime.
+    # To use the Unified backend, do the following:
     target_link_libraries(<my_executable> ArrayFire::af)
 
-where `<my_executable>` is the name of the executable you wish to create. See the
-[CMake documentation](https://cmake.org/documentation/) for more information on
-how to use CMake. To link with a specific backend directly, replace the
+, where `<my_executable>` is the name of the executable to create. See the
+[CMake documentation](https://cmake.org/documentation/) for more information
+on how to use CMake. To link with a specific backend directly, replace the
 `ArrayFire::af` with the following for their respective backends.
 
 * `ArrayFire::afcpu` for CPU backend.
@@ -144,13 +138,13 @@ how to use CMake. To link with a specific backend directly, replace the
 * `ArrayFire::afoneapi` for oneAPI backend.
 * `ArrayFire::afopencl` for OpenCL backend.
 
-Next we need to instruct CMake to create build instructions and then compile. We
-suggest using CMake's out-of-source build functionality to keep your build and
-source files cleanly separated. To do this open the CMake GUI.
+Next, instruct CMake to create build instructions and compile them. We suggest
+using CMake's out-of-source build functionality to keep your build and source
+files cleanly separated. To do this, open the CMake GUI.
 
-* Under source directory, add the path to your project
-* Under build directory, add the path to your project and append /build
-* Click configure and choose a 64 bit Visual Studio generator.
-* If configuration was successful, click generate. This will create a
-  my-project.sln file under build. Click `Open Project` in CMake-GUI to open the
-  solution and compile the ALL_BUILD project.
+* Under "source directory", add the path to your project.
+* Under "build directory", add the path to your project and append /build.
+* Click "configure" and choose a 64-bit Visual Studio generator.
+* If the configuration was successful, click "generate". This will create a
+  my-project.sln file under build. Click `Open Project` in CMake-GUI to open
+  the solution and compile the ALL_BUILD project.
diff --git a/include/af/arith.h b/include/af/arith.h
index 0dd2eb2c1f..c75544a5ab 100644
--- a/include/af/arith.h
+++ b/include/af/arith.h
@@ -80,7 +80,7 @@ namespace af
     /// \param[in] lo lower limit; can be an array or a scalar
     /// \param[in] hi upper limit; can be an array or a scalar
     /// \return       clamped array
-    /// 
+    ///
     /// \ingroup arith_func_clamp
     AFAPI array clamp(const array &in, const array &lo, const array &hi);
 #endif
@@ -110,7 +110,7 @@ namespace af
     /// \param[in] lhs numerator; can be an array or a scalar
     /// \param[in] rhs denominator; can be an array or a scalar
     /// \return        remainder
-    /// 
+    ///
     /// \ingroup arith_func_rem
     AFAPI array rem    (const array &lhs, const array &rhs);
 
@@ -130,7 +130,7 @@ namespace af
     /// \param[in] lhs dividend; can be an array or a scalar
     /// \param[in] rhs divisor; can be an array or a scalar
     /// \return        modulus
-    /// 
+    ///
     /// \ingroup arith_func_mod
     AFAPI array mod    (const array &lhs, const array &rhs);
 
@@ -154,7 +154,7 @@ namespace af
     ///
     /// \param[in] in input array, typically complex
     /// \return       phase angle (in radians)
-    /// 
+    ///
     /// \ingroup arith_func_arg
     AFAPI array arg    (const array &in);
 
@@ -162,7 +162,7 @@ namespace af
     ///
     /// \param[in] in input array
     /// \return       array containing 1's for negative values; 0's otherwise
-    /// 
+    ///
     /// \ingroup arith_func_sign
     AFAPI array sign  (const array &in);
 
@@ -178,7 +178,7 @@ namespace af
     ///
     /// \param[in] in input array
     /// \return       nearest integer not greater in magnitude than `in`
-    /// 
+    ///
     /// \ingroup arith_func_trunc
     AFAPI array trunc  (const array &in);
 
@@ -336,7 +336,7 @@ namespace af
     /// \param[in] in input array
     /// \return       complex array
     AFAPI array complex(const array& in);
- 
+
     /// C++ Interface to create a complex array from two real arrays.
     ///
     /// \param[in] real_ input array to be assigned as the real component of
@@ -418,7 +418,6 @@ namespace af
     /// \ingroup arith_func_root
     AFAPI array root    (const double nth_root, const array &value);
 
-
     /// \ingroup arith_func_pow
     /// @{
     /// C++ Interface to raise a base to a power (or exponent).
@@ -441,7 +440,6 @@ namespace af
     ///
     /// \param[in] in power
     /// \return       2 raised to the power
-    ///
     AFAPI array pow2    (const array &in);
     /// @}
 
@@ -449,7 +447,7 @@ namespace af
     /// C++ Interface to evaluate the logistical sigmoid function.
     ///
     /// Computes \f$\frac{1}{1+e^{-x}}\f$.
-    /// 
+    ///
     /// \param[in] in input
     /// \return       sigmoid
     ///
@@ -469,7 +467,7 @@ namespace af
     /// `exp(in) - 1`.
     ///
     /// This function is useful when `in` is small.
-    /// 
+    ///
     /// \param[in] in exponent
     /// \return       exponential minus 1
     ///
@@ -502,9 +500,9 @@ namespace af
 
     /// C++ Interface to evaluate the natural logarithm of 1 + input,
     /// `ln(1+in)`.
-    /// 
+    ///
     /// This function is useful when `in` is small.
-    /// 
+    ///
     /// \param[in] in input
     /// \return natural logarithm of `1 + input`
     ///

From c6269a6bf529a13a69c18373f172f6c04415883c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Fri, 21 Mar 2025 15:24:16 -0700
Subject: [PATCH 440/473] Add Lattice Boltzmann Fluid Simulation Example
 (#3455)

* Added cfd simulation example

* Added cfd simulation code

* Applied clang-format

* Fixed ambiguous call error

* Fixed scaling and image search issues
---
 examples/pde/CMakeLists.txt    |  16 +-
 examples/pde/boltzmann_cfd.cpp | 570 +++++++++++++++++++++++++++++++++
 2 files changed, 585 insertions(+), 1 deletion(-)
 create mode 100644 examples/pde/boltzmann_cfd.cpp

diff --git a/examples/pde/CMakeLists.txt b/examples/pde/CMakeLists.txt
index bceb38665a..4a20caf5f9 100644
--- a/examples/pde/CMakeLists.txt
+++ b/examples/pde/CMakeLists.txt
@@ -12,23 +12,37 @@ project(ArrayFire-Example-PDE
 
 find_package(ArrayFire REQUIRED)
 
+add_definitions("-DASSETS_DIR=\"${ASSETS_DIR}\"")
+
 if(ArrayFire_CPU_FOUND)
   # Shallow Water simulation example
   add_executable(swe_cpu swe.cpp)
   target_link_libraries(swe_cpu ArrayFire::afcpu)
+
+  add_executable(boltzmann_cfd_cpu boltzmann_cfd.cpp)
+  target_link_libraries(boltzmann_cfd_cpu ArrayFire::afcpu)
 endif()
 
 if(ArrayFire_CUDA_FOUND)
   add_executable(swe_cuda swe.cpp)
   target_link_libraries(swe_cuda ArrayFire::afcuda)
+
+  add_executable(boltzmann_cfd_cuda boltzmann_cfd.cpp)
+  target_link_libraries(boltzmann_cfd_cuda ArrayFire::afcuda)
 endif()
 
 if(ArrayFire_OpenCL_FOUND)
   add_executable(swe_opencl swe.cpp)
   target_link_libraries(swe_opencl ArrayFire::afopencl)
+
+  add_executable(boltzmann_cfd_opencl boltzmann_cfd.cpp)
+  target_link_libraries(boltzmann_cfd_opencl ArrayFire::afopencl)
 endif()
 
 if(ArrayFire_oneAPI_FOUND)
   add_executable(swe_oneapi swe.cpp)
   target_link_libraries(swe_oneapi ArrayFire::afoneapi)
-endif()
+
+  add_executable(boltzmann_cfd_oneapi boltzmann_cfd.cpp)
+  target_link_libraries(boltzmann_cfd_oneapi ArrayFire::afoneapi)
+endif()
\ No newline at end of file
diff --git a/examples/pde/boltzmann_cfd.cpp b/examples/pde/boltzmann_cfd.cpp
new file mode 100644
index 0000000000..38882f3c5c
--- /dev/null
+++ b/examples/pde/boltzmann_cfd.cpp
@@ -0,0 +1,570 @@
+/*******************************************************
+ * Copyright (c) 2023, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+/*
+    This is a Computational Fluid Dynamics Simulation using the Lattice
+   Boltzmann Method For this simulation we are using D2N9 (2 dimensions, 9
+   neighbors) with bounce-back boundary conditions For more information on the
+   simulation equations, check out
+   https://en.wikipedia.org/wiki/Lattice_Boltzmann_methods#Mathematical_equations_for_simulations
+
+    The initial conditions of the fluid are obtained from three images that
+   specify their properties using the function read_initial_condition_arrays.
+   These images can be modified to simulate different cases
+*/
+
+#include <arrayfire.h>
+#include <chrono>
+#include <iostream>
+#include <thread>
+
+/*
+    Values of the D2N9 grid follow the following order structure:
+
+
+          -1      0       1
+      * ----------------------> x
+  -1   |   6      3       0
+       |
+   0   |   7      4       1
+       |
+   1   |   8      5       2
+       |
+       v
+       y
+
+    The (-1, 0, 1) refer to the x and y offsets with respect to a single cell
+    and the (0-8) refer to indices of each cell in the 3x3 grid
+
+    Eg. Element with index 4 is the center of the grid which has an x-offset =
+  ex_vals[4] = 0 and y-offset = ey_vals[4] = 0 with its quantities being
+  weighted with weight wt_vals[4] = 16/36
+*/
+
+static const float ex_vals[] = {1.0, 1.0, 1.0, 0.0, 0.0, 0.0, -1.0, -1.0, -1.0};
+
+static const float ey_vals[] = {1.0, 0.0, -1.0, 1.0, 0.0, -1.0, 1.0, 0.0, -1.0};
+
+static const float wt_vals[] = {1.0f / 36.0f, 4.0f / 36.0f,  1.0f / 36.0f,
+                                4.0f / 36.0f, 16.0f / 36.0f, 4.0f / 36.0f,
+                                1.0f / 36.0f, 4.0f / 36.0f,  1.0f / 36.0f};
+
+static const int opposite_indices[] = {8, 7, 6, 5, 4, 3, 2, 1, 0};
+
+struct Simulation {
+    // Fluid quantities
+    af::array ux;
+    af::array uy;
+    af::array rho;
+    af::array sigma;
+    af::array f;
+    af::array feq;
+
+    // Constant velocity boundary conditions positions
+    af::array set_boundaries;
+
+    // Simulation Parameters
+    size_t grid_width;
+    size_t grid_height;
+    float density;
+    float velocity;
+    float reynolds;
+
+    // Helper arrays stored for computation
+    af::array ex;
+    af::array ey;
+    af::array wt;
+
+    af::array ex_T;
+    af::array ey_T;
+    af::array wt_T;
+
+    af::array ex_;
+    af::array ey_;
+};
+
+/**
+ * @brief Create a simulation object containing all the initial parameters and
+ * condition of the simulation
+ *
+ * @details
+ * For the ux, uy, and boundary images, we use RGB values for to define the
+ * specific quantites for each grid cell/pixel
+ *
+ * /// R & B for ux & uy
+ *
+ * For ux and uy, Red means positive value while Blue means negative value. The
+ * speed value for both ux and uy is computed as $(R - B) * velocity / 255$.
+ *
+ * For example, for the same pixel in the two images if we had ux = RGB(255,0,0)
+ * and uy = RGB(0,0,255) means that cell's fluid has an x-velocity of +v and
+ * y-velocity of -v where v is the velocity quantity pass to this function.
+ *
+ * Note that having the same value in the R and B components will cancel each
+ * other out, i.e., have the fluid has 0 velocity in that direction similar to
+ * having it be 0.
+ *
+ * /// G for ux & uy
+ *
+ * The G component is reserved for an object or obstacle. Any non-zero value for
+ * the green component represents a hard boundary in the simulation
+ *
+ * /// RGB for boundary
+ *
+ * Any non-zero value for any of the components in the RGB value of the pixel
+ * means that the initial values passed for ux and uy will remain constant
+ * throught the simulation
+ *
+ */
+Simulation create_simulation(uint32_t grid_width, uint32_t grid_height,
+                             float density, float velocity, float reynolds,
+                             const char* ux_image_filename,
+                             const char* uy_image_filename,
+                             const char* boundaries_filename) {
+    Simulation sim;
+
+    sim.grid_width  = grid_width;
+    sim.grid_height = grid_height;
+    sim.velocity    = velocity;
+    sim.density     = density;
+    sim.reynolds    = reynolds;
+
+    try {
+        sim.ux = af::loadImage(ux_image_filename, true);
+    } catch (const af::exception& e) {
+        std::cerr << e.what() << std::endl;
+        sim.ux = af::constant(0, grid_width, grid_height, 3);
+    }
+
+    auto ux_dim = sim.ux.dims();
+    if (ux_dim[0] != grid_width || ux_dim[1] != grid_height) {
+        std::cerr
+            << "Fluid flow ux image has dimensions different to the simulation"
+            << std::endl;
+        throw std::runtime_error{
+            "Fluid flow ux image has dimensions different to the simulation"};
+    }
+
+    try {
+        sim.uy = af::loadImage(uy_image_filename, true);
+    } catch (const af::exception& e) {
+        std::cerr << e.what() << std::endl;
+        sim.uy = af::constant(0, grid_width, grid_height, 3);
+    }
+
+    auto uy_dim = sim.uy.dims();
+    if (uy_dim[0] != grid_width || uy_dim[1] != grid_height) {
+        std::cerr
+            << "Fluid flow uy image has dimensions different to the simulation"
+            << std::endl;
+        throw std::runtime_error{
+            "Fluid flow uy image has dimensions different to the simulation"};
+    }
+
+    try {
+        sim.set_boundaries = af::loadImage(boundaries_filename, false);
+    } catch (const af::exception& e) {
+        std::cerr << e.what() << std::endl;
+        sim.set_boundaries = af::constant(0, grid_width, grid_height);
+    }
+
+    auto b_dim = sim.set_boundaries.dims();
+    if (b_dim[0] != grid_width || b_dim[1] != grid_height) {
+        std::cerr
+            << "Fluid boundary image has dimensions different to the simulation"
+            << std::endl;
+        throw std::runtime_error{
+            "Fluid boundary image has dimensions different to the simulation"};
+    }
+
+    sim.ux = (sim.ux(af::span, af::span, 0).T() -
+              sim.ux(af::span, af::span, 2).T()) *
+             velocity / 255.f;
+    sim.uy = (sim.uy(af::span, af::span, 0).T() -
+              sim.uy(af::span, af::span, 2).T()) *
+             velocity / 255.f;
+    sim.set_boundaries = sim.set_boundaries.T() > 0;
+
+    return sim;
+}
+
+/**
+ * @brief Initializes internal values used for computation
+ *
+ */
+void initialize(Simulation& sim) {
+    auto& ux    = sim.ux;
+    auto& uy    = sim.uy;
+    auto& rho   = sim.rho;
+    auto& sigma = sim.sigma;
+    auto& f     = sim.f;
+    auto& feq   = sim.feq;
+
+    auto& ex   = sim.ex;
+    auto& ey   = sim.ey;
+    auto& wt   = sim.wt;
+    auto& ex_  = sim.ex_;
+    auto& ey_  = sim.ey_;
+    auto& ex_T = sim.ex_T;
+    auto& ey_T = sim.ey_T;
+    auto& wt_T = sim.wt_T;
+
+    auto density  = sim.density;
+    auto velocity = sim.velocity;
+    auto xcount   = sim.grid_width;
+    auto ycount   = sim.grid_height;
+
+    ex = af::array(1, 1, 9, ex_vals);
+    ey = af::array(1, 1, 9, ey_vals);
+    wt = af::array(1, 1, 9, wt_vals);
+
+    ex_T = af::array(1, 9, ex_vals);
+    ey_T = af::array(1, 9, ey_vals);
+    wt_T = af::moddims(wt, af::dim4(1, 9));
+
+    rho   = af::constant(density, xcount, ycount, f32);
+    sigma = af::constant(0, xcount, ycount, f32);
+
+    f = af::constant(0, xcount, ycount, 9, f32);
+
+    ex_ = af::tile(ex, xcount, ycount, 1);
+    ey_ = af::tile(ey, xcount, ycount, 1);
+
+    // Initialization of the distribution function
+    auto edotu = ex_ * ux + ey_ * uy;
+    auto udotu = ux * ux + uy * uy;
+
+    feq = rho * wt *
+          ((edotu * edotu * 4.5f) - (udotu * 1.5f) + (edotu * 3.0f) + 1.0f);
+    f = feq;
+}
+
+/**
+ * @brief Updates the particle distribution functions for the new simulation
+ * frame
+ *
+ */
+void collide_stream(Simulation& sim) {
+    auto& ux             = sim.ux;
+    auto& uy             = sim.uy;
+    auto& rho            = sim.rho;
+    auto& sigma          = sim.sigma;
+    auto& f              = sim.f;
+    auto& feq            = sim.feq;
+    auto& set_boundaries = sim.set_boundaries;
+
+    auto& ex   = sim.ex;
+    auto& ey   = sim.ey;
+    auto& wt   = sim.wt;
+    auto& ex_  = sim.ex_;
+    auto& ey_  = sim.ey_;
+    auto& ex_T = sim.ex_T;
+    auto& ey_T = sim.ey_T;
+    auto& wt_T = sim.wt_T;
+
+    auto density  = sim.density;
+    auto velocity = sim.velocity;
+    auto reynolds = sim.reynolds;
+    auto xcount   = sim.grid_width;
+    auto ycount   = sim.grid_height;
+
+    const float viscosity =
+        velocity * std::sqrt(static_cast<float>(xcount * ycount)) / reynolds;
+    const float tau  = 0.5f + 3.0f * viscosity;
+    const float csky = 0.16f;
+
+    auto edotu = ex_ * ux + ey_ * uy;
+    auto udotu = ux * ux + uy * uy;
+
+    // Compute the new distribution function
+    feq =
+        rho * wt * (edotu * edotu * 4.5f - udotu * 1.5f + edotu * 3.0f + 1.0f);
+
+    auto taut =
+        af::sqrt(sigma * (csky * csky * 18.0f * 0.25f) + (tau * tau * 0.25f)) -
+        (tau * 0.5f);
+
+    // Compute the shifted distribution functions
+    auto fplus = f - (f - feq) / (taut + tau);
+
+    // Compute new particle distribution according to the corresponding D2N9
+    // weights
+    for (int i = 0; i < 9; ++i) {
+        int xshift = static_cast<int>(ex_vals[i]);
+        int yshift = static_cast<int>(ey_vals[i]);
+
+        fplus(af::span, af::span, i) =
+            af::shift(fplus(af::span, af::span, i), xshift, yshift);
+    }
+
+    // Keep the boundary conditions at the borders the same
+    af::replace(fplus, af::tile(!set_boundaries, af::dim4(1, 1, 9)), f);
+
+    // Update the particle distribution
+    f = fplus;
+
+    // Computing u dot e at the each of the boundaries
+    af::array ux_top = ux.rows(0, 2);
+    ux_top =
+        af::moddims(af::tile(ux_top, af::dim4(1, 3)).T(), af::dim4(ycount, 9));
+    af::array ux_bot = ux.rows(xcount - 3, xcount - 1);
+    ux_bot =
+        af::moddims(af::tile(ux_bot, af::dim4(1, 3)).T(), af::dim4(ycount, 9));
+
+    af::array uy_top = uy.rows(0, 2);
+    uy_top =
+        af::moddims(af::tile(uy_top, af::dim4(1, 3)).T(), af::dim4(ycount, 9));
+    af::array uy_bot = uy.rows(xcount - 3, xcount - 1);
+    uy_bot =
+        af::moddims(af::tile(uy_bot, af::dim4(1, 3)).T(), af::dim4(ycount, 9));
+
+    auto ux_lft = af::tile(ux.cols(0, 2), af::dim4(1, 3));
+    auto uy_lft = af::tile(uy.cols(0, 2), af::dim4(1, 3));
+    auto ux_rht = af::tile(ux.cols(ycount - 3, ycount - 1), af::dim4(1, 3));
+    auto uy_rht = af::tile(uy.cols(ycount - 3, ycount - 1), af::dim4(1, 3));
+
+    auto ubdoute_top = ux_top * ex_T + uy_top * ey_T;
+    auto ubdoute_bot = ux_bot * ex_T + uy_bot * ey_T;
+    auto ubdoute_lft = ux_lft * ex_T + uy_lft * ey_T;
+    auto ubdoute_rht = ux_rht * ex_T + uy_rht * ey_T;
+
+    // Computing bounce-back boundary conditions
+    auto fnew_top = af::moddims(fplus.row(1), af::dim4(ycount, 9)) -
+                    6.0 * density * wt_T * ubdoute_top;
+    auto fnew_bot = af::moddims(fplus.row(xcount - 2), af::dim4(ycount, 9)) -
+                    6.0 * density * wt_T * ubdoute_bot;
+    auto fnew_lft = af::moddims(fplus.col(1), af::dim4(xcount, 9)) -
+                    6.0 * density * wt_T * ubdoute_lft;
+    auto fnew_rht = af::moddims(fplus.col(ycount - 2), af::dim4(xcount, 9)) -
+                    6.0 * density * wt_T * ubdoute_rht;
+
+    // Update the values near the boundaries with the correct bounce-back
+    // boundary
+    for (int i = 0; i < 9; ++i) {
+        int xshift = static_cast<int>(ex_vals[i]);
+        int yshift = static_cast<int>(ey_vals[i]);
+        if (xshift == 1)
+            f(1, af::span, opposite_indices[i]) = fnew_top(af::span, i);
+        if (xshift == -1)
+            f(xcount - 2, af::span, opposite_indices[i]) =
+                fnew_bot(af::span, i);
+        if (yshift == 1)
+            f(af::span, 1, opposite_indices[i]) = fnew_lft(af::span, i);
+        if (yshift == -1)
+            f(af::span, ycount - 2, opposite_indices[i]) =
+                fnew_rht(af::span, i);
+    }
+}
+
+/**
+ * @brief Updates the velocity field, density and strain at each point in the
+ * grid
+ *
+ */
+void update(Simulation& sim) {
+    auto& ux    = sim.ux;
+    auto& uy    = sim.uy;
+    auto& rho   = sim.rho;
+    auto& sigma = sim.sigma;
+    auto& f     = sim.f;
+    auto& feq   = sim.feq;
+    auto& ex    = sim.ex;
+    auto& ey    = sim.ey;
+
+    auto e_tile = af::join(3, af::constant(1, 1, 1, 9), ex, ey);
+    auto result = af::sum(f * e_tile, 2);
+
+    rho = result(af::span, af::span, af::span, 0);
+    result /= rho;
+    ux = result(af::span, af::span, af::span, 1);
+    uy = result(af::span, af::span, af::span, 2);
+
+    // Above code equivalent to
+    // rho = af::sum(f, 2);
+    // ux = af::sum(f * ex, 2) / rho;
+    // uy = af::sum(f * ey, 2) / rho;
+
+    auto product   = f - feq;
+    auto e_product = af::join(3, ex * ex, ex * ey * std::sqrt(2), ey * ey);
+
+    sigma = af::sqrt(af::sum(af::pow(af::sum(product * e_product, 2), 2), 3));
+
+    // Above code equivalent to
+
+    // auto xx = af::sum(product * ex * ex, 2);
+    // auto xy = af::sum(product * ex * ey, 2);
+    // auto yy = af::sum(product * ey * ey, 2);
+
+    // sigma = af::sqrt(xx * xx + xy * xy * 2 + yy * yy);
+}
+
+af::array generate_image(size_t width, size_t height, const Simulation& sim) {
+    const auto& ux         = sim.ux;
+    const auto& uy         = sim.uy;
+    const auto& boundaries = sim.set_boundaries;
+    auto velocity          = sim.velocity;
+
+    float image_scale =
+        static_cast<float>(width) / static_cast<float>(sim.grid_width - 1);
+
+    // Relative Flow speed at each cell
+    auto val = af::sqrt(ux * ux + uy * uy) / velocity;
+
+    af::replace(val, val != 0 || !boundaries, -1.0);
+
+    // Scaling and interpolating flow speed to the window size
+    if (width != sim.grid_width || height != sim.grid_height)
+        val =
+            af::approx2(val, af::iota(width, af::dim4(1, height)) / image_scale,
+                        af::iota(height, af::dim4(1, width)).T() / image_scale);
+
+    // Flip image
+    val = val.T();
+
+    auto image  = af::constant(0, height, width, 3);
+    auto image2 = image;
+
+    // Add custom coloring
+    image(af::span, af::span, 0) = val * 2;
+    image(af::span, af::span, 1) = val * 2;
+    image(af::span, af::span, 2) = 1.0 - val * 2;
+
+    image2(af::span, af::span, 0) = 1;
+    image2(af::span, af::span, 1) = -2 * val + 2;
+    image2(af::span, af::span, 2) = 0;
+
+    auto tile_val = af::tile(val, 1, 1, 3);
+    af::replace(image, tile_val < 0.5, image2);
+    af::replace(image, tile_val >= 0, 0.0);
+
+    return image;
+}
+
+void lattice_boltzmann_cfd_demo() {
+    // Define the lattice for the simulation
+    const size_t len         = 128;
+    const size_t grid_width  = len;
+    const size_t grid_height = len;
+
+    // Specify the image scaling displayed
+    float scale = 4.0f;
+
+    // Forge window initialization
+    int height = static_cast<int>(grid_width * scale);
+    int width  = static_cast<int>(grid_height * scale);
+    af::Window window(height, width, "Driven Cavity Flow");
+
+    int frame_count       = 0;
+    int max_frames        = 20000;
+    int simulation_frames = 100;
+    float total_time      = 0;
+    float total_time2     = 0;
+
+    // CFD fluid parameters
+    const float density  = 2.7f;
+    const float velocity = 0.35f;
+    const float reynolds = 1e5f;
+
+    const char* ux_image = ASSETS_DIR "/examples/images/default_ux.bmp";
+    const char* uy_image = ASSETS_DIR "/examples/images/default_uy.bmp";
+    const char* set_boundary_image =
+        ASSETS_DIR "/examples/images/default_boundary.bmp";
+
+    // Tesla Valve Fluid Simulation - entering from constricted side
+    {
+        //           ux_image = ASSETS_DIR "/examples/images/left_tesla_ux.bmp";
+        //           uy_image = ASSETS_DIR "/examples/images/left_tesla_uy.bmp";
+        // set_boundary_image = ASSETS_DIR
+        // "/examples/images/left_tesla_boundary.bmp";
+    }
+
+    // Tesla Valve Fluid Simulation - entering from transfer side
+    {
+        //           ux_image = ASSETS_DIR
+        //           "/examples/images/right_tesla_ux.bmp"; uy_image =
+        //           ASSETS_DIR "/examples/images/right_tesla_uy.bmp";
+        // set_boundary_image = ASSETS_DIR
+        // "/examples/images/right_tesla_boundary.bmp";
+    }
+
+    // Reads the initial values of fluid quantites and simulation parameters
+    Simulation sim =
+        create_simulation(grid_width, grid_height, density, velocity, reynolds,
+                          ux_image, uy_image, set_boundary_image);
+
+    // Initializes the simulation quantites
+    initialize(sim);
+
+    while (!window.close() && frame_count != max_frames) {
+        af::sync();
+        auto begin = std::chrono::high_resolution_clock::now();
+
+        // Computes the new particle distribution functions for the new
+        // simulation frame
+        collide_stream(sim);
+
+        // Updates the velocity, density, and stress fields
+        update(sim);
+
+        af::sync();
+        auto end = std::chrono::high_resolution_clock::now();
+
+        // Calculate computation time of 1 simulation frame
+        auto duration =
+            std::chrono::duration_cast<std::chrono::microseconds>(end - begin)
+                .count();
+
+        // Used for computing the distribution of frame computation time
+        total_time += duration;
+        total_time2 += duration * duration;
+
+        // Every number of `simulation_frames` display the last computed frame
+        // to the screen
+        if (frame_count % simulation_frames == 0) {
+            auto image = generate_image(width, height, sim);
+
+            // Display colored image
+            window.image(image);
+
+            float avg_time  = total_time / (float)simulation_frames;
+            float stdv_time = std::sqrt(total_time2 * simulation_frames -
+                                        total_time * total_time) /
+                              (float)simulation_frames;
+
+            std::cout << "Average Simulation Step Time: (" << avg_time
+                      << " +/- " << stdv_time
+                      << ") us; Total simulation time: " << total_time
+                      << " us; Simulation Frames: " << simulation_frames
+                      << std::endl;
+
+            total_time  = 0;
+            total_time2 = 0;
+        }
+
+        frame_count++;
+    }
+}
+
+int main(int argc, char** argv) {
+    int device = argc > 1 ? std::atoi(argv[1]) : 0;
+
+    try {
+        af::setDevice(device);
+        af::info();
+
+        std::cout << "** ArrayFire CFD Simulation Demo\n\n";
+
+        lattice_boltzmann_cfd_demo();
+    } catch (const af::exception& e) {
+        std::cerr << e.what() << std::endl;
+        return -1;
+    }
+
+    return 0;
+}
\ No newline at end of file

From 651988abf69d2f17a69a9d0c3d6beb8b00df4683 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Fri, 28 Mar 2025 11:41:03 -0700
Subject: [PATCH 441/473] Added Black Hole Raytracing Example (#3530)

* Added Black Hole Raytracing Example

* Fixed clang format issues

* Fixed compilation error

* Implemented adaptive rk, improved runtime and memory footprint, fixed math errors

* Fixed compilation issues

* Improved sample parameters for example

* Removed structure binding to comply with c++14

* Fix merge

* Remove black hole raytracing examples from oneapi backend

---------

Co-authored-by: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
---
 examples/pde/CMakeLists.txt |   12 +-
 examples/pde/bhrt.cpp       | 1139 +++++++++++++++++++++++++++++++++++
 2 files changed, 1150 insertions(+), 1 deletion(-)
 create mode 100644 examples/pde/bhrt.cpp

diff --git a/examples/pde/CMakeLists.txt b/examples/pde/CMakeLists.txt
index 4a20caf5f9..57f689a9e9 100644
--- a/examples/pde/CMakeLists.txt
+++ b/examples/pde/CMakeLists.txt
@@ -19,6 +19,10 @@ if(ArrayFire_CPU_FOUND)
   add_executable(swe_cpu swe.cpp)
   target_link_libraries(swe_cpu ArrayFire::afcpu)
 
+  # Black Hole Raytracing example
+  add_executable(bhrt_cpu bhrt.cpp)
+  target_link_libraries(bhrt_cpu ArrayFire::afcpu)
+
   add_executable(boltzmann_cfd_cpu boltzmann_cfd.cpp)
   target_link_libraries(boltzmann_cfd_cpu ArrayFire::afcpu)
 endif()
@@ -27,6 +31,9 @@ if(ArrayFire_CUDA_FOUND)
   add_executable(swe_cuda swe.cpp)
   target_link_libraries(swe_cuda ArrayFire::afcuda)
 
+  add_executable(bhrt_cuda bhrt.cpp)
+  target_link_libraries(bhrt_cuda ArrayFire::afcuda)
+
   add_executable(boltzmann_cfd_cuda boltzmann_cfd.cpp)
   target_link_libraries(boltzmann_cfd_cuda ArrayFire::afcuda)
 endif()
@@ -35,6 +42,9 @@ if(ArrayFire_OpenCL_FOUND)
   add_executable(swe_opencl swe.cpp)
   target_link_libraries(swe_opencl ArrayFire::afopencl)
 
+  add_executable(bhrt_opencl bhrt.cpp)
+  target_link_libraries(bhrt_opencl ArrayFire::afopencl)
+
   add_executable(boltzmann_cfd_opencl boltzmann_cfd.cpp)
   target_link_libraries(boltzmann_cfd_opencl ArrayFire::afopencl)
 endif()
@@ -45,4 +55,4 @@ if(ArrayFire_oneAPI_FOUND)
 
   add_executable(boltzmann_cfd_oneapi boltzmann_cfd.cpp)
   target_link_libraries(boltzmann_cfd_oneapi ArrayFire::afoneapi)
-endif()
\ No newline at end of file
+endif()
diff --git a/examples/pde/bhrt.cpp b/examples/pde/bhrt.cpp
new file mode 100644
index 0000000000..55e116a330
--- /dev/null
+++ b/examples/pde/bhrt.cpp
@@ -0,0 +1,1139 @@
+/*******************************************************
+ * Copyright (c) 2024, ArrayFire
+ * All rights reserved.
+ *
+ * This file is distributed under 3-clause BSD license.
+ * The complete license agreement can be obtained at:
+ * http://arrayfire.com/licenses/BSD-3-Clause
+ ********************************************************/
+
+/*
+    This is a Black Hole Raytracer.
+    For this raytracer we are using backwards path tracing to compute the
+   resulting image The path of the rays shot from the camera are simulated step
+   by step from the null geodesics light follows in spacetime. The geodesics are
+   computed from the spacetime metric of the space. This project has three
+   metrics that can be used: Schwarzchild, Kerr, and Ellis.
+
+    For more information on the black hole raytracing, check out
+    Riazuelo, A. (2015). Seeing relativity -- I. Ray tracing in a Schwarzschild
+   metric to explore the maximal analytic extension of the metric and making a
+   proper rendering of the stars. ArXiv.
+   https://doi.org/10.1142/S0218271819500421
+
+    For more information on raytracing, check out
+    Raytracing in a Weekend Series, https://raytracing.github.io/
+
+    Image being used for the background is Westerlund 2 from
+    NASA, ESA, the Hubble Heritage Team (STScI/AURA), A. Nota (ESA/STScI), and
+   the Westerlund 2 Science Team See
+   http://www.spacetelescope.org/images/heic1509a/ for details.
+
+    The default scene is the rotating black hole using the Kerr metric set by
+   the global variable 'scene' The parameters of the blackholes/wormholes may be
+   changed at the top with the simulation constants The parameters of the image
+   may be changed in the 'raytracing' function.
+*/
+#include <arrayfire.h>
+
+#include <chrono>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+
+enum class Scene { ROTATE_BH, STATIC_BH, WORMHOLE };
+
+// Scene being computed
+static constexpr Scene scene = Scene::ROTATE_BH;
+
+// **** Simulation Constants ****
+static constexpr double M = 0.5;    // Black Hole Mass
+static constexpr double J = 0.249;  // Black Hole Rotation (J < M^2)
+static constexpr double b = 3.0;    // Wormhole drainhole parameter
+
+/**
+ * @brief Generates a string progress bar
+ *
+ * @param current current job
+ * @param total total number of jobs
+ * @param start_info progress bar prior info
+ */
+void status_bar(int64_t current, int64_t total, const std::string& start_info) {
+    auto precision         = std::cout.precision();
+    static auto prev_time  = std::chrono::high_resolution_clock::now();
+    static auto prev       = current - 1;
+    static auto prev2      = prev;
+    static auto prev2_time = prev_time;
+
+    auto curr_time = std::chrono::high_resolution_clock::now();
+
+    double percent  = 100.0 * (double)(current + 1) / (double)total;
+    std::string str = "[";
+    for (int i = 0; i < 50; ++i) {
+        if (percent >= i * 2)
+            str += "=";
+        else
+            str += " ";
+    }
+    str += "]";
+
+    auto time =
+        current != prev
+            ? (total - current) * (curr_time - prev_time) / (current - prev)
+            : (total - current) * (curr_time - prev2_time) / (current - prev2);
+
+    if (current != prev && prev != prev2) {
+        prev2      = prev;
+        prev2_time = prev_time;
+    }
+    prev      = current;
+    prev_time = curr_time;
+
+    if (current != total) {
+        using namespace std::chrono_literals;
+        std::cout << start_info << " " << std::fixed << std::setprecision(1)
+                  << percent << "%  " << str << " Time Remaining: ";
+        if (std::chrono::duration_cast<std::chrono::seconds>(time).count() >
+            300)
+            std::cout << std::chrono::duration_cast<std::chrono::minutes>(time)
+                             .count()
+                      << " min";
+        else
+            std::cout << std::chrono::duration_cast<std::chrono::seconds>(time)
+                             .count()
+                      << " s";
+
+        std::cout << std::string(5, ' ') << '\r';
+    } else
+        std::cout << "\rDone!" << std::string(120, ' ') << std::endl;
+
+    std::cout << std::setprecision(precision) << std::defaultfloat;
+}
+
+/**
+ * @brief Returns the euclidean dot product for two cartesian vectors with 3
+ * coords
+ *
+ * @param lhs
+ * @param rhs
+ * @return af::array
+ */
+af::array dot3(const af::array& lhs, const af::array& rhs) {
+    return af::sum(lhs * rhs, 0);
+}
+
+/**
+ * @brief Returns the euclidean norm for a cartesian vector with 3 coords
+ *
+ * @param vector
+ * @return af::array
+ */
+af::array norm3(const af::array& vector) {
+    return af::sqrt(dot3(vector, vector));
+}
+
+/**
+ * @brief Returns the normalized vector for a cartesian vector with 3 coords
+ *
+ * @param vector
+ * @return af::array
+ */
+af::array normalize3(const af::array& vector) { return vector / norm3(vector); }
+
+af::exception make_error(const char* string) {
+    std::cout << string << std::endl;
+    return af::exception(string);
+}
+
+/**
+ * @brief Transforms degrees to radians
+ *
+ * @param degrees
+ * @return double
+ */
+double radians(double degrees) { return degrees * af::Pi / 180.0; }
+
+/**
+ * @brief Computes the cross_product of two euclidean vectors
+ *
+ * @param lhs
+ * @param rhs
+ * @return af::array
+ */
+af::array cross_product(const af::array& lhs, const af::array& rhs) {
+    if (lhs.dims() != rhs.dims())
+        throw make_error("Arrays must have the same dimensions");
+    else if (lhs.dims()[0] != 3)
+        throw make_error("Arrays must have 3 principal coordintes");
+
+    return af::join(
+        0,
+        lhs(1, af::span, af::span) * rhs(2, af::span, af::span) -
+            lhs(2, af::span, af::span) * rhs(1, af::span, af::span),
+        lhs(2, af::span, af::span) * rhs(0, af::span, af::span) -
+            lhs(0, af::span, af::span) * rhs(2, af::span, af::span),
+        lhs(0, af::span, af::span) * rhs(1, af::span, af::span) -
+            lhs(1, af::span, af::span) * rhs(0, af::span, af::span));
+}
+
+/**
+ * @brief Transform the position vectors from cartesian to spherical coordinates
+ *
+ * @param pos
+ * @return af::array
+ */
+af::array cart_to_sph_position(const af::array& pos) {
+    if (pos.dims()[0] != 3)
+        throw make_error("Arrays must have 3 principal coordintes");
+
+    af::array x = pos(0, af::span);
+    af::array y = pos(1, af::span);
+    af::array z = pos(2, af::span);
+
+    af::array r = af::sqrt(x * x + y * y + z * z);
+    af::array o = af::acos(z / r);
+    af::array p = af::atan2(y, x);
+
+    af::array transformed_pos = af::join(0, r, o, p);
+
+    return transformed_pos;
+}
+
+/**
+ * @brief Transform the velocity vectors from cartesian to spherical coordinates
+ *
+ * @param vel
+ * @param pos
+ * @return af::array
+ */
+af::array cart_to_sph_velocity(const af::array& vel, const af::array& pos) {
+    if (vel.dims() != pos.dims())
+        throw make_error("Arrays must have the same dimensions");
+    else if (pos.dims()[0] != 3)
+        throw make_error("Arrays must have 3 principal coordintes");
+
+    af::array x = pos(0, af::span);
+    af::array y = pos(1, af::span);
+    af::array z = pos(2, af::span);
+
+    af::array r = af::sqrt(x * x + y * y + z * z);
+    af::array o = af::acos(z / r);
+    af::array p = af::atan2(y, x);
+
+    af::array ux = vel(0, af::span);
+    af::array uy = vel(1, af::span);
+    af::array uz = vel(2, af::span);
+
+    af::array ur = (ux * x + uy * y + uz * z) / r;
+    af::array up = (uy * af::cos(p) - ux * af::sin(p)) / (r * af::sin(o));
+    af::array uo =
+        (af::cos(o) * (ux * af::cos(p) + uy * af::sin(p)) - uz * af::sin(o)) /
+        r;
+    af::array transformed_vel = af::join(0, ur, uo, up);
+
+    return transformed_vel;
+}
+
+/**
+ * @brief Transform the velocity vectors from cartesian to spherical coordinates
+ *
+ * @param vel
+ * @param pos
+ * @return af::array
+ */
+af::array sph_to_cart_velocity(const af::array& vel, const af::array& pos) {
+    if (vel.dims() != pos.dims())
+        throw make_error("Arrays must have the same dimensions");
+    else if (pos.dims()[0] != 3)
+        throw make_error("Arrays must have 3 principal coordintes");
+
+    af::array r = pos(0, af::span);
+    af::array o = pos(1, af::span);
+    af::array p = pos(2, af::span);
+
+    af::array ur = vel(0, af::span);
+    af::array uo = vel(1, af::span);
+    af::array up = vel(2, af::span);
+
+    af::array ux = (ur * af::sin(o) + uo * r * af::cos(o)) * af::cos(p) -
+                   up * r * af::sin(o) * af::sin(p);
+    af::array uy = (ur * af::sin(o) + uo * r * af::cos(o)) * af::sin(p) +
+                   up * r * af::sin(o) * af::cos(p);
+    af::array uz              = ur * af::cos(o) - uo * r * af::sin(o);
+    af::array transformed_vel = af::join(0, ux, uy, uz);
+
+    return transformed_vel;
+}
+
+/**
+ * @brief Transform the position vectors from cartesian to oblate coordinates
+ *
+ * @param vel
+ * @param pos
+ * @return af::array
+ */
+af::array cart_to_oblate_position(const af::array& pos) {
+    if (pos.dims()[0] != 3)
+        throw make_error("Arrays must have 3 principal coordintes");
+
+    af::array x = pos(0, af::span);
+    af::array y = pos(1, af::span);
+    af::array z = pos(2, af::span);
+    auto a      = J / M;
+    auto diff   = x * x + y * y + z * z - a * a;
+
+    af::array r =
+        af::sqrt((diff + af::sqrt(diff * diff + z * z * a * a * 4.0)) / 2.0);
+    af::array o = af::acos(z / r);
+    af::array p = af::atan2(y, x);
+
+    af::array transformed_pos = af::join(0, r, o, p);
+
+    return transformed_pos;
+}
+
+/**
+ * @brief Transform the position vectors from oblate to cartesian coordinates
+ *
+ * @param vel
+ * @param pos
+ * @return af::array
+ */
+af::array oblate_to_cart_position(const af::array& pos) {
+    if (pos.dims()[0] != 3)
+        throw make_error("Arrays must have 3 principal coordintes");
+
+    af::array r = pos(0, af::span);
+    af::array o = pos(1, af::span);
+    af::array p = pos(2, af::span);
+    auto a      = J / M;
+    auto R      = af::sqrt(r * r + a * a);
+
+    af::array x = R * af::sin(o) * af::cos(p);
+    af::array y = R * af::sin(o) * af::sin(p);
+    af::array z = r * af::cos(o);
+
+    af::array transformed_pos = af::join(0, x, y, z);
+
+    return transformed_pos;
+}
+
+/**
+ * @brief Transform the velocity vectors from oblate to cartesian coordinates
+ *
+ * @param vel
+ * @param pos
+ * @return af::array
+ */
+af::array oblate_to_cart_velocity(const af::array& vel, const af::array& pos) {
+    if (vel.dims() != pos.dims())
+        throw make_error("Arrays must have the same dimensions");
+    else if (pos.dims()[0] != 3)
+        throw make_error("Arrays must have 3 principal coordintes");
+
+    af::array r = pos(0, af::span);
+    af::array o = pos(1, af::span);
+    af::array p = pos(2, af::span);
+
+    af::array ur = vel(0, af::span);
+    af::array uo = vel(1, af::span);
+    af::array up = vel(2, af::span);
+
+    double a     = J / M;
+    af::array ra = af::sqrt(r * r + a * a);
+
+    af::array ux =
+        (ur * r * af::sin(o) / ra + uo * ra * af::cos(o)) * af::cos(p) -
+        up * r * af::sin(o) * af::sin(p);
+    af::array uy =
+        (ur * r * af::sin(o) / ra + uo * ra * af::cos(o)) * af::sin(p) +
+        up * r * af::sin(o) * af::cos(p);
+    af::array uz              = ur * af::cos(o) - uo * r * af::sin(o);
+    af::array transformed_vel = af::join(0, ux, uy, uz);
+
+    return transformed_vel;
+}
+
+/**
+ * @brief Transform the velocity vectors from cartesian to oblate coordinates
+ *
+ * @param vel
+ * @param pos
+ * @return af::array
+ */
+af::array cart_to_oblate_velocity(const af::array& vel, const af::array& pos) {
+    if (vel.dims() != pos.dims())
+        throw make_error("Arrays must have the same dimensions");
+    else if (pos.dims()[0] != 3)
+        throw make_error("Arrays must have 3 principal coordintes");
+
+    af::array x = pos(0, af::span);
+    af::array y = pos(1, af::span);
+    af::array z = pos(2, af::span);
+
+    auto a    = J / M;
+    auto diff = x * x + y * y + z * z - a * a;
+
+    af::array r =
+        af::sqrt((diff + af::sqrt(diff * diff + z * z * a * a * 4.0)) / 2.0);
+    af::array o = af::acos(z / r);
+    af::array p = af::atan2(y, x);
+
+    af::array ux = vel(0, af::span);
+    af::array uy = vel(1, af::span);
+    af::array uz = vel(2, af::span);
+
+    af::array ra = r * r + a * a;
+    af::array ur = ((ux * x + uy * y) * r + uz * ra * z / r) /
+                   (r * r + af::pow(a * af::cos(o), 2.0));
+    af::array up = (uy * x - ux * y) / (x * x + y * y);
+    af::array uo = ((ux * x + uy * y) / af::tan(o) - uz * z * af::tan(o)) /
+                   (r * r + af::pow(a * af::cos(o), 2.0));
+    af::array transformed_vel = af::join(0, ur, uo, up);
+
+    return transformed_vel;
+}
+
+/**
+ * @brief Transform the position vectors from spherical to cartesian coordinates
+ *
+ * @param pos
+ * @return af::array
+ */
+af::array sph_to_cart_position(const af::array& pos) {
+    af::array r = pos(0, af::span);
+    af::array o = pos(1, af::span);
+    af::array p = pos(2, af::span);
+
+    af::array x = r * af::sin(o) * af::cos(p);
+    af::array y = r * af::sin(o) * af::sin(p);
+    af::array z = r * af::cos(o);
+
+    af::array transformed_pos = af::join(0, x, y, z);
+
+    return transformed_pos;
+}
+
+/**
+ * @brief Computes the inverse of a 4x4 matrix with the layout
+ *          [ a 0 0 b ]
+ *          [ 0 c 0 0 ]
+ *          [ 0 0 d 0 ]
+ *          [ b 0 0 e ]
+ *
+ * @param metric af::array with the shape af::dims4(4, 4, M, N)
+ *
+ * @return af::array with the shape af::dims4(4, 4, M, N)
+ */
+af::array inv_metric(const af::array& metric) {
+    af::array a = metric(0, 0, af::span);
+    af::array b = metric(3, 0, af::span);
+    af::array c = metric(1, 1, af::span);
+    af::array d = metric(2, 2, af::span);
+    af::array e = metric(3, 3, af::span);
+
+    af::array det = b * b - a * e;
+
+    auto res = af::constant(0, 4, 4, metric.dims()[2], metric.dims()[3], f64);
+
+    res(0, 0, af::span) = -e / det;
+    res(0, 3, af::span) = b / det;
+    res(3, 0, af::span) = b / det;
+    res(1, 1, af::span) = 1.0 / c;
+    res(2, 2, af::span) = 1.0 / d;
+    res(3, 3, af::span) = -a / det;
+
+    return res;
+}
+
+/**
+ * @brief Computes the 4x4 metric matrix for the given 4-vector positions
+ *
+ * @param pos af::dim4(4, N)
+ * @return af::array af::dim4(4, 4, 1, N)
+ */
+af::array metric4(const af::array& pos) {
+    if (pos.dims()[0] != 4)
+        throw make_error("Arrays must have 4 principal coordinates");
+
+    auto dims = pos.dims();
+
+    af::array t = af::moddims(pos(0, af::span), 1, 1, dims[1]);
+    af::array r = af::moddims(pos(1, af::span), 1, 1, dims[1]);
+    af::array o = af::moddims(pos(2, af::span), 1, 1, dims[1]);
+    af::array p = af::moddims(pos(3, af::span), 1, 1, dims[1]);
+
+    af::array gtt, gtr, gto, gtp, grt, grr, gro, grp, got, gor, goo, gop, gpt,
+        gpr, gpo, gpp;
+
+    switch (scene) {
+        // ******* Kerr Black Hole Metric *******
+        case Scene::ROTATE_BH: {
+            auto rs    = 2.0 * M;
+            auto a     = J / M;
+            auto delta = (r - rs) * r + a * a;
+            auto sigma = r * r + af::pow(a * af::cos(o), 2);
+
+            gtt = 1.0 - r * rs / sigma;
+            gtr = af::constant(0.0, 1, 1, dims[1], f64);
+            gto = af::constant(0.0, 1, 1, dims[1], f64);
+            gtp = rs * r * a * af::pow(af::sin(o), 2.0) / sigma;
+            grr = -sigma / delta;
+            gro = af::constant(0.0, 1, 1, dims[1], f64);
+            grp = af::constant(0.0, 1, 1, dims[1], f64);
+            goo = -sigma;
+            gop = af::constant(0.0, 1, 1, dims[1], f64);
+            gpp =
+                -(r * r + a * a + rs * r * af::pow(a * af::sin(o), 2) / sigma) *
+                af::pow(af::sin(o), 2);
+
+            break;
+        }
+
+        // ******* Schwarzchild Black Hole Metric *******
+        case Scene::STATIC_BH: {
+            gtt = 1.0 - 2.0 * M / r;
+            gtr = af::constant(0.0, 1, 1, dims[1], f64);
+            gto = af::constant(0.0, 1, 1, dims[1], f64);
+            gtp = af::constant(0.0, 1, 1, dims[1], f64);
+            grr = -1.0 / (1.0 - 2.0 * M / r);
+            gro = af::constant(0.0, 1, 1, dims[1], f64);
+            grp = af::constant(0.0, 1, 1, dims[1], f64);
+            goo = -r * r;
+            gop = af::constant(0.0, 1, 1, dims[1], f64);
+            gpp = -af::pow(r * af::sin(o), 2);
+
+            break;
+        }
+
+        // ******* Ellis Wormhole Metric *******
+        case Scene::WORMHOLE: {
+            gtt = af::constant(1.0, 1, 1, dims[1], f64);
+            gtr = af::constant(0.0, 1, 1, dims[1], f64);
+            gto = af::constant(0.0, 1, 1, dims[1], f64);
+            gtp = af::constant(0.0, 1, 1, dims[1], f64);
+            grr = -af::constant(1.0, 1, 1, dims[1], f64);
+            gro = af::constant(0.0, 1, 1, dims[1], f64);
+            grp = af::constant(0.0, 1, 1, dims[1], f64);
+            goo = -(r * r + b * b);
+            gop = af::constant(0.0, 1, 1, dims[1], f64);
+            gpp = -(r * r + b * b) * af::pow(af::sin(o), 2);
+
+            break;
+        }
+
+        default: throw;
+    }
+
+    auto res = af::join(
+        0, af::join(1, gtt, gtr, gto, gtp), af::join(1, gtr, grr, gro, grp),
+        af::join(1, gto, gro, goo, gop), af::join(1, gtp, grp, gop, gpp));
+
+    return res;
+}
+
+/**
+ * @brief Computes the dot product as defined by a metric between two 4-vector
+ * velocities
+ *
+ * @param pos
+ * @param lhs
+ * @param rhs
+ * @return af::array
+ */
+af::array dot_product(const af::array& pos, const af::array& lhs,
+                      const af::array& rhs) {
+    if (pos.dims() != lhs.dims())
+        throw make_error(
+            "Position and lhs velocity must have the same dimensions");
+    else if (lhs.dims() != rhs.dims())
+        throw make_error(
+            "Position and rhs velocity must have the same dimensions");
+    else if (rhs.dims()[0] != 4)
+        throw make_error("Arrays must have 4 principal coordinates");
+
+    return af::matmul(af::moddims(lhs, 1, 4, lhs.dims()[1]), metric4(pos),
+                      af::moddims(rhs, 4, 1, rhs.dims()[1]));
+}
+
+af::array norm4(const af::array& pos, const af::array& vel) {
+    return dot_product(pos, vel, vel);
+}
+
+af::array partials(const af::array& pos4, uint32_t index, double rel_diff,
+                   double abs_diff) {
+    double arr[4] = {0.0};
+    arr[index]    = 1.0;
+
+    auto pos_diff = pos4 * rel_diff + abs_diff;
+    auto h4       = pos_diff * af::array(af::dim4(4, 1), arr);
+    af::array h =
+        af::moddims(pos_diff(index, af::span), af::dim4(1, 1, pos4.dims()[1]));
+
+    return (-metric4(pos4 + h4 * 2.0) + metric4(pos4 + h4) * 8.0 -
+            metric4(pos4 - h4) * 8.0 + metric4(pos4 - h4 * 2.0)) /
+           (h * 12.0);
+}
+
+/**
+ * @brief Computes the geodesics from the established metric, 4-vector positions
+ * and velocities
+ *
+ * @param pos4
+ * @param vel4
+ * @return af::array
+ */
+af::array geodesics(const af::array& pos4, const af::array& vel4) {
+    auto N = vel4.dims()[1];
+
+    af::array uu = af::matmul(af::moddims(vel4, af::dim4(4, 1, N)),
+                              af::moddims(vel4, af::dim4(1, 4, N)));
+    uu           = af::moddims(uu, af::dim4(1, 4, 4, N));
+
+    af::array metric    = metric4(pos4);
+    af::array invmetric = af::moddims(inv_metric(metric), af::dim4(4, 4, 1, N));
+
+    // Compute the partials of the metric with respect to coordinates indices
+    af::array dt = af::constant(0, 4, 4, 1, N, f64);
+
+    auto dr     = partials(pos4, 1, 1e-6, 1e-12);
+    auto dtheta = partials(pos4, 2, 1e-6, 1e-12);
+    auto dphi   = partials(pos4, 3, 1e-6, 1e-12);
+
+    dr     = af::moddims(dr, af::dim4(4, 4, 1, N));
+    dtheta = af::moddims(dtheta, af::dim4(4, 4, 1, N));
+    dphi   = af::moddims(dphi, af::dim4(4, 4, 1, N));
+
+    // Compute the einsum for each of the christoffel terms
+    af::array partials = af::join(2, dt, dr, dtheta, dphi);
+    af::array p1       = af::matmul(invmetric, partials);
+    af::array p2       = af::reorder(p1, 0, 2, 1, 3);
+    af::array p3 = af::matmul(invmetric, af::reorder(partials, 2, 0, 1, 3));
+
+    auto christoffels = -0.5 * (p1 + p2 - p3);
+
+    // Use the geodesics equation to find the 4-vector acceleration
+    return af::moddims(af::sum(af::sum(christoffels * uu, 1), 2),
+                       af::dim4(4, N));
+}
+
+/**
+ * @brief Camera struct
+ *
+ * Contains all the data pertaining to the parameters for the image as seen from
+ * the camera
+ *
+ */
+struct Camera {
+    af::array position;
+    af::array lookat;
+    double fov;
+    double focal_length;
+    uint32_t width;
+    uint32_t height;
+
+    af::array direction;
+    af::array vertical;
+    af::array horizontal;
+    double aspect_ratio;
+
+    Camera(const af::array& position_, const af::array& lookat_, double fov_,
+           double focal_length_, uint32_t viewport_width_,
+           uint32_t viewport_height_)
+        : position(position_)
+        , lookat(lookat_)
+        , fov(fov_)
+        , focal_length(focal_length_)
+        , width(viewport_width_)
+        , height(viewport_height_) {
+        auto global_vertical = af::array(3, {0.0, 0.0, 1.0});
+
+        // Compute the camera three main axes
+        direction  = normalize3(lookat - position);
+        horizontal = normalize3(cross_product(direction, global_vertical));
+        vertical   = normalize3(cross_product(direction, horizontal));
+
+        aspect_ratio = (double)width / (double)height;
+    }
+
+    /**
+     * @brief Generates the initial rays 4-vector position and velocities
+     * (direction) for the simulation
+     *
+     * @return std::pair<af::array, af::array> (pos4, vel4)
+     */
+    std::pair<af::array, af::array> generate_viewport_4rays() {
+        auto& camera_direction  = direction;
+        auto& camera_horizontal = horizontal;
+        auto& camera_vertical   = vertical;
+        auto& camera_position   = position;
+        auto vfov               = fov;
+
+        double viewport_height = 2.0 * focal_length * std::tan(vfov / 2.0);
+        double viewport_width  = aspect_ratio * viewport_height;
+
+        // Create rays in equally spaced directions of the viewport
+        af::array viewport_rays = af::constant(0, 3, width, height, f64);
+        viewport_rays +=
+            (af::iota(af::dim4(1, width, 1), af::dim4(1, 1, height), f64) /
+                 (width - 1) -
+             0.5) *
+            viewport_width * camera_horizontal;
+        viewport_rays +=
+            (af::iota(af::dim4(1, 1, height), af::dim4(1, width, 1), f64) /
+                 (height - 1) -
+             0.5) *
+            viewport_height * camera_vertical;
+        viewport_rays += focal_length * camera_direction;
+        viewport_rays = af::moddims(af::reorder(viewport_rays, 1, 2, 0),
+                                    af::dim4(width * height, 3))
+                            .T();
+
+        // Compute the initial position from which the rays are launched
+        af::array viewport_position = viewport_rays + camera_position;
+        af::array viewport_sph_pos;
+        if (scene != Scene::ROTATE_BH)
+            viewport_sph_pos = cart_to_sph_position(viewport_position);
+        else
+            viewport_sph_pos = cart_to_oblate_position(viewport_position);
+
+        // Normalize the ray directions
+        viewport_rays = normalize3(viewport_rays);
+
+        // Generate the position 4-vector
+        af::array camera_sph_pos;
+        if (scene != Scene::ROTATE_BH)
+            camera_sph_pos = cart_to_sph_position(camera_position);
+        else
+            camera_sph_pos = cart_to_oblate_position(camera_position);
+
+        af::array camera_pos4 =
+            af::join(0, af::constant(0.0, 1, f64), camera_sph_pos);
+        double camera_velocity =
+            1.0 /
+            af::sqrt(norm4(camera_pos4, af::array(4, {1.0, 0.0, 0.0, 0.0})))
+                .scalar<double>();
+        af::array camera_vel4 = af::array(4, {camera_velocity, 0.0, 0.0, 0.0});
+
+        af::array viewport_rays_pos4 = af::join(
+            0, af::constant(0.0, 1, width * height, f64), viewport_sph_pos);
+
+        // Generate the velocity 4-vector by setting the camera to be stationary
+        // with respect to an observer at infinity
+        af::array vv;
+        if (scene != Scene::ROTATE_BH)
+            vv = cart_to_sph_velocity(viewport_rays, viewport_position);
+        else
+            vv = cart_to_oblate_velocity(viewport_rays, viewport_position);
+
+        af::array vvr = vv(0, af::span);
+        af::array vvo = vv(1, af::span);
+        af::array vvp = vv(2, af::span);
+        auto viewport_sph_rays4 =
+            af::join(0, af::constant(1, 1, width * height, f64), vvr, vvo, vvp);
+
+        af::array dot = af::moddims(
+            af::matmul(metric4(viewport_rays_pos4),
+                       af::moddims(viewport_sph_rays4 * viewport_sph_rays4,
+                                   af::dim4(4, 1, width * height))),
+            af::dim4(4, width * height));
+
+        // Normalize the 4-velocity vectors
+        af::array viewport_vel =
+            af::sqrt(-af::array(dot(0, af::span)) /
+                     (dot(1, af::span) + dot(2, af::span) + dot(3, af::span)));
+        af::array viewport_rays_vel4 =
+            af::join(0, af::constant(camera_velocity, 1, width * height, f64),
+                     vv * viewport_vel * camera_velocity);
+
+        return {viewport_rays_pos4, viewport_rays_vel4};
+    }
+};
+
+/**
+ * @brief Object struct
+ *
+ * Contains the methods for testing if a ray has collided with the object
+ *
+ */
+struct Object {
+    using HasHit = af::array;
+    using HitPos = af::array;
+
+    /**
+     * @brief Gets the color of the pixel that correspond to the ray that has
+     * intersected with the object
+     *
+     * @param ray_begin begining
+     * @param ray_end
+     * @return af::array
+     */
+    virtual af::array get_color(const af::array& ray_begin,
+                                const af::array& ray_end) const = 0;
+
+    /**
+     * @brief Returns a bool array if the rays have hit the object and the
+     * correspoding position where the ray has hit
+     *
+     * @param ray_begin
+     * @param ray_end
+     * @return std::pair<HasHit, HitPos>
+     */
+    virtual std::pair<HasHit, HitPos> intersect(
+        const af::array& ray_begin, const af::array& ray_end) const = 0;
+};
+
+struct AccretionDisk : public Object {
+    af::array disk_color;
+    af::array center;
+    af::array normal;
+    double inner_radius;
+    double outter_radius;
+
+    AccretionDisk(const af::array& center, const af::array& normal,
+                  double inner_radius, double outter_radius)
+        : disk_color(af::array(3, {209.f, 77.f, 0.f}))
+        , center(center)
+        , normal(normal)
+        , inner_radius(inner_radius)
+        , outter_radius(outter_radius) {
+        // disk_color = af::array(3, {254.f, 168.f, 29.f});
+    }
+
+    std::pair<HasHit, HitPos> intersect(
+        const af::array& ray_begin, const af::array& ray_end) const override {
+        uint32_t count = ray_begin.dims()[1];
+
+        // Compute intersection of ray with a plane
+        af::array has_hit = af::constant(0, count).as(b8);
+        af::array hit_pos = ray_end;
+        af::array a       = dot3(normal, center - ray_begin);
+        af::array b       = dot3(normal, ray_end - ray_begin);
+        af::array t       = af::select(b != 0.0, a / b, (double)0.0);
+
+        af::array plane_intersect = (ray_end - ray_begin) * t + ray_begin;
+        af::array dist            = norm3(plane_intersect - center);
+
+        t = af::abs(t);
+
+        // Determine if the intersection falls inside the disk radius and occurs
+        // with the current ray segment
+        has_hit = af::moddims((dist < outter_radius) && (t <= 1.0) &&
+                                  (t > 0.0) && (dist > inner_radius),
+                              af::dim4(count));
+        hit_pos = plane_intersect;
+
+        return {has_hit, hit_pos};
+    }
+
+    af::array get_color(const af::array& ray_begin,
+                        const af::array& ray_end) const override {
+        auto pair = intersect(ray_begin, ray_end);
+        af::array hit = pair.first;
+        af::array pos = pair.second;
+
+        auto val = 1.f - (norm3(pos - center).T() - inner_radius) /
+                             (outter_radius - inner_radius);
+
+        af::array color =
+            disk_color.T() * 1.5f * (val * val * (val * -2.f + 3.f)).as(f32);
+
+        return af::select(af::tile(hit, af::dim4(1, 3)), color, 0.f);
+    }
+};
+/**
+ * @brief Background struct
+ *
+ * Contains the methods for getting the color of background image
+ *
+ */
+struct Background {
+    af::array image;
+
+    Background(const af::array& image_) { image = image_; }
+
+    af::array get_color(const af::array& ray_dir) const {
+        auto spherical_dir = cart_to_sph_position(ray_dir);
+
+        auto img_height = image.dims()[0];
+        auto img_width  = image.dims()[1];
+        auto count      = ray_dir.dims()[1];
+
+        // Spherical mapping of the direction to a pixel of the image
+        af::array o = spherical_dir(1, af::span);
+        af::array p = spherical_dir(2, af::span);
+
+        auto x = (p / af::Pi + 1.0) * img_width / 2.0;
+        auto y = (o / af::Pi) * img_height;
+
+        // Interpolate the colors of the image from the calculated pixel
+        // positions
+        af::array colors = af::approx2(image, af::moddims(y.as(f32), count),
+                                       af::moddims(x.as(f32), count),
+                                       af::interpType::AF_INTERP_CUBIC_SPLINE);
+
+        // Zero out the color of any null rays
+        colors = af::moddims(colors, af::dim4(count, 3));
+        af::replace(colors, !af::isNaN(colors), 0.f);
+
+        return colors;
+    }
+};
+
+/**
+ * @brief Transform the array of pixels to the correct image format to display
+ *
+ * @param image
+ * @param width
+ * @param height
+ * @return af::array
+ */
+af::array rearrange_image(const af::array& image, uint32_t width,
+                          uint32_t height) {
+    return af::clamp(af::moddims(image, af::dim4(width, height, 3)).T(), 0.0,
+                     255.0)
+               .as(f32) /
+           255.f;
+}
+
+/**
+ * @brief Returns an rgb image containing the raytraced black hole from the
+ * camera rays, spacetime metric, objects living in the space, and background
+ *
+ * @param initial_pos initial position from where the rays are launched
+ * @param initial_vel initial velocities (directions) the rays have
+ * @param objects the objects the rays can collide with
+ * @param background the background of the scene
+ * @param time how long are the rays traced through space
+ * @param steps how many steps should be taken to trace the rays path
+ * @param width width of the image the camera produces
+ * @param height height of the image the camera produces
+ * @param checks the intervals between steps to check if the rays have collided
+ * with an object
+ * @return af::array
+ */
+af::array generate_image(const af::array& initial_pos,
+                         const af::array& initial_vel,
+                         const std::vector<std::unique_ptr<Object> >& objects,
+                         const Background& background, uint32_t width,
+                         uint32_t height, double time, double tol,
+                         uint32_t checks = 10) {
+    uint32_t lines = initial_pos.dims()[1];
+
+    auto def_step = 0.5 * pow(tol, 0.25);
+    auto dt       = af::constant(def_step, 1, lines, f64);
+    auto t        = af::constant(0.0, 1, lines, f64);
+    auto index    = af::iota(lines);
+    auto selected = t < time;
+
+    auto result = af::constant(0, lines, 3, f32);
+
+    auto pos = initial_pos;
+    auto vel = initial_vel;
+
+    af::Window window{(int)width, (int)height, "Black Hole Raytracing"};
+
+    af::array bg_col = af::constant(0.f, lines, 3);
+    af::array begin_pos, end_pos;
+    af::array bh_nohit;
+
+    if (scene != Scene::ROTATE_BH)
+        begin_pos = sph_to_cart_position(pos(af::seq(1, 3), af::span));
+    else
+        begin_pos = oblate_to_cart_position(pos(af::seq(1, 3), af::span));
+    end_pos = begin_pos;
+
+    int i = 0;
+
+    while (t.dims()[1] != 0 && af::anyTrue<bool>(t < time) &&
+           af::anyTrue<bool>(dt != 0.0)) {
+        // Displays the current progress and approximate time needed to finish
+        // it
+        status_bar((lines - t.dims()[1]) * time +
+                       af::sum<double>(af::clamp(t, 0.0, time)),
+                   time * lines, "Progress:");
+
+        // RK34 method for second order differential equation
+        auto dt2 = dt * dt;
+        auto k1  = geodesics(pos, vel);
+        auto k2  = geodesics(pos + vel * dt / 4.0 + k1 * dt2 / 32.0,
+                             vel + k1 * dt / 4.0);
+        auto k3  = geodesics(pos + vel * dt / 2.0 + (k1 + k2) * dt2 / 16.0,
+                             vel + k2 * dt / 2.0);
+        auto k4  = geodesics(pos + vel * dt + (k1 - k2 + k3 * 2.0) * dt2 / 4.0,
+                             vel + (k1 - k2 * 2.0 + 2.0 * k3) * dt);
+
+        auto diff4 = (k1 + k2 * 8.0 + k3 * 2.0 + k4) / 24.0;
+        auto diff3 = (k2 * 8.0 + k4) / 18.0;
+
+        auto err    = (af::max)(af::abs(diff4 - diff3), 0) * dt2;
+        auto maxerr = tol * (1.0 + (af::max)(af::abs(pos), 0));
+
+        auto rdt = af::constant(0, 1, dt.dims()[1], f64);
+        af::replace(rdt, err > maxerr, dt);
+
+        auto rdt2 = rdt * rdt;
+
+        pos += vel * rdt + (k1 + k2 * 8.0 + k3 * 2.0 + k4) * rdt2 / 24.0;
+        vel += (k1 + k3 * 4.0 + k4) * rdt / 6.0;
+        t += rdt;
+
+        auto q = af::clamp(0.8 * af::pow(maxerr / err, 0.25), 0.0, 5.0);
+
+        // Select the next time step
+        dt = af::select(q * dt < (time - t), q * dt, af::abs(time - t));
+
+        // Update image
+        if (i % checks == (checks - 1)) {
+            af::array ray_dir;
+            if (scene != Scene::ROTATE_BH) {
+                end_pos(af::span, index) =
+                    sph_to_cart_position(pos(af::seq(1, 3), af::span));
+                ray_dir = sph_to_cart_velocity(vel(af::seq(1, 3), af::span),
+                                               pos(af::seq(1, 3), af::span));
+            } else {
+                end_pos(af::span, index) =
+                    oblate_to_cart_position(pos(af::seq(1, 3), af::span));
+                ray_dir = oblate_to_cart_velocity(vel(af::seq(1, 3), af::span),
+                                                  pos(af::seq(1, 3), af::span));
+            }
+
+            af::array s_begin_pos = begin_pos(af::span, index);
+            af::array s_end_pos   = end_pos(af::span, index);
+
+            // Check if light ray intersect an object
+            for (const auto& obj : objects) {
+                result(index, af::span) +=
+                    obj->get_color(s_begin_pos, s_end_pos);
+            }
+
+            // Update background colors from rays
+            bg_col(index, af::span) = background.get_color(ray_dir);
+
+            // Display image
+            window.image(rearrange_image(result + bg_col, width, height));
+
+            begin_pos = end_pos;
+        }
+
+        // Stop rays entering the event horizon
+        switch (scene) {
+            case Scene::ROTATE_BH: {
+                auto a = J / M;
+                bh_nohit =
+                    (pos(1, af::span) > 1.01 * (M + std::sqrt(M * M - a * a)));
+                selected = bh_nohit && (t < time);
+
+                break;
+            }
+
+            case Scene::STATIC_BH: {
+                bh_nohit = pos(1, af::span) > 2.0 * M * 1.01;
+                selected = bh_nohit && (t < time);
+
+                break;
+            }
+
+            case Scene::WORMHOLE: {
+                selected = (t < time);
+            }
+            default: break;
+        }
+
+        // Remove finished rays from computation
+        if (af::sum<float>(selected.as(f32)) / (float)index.dims()[0] < 0.75) {
+            if (scene == Scene::STATIC_BH || scene == Scene::ROTATE_BH)
+                bg_col(af::array(index(!bh_nohit)), af::span) = 0.f;
+
+            index = index(selected);
+            pos   = pos(af::span, selected);
+            vel   = vel(af::span, selected);
+            dt    = dt(af::span, selected);
+            t     = t(af::span, selected);
+
+            // Free finished rays memory
+            af::deviceGC();
+        }
+
+        ++i;
+    }
+
+    result += bg_col;
+
+    return rearrange_image(result, width, height);
+}
+
+void raytracing(uint32_t width, uint32_t height) {
+    // Set the parameters of the raytraced image
+    double vfov         = radians(90.0);
+    double focal_length = 0.01;
+
+    // Set the parameters of the camera
+    af::array global_vertical            = af::array(3, {0.0, 0.0, 1.0});
+    af::array camera_position            = af::array(3, {-7.0, 6.0, 2.0});
+    af::array camera_lookat              = af::array(3, {0.0, 0.0, 0.0});
+    double accretion_inner_radius        = M * 3.0;
+    double accretion_outter_radius       = M * 8.0;
+    double simulation_tolerance          = 1e-6;
+    double max_simulation_time           = 12.;
+    uint32_t num_steps_per_collide_check = 1;
+
+    // Set the background of the scene
+    auto bg_image =
+        af::loadimage(ASSETS_DIR "/examples/images/westerlund.jpg", true);
+    auto background = Background(bg_image);
+
+    // Set the objects living in the scene
+    std::vector<std::unique_ptr<Object> > objects;
+    if (scene != Scene::WORMHOLE)
+        objects.push_back(std::make_unique<AccretionDisk>(
+            af::array(3, {0.0, 0.0, 0.0}), af::array(3, {0.0, 0.0, 1.0}),
+            accretion_inner_radius, accretion_outter_radius));
+
+    // Generate rays from the camera
+    auto camera = Camera(camera_position, camera_lookat, vfov, focal_length,
+                         width, height);
+    auto pair   = camera.generate_viewport_4rays();
+
+    auto ray4_pos = pair.first;
+    auto ray4_vel = pair.second;
+
+    auto begin = std::chrono::high_resolution_clock::now();
+    // Generate raytraced image
+    auto image = generate_image(
+        ray4_pos, ray4_vel, objects, background, width, height,
+        max_simulation_time, simulation_tolerance, num_steps_per_collide_check);
+
+    auto end = std::chrono::high_resolution_clock::now();
+
+    std::cout
+        << "\nSimulation took: "
+        << std::chrono::duration_cast<std::chrono::seconds>(end - begin).count()
+        << " s" << std::endl;
+
+    // Save image
+    af::saveImage("result.png", image);
+}
+
+int main(int argc, char** argv) {
+    int device = argc > 1 ? std::atoi(argv[1]) : 0;
+
+    int width  = argc > 2 ? std::atoi(argv[2]) : 200;
+    int height = argc > 3 ? std::atoi(argv[3]) : 200;
+
+    try {
+        af::setDevice(device);
+        af::info();
+
+        std::cout << "** ArrayFire Black Hole Raytracing Demo\n\n";
+
+        raytracing(width, height);
+    } catch (const af::exception& e) {
+        std::cerr << e.what() << std::endl;
+        return -1;
+    }
+
+    return 0;
+}
\ No newline at end of file

From ccac73e86ac6f761770ed594255c04a01087732d Mon Sep 17 00:00:00 2001
From: verstatx <ayer.3d@gmail.com>
Date: Wed, 4 Oct 2023 05:50:04 -0400
Subject: [PATCH 442/473] Add int8 matmul support to the CUDA backend

changes to gemm account for differing input/output types
---
 docs/details/blas.dox       |   4 ++
 include/af/blas.h           |   8 +++
 src/api/c/blas.cpp          |  35 +++++++++----
 src/backend/cpu/blas.cpp    |  57 +++++++++++---------
 src/backend/cpu/blas.hpp    |   7 +--
 src/backend/cuda/blas.cu    | 102 ++++++++++++++++++++----------------
 src/backend/cuda/blas.hpp   |   7 +--
 src/backend/oneapi/blas.cpp |  37 ++++++++-----
 src/backend/oneapi/blas.hpp |   7 +--
 src/backend/opencl/blas.cpp |  21 +++++---
 src/backend/opencl/blas.hpp |   7 +--
 test/blas.cpp               |  30 +++++++++++
 12 files changed, 212 insertions(+), 110 deletions(-)

diff --git a/docs/details/blas.dox b/docs/details/blas.dox
index 943e77a502..ac0aa99673 100644
--- a/docs/details/blas.dox
+++ b/docs/details/blas.dox
@@ -32,6 +32,10 @@ memory allocations either on host or device.
 for Sparse-Dense matrix multiplication. See the notes of the function for usage
 and restrictions.
 
+\par
+\note Limited support for \ref s8 was added to the CUDA backend in ArrayFire
+v3.10.0. See \ref af_gemm "s8 Support" notes for details.
+
 \ingroup blas_mat
 
 =======================================================================
diff --git a/include/af/blas.h b/include/af/blas.h
index 4580ea2112..05434ee861 100644
--- a/include/af/blas.h
+++ b/include/af/blas.h
@@ -242,6 +242,14 @@ extern "C" {
 
         \snippet test/blas.cpp ex_af_gemm_overwrite
 
+        \note <b>s8 Support</b>
+        \note Starting with ArrayFire version v3.10.0, the CUDA backend supports
+        \p A, \p B input arrays of type \ref s8.
+        \note Scalars \p alpha, \p beta must be of type \ref f32.
+        \note Output array \p C will be of type \ref f32.
+        \note <br><b>Requires</b>
+        \note CUDA version >= 10 on devices with compute capability >= 5.0
+
         \param[in,out] C     `A` * `B` = `C`
         \param[in]     opA   operation to perform on A before the multiplication
         \param[in]     opB   operation to perform on B before the multiplication
diff --git a/src/api/c/blas.cpp b/src/api/c/blas.cpp
index 0cd8fddd8d..f42bc7d57c 100644
--- a/src/api/c/blas.cpp
+++ b/src/api/c/blas.cpp
@@ -33,6 +33,7 @@ using detail::cdouble;
 using detail::cfloat;
 using detail::gemm;
 using detail::matmul;
+using detail::schar;
 
 namespace {
 template<typename T>
@@ -42,12 +43,12 @@ static inline af_array sparseMatmul(const af_array lhs, const af_array rhs,
         matmul<T>(getSparseArray<T>(lhs), getArray<T>(rhs), optLhs, optRhs));
 }
 
-template<typename T>
+template<typename Ti, typename To = Ti>
 static inline void gemm(af_array *out, af_mat_prop optLhs, af_mat_prop optRhs,
-                        const T *alpha, const af_array lhs, const af_array rhs,
-                        const T *betas) {
-    gemm<T>(getArray<T>(*out), optLhs, optRhs, alpha, getArray<T>(lhs),
-            getArray<T>(rhs), betas);
+                        const To *alpha, const af_array lhs, const af_array rhs,
+                        const To *betas) {
+    gemm<Ti, To>(getArray<To>(*out), optLhs, optRhs, alpha, getArray<Ti>(lhs),
+                 getArray<Ti>(rhs), betas);
 }
 
 template<typename T>
@@ -178,6 +179,8 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs,
         if (*out) {
             output = *out;
         } else {
+            af_dtype out_type = (lhs_type != s8) ? lhs_type : f32;
+
             const int aRowDim    = (optLhs == AF_MAT_NONE) ? 0 : 1;
             const int bColDim    = (optRhs == AF_MAT_NONE) ? 1 : 0;
             const int M          = lDims[aRowDim];
@@ -186,7 +189,7 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs,
             const dim_t d3       = std::max(lDims[3], rDims[3]);
             const af::dim4 oDims = af::dim4(M, N, d2, d3);
             AF_CHECK(af_create_handle(&output, lhsInfo.ndims(), oDims.get(),
-                                      lhs_type));
+                                      out_type));
         }
 
         switch (lhs_type) {
@@ -215,6 +218,11 @@ af_err af_gemm(af_array *out, const af_mat_prop optLhs,
                            static_cast<const half *>(alpha), lhs, rhs,
                            static_cast<const half *>(beta));
                 break;
+            case s8:
+                gemm<schar, float>(&output, optLhs, optRhs,
+                                   static_cast<const float *>(alpha), lhs, rhs,
+                                   static_cast<const float *>(beta));
+                break;
             default: TYPE_ERROR(3, lhs_type);
         }
 
@@ -246,11 +254,13 @@ af_err af_matmul(af_array *out, const af_array lhs, const af_array rhs,
         const dim_t d3       = std::max(lDims[3], rDims[3]);
         const af::dim4 oDims = af::dim4(M, N, d2, d3);
 
-        af_array gemm_out = 0;
+        af_dtype lhs_type = lhsInfo.getType();
+
+        af_array gemm_out      = 0;
+        af_dtype gemm_out_type = (lhs_type != s8) ? lhs_type : f32;
         AF_CHECK(af_create_handle(&gemm_out, oDims.ndims(), oDims.get(),
-                                  lhsInfo.getType()));
+                                  gemm_out_type));
 
-        af_dtype lhs_type = lhsInfo.getType();
         switch (lhs_type) {
             case f16: {
                 static const half alpha(1.0f);
@@ -288,6 +298,13 @@ af_err af_matmul(af_array *out, const af_array lhs, const af_array rhs,
                                  &beta));
                 break;
             }
+            case s8: {
+                float alpha = 1.0;
+                float beta  = 0.0;
+                AF_CHECK(af_gemm(&gemm_out, optLhs, optRhs, &alpha, lhs, rhs,
+                                 &beta));
+                break;
+            }
             default: TYPE_ERROR(1, lhs_type);
         }
 
diff --git a/src/backend/cpu/blas.cpp b/src/backend/cpu/blas.cpp
index b7d158eb21..60cd9be655 100644
--- a/src/backend/cpu/blas.cpp
+++ b/src/backend/cpu/blas.cpp
@@ -219,9 +219,10 @@ toCblasTranspose(af_mat_prop opt) {
     return out;
 }
 
-template<typename T>
-void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
-          const Array<T> &lhs, const Array<T> &rhs, const T *beta) {
+template<typename Ti, typename To>
+void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+          const To *alpha, const Array<Ti> &lhs, const Array<Ti> &rhs,
+          const To *beta) {
     const CBLAS_TRANSPOSE lOpts = toCblasTranspose(optLhs);
     const CBLAS_TRANSPOSE rOpts = toCblasTranspose(optRhs);
 
@@ -236,17 +237,17 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
     const int K       = lDims[aColDim];
     const dim4 oDims  = out.dims();
 
-    using BT  = typename blas_base<T>::type;
-    using CBT = const typename blas_base<T>::type;
+    using BT  = typename blas_base<Ti>::type;
+    using CBT = const typename blas_base<Ti>::type;
 
-    auto alpha_ = scale_type<T, false>(alpha);
-    auto beta_  = scale_type<T, false>(beta);
+    auto alpha_ = scale_type<Ti, false>(alpha);
+    auto beta_  = scale_type<Ti, false>(beta);
 #ifdef USE_MKL
-    auto alpha_batched = scale_type<T, true>(alpha);
-    auto beta_batched  = scale_type<T, true>(beta);
+    auto alpha_batched = scale_type<Ti, true>(alpha);
+    auto beta_batched  = scale_type<Ti, true>(beta);
 #endif
 
-    auto func = [=](Param<T> output, CParam<T> left, CParam<T> right) {
+    auto func = [=](Param<Ti> output, CParam<Ti> left, CParam<Ti> right) {
         dim4 lStrides = left.strides();
         dim4 rStrides = right.strides();
         dim4 oStrides = output.strides();
@@ -255,14 +256,14 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
             if (right.dims()[bColDim] == 1) {
                 dim_t incr =
                     (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
-                gemv_func<T>()(
+                gemv_func<Ti>()(
                     CblasColMajor, lOpts, lDims[0], lDims[1], alpha_.getScale(),
                     reinterpret_cast<CBT *>(left.get()), lStrides[1],
                     reinterpret_cast<CBT *>(right.get()), incr,
                     beta_.getScale(), reinterpret_cast<BT *>(output.get()),
                     oStrides[0]);
             } else {
-                gemm_func<T>()(
+                gemm_func<Ti>()(
                     CblasColMajor, lOpts, rOpts, M, N, K, alpha_.getScale(),
                     reinterpret_cast<CBT *>(left.get()), lStrides[1],
                     reinterpret_cast<CBT *>(right.get()), rStrides[1],
@@ -303,24 +304,24 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
             const MKL_INT ldb = rStrides[1];
             const MKL_INT ldc = oStrides[1];
 
-            gemm_batch_func<T>()(CblasColMajor, &lOpts, &rOpts, &M, &N, &K,
-                                 alpha_batched.getScale(), lptrs.data(), &lda,
-                                 rptrs.data(), &ldb, beta_batched.getScale(),
-                                 optrs.data(), &ldc, 1, &batchSize);
+            gemm_batch_func<Ti>()(CblasColMajor, &lOpts, &rOpts, &M, &N, &K,
+                                  alpha_batched.getScale(), lptrs.data(), &lda,
+                                  rptrs.data(), &ldb, beta_batched.getScale(),
+                                  optrs.data(), &ldc, 1, &batchSize);
 #else
             for (int n = 0; n < batchSize; n++) {
                 if (rDims[bColDim] == 1) {
                     dim_t incr =
                         (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
-                    gemv_func<T>()(CblasColMajor, lOpts, lDims[0], lDims[1],
-                                   alpha_.getScale(), lptrs[n], lStrides[1],
-                                   rptrs[n], incr, beta_.getScale(), optrs[n],
-                                   oStrides[0]);
+                    gemv_func<Ti>()(CblasColMajor, lOpts, lDims[0], lDims[1],
+                                    alpha_.getScale(), lptrs[n], lStrides[1],
+                                    rptrs[n], incr, beta_.getScale(), optrs[n],
+                                    oStrides[0]);
                 } else {
-                    gemm_func<T>()(CblasColMajor, lOpts, rOpts, M, N, K,
-                                   alpha_.getScale(), lptrs[n], lStrides[1],
-                                   rptrs[n], rStrides[1], beta_.getScale(),
-                                   optrs[n], oStrides[1]);
+                    gemm_func<Ti>()(CblasColMajor, lOpts, rOpts, M, N, K,
+                                    alpha_.getScale(), lptrs[n], lStrides[1],
+                                    rptrs[n], rStrides[1], beta_.getScale(),
+                                    optrs[n], oStrides[1]);
                 }
             }
 #endif
@@ -341,6 +342,14 @@ void gemm<half>(Array<half> &out, af_mat_prop optLhs, af_mat_prop optRhs,
     copyArray(out, outArr);
 }
 
+template<>
+void gemm<schar, float>(Array<float> &out, af_mat_prop optLhs,
+                        af_mat_prop optRhs, const float *alpha,
+                        const Array<schar> &lhs, const Array<schar> &rhs,
+                        const float *beta) {
+    TYPE_ERROR(3, af_dtype::s8);
+}
+
 template<typename T>
 Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
              af_mat_prop optRhs) {
diff --git a/src/backend/cpu/blas.hpp b/src/backend/cpu/blas.hpp
index 1043a567e9..c16916dafb 100644
--- a/src/backend/cpu/blas.hpp
+++ b/src/backend/cpu/blas.hpp
@@ -13,9 +13,10 @@
 namespace arrayfire {
 namespace cpu {
 
-template<typename T>
-void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
-          const Array<T> &lhs, const Array<T> &rhs, const T *beta);
+template<typename Ti, typename To = Ti>
+void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+          const To *alpha, const Array<Ti> &lhs, const Array<Ti> &rhs,
+          const To *beta);
 
 template<typename T>
 Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
diff --git a/src/backend/cuda/blas.cu b/src/backend/cuda/blas.cu
index 6c88ea002a..08df398a8d 100644
--- a/src/backend/cuda/blas.cu
+++ b/src/backend/cuda/blas.cu
@@ -91,6 +91,17 @@ BLAS_FUNC(gemmBatched, double, D)
 BLAS_FUNC(gemmBatched, cdouble, Z)
 BLAS_FUNC(gemmBatched, __half, H)
 
+template<>
+gemm_func_def<schar> gemm_func<schar>() {
+    TYPE_ERROR(3, af_dtype::s8);
+    return gemm_func_def<schar>();
+}
+template<>
+gemmBatched_func_def<schar> gemmBatched_func<schar>() {
+    TYPE_ERROR(3, af_dtype::s8);
+    return gemmBatched_func_def<schar>();
+}
+
 BLAS_FUNC_DEF(trsm)
 BLAS_FUNC(trsm, float, S)
 BLAS_FUNC(trsm, cfloat, C)
@@ -161,20 +172,20 @@ cublasGemmAlgo_t selectGEMMAlgorithm<__half>() {
     return selectGEMMAlgorithm<common::half>();
 }
 
-template<typename T>
+template<typename Ti, typename To = Ti>
 cublasStatus_t gemmDispatch(BlasHandle handle, cublasOperation_t lOpts,
                             cublasOperation_t rOpts, int M, int N, int K,
-                            const T *alpha, const Array<T> &lhs, dim_t lStride,
-                            const Array<T> &rhs, dim_t rStride, const T *beta,
-                            Array<T> &out, dim_t oleading) {
+                            const To *alpha, const Array<Ti> &lhs, dim_t lStride,
+                            const Array<Ti> &rhs, dim_t rStride, const To *beta,
+                            Array<To> &out, dim_t oleading) {
     auto prop = getDeviceProp(getActiveDeviceId());
 #if __CUDACC_VER_MAJOR__ >= 10
     if (prop.major > 3 && __CUDACC_VER_MAJOR__ >= 10) {
         return cublasGemmEx(
-            blasHandle(), lOpts, rOpts, M, N, K, alpha, lhs.get(), getType<T>(),
-            lStride, rhs.get(), getType<T>(), rStride, beta, out.get(),
-            getType<T>(), out.strides()[1],
-            getComputeType<T>(),  // Compute type
+            blasHandle(), lOpts, rOpts, M, N, K, alpha, lhs.get(), getType<Ti>(),
+            lStride, rhs.get(), getType<Ti>(), rStride, beta, out.get(),
+            getType<To>(), out.strides()[1],
+            getComputeType<To>(),  // Compute type
 
             // NOTE: When using the CUBLAS_GEMM_DEFAULT_TENSOR_OP algorithm
             // for the cublasGemm*Ex functions, the performance of the
@@ -184,10 +195,10 @@ cublasStatus_t gemmDispatch(BlasHandle handle, cublasOperation_t lOpts,
             // this change. Does this imply that the TENSOR_OP function
             // performs the computation in fp16 bit even when the compute
             // type is CUDA_R_32F?
-            selectGEMMAlgorithm<T>());
+            selectGEMMAlgorithm<Ti>());
     } else {
 #endif
-        using Nt = typename common::kernel_type<T>::native;
+        using Nt = typename common::kernel_type<Ti>::native;
         return gemm_func<Nt>()(blasHandle(), lOpts, rOpts, M, N, K, (Nt *)alpha,
                                (Nt *)lhs.get(), lStride, (Nt *)rhs.get(),
                                rStride, (Nt *)beta, (Nt *)out.get(), oleading);
@@ -197,21 +208,21 @@ cublasStatus_t gemmDispatch(BlasHandle handle, cublasOperation_t lOpts,
 #endif
 }
 
-template<typename T>
+template<typename Ti, typename To = Ti>
 cublasStatus_t gemmBatchedDispatch(BlasHandle handle, cublasOperation_t lOpts,
                                    cublasOperation_t rOpts, int M, int N, int K,
-                                   const T *alpha, const T **lptrs,
-                                   int lStrides, const T **rptrs, int rStrides,
-                                   const T *beta, T **optrs, int oStrides,
+                                   const To *alpha, const Ti **lptrs,
+                                   int lStrides, const Ti **rptrs, int rStrides,
+                                   const To *beta, To **optrs, int oStrides,
                                    int batchSize) {
     auto prop = getDeviceProp(getActiveDeviceId());
 #if __CUDACC_VER_MAJOR__ >= 10
     if (prop.major > 3) {
         return cublasGemmBatchedEx(
             blasHandle(), lOpts, rOpts, M, N, K, alpha, (const void **)lptrs,
-            getType<T>(), lStrides, (const void **)rptrs, getType<T>(),
-            rStrides, beta, (void **)optrs, getType<T>(), oStrides, batchSize,
-            getComputeType<T>(),  // compute type
+            getType<Ti>(), lStrides, (const void **)rptrs, getType<Ti>(),
+            rStrides, beta, (void **)optrs, getType<Ti>(), oStrides, batchSize,
+            getComputeType<Ti>(),  // compute type
             // NOTE: When using the CUBLAS_GEMM_DEFAULT_TENSOR_OP algorithm
             // for the cublasGemm*Ex functions, the performance of the
             // fp32 numbers seem to increase dramatically. Their numerical
@@ -220,10 +231,10 @@ cublasStatus_t gemmBatchedDispatch(BlasHandle handle, cublasOperation_t lOpts,
             // this change. Does this imply that the TENSOR_OP function
             // performs the computation in fp16 bit even when the compute
             // type is CUDA_R_32F?
-            selectGEMMAlgorithm<T>());
+            selectGEMMAlgorithm<Ti>());
     } else {
 #endif
-        using Nt = typename common::kernel_type<T>::native;
+        using Nt = typename common::kernel_type<Ti>::native;
         return gemmBatched_func<Nt>()(
             blasHandle(), lOpts, rOpts, M, N, K, (const Nt *)alpha,
             (const Nt **)lptrs, lStrides, (const Nt **)rptrs, rStrides,
@@ -233,9 +244,9 @@ cublasStatus_t gemmBatchedDispatch(BlasHandle handle, cublasOperation_t lOpts,
 #endif
 }
 
-template<typename T>
-void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
-          const Array<T> &lhs, const Array<T> &rhs, const T *beta) {
+template<typename Ti, typename To>
+void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs, const To *alpha,
+          const Array<Ti> &lhs, const Array<Ti> &rhs, const To *beta) {
     const cublasOperation_t lOpts = toCblasTranspose(optLhs);
     const cublasOperation_t rOpts = toCblasTranspose(optRhs);
 
@@ -255,14 +266,14 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
     dim4 oStrides = out.strides();
 
     if (oDims.ndims() <= 2) {
-        CUBLAS_CHECK(gemmDispatch<T>(blasHandle(), lOpts, rOpts, M, N, K, alpha,
-                                     lhs, lStrides[1], rhs, rStrides[1], beta,
-                                     out, oStrides[1]));
+        CUBLAS_CHECK((gemmDispatch<Ti, To>(blasHandle(), lOpts, rOpts, M, N, K, alpha,
+                                           lhs, lStrides[1], rhs, rStrides[1], beta,
+                                           out, oStrides[1])));
     } else {
         int batchSize = oDims[2] * oDims[3];
-        vector<const T *> lptrs(batchSize);
-        vector<const T *> rptrs(batchSize);
-        vector<T *> optrs(batchSize);
+        vector<const Ti *> lptrs(batchSize);
+        vector<const Ti *> rptrs(batchSize);
+        vector<To *> optrs(batchSize);
 
         bool is_l_d2_batched = oDims[2] == lDims[2];
         bool is_l_d3_batched = oDims[3] == lDims[3];
@@ -270,9 +281,9 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
         bool is_r_d2_batched = oDims[2] == rDims[2];
         bool is_r_d3_batched = oDims[3] == rDims[3];
 
-        const T *lptr = lhs.get();
-        const T *rptr = rhs.get();
-        T *optr       = out.get();
+        const Ti *lptr = lhs.get();
+        const Ti *rptr = rhs.get();
+        To *optr    = out.get();
 
         for (int n = 0; n < batchSize; n++) {
             int w    = n / oDims[2];
@@ -286,7 +297,7 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
             optrs[n] = optr + z * oStrides[2] + w * oStrides[3];
         }
 
-        size_t bytes = batchSize * sizeof(T **);
+        size_t bytes = batchSize * sizeof(Ti **);
         auto d_lptrs = memAlloc<uchar>(bytes);
         auto d_rptrs = memAlloc<uchar>(bytes);
         auto d_optrs = memAlloc<uchar>(bytes);
@@ -302,11 +313,11 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
         // afterwards
         CUDA_CHECK(cudaStreamSynchronize(getActiveStream()));
 
-        using Nt = typename common::kernel_type<T>::native;
+        using Nt = typename common::kernel_type<Ti>::native;
         CUBLAS_CHECK(gemmBatchedDispatch(
             blasHandle(), lOpts, rOpts, M, N, K, alpha,
-            (const T **)d_lptrs.get(), lStrides[1], (const T **)d_rptrs.get(),
-            rStrides[1], beta, (T **)d_optrs.get(), oStrides[1], batchSize));
+            (const Ti **)d_lptrs.get(), lStrides[1], (const Ti **)d_rptrs.get(),
+            rStrides[1], beta, (To **)d_optrs.get(), oStrides[1], batchSize));
     }
 }
 
@@ -340,17 +351,18 @@ void trsm(const Array<T> &lhs, Array<T> &rhs, af_mat_prop trans, bool is_upper,
         lhs.get(), lStrides[1], rhs.get(), rStrides[1]));
 }
 
-#define INSTANTIATE_GEMM(TYPE)                                               \
-    template void gemm<TYPE>(Array<TYPE> & out, af_mat_prop optLhs,          \
-                             af_mat_prop optRhs, const TYPE *alpha,          \
+#define INSTANTIATE_GEMM(TYPE, OUTTYPE)                                      \
+    template void gemm<TYPE>(Array<OUTTYPE> & out, af_mat_prop optLhs,       \
+                             af_mat_prop optRhs, const OUTTYPE *alpha,       \
                              const Array<TYPE> &lhs, const Array<TYPE> &rhs, \
-                             const TYPE *beta);
-
-INSTANTIATE_GEMM(float)
-INSTANTIATE_GEMM(cfloat)
-INSTANTIATE_GEMM(double)
-INSTANTIATE_GEMM(cdouble)
-INSTANTIATE_GEMM(half)
+                             const OUTTYPE *beta);
+
+INSTANTIATE_GEMM(float, float)
+INSTANTIATE_GEMM(cfloat, cfloat)
+INSTANTIATE_GEMM(double, double)
+INSTANTIATE_GEMM(cdouble, cdouble)
+INSTANTIATE_GEMM(half, half)
+INSTANTIATE_GEMM(schar, float)
 
 #define INSTANTIATE_DOT(TYPE)                                                  \
     template Array<TYPE> dot<TYPE>(const Array<TYPE> &lhs,                     \
diff --git a/src/backend/cuda/blas.hpp b/src/backend/cuda/blas.hpp
index dc4382d013..37432911e2 100644
--- a/src/backend/cuda/blas.hpp
+++ b/src/backend/cuda/blas.hpp
@@ -11,9 +11,10 @@
 
 namespace arrayfire {
 namespace cuda {
-template<typename T>
-void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
-          const Array<T> &lhs, const Array<T> &rhs, const T *beta);
+template<typename Ti, typename To = Ti>
+void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+          const To *alpha, const Array<Ti> &lhs, const Array<Ti> &rhs,
+          const To *beta);
 
 template<typename T>
 Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
diff --git a/src/backend/oneapi/blas.cpp b/src/backend/oneapi/blas.cpp
index 37495957e9..93ae6559a4 100644
--- a/src/backend/oneapi/blas.cpp
+++ b/src/backend/oneapi/blas.cpp
@@ -97,9 +97,10 @@ bool isStrideMonotonic(const af::dim4 &dim) {
     return (dim[0] <= dim[1]) && (dim[1] <= dim[2]) && (dim[2] <= dim[3]);
 }
 
-template<typename T>
-void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
-          const Array<T> &lhs, const Array<T> &rhs, const T *beta) {
+template<typename Ti, typename To>
+void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+          const To *alpha, const Array<Ti> &lhs, const Array<Ti> &rhs,
+          const To *beta) {
     const auto lOpts = toBlasTranspose(optLhs);
     const auto rOpts = toBlasTranspose(optRhs);
 
@@ -120,25 +121,25 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
 
     if (oDims.ndims() <= 2) {  // if non-batched
         if (rhs.dims()[bColDim] == 1) {
-            if constexpr (std::is_same_v<T, arrayfire::common::half>) {
+            if constexpr (std::is_same_v<Ti, arrayfire::common::half>) {
                 // currently no half support for gemv, use gemm instead
-                gemmDispatch<T>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
-                                lStrides[1], rhs, rStrides[1], beta, out,
-                                oStrides[1]);
+                gemmDispatch<Ti>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
+                                 lStrides[1], rhs, rStrides[1], beta, out,
+                                 oStrides[1]);
             } else {
                 dim_t incr =
                     (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
-                gemvDispatch<T>(getQueue(), lOpts, rOpts, lDims[0], lDims[1],
-                                alpha, lhs, lStrides[1], rhs, incr, beta, out,
-                                oStrides[0]);
+                gemvDispatch<Ti>(getQueue(), lOpts, rOpts, lDims[0], lDims[1],
+                                 alpha, lhs, lStrides[1], rhs, incr, beta, out,
+                                 oStrides[0]);
             }
         } else {
-            gemmDispatch<T>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
-                            lStrides[1], rhs, rStrides[1], beta, out,
-                            oStrides[1]);
+            gemmDispatch<Ti>(getQueue(), lOpts, rOpts, M, N, K, alpha, lhs,
+                             lStrides[1], rhs, rStrides[1], beta, out,
+                             oStrides[1]);
         }
     } else {  // if batched
-        using Dt = arrayfire::oneapi::data_t<T>;
+        using Dt = arrayfire::oneapi::data_t<Ti>;
 
         int64_t batchSize = static_cast<int64_t>(oDims[2] * oDims[3]);
 
@@ -206,6 +207,14 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
     ONEAPI_DEBUG_FINISH(getQueue());
 }
 
+template<>
+void gemm<schar, float>(Array<float> &out, af_mat_prop optLhs,
+                        af_mat_prop optRhs, const float *alpha,
+                        const Array<schar> &lhs, const Array<schar> &rhs,
+                        const float *beta) {
+    TYPE_ERROR(3, af_dtype::s8);
+}
+
 template<typename T>
 Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
              af_mat_prop optRhs) {
diff --git a/src/backend/oneapi/blas.hpp b/src/backend/oneapi/blas.hpp
index 9e2381c336..af65f56d12 100644
--- a/src/backend/oneapi/blas.hpp
+++ b/src/backend/oneapi/blas.hpp
@@ -20,9 +20,10 @@ namespace oneapi {
 void initBlas();
 void deInitBlas();
 
-template<typename T>
-void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
-          const Array<T> &lhs, const Array<T> &rhs, const T *beta);
+template<typename Ti, typename To = Ti>
+void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+          const To *alpha, const Array<Ti> &lhs, const Array<Ti> &rhs,
+          const To *beta);
 
 template<typename T>
 Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
diff --git a/src/backend/opencl/blas.cpp b/src/backend/opencl/blas.cpp
index 45b4149599..8010fe555d 100644
--- a/src/backend/opencl/blas.cpp
+++ b/src/backend/opencl/blas.cpp
@@ -62,13 +62,14 @@ void gemm_fallback<half>(Array<half> & /*out*/, af_mat_prop /*optLhs*/,
     assert(false && "CPU fallback not implemented for f16");
 }
 
-template<typename T>
-void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
-          const Array<T> &lhs, const Array<T> &rhs, const T *beta) {
+template<typename Ti, typename To>
+void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+          const To *alpha, const Array<Ti> &lhs, const Array<Ti> &rhs,
+          const To *beta) {
 #if defined(WITH_LINEAR_ALGEBRA)
     // Do not force offload gemm on OSX Intel devices
     if (OpenCLCPUOffload(false) &&
-        static_cast<af_dtype>(dtype_traits<T>::af_type) != f16) {
+        static_cast<af_dtype>(dtype_traits<Ti>::af_type) != f16) {
         gemm_fallback(out, optLhs, optRhs, alpha, lhs, rhs, beta);
         return;
     }
@@ -114,14 +115,14 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
         cl::Event event;
         if (rDims[bColDim] == 1) {
             dim_t incr = (optRhs == AF_MAT_NONE) ? rStrides[0] : rStrides[1];
-            gpu_blas_gemv_func<T> gemv;
+            gpu_blas_gemv_func<Ti> gemv;
             OPENCL_BLAS_CHECK(gemv(lOpts, lDims[0], lDims[1], *alpha,
                                    (*lhs.get())(), lOffset, lStrides[1],
                                    (*rhs.get())(), rOffset, incr, *beta,
                                    (*out.get())(), oOffset, oStrides[0], 1,
                                    &getQueue()(), 0, nullptr, &event()));
         } else {
-            gpu_blas_gemm_func<T> gemm;
+            gpu_blas_gemm_func<Ti> gemm;
             OPENCL_BLAS_CHECK(gemm(lOpts, rOpts, M, N, K, *alpha,
                                    (*lhs.get())(), lOffset, lStrides[1],
                                    (*rhs.get())(), rOffset, rStrides[1], *beta,
@@ -131,6 +132,14 @@ void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
     }
 }
 
+template<>
+void gemm<schar, float>(Array<float> &out, af_mat_prop optLhs,
+                        af_mat_prop optRhs, const float *alpha,
+                        const Array<schar> &lhs, const Array<schar> &rhs,
+                        const float *beta) {
+    TYPE_ERROR(3, af_dtype::s8);
+}
+
 template<typename T>
 Array<T> dot(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
              af_mat_prop optRhs) {
diff --git a/src/backend/opencl/blas.hpp b/src/backend/opencl/blas.hpp
index 4416960f46..fc4571d4b5 100644
--- a/src/backend/opencl/blas.hpp
+++ b/src/backend/opencl/blas.hpp
@@ -20,9 +20,10 @@ namespace opencl {
 void initBlas();
 void deInitBlas();
 
-template<typename T>
-void gemm(Array<T> &out, af_mat_prop optLhs, af_mat_prop optRhs, const T *alpha,
-          const Array<T> &lhs, const Array<T> &rhs, const T *beta);
+template<typename Ti, typename To = Ti>
+void gemm(Array<To> &out, af_mat_prop optLhs, af_mat_prop optRhs,
+          const To *alpha, const Array<Ti> &lhs, const Array<Ti> &rhs,
+          const To *beta);
 
 template<typename T>
 Array<T> matmul(const Array<T> &lhs, const Array<T> &rhs, af_mat_prop optLhs,
diff --git a/test/blas.cpp b/test/blas.cpp
index 6b0590d73b..6f77c10160 100644
--- a/test/blas.cpp
+++ b/test/blas.cpp
@@ -492,6 +492,36 @@ TEST(MatrixMultiply, half) {
     }
 }
 
+TEST(MatrixMultiply, schar) {
+    array A8         = array(3, 3, h_lhs).as(s8);
+    array B8         = array(3, 3, h_rhs).as(s8);
+    array expected32 = array(3, 3, h_gold).as(f32);
+
+    {
+        af_array C32 = 0;
+        const float alpha32(1.0f);
+        const float beta32(0.0f);
+        af_backend backend;
+        af_get_active_backend(&backend);
+        if (backend == AF_BACKEND_CUDA) {
+            ASSERT_SUCCESS(af_gemm(&C32, AF_MAT_NONE, AF_MAT_NONE, &alpha32,
+                                   A8.get(), B8.get(), &beta32));
+        } else {
+            ASSERT_EQ(AF_ERR_TYPE,
+                      af_gemm(&C32, AF_MAT_NONE, AF_MAT_NONE, &alpha32,
+                              A8.get(), B8.get(), &beta32));
+            SUCCEED();
+            return;
+        }
+        af::array C(C32);
+        ASSERT_ARRAYS_NEAR(expected32, C, 0.00001);
+    }
+    {
+        array C32 = matmul(A8, B8);
+        ASSERT_ARRAYS_NEAR(expected32, C32, 0.00001);
+    }
+}
+
 struct test_params {
     af_mat_prop opt_lhs;
     af_mat_prop opt_rhs;

From 65ad9105b811b6011bfd3a9a9f69c6d7f515b3e6 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Fri, 28 Mar 2025 15:39:18 -0400
Subject: [PATCH 443/473] Update FindcuDNN for version 9 (#3641)

* cuDNN library naming has changed in cuDNN 9. Update FindcuDNN to match the new pattern.

* Update cuDNN dependency for v9
---
 CMakeModules/CPackProjectConfig.cmake | 2 +-
 CMakeModules/FindcuDNN.cmake          | 6 +++++-
 src/backend/cuda/CMakeLists.txt       | 6 +++++-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/CMakeModules/CPackProjectConfig.cmake b/CMakeModules/CPackProjectConfig.cmake
index f85dcaa556..ec5df2ee11 100644
--- a/CMakeModules/CPackProjectConfig.cmake
+++ b/CMakeModules/CPackProjectConfig.cmake
@@ -287,7 +287,7 @@ af_component(
   DEB_USE_SHLIBDEPS
   DEB_PROVIDES "arrayfire-cuda (= ${CPACK_PACKAGE_VERSION}), arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION}), libarrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (= ${CPACK_PACKAGE_VERSION})"
   DEB_REPLACES "arrayfire-cuda (<< ${CPACK_PACKAGE_VERSION}), arrayfire-cuda${CPACK_PACKAGE_VERSION_MAJOR} (<< ${CPACK_PACKAGE_VERSION})"
-  DEB_OPTIONAL libcudnn8 forge libfreeimage3
+  DEB_OPTIONAL cudnn9-cuda-${CPACK_CUDA_VERSION_MAJOR}-${CPACK_CUDA_VERSION_MINOR} forge libfreeimage3
 )
 
 af_component(
diff --git a/CMakeModules/FindcuDNN.cmake b/CMakeModules/FindcuDNN.cmake
index 4c28d3c854..98641f4198 100644
--- a/CMakeModules/FindcuDNN.cmake
+++ b/CMakeModules/FindcuDNN.cmake
@@ -169,13 +169,17 @@ if(cuDNN_INCLUDE_DIRS)
   endmacro()
 
   af_find_cudnn_libs("") # gets base cudnn shared library
-  if(cuDNN_VERSION_MAJOR VERSION_GREATER 8 OR cuDNN_VERSION_MAJOR VERSION_EQUAL 8)
+  if(cuDNN_VERSION_MAJOR VERSION_EQUAL 8)
     af_find_cudnn_libs("_adv_infer")
     af_find_cudnn_libs("_adv_train")
     af_find_cudnn_libs("_cnn_infer")
     af_find_cudnn_libs("_cnn_train")
     af_find_cudnn_libs("_ops_infer")
     af_find_cudnn_libs("_ops_train")
+  elseif(cuDNN_VERSION_MAJOR VERSION_GREATER_EQUAL 9)
+    af_find_cudnn_libs("_adv")
+    af_find_cudnn_libs("_cnn")
+    af_find_cudnn_libs("_ops")
   endif()
 endif()
 
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 6d8731e1e1..6d023f3cb8 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -838,13 +838,17 @@ endfunction()
 if(AF_INSTALL_STANDALONE)
   if(AF_WITH_CUDNN)
     afcu_collect_cudnn_libs("")
-    if(cuDNN_VERSION_MAJOR VERSION_GREATER 8 OR cuDNN_VERSION_MAJOR VERSION_EQUAL 8)
+    if(cuDNN_VERSION_MAJOR VERSION_EQUAL 8)
       # cudnn changed how dlls are shipped starting major version 8
       # except the main dll a lot of the other DLLs are loaded upon demand
       afcu_collect_cudnn_libs(cnn_infer)
       afcu_collect_cudnn_libs(cnn_train)
       afcu_collect_cudnn_libs(ops_infer)
       afcu_collect_cudnn_libs(ops_train)
+    elseif(cuDNN_VERSION_MAJOR VERSION_GREATER_EQUAL 9)
+      # infer and train libraries are now combined in version 9
+      afcu_collect_cudnn_libs(cnn)
+      afcu_collect_cudnn_libs(ops)
     endif()
   endif()
 

From 9ae75d768008c1372896387c1e7e4348724c4546 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Fri, 28 Mar 2025 15:39:44 -0400
Subject: [PATCH 444/473] Add clang flag to include build id in elf data
 (#3644)

* Add clang flag to include build id in elf data

The build id is not generated by default when using intel oneapi. A flag has been added to enable it. This is to satisfy cpack when it is generating the deb file for the oneapi backend.

* Update LICENSE Copyright
---
 LICENSE                           | 2 +-
 src/backend/oneapi/CMakeLists.txt | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/LICENSE b/LICENSE
index 3d960db185..d63051d62b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2014-2024, ArrayFire
+Copyright (c) 2014-2025, ArrayFire
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 702abd3125..a8a1c3aca6 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -358,6 +358,7 @@ target_link_libraries(afoneapi
     $<$<PLATFORM_ID:Linux>:-flink-huge-device-code>
     $<$<PLATFORM_ID:Linux>:-fvisibility-inlines-hidden>
     $<$<PLATFORM_ID:Linux>:-fno-sycl-rdc>
+    $<$<PLATFORM_ID:Linux>:-Wl,--build-id>
     -fsycl-max-parallel-link-jobs=${NumberOfThreads}
     MKL::MKL_SYCL
   )

From a13dcb64287355ee100351827f069d4bbf5fe471 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 1 Apr 2025 19:21:43 -0400
Subject: [PATCH 445/473] Update version numbers for CUDA libraries to be
 collected (#3645)

* Update version numbers for CUDA libraries to be collected

* The nvrtc-builtins library needs to be included in the target link libraries so that the runpath is set.
---
 src/backend/cuda/CMakeLists.txt | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index 6d023f3cb8..a4783b4936 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -50,7 +50,7 @@ set(CUDA_architecture_build_targets "Auto" CACHE
 
 find_cuda_helper_libs(nvrtc)
 find_cuda_helper_libs(nvrtc-builtins)
-list(APPEND nvrtc_libs ${CUDA_nvrtc_LIBRARY})
+list(APPEND nvrtc_libs ${CUDA_nvrtc_LIBRARY} ${CUDA_nvrtc-builtins_LIBRARY})
 
 if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   # The libraries that may be staticly linked or may be loaded at runtime
@@ -853,7 +853,9 @@ if(AF_INSTALL_STANDALONE)
   endif()
 
   if(WIN32 OR NOT AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
-    if(CUDA_VERSION_MAJOR VERSION_EQUAL 11)
+    if(CUDA_VERSION_MAJOR VERSION_EQUAL 12)
+        afcu_collect_libs(cufft LIB_MAJOR 11 LIB_MINOR 3)
+    elseif(CUDA_VERSION_MAJOR VERSION_EQUAL 11)
         afcu_collect_libs(cufft LIB_MAJOR 10 LIB_MINOR 4)
     else()
         afcu_collect_libs(cufft)
@@ -862,17 +864,25 @@ if(AF_INSTALL_STANDALONE)
     if(CUDA_VERSION VERSION_GREATER 10.0)
       afcu_collect_libs(cublasLt)
     endif()
-    afcu_collect_libs(cusolver)
+    if(CUDA_VERSION_MAJOR VERSION_EQUAL 12)
+        afcu_collect_libs(cusolver LIB_MAJOR 11 LIB_MINOR 7)
+    else()
+        afcu_collect_libs(cusolver)
+    endif()
     afcu_collect_libs(cusparse)
     if(CUDA_VERSION VERSION_GREATER 12.0)
       afcu_collect_libs(nvJitLink)
     endif()
   elseif(NOT ${use_static_cuda_lapack})
-    afcu_collect_libs(cusolver)
+    if(CUDA_VERSION_MAJOR VERSION_EQUAL 12)
+        afcu_collect_libs(cusolver LIB_MAJOR 11 LIB_MINOR 7)
+    else()
+        afcu_collect_libs(cusolver)
+    endif()
   endif()
 
   if(WIN32 OR CUDA_VERSION VERSION_LESS 11.5 OR NOT AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
-    afcu_collect_libs(nvrtc FULL_VERSION)
+    afcu_collect_libs(nvrtc)
     if(CUDA_VERSION VERSION_GREATER 10.0)
       afcu_collect_libs(nvrtc-builtins FULL_VERSION)
     else()

From c2e76f14605f696c7171a1384b3a4886bbe7fb94 Mon Sep 17 00:00:00 2001
From: willyborn <sabine.willy.born@gmail.com>
Date: Tue, 1 Oct 2024 16:28:22 +0200
Subject: [PATCH 446/473] Fixed missing offset handling in lookup

---
 src/backend/opencl/kernel/lookup.cl | 2 +-
 src/backend/opencl/lookup.cpp       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/backend/opencl/kernel/lookup.cl b/src/backend/opencl/kernel/lookup.cl
index 622a47e8f6..7ed4bc1cfa 100644
--- a/src/backend/opencl/kernel/lookup.cl
+++ b/src/backend/opencl/kernel/lookup.cl
@@ -31,7 +31,7 @@ kernel void lookupND(global in_t *out, KParam oInfo, global const in_t *in,
     int gx = get_local_size(0) * (get_group_id(0) - gz * nBBS0) + lx;
     int gy = get_local_size(1) * (get_group_id(1) - gw * nBBS1) + ly;
 
-    global const idx_t *idxPtr = indices;
+    global const idx_t *idxPtr = indices + idxInfo.offset;
 
     int i = iInfo.strides[0] *
             (DIM == 0 ? trimIndex((int)idxPtr[gx], iInfo.dims[0]) : gx);
diff --git a/src/backend/opencl/lookup.cpp b/src/backend/opencl/lookup.cpp
index 36b5929f1f..83bca0ac44 100644
--- a/src/backend/opencl/lookup.cpp
+++ b/src/backend/opencl/lookup.cpp
@@ -25,8 +25,8 @@ Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
     const dim4 &iDims = input.dims();
 
     dim4 oDims(1);
-    for (int d = 0; d < 4; ++d) {
-        oDims[d] = (d == int(dim) ? indices.elements() : iDims[d]);
+    for (dim_t d = 0; d < 4; ++d) {
+        oDims[d] = (d == dim ? indices.elements() : iDims[d]);
     }
 
     Array<in_t> out = createEmptyArray<in_t>(oDims);

From 83f4bb64a813f19e2a7c233e9dcd03e054ea4fcb Mon Sep 17 00:00:00 2001
From: Edwin Solis <edwinsolisf12@gmail.com>
Date: Mon, 31 Mar 2025 16:14:42 -0700
Subject: [PATCH 447/473] Added tests for lookup with indices with offsets and
 lying in different dimensions (with fix)

Tests added were for lookup with indices being subarrays with non-zero offsets and with indices lying in second, third, and fourth dimension
A fix was added for supporting indices that do not lie in the first dimension by making a copy and flattening the indices
---
 src/api/c/index.cpp |  32 ++++++++----
 test/index.cpp      | 121 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 143 insertions(+), 10 deletions(-)

diff --git a/src/api/c/index.cpp b/src/api/c/index.cpp
index a697f8457c..792a5a5af7 100644
--- a/src/api/c/index.cpp
+++ b/src/api/c/index.cpp
@@ -178,22 +178,34 @@ af_err af_lookup(af_array* out, const af_array in, const af_array indices,
         ARG_ASSERT(2, (idxType != b8));
 
         af_array output = 0;
+        af_array idx = 0;
+
+        if (!idxInfo.isColumn()) {
+            // Force a deep copy to flatten the array and handle subarrays of not column vector arrays correctly
+            AF_CHECK(af_copy_array(&idx, indices)); 
+        } else {
+            idx = indices;
+        }
 
         switch (idxType) {
-            case f32: output = lookup<float>(in, indices, dim); break;
-            case f64: output = lookup<double>(in, indices, dim); break;
+            case f32: output = lookup<float>(in, idx, dim); break;
+            case f64: output = lookup<double>(in, idx, dim); break;
             case s32: output = lookup<int>(in, indices, dim); break;
-            case u32: output = lookup<unsigned>(in, indices, dim); break;
-            case s16: output = lookup<short>(in, indices, dim); break;
-            case u16: output = lookup<ushort>(in, indices, dim); break;
-            case s64: output = lookup<intl>(in, indices, dim); break;
-            case u64: output = lookup<uintl>(in, indices, dim); break;
-            case s8: output = lookup<schar>(in, indices, dim); break;
-            case u8: output = lookup<uchar>(in, indices, dim); break;
-            case f16: output = lookup<half>(in, indices, dim); break;
+            case u32: output = lookup<unsigned>(in, idx, dim); break;
+            case s16: output = lookup<short>(in, idx, dim); break;
+            case u16: output = lookup<ushort>(in, idx, dim); break;
+            case s64: output = lookup<intl>(in, idx, dim); break;
+            case u64: output = lookup<uintl>(in, idx, dim); break;
+            case s8: output = lookup<schar>(in, idx, dim); break;
+            case u8: output = lookup<uchar>(in, idx, dim); break;
+            case f16: output = lookup<half>(in, idx, dim); break;
             default: TYPE_ERROR(1, idxType);
         }
         std::swap(*out, output);
+
+        if (idx != indices) {
+            AF_CHECK(af_release_array(idx)); // Release indices array if a copy has been made
+        }
     }
     CATCHALL;
     return AF_SUCCESS;
diff --git a/test/index.cpp b/test/index.cpp
index 39491453e7..d5d010ffb1 100644
--- a/test/index.cpp
+++ b/test/index.cpp
@@ -809,6 +809,127 @@ TEST(lookup, Issue2009) {
     ASSERT_ARRAYS_EQ(a, b);
 }
 
+TEST(lookup, Issue3613_FirstDimLookupWithOffset) {
+    dim4 dims(1);
+    const int selected_dim = 0; // selected span dimension
+    dims[selected_dim] = 125; // input size
+
+    array a = iota(dims);
+    array idxs = iota(dim4(5, 4, 3, 2));
+    array selected_idx = idxs(af::span, 3, 2, 1); // Offsets in second, third, & fourth dimension
+
+    array expected_selected_idx = range(dim4(5)) * 1 + 3 * 5 + 2 * (5 * 4) + 1 * (5 * 4 * 3);
+    ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx);
+
+    array b = af::lookup(a, selected_idx, selected_dim);
+    dim4 output_dims(1);
+    output_dims[selected_dim] = 5; // output size
+    ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, output_dims), b); // lookup output should be the same as looked up indices
+}
+
+TEST(lookup, Issue3613_SecondDimLookupWithOffset) {
+    dim4 dims(1);
+    const int selected_dim = 1; // selected span dimension
+    dims[selected_dim] = 125; // input size
+
+    array a = iota(dims);
+    array idxs = iota(dim4(5, 4, 3, 2));
+    array selected_idx = idxs(af::span, 3, 2, 1); // Offsets in second, third, & fourth dimension
+
+    array expected_selected_idx = range(dim4(5)) * 1 + 3 * 5 + 2 * (5 * 4) + 1 * (5 * 4 * 3);
+    ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx);
+
+    array b = af::lookup(a, selected_idx, selected_dim);
+    dim4 output_dims(1);
+    output_dims[selected_dim] = 5; // output size
+    ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, output_dims), b); // lookup output should be the same as looked up indices
+}
+
+
+TEST(lookup, Issue3613_ThirdDimLookupWithOffset) {
+    dim4 dims(1);
+    const int selected_dim = 2; // selected span dimension
+    dims[selected_dim] = 125; // input size
+    
+    array a = iota(dims);
+    array idxs = iota(dim4(5, 4, 3, 2));
+    array selected_idx = idxs(af::span, 3, 2, 1); // Offsets in second, third, & fourth dimension
+    
+    array expected_selected_idx = range(dim4(5)) * 1 + 3 * 5 + 2 * (5 * 4) + 1 * (5 * 4 * 3);
+    ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx);
+    
+    array b = af::lookup(a, selected_idx, selected_dim);
+    dim4 output_dims(1);
+    output_dims[selected_dim] = 5; // output size
+    ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, output_dims), b); // lookup output should be the same as looked up indices
+}
+
+TEST(lookup, Issue3613_FourthDimLookupWithOffset) {
+    dim4 dims(1);
+    const int selected_dim = 3; // selected span dimension
+    dims[selected_dim] = 125; // input size
+    
+    array a = iota(dims);
+    array idxs = iota(dim4(5, 4, 3, 2));
+    array selected_idx = idxs(af::span, 3, 2, 1); // Offsets in second, third, & fourth dimension
+    
+    array expected_selected_idx = range(dim4(5)) * 1 + 3 * 5 + 2 * (5 * 4) + 1 * (5 * 4 * 3);
+    ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx);
+    
+    array b = af::lookup(a, selected_idx, selected_dim);
+    dim4 output_dims(1);
+    output_dims[selected_dim] = 5; // output size
+    ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, output_dims), b); // lookup output should be the same as looked up indices
+}
+
+TEST(lookup, IndicesInSecondDimension) {
+    const int selected_dim = 1; // selected span dimension
+    dim4 dims(1);
+    dims[selected_dim] = 3;
+
+    array a = iota(dim4(100));
+    array idxs = iota(dim4(3, 3, 3, 3));
+    array selected_idx = idxs(0, af::span, 0, 0); // Indices along the second dimension
+
+    array expected_selected_idx = iota(dims) * pow(3, selected_dim);
+    ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx);
+
+    array b = af::lookup(a, selected_idx);
+    ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, dim4(3)), b);
+}
+
+TEST(lookup, IndicesInThirdDimension) {
+    const int selected_dim = 2; // selected span dimension
+    dim4 dims(1);
+    dims[selected_dim] = 3;
+
+    array a = iota(dim4(100));
+    array idxs = iota(dim4(3, 3, 3, 3));
+    array selected_idx = idxs(0, 0, af::span, 0); // Indices along the third dimension
+
+    array expected_selected_idx = iota(dims) * pow(3, selected_dim);
+    ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx);
+
+    array b = af::lookup(a, selected_idx);
+    ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, dim4(3)), b);
+}
+
+TEST(lookup, IndicesInFourthDimension) {
+    const int selected_dim = 3; // selected span dimension
+    dim4 dims(1);
+    dims[selected_dim] = 3;
+
+    array a = iota(dim4(100));
+    array idxs = iota(dim4(3, 3, 3, 3));
+    array selected_idx = idxs(0, 0, 0, af::span); // Indices along the fourth dimension
+
+    array expected_selected_idx = iota(dims) * pow(3, selected_dim);
+    ASSERT_ARRAYS_EQ(expected_selected_idx, selected_idx);
+
+    array b = af::lookup(a, selected_idx);
+    ASSERT_ARRAYS_EQ(af::moddims(expected_selected_idx, dim4(3)), b);
+}
+
 TEST(lookup, SNIPPET_lookup1d) {
     //! [ex_index_lookup1d]
 

From 59791340b750dc7e32666f05b3866d5ee146bc65 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Fri, 4 Apr 2025 15:49:25 -0400
Subject: [PATCH 448/473] Add minimum driver version check to allow minor
 version compatibility. (#3648)

* Add minimum driver version check to allow minor version compatibility.

* Simpler method of checking for nvidia driver cuda minor version compatibility.
---
 src/backend/cuda/device_manager.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index c445d5784b..88cbe487a8 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -511,7 +511,10 @@ void DeviceManager::checkCudaVsDriverVersion() {
 
     debugRuntimeCheck(getLogger(), runtime, driver);
 
-    if (runtime > driver) {
+    int runtime_major = runtime / 1000;
+    int driver_major = driver / 1000;
+
+    if (runtime_major > driver_major) {
         string msg =
             "ArrayFire was built with CUDA {} which requires GPU driver "
             "version {} or later. Please download and install the latest "

From f50057bc3b61ea11390c353da663baf0fa1496bd Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 3 Jun 2025 17:38:26 -0400
Subject: [PATCH 449/473] Additional libraries for standalone installer and fix
 ilp consistency with find AF_MKL. (#3646)

* Additional libraries for standalone installer and fix ilp consistency with find AF_MKL.

* Yet More oneAPI libraries and name typo fixes

* Remove libraries already provided by runtime

* Move additional mkl libraries collection into block for non-static build.

* Modifications to CMake files to use LP64 interface for CPU and OpenCL back ends and ILP64 for oneAPI back end.

* Both ILP64 and LP64 MKL interface libraries are needed for the oneAPI and CPU & OpenCL back ends respectively. So both need to be installed.
---
 CMakeLists.txt                    | 33 ++++++++++++++++++++++++++++---
 CMakeModules/FindAF_MKL.cmake     | 23 +++++++++++++++------
 src/api/c/CMakeLists.txt          |  4 ----
 src/backend/cpu/CMakeLists.txt    |  5 +++++
 src/backend/oneapi/CMakeLists.txt |  4 ++++
 src/backend/opencl/CMakeLists.txt |  5 +++++
 6 files changed, 61 insertions(+), 13 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca942f301a..b1cd049b64 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -143,8 +143,6 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.13)
     set(MKL_ROOT "$ENV{MKLROOT}")
   endif()
   set(SYCL_COMPILER ON)
-  set(MKL_THREADING "tbb_thread")
-  set(MKL_INTERFACE "ilp64")
   find_package(MKL)
 endif()
 
@@ -552,13 +550,21 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE)
         ${mkl_int}
         DESTINATION ${AF_INSTALL_LIB_DIR}
         COMPONENT mkl_dependencies)
+
+      # LP64 library is required for the CPU and OpenCL back ends, so install it too
+      if(MKL_INTERFACE_INTEGER_SIZE EQUAL 8)
+        get_filename_component(mkl_int_lp ${MKL_InterfaceLP_LINK_LIBRARY} REALPATH)
+        install(FILES
+          ${mkl_int_lp}
+          DESTINATION ${AF_INSTALL_LIB_DIR}
+          COMPONENT mkl_dependencies)
+      endif()
     endif()
 
     get_filename_component(mkl_rnt ${MKL_RT_LINK_LIBRARY} REALPATH)
     get_filename_component(mkl_shd ${MKL_Core_LINK_LIBRARY} REALPATH)
     get_filename_component(mkl_tly ${MKL_ThreadLayer_LINK_LIBRARY} REALPATH)
     install(FILES
-      ${mkl_sycl}
       ${mkl_rnt}
       ${mkl_shd}
       ${mkl_tly}
@@ -573,6 +579,27 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE)
       ${AF_ADDITIONAL_MKL_LIBRARIES}
       DESTINATION ${AF_INSTALL_LIB_DIR}
       COMPONENT mkl_dependencies)
+    if(AF_BUILD_ONEAPI)
+      get_filename_component(mkl_sycl_lapack ${MKL_SyclLapack_LINK_LIBRARY} REALPATH)
+      get_filename_component(mkl_sycl_dft ${MKL_SyclDft_LINK_LIBRARY} REALPATH)
+      get_filename_component(mkl_sycl_blas ${MKL_SyclBlas_LINK_LIBRARY} REALPATH)
+      get_filename_component(mkl_sycl_sparse ${MKL_SyclSparse_LINK_LIBRARY} REALPATH)
+      get_filename_component(mkl_sycl_data ${MKL_SyclDataFitting_LINK_LIBRARY} REALPATH)
+      get_filename_component(mkl_sycl_rng ${MKL_SyclRNG_LINK_LIBRARY} REALPATH)
+      get_filename_component(mkl_sycl_stats ${MKL_SyclStats_LINK_LIBRARY} REALPATH)
+      get_filename_component(mkl_sycl_vm ${MKL_SyclVM_LINK_LIBRARY} REALPATH)
+      install(FILES
+        ${mkl_sycl_lapack}
+        ${mkl_sycl_dft}
+        ${mkl_sycl_blas}
+        ${mkl_sycl_sparse}
+        ${mkl_sycl_data}
+        ${mkl_sycl_rng}
+        ${mkl_sycl_stats}
+        ${mkl_sycl_vm}
+        DESTINATION ${AF_INSTALL_LIB_DIR}
+        COMPONENT mkl_dependencies)
+    endif()
   endif()
 endif()
 
diff --git a/CMakeModules/FindAF_MKL.cmake b/CMakeModules/FindAF_MKL.cmake
index 88037c4519..123b6bee61 100644
--- a/CMakeModules/FindAF_MKL.cmake
+++ b/CMakeModules/FindAF_MKL.cmake
@@ -74,8 +74,12 @@
 include(CheckTypeSize)
 include(FindPackageHandleStandardArgs)
 
-check_type_size("int" INT_SIZE
-  BUILTIN_TYPES_ONLY LANGUAGE C)
+if(DEFINED MKL_INTERFACE_INTEGER_SIZE)
+  set(INT_SIZE ${MKL_INTERFACE_INTEGER_SIZE})
+else()
+  check_type_size("int" INT_SIZE
+    BUILTIN_TYPES_ONLY LANGUAGE C)
+endif()
 
 set(MKL_THREAD_LAYER "TBB" CACHE STRING "The thread layer to choose for MKL")
 set_property(CACHE MKL_THREAD_LAYER PROPERTY STRINGS "TBB" "GNU OpenMP" "Intel OpenMP" "Sequential")
@@ -323,10 +327,14 @@ find_mkl_library(NAME RT LIBRARY_NAME mkl_rt)
 
 if(AF_BUILD_ONEAPI)
     find_mkl_library(NAME Sycl LIBRARY_NAME sycl DLL_ONLY)
-	find_mkl_library(NAME SyclLapack LIBRARY_NAME sycl_lapack DLL_ONLY)
-	find_mkl_library(NAME SyclDft LIBRARY_NAME sycl_dft DLL_ONLY)
-	find_mkl_library(NAME SyclBlas LIBRARY_NAME sycl_blas DLL_ONLY)
-	find_mkl_library(NAME SyclSparse LIBRARY_NAME sycl_sparse DLL_ONLY)
+    find_mkl_library(NAME SyclLapack LIBRARY_NAME mkl_sycl_lapack DLL_ONLY)
+    find_mkl_library(NAME SyclDft LIBRARY_NAME mkl_sycl_dft DLL_ONLY)
+    find_mkl_library(NAME SyclBlas LIBRARY_NAME mkl_sycl_blas DLL_ONLY)
+    find_mkl_library(NAME SyclSparse LIBRARY_NAME mkl_sycl_sparse DLL_ONLY)
+    find_mkl_library(NAME SyclDataFitting LIBRARY_NAME mkl_sycl_data_fitting DLL_ONLY)
+    find_mkl_library(NAME SyclRNG LIBRARY_NAME mkl_sycl_rng DLL_ONLY)
+    find_mkl_library(NAME SyclStats LIBRARY_NAME mkl_sycl_stats DLL_ONLY)
+    find_mkl_library(NAME SyclVM LIBRARY_NAME mkl_sycl_vm DLL_ONLY)
 endif()
 
 # MKL can link against Intel OpenMP, GNU OpenMP, TBB, and Sequential
@@ -356,10 +364,13 @@ endif()
 
 if("${INT_SIZE}" EQUAL 4)
   set(MKL_INTERFACE_INTEGER_SIZE 4)
+  set(MKL_INTERFACE "lp64")
   find_mkl_library(NAME Interface LIBRARY_NAME mkl_intel_lp64 SEARCH_STATIC)
 else()
   set(MKL_INTERFACE_INTEGER_SIZE 8)
+  set(MKL_INTERFACE "ilp64")
   find_mkl_library(NAME Interface LIBRARY_NAME mkl_intel_ilp64 SEARCH_STATIC)
+  find_mkl_library(NAME InterfaceLP LIBRARY_NAME mkl_intel_lp64 SEARCH_STATIC)
 endif()
 
 set(MKL_KernelLibraries "mkl_def;mkl_mc;mkl_mc3;mkl_avx;mkl_avx2;mkl_avx512")
diff --git a/src/api/c/CMakeLists.txt b/src/api/c/CMakeLists.txt
index 870d687382..d374b9a669 100644
--- a/src/api/c/CMakeLists.txt
+++ b/src/api/c/CMakeLists.txt
@@ -186,10 +186,6 @@ if(FreeImage_FOUND AND AF_WITH_IMAGEIO)
 endif()
 
 if(BUILD_WITH_MKL)
-  target_compile_definitions(c_api_interface
-    INTERFACE
-      AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE}
-    )
   # Create mkl thread layer compile option based on cmake cache variable
   if(MKL_THREAD_LAYER STREQUAL "Sequential")
     target_compile_definitions(c_api_interface INTERFACE AF_MKL_THREAD_LAYER=0)
diff --git a/src/backend/cpu/CMakeLists.txt b/src/backend/cpu/CMakeLists.txt
index b8025d53a2..8a83a55894 100644
--- a/src/backend/cpu/CMakeLists.txt
+++ b/src/backend/cpu/CMakeLists.txt
@@ -15,6 +15,10 @@ generate_product_version(af_cpu_ver_res_file
 add_library(afcpu "")
 add_library(ArrayFire::afcpu ALIAS afcpu)
 
+# CPU back end needs to use MKL LP64 interface
+set(MKL_INTERFACE_INTEGER_SIZE 4)
+set(MKL_INTERFACE "lp64")
+
 # CPU backend source files
 target_sources(afcpu
   PRIVATE
@@ -313,6 +317,7 @@ target_link_libraries(afcpu
   )
 if(BUILD_WITH_MKL)
   target_compile_definitions(afcpu PRIVATE USE_MKL)
+  target_compile_definitions(afcpu PRIVATE AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE})
 
   if(MKL_BATCH)
     target_compile_definitions(afcpu PRIVATE AF_USE_MKL_BATCH)
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index a8a1c3aca6..210c8f59a9 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -341,7 +341,11 @@ target_compile_definitions(afoneapi
     CL_HPP_TARGET_OPENCL_VERSION=300
     CL_HPP_MINIMUM_OPENCL_VERSION=110
     CL_HPP_ENABLE_EXCEPTIONS
+    AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE}
   )
+if(MKL_INTERFACE_INTEGER_SIZE EQUAL 8)
+  target_compile_definitions(afoneapi PRIVATE MKL_ILP64)
+endif()
 
 cmake_host_system_information(RESULT NumberOfThreads
   QUERY NUMBER_OF_LOGICAL_CORES)
diff --git a/src/backend/opencl/CMakeLists.txt b/src/backend/opencl/CMakeLists.txt
index a02ae6781d..23bedeedab 100644
--- a/src/backend/opencl/CMakeLists.txt
+++ b/src/backend/opencl/CMakeLists.txt
@@ -7,6 +7,10 @@
 
 dependency_check(OpenCL_FOUND "OpenCL not found.")
 
+# OpenCL back end needs to use MKL LP64 interface
+set(MKL_INTERFACE_INTEGER_SIZE 4)
+set(MKL_INTERFACE "lp64")
+
 include(InternalUtils)
 include(build_cl2hpp)
 include(build_CLBlast)
@@ -578,6 +582,7 @@ if(LAPACK_FOUND OR BUILD_WITH_MKL)
 
   if(BUILD_WITH_MKL)
     target_compile_definitions(afopencl PRIVATE USE_MKL)
+    target_compile_definitions(afopencl PRIVATE AF_MKL_INTERFACE_SIZE=${MKL_INTERFACE_INTEGER_SIZE})
     if(MKL_BATCH)
       target_compile_definitions(afopencl PRIVATE AF_USE_MKL_BATCH)
     endif()

From 20b8f6720dde399f2a620a2cfa853bb1458fd31f Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 3 Jun 2025 17:40:43 -0400
Subject: [PATCH 450/473] Update dependencies built with ArrayFire (#3651)

* Update dependencies built with ArrayFire

Update dependencies cloned from git and built along with ArrayFire:
- CLBlast (note that the minimum supported version needs to be overriden to 3.5 to work with the latest version of Cmake (v4.0)
- CL2HPP
- GoogleTest

* Remove version override for Boost, use the newest one available. Update baseline to use a more recent version of VCPKG package list. Remove MKL VCPKG package because it is old and is conflicting with the oneAPI system installed version.
---
 CMakeModules/AF_vcpkg_options.cmake |  5 +----
 CMakeModules/build_CLBlast.cmake    |  3 ++-
 CMakeModules/build_cl2hpp.cmake     |  2 +-
 test/CMakeLists.txt                 |  2 +-
 vcpkg.json                          | 12 +-----------
 5 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/CMakeModules/AF_vcpkg_options.cmake b/CMakeModules/AF_vcpkg_options.cmake
index 09701af274..c84adcee82 100644
--- a/CMakeModules/AF_vcpkg_options.cmake
+++ b/CMakeModules/AF_vcpkg_options.cmake
@@ -6,7 +6,6 @@
 # http://arrayfire.com/licenses/BSD-3-Clause
 
 set(ENV{VCPKG_FEATURE_FLAGS} "versions")
-set(ENV{VCPKG_KEEP_ENV_VARS} "MKLROOT")
 set(VCPKG_MANIFEST_NO_DEFAULT_FEATURES ON)
 
 set(VCPKG_OVERLAY_TRIPLETS ${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/vcpkg/vcpkg-triplets)
@@ -28,9 +27,7 @@ if(BUILD_TESTING)
   list(APPEND VCPKG_MANIFEST_FEATURES "tests")
 endif()
 
-if(AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL")
-  list(APPEND VCPKG_MANIFEST_FEATURES "mkl")
-else()
+if(NOT AF_COMPUTE_LIBRARY STREQUAL "Intel-MKL")
   list(APPEND VCPKG_MANIFEST_FEATURES "openblasfftw")
 endif()
 
diff --git a/CMakeModules/build_CLBlast.cmake b/CMakeModules/build_CLBlast.cmake
index 933531cdf2..a0d9fab435 100644
--- a/CMakeModules/build_CLBlast.cmake
+++ b/CMakeModules/build_CLBlast.cmake
@@ -26,7 +26,7 @@ if(TARGET clblast OR AF_WITH_EXTERNAL_PACKAGES_ONLY)
 else()
   af_dep_check_and_populate(${clblast_prefix}
     URI https://github.com/cnugteren/CLBlast.git
-    REF 4500a03440e2cc54998c0edab366babf5e504d67
+    REF 1.6.3
   )
 
   include(ExternalProject)
@@ -69,6 +69,7 @@ else()
       BUILD_BYPRODUCTS ${CLBlast_location}
       CONFIGURE_COMMAND ${CMAKE_COMMAND} ${extproj_gen_opts}
         -Wno-dev <SOURCE_DIR>
+        -DCMAKE_POLICY_VERSION_MINIMUM=3.5
         -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}
         "-DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}"
         -DOVERRIDE_MSVC_FLAGS_TO_MT:BOOL=OFF
diff --git a/CMakeModules/build_cl2hpp.cmake b/CMakeModules/build_cl2hpp.cmake
index 0a3fef2de0..b38c4bc1d1 100644
--- a/CMakeModules/build_cl2hpp.cmake
+++ b/CMakeModules/build_cl2hpp.cmake
@@ -27,7 +27,7 @@ if(NOT TARGET OpenCL::cl2hpp)
   elseif (NOT TARGET OpenCL::cl2hpp OR NOT TARGET cl2hpp)
     af_dep_check_and_populate(${cl2hpp_prefix}
       URI https://github.com/KhronosGroup/OpenCL-CLHPP.git
-      REF v2022.09.30)
+      REF v2024.10.24)
 
     find_path(cl2hpp_var
       NAMES CL/cl2.hpp
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 8107f3c063..64e1feb777 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -24,7 +24,7 @@ if(AF_WITH_EXTERNAL_PACKAGES_ONLY)
 elseif(NOT TARGET GTest::gtest)
   af_dep_check_and_populate(${gtest_prefix}
     URI https://github.com/google/googletest.git
-    REF release-1.12.1
+    REF v1.16.0
   )
   if(WIN32)
     set(gtest_force_shared_crt ON
diff --git a/vcpkg.json b/vcpkg.json
index fe16a0aa6d..063e402a02 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -23,10 +23,6 @@
         {
             "name": "jasper",
             "version": "4.2.0"
-        },
-        {
-            "name": "boost-modular-build-helper",
-            "version": "1.84.0#3"
         }
     ],
     "features": {
@@ -78,12 +74,6 @@
                 "opencl"
             ]
         },
-        "mkl": {
-            "description": "Build with MKL",
-            "dependencies": [
-                "intel-mkl"
-            ]
-        },
         "cudnn": {
             "description": "Build CUDA with support for cuDNN",
             "dependencies": [
@@ -91,5 +81,5 @@
             ]
         }
     },
-    "builtin-baseline": "9d47b24eacbd1cd94f139457ef6cd35e5d92cc84"
+    "builtin-baseline": "b02e341c927f16d991edbd915d8ea43eac52096c"
 }

From e8e30bdbe568044c80e4e708ded3ea600c605ad2 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 3 Jun 2025 17:49:26 -0400
Subject: [PATCH 451/473] Windows installer fixes (#3655)

* Additional libraries for standalone installer and fix ilp consistency with find AF_MKL.

* Yet More oneAPI libraries and name typo fixes

* Remove libraries already provided by runtime

* Move additional mkl libraries collection into block for non-static build.

* Fixes to enable SYCL language to work with MSVC runtime library variables

* Fixes to find FreeImage for Windows build

* Fixes to allow Nvidia libraries to be found for Windows build

* Fixes to allow all oneAPI and MKL libraries to be found for Windows build

* Fixes for CPack scripts to work with NSIS installer on Windows

* Fixes to find tbb libraries.

* Add missing Debug/Release flags for SYCL compiler on Windows

* Add boost program-options dependency required for clFFT

* fix typos in CMakeSYCLInformation

* Add Windows defines

* Only attempt to include debug data files in the installer package for debug builds.

* Revert some changes that removed some library installs which are needed for building on Linux

* Install Visual C++ redistributable as part of the Windows installer.
---
 CMakeLists.txt                                | 42 +++++++--
 CMakeModules/CMakeSYCLInformation.cmake       | 20 +++++
 CMakeModules/CPackConfig.cmake                |  7 +-
 CMakeModules/CPackProjectConfig.cmake         | 90 ++++++++++---------
 CMakeModules/FindAF_MKL.cmake                 |  6 ++
 CMakeModules/FindFreeImage.cmake              |  2 +
 CMakeModules/nsis/NSIS.InstallOptions.ini.in  |  6 +-
 CMakeModules/nsis/NSIS.definitions.nsh.in     | 12 +--
 CMakeModules/nsis/NSIS.template.in            |  5 ++
 src/backend/cuda/CMakeLists.txt               |  7 +-
 src/backend/oneapi/CMakeLists.txt             |  5 ++
 .../oneapi/kernel/sort_by_key/CMakeLists.txt  |  1 +
 vcpkg.json                                    |  1 +
 13 files changed, 142 insertions(+), 62 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b1cd049b64..4ce33555e1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -56,6 +56,7 @@ endif()
 if(SYCL_COMPILER_NAME STREQUAL "dpcpp" OR SYCL_COMPILER_NAME STREQUAL "dpcpp.exe"
    OR SYCL_COMPILER_NAME STREQUAL "icpx" OR SYCL_COMPILER_NAME STREQUAL "icx.exe")
   set(MKL_THREAD_LAYER "TBB" CACHE STRING "The thread layer to choose for MKL")
+  set(TBB_ROOT "$ENV{TBBROOT}")
   set(MKL_INTERFACE "ilp64")
   set(MKL_INTERFACE_INTEGER_SIZE 8)
 else()
@@ -532,6 +533,14 @@ install(FILES ${ArrayFire_BINARY_DIR}/cmake/install/ArrayFireConfig.cmake
               DESTINATION ${AF_INSTALL_CMAKE_DIR}
               COMPONENT cmake)
 
+if(WIN32 AND AF_INSTALL_STANDALONE)
+  find_program(MSVC_REDIST NAMES vc_redist.x64.exe
+          PATHS "$ENV{VCINSTALLDIR}Redist\\MSVC\\v${MSVC_TOOLSET_VERSION}")
+  get_filename_component(MSVC_REDIST_INSTALLER ${MSVC_REDIST} NAME)
+  install(PROGRAMS ${MSVC_REDIST} COMPONENT common_backend_dependencies
+          DESTINATION ${AF_INSTALL_BIN_DIR})
+endif()
+
 if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE)
   if(TARGET MKL::ThreadingLibrary)
     get_filename_component(mkl_tl ${MKL_ThreadingLibrary_LINK_LIBRARY} REALPATH)
@@ -561,6 +570,7 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE)
       endif()
     endif()
 
+  if(UNIX)
     get_filename_component(mkl_rnt ${MKL_RT_LINK_LIBRARY} REALPATH)
     get_filename_component(mkl_shd ${MKL_Core_LINK_LIBRARY} REALPATH)
     get_filename_component(mkl_tly ${MKL_ThreadLayer_LINK_LIBRARY} REALPATH)
@@ -568,6 +578,11 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE)
       ${mkl_rnt}
       ${mkl_shd}
       ${mkl_tly}
+      DESTINATION ${AF_INSTALL_LIB_DIR}
+      COMPONENT mkl_dependencies)
+  endif()
+
+    install(FILES
       $<TARGET_FILE:MKL::RT>
       $<TARGET_FILE:MKL::Shared>
       $<TARGET_FILE:MKL::ThreadLayer>
@@ -580,14 +595,25 @@ if(BUILD_WITH_MKL AND AF_INSTALL_STANDALONE)
       DESTINATION ${AF_INSTALL_LIB_DIR}
       COMPONENT mkl_dependencies)
     if(AF_BUILD_ONEAPI)
-      get_filename_component(mkl_sycl_lapack ${MKL_SyclLapack_LINK_LIBRARY} REALPATH)
-      get_filename_component(mkl_sycl_dft ${MKL_SyclDft_LINK_LIBRARY} REALPATH)
-      get_filename_component(mkl_sycl_blas ${MKL_SyclBlas_LINK_LIBRARY} REALPATH)
-      get_filename_component(mkl_sycl_sparse ${MKL_SyclSparse_LINK_LIBRARY} REALPATH)
-      get_filename_component(mkl_sycl_data ${MKL_SyclDataFitting_LINK_LIBRARY} REALPATH)
-      get_filename_component(mkl_sycl_rng ${MKL_SyclRNG_LINK_LIBRARY} REALPATH)
-      get_filename_component(mkl_sycl_stats ${MKL_SyclStats_LINK_LIBRARY} REALPATH)
-      get_filename_component(mkl_sycl_vm ${MKL_SyclVM_LINK_LIBRARY} REALPATH)
+      if(WIN32)
+        get_filename_component(mkl_sycl_lapack ${MKL_SyclLapack_DLL_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_dft ${MKL_SyclDft_DLL_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_blas ${MKL_SyclBlas_DLL_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_sparse ${MKL_SyclSparse_DLL_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_data ${MKL_SyclDataFitting_DLL_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_rng ${MKL_SyclRNG_DLL_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_stats ${MKL_SyclStats_DLL_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_vm ${MKL_SyclVM_DLL_LIBRARY} REALPATH)
+      else()
+        get_filename_component(mkl_sycl_lapack ${MKL_SyclLapack_LINK_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_dft ${MKL_SyclDft_LINK_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_blas ${MKL_SyclBlas_LINK_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_sparse ${MKL_SyclSparse_LINK_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_data ${MKL_SyclDataFitting_LINK_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_rng ${MKL_SyclRNG_LINK_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_stats ${MKL_SyclStats_LINK_LIBRARY} REALPATH)
+        get_filename_component(mkl_sycl_vm ${MKL_SyclVM_LINK_LIBRARY} REALPATH)
+      endif()
       install(FILES
         ${mkl_sycl_lapack}
         ${mkl_sycl_dft}
diff --git a/CMakeModules/CMakeSYCLInformation.cmake b/CMakeModules/CMakeSYCLInformation.cmake
index df850959f1..b5ec7876db 100644
--- a/CMakeModules/CMakeSYCLInformation.cmake
+++ b/CMakeModules/CMakeSYCLInformation.cmake
@@ -41,6 +41,11 @@ include(Compiler/${CMAKE_SYCL_COMPILER_ID} OPTIONAL)
 __compiler_intel_llvm(SYCL)
 
 if("x${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "xMSVC")
+  string(APPEND CMAKE_SYCL_FLAGS_INIT " /DWIN32 /D_WINDOWS")
+  string(APPEND CMAKE_SYCL_FLAGS_DEBUG_INIT " /Zi /Ob0 /Od /RTC1")
+  string(APPEND CMAKE_SYCL_FLAGS_MINSIZEREL_INIT " /O1 /Ob1 /DNDEBUG")
+  string(APPEND CMAKE_SYCL_FLAGS_RELEASE_INIT " /O2 /Ob2 /DNDEBUG")
+  string(APPEND CMAKE_SYCL_FLAGS_RELWITHDEBINFO_INIT " /Zi /O2 /Ob1 /DNDEBUG")
   set(CMAKE_SYCL_COMPILE_OPTIONS_EXPLICIT_LANGUAGE -TP)
   set(CMAKE_SYCL_CLANG_TIDY_DRIVER_MODE "cl")
   set(CMAKE_SYCL_INCLUDE_WHAT_YOU_USE_DRIVER_MODE "cl")
@@ -353,6 +358,21 @@ if(NOT CMAKE_SYCL_LINK_EXECUTABLE)
     "<CMAKE_SYCL_COMPILER> <FLAGS> <CMAKE_SYCL_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
 endif()
 
+if(CMAKE_HOST_WIN32)
+  set(MSVC_RUNTIME "")
+  if("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL "MultiThreaded")
+    set(MSVC_RUNTIME "-MT")
+  elseif("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL "MultiThreadedDLL")
+    set(MSVC_RUNTIME "-MD")
+  elseif("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL "MultiThreadedDebug")
+    set(MSVC_RUNTIME "-MTd")
+  elseif("${CMAKE_MSVC_RUNTIME_LIBRARY}" STREQUAL "MultiThreadedDebugDLL")
+    set(MSVC_RUNTIME "-MDd")
+  else()
+    set(MSVC_RUNTIME "-MD$<$<CONFIG:Debug>:d>")
+  endif()
+  set(CMAKE_MSVC_RUNTIME_LIBRARY "")
+endif()
 
 mark_as_advanced(
 CMAKE_VERBOSE_MAKEFILE
diff --git a/CMakeModules/CPackConfig.cmake b/CMakeModules/CPackConfig.cmake
index 6cd13a1d71..8cf0880faa 100644
--- a/CMakeModules/CPackConfig.cmake
+++ b/CMakeModules/CPackConfig.cmake
@@ -43,9 +43,9 @@ set(CPACK_PACKAGE_NAME "${LIBRARY_NAME}")
 set(CPACK_PACKAGE_VENDOR "${VENDOR_NAME}")
 set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY ${LIBRARY_NAME})
 set(CPACK_PACKAGE_CONTACT "ArrayFire <technical@arrayfire.com>")
-set(MY_CPACK_PACKAGE_ICON "${CMAKE_SOURCE_DIR}/assets/${APP_LOW_NAME}.ico")
+set(MY_CPACK_PACKAGE_ICON "${ASSETS_DIR}/${APP_LOW_NAME}.ico")
 
-file(TO_NATIVE_PATH "${CMAKE_SOURCE_DIR}/assets/" NATIVE_ASSETS_PATH)
+file(TO_NATIVE_PATH "${ASSETS_DIR}/" NATIVE_ASSETS_PATH)
 string(REPLACE "\\" "\\\\" NATIVE_ASSETS_PATH  ${NATIVE_ASSETS_PATH})
 set(CPACK_AF_ASSETS_DIR "${NATIVE_ASSETS_PATH}")
 
@@ -137,6 +137,9 @@ elseif(WIN32)
   else (CMAKE_CL_64)
     set(CPACK_NSIS_INSTALL_ROOT "$PROGRAMFILES")
   endif (CMAKE_CL_64)
+  configure_file(
+      ${PROJECT_SOURCE_DIR}/CMakeModules/nsis/NSIS.definitions.nsh.in
+      ${CMAKE_CURRENT_BINARY_DIR}/NSIS.definitions.nsh)
 else()
   set(CPACK_RESOURCE_FILE_LICENSE "${ArrayFire_SOURCE_DIR}/LICENSE")
   set(CPACK_RESOURCE_FILE_README "${ArrayFire_SOURCE_DIR}/README.md")
diff --git a/CMakeModules/CPackProjectConfig.cmake b/CMakeModules/CPackProjectConfig.cmake
index ec5df2ee11..f75591f8bb 100644
--- a/CMakeModules/CPackProjectConfig.cmake
+++ b/CMakeModules/CPackProjectConfig.cmake
@@ -161,9 +161,12 @@ if(NOT CPACK_GENERATOR MATCHES "DEB")
     DESCRIPTION "ArrayFire development files including headers and configuration files"
     EXPANDED)
 
-  cpack_add_component_group(debug
-    DISPLAY_NAME "ArrayFire Debug Symbols"
-    DESCRIPTION "ArrayFire Debug symbols")
+  if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR
+     CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+    cpack_add_component_group(debug
+      DISPLAY_NAME "ArrayFire Debug Symbols"
+      DESCRIPTION "ArrayFire Debug symbols")
+  endif()
 endif()
 
 set(arrayfire_cuda_runtime_name "CUDA Runtime(${CPACK_CUDA_VERSION_MAJOR}.${CPACK_CUDA_VERSION_MINOR})")
@@ -473,45 +476,48 @@ endif()
 # Debug symbols in debian installers are created using the DEBINFO property
 if(NOT APPLE AND
    NOT CPACK_GENERATOR MATCHES "DEB")
-  af_component(
-    COMPONENT afoneapi_debug_symbols
-    DISPLAY_NAME "oneAPI Debug Symbols"
-    DESCRIPTION "Debug symbols for the oneAPI backend."
-    GROUP debug
-    DISABLED
-    INSTALL_TYPES Development)
-
-  af_component(
-    COMPONENT afopencl_debug_symbols
-    DISPLAY_NAME "OpenCL Debug Symbols"
-    DESCRIPTION "Debug symbols for the OpenCL backend."
-    GROUP debug
-    DISABLED
-    INSTALL_TYPES Development)
-
-  af_component(
-    COMPONENT afcuda_debug_symbols
-    DISPLAY_NAME "CUDA Debug Symbols"
-    DESCRIPTION "Debug symbols for CUDA backend backend."
-    GROUP debug
-    DISABLED
-    INSTALL_TYPES Development)
-
-  af_component(
-    COMPONENT afcpu_debug_symbols
-    DISPLAY_NAME "CPU Debug Symbols"
-    DESCRIPTION "Debug symbols for CPU backend backend."
-    GROUP debug
-    DISABLED
-    INSTALL_TYPES Development)
-
-  af_component(
-    COMPONENT af_debug_symbols
-    DISPLAY_NAME "Unified Debug Symbols"
-    DESCRIPTION "Debug symbols for the Unified backend."
-    GROUP debug
-    DISABLED
-    INSTALL_TYPES Development)
+  if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR
+     CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+    af_component(
+      COMPONENT afoneapi_debug_symbols
+      DISPLAY_NAME "oneAPI Debug Symbols"
+      DESCRIPTION "Debug symbols for the oneAPI backend."
+      GROUP debug
+      DISABLED
+      INSTALL_TYPES Development)
+  
+    af_component(
+      COMPONENT afopencl_debug_symbols
+      DISPLAY_NAME "OpenCL Debug Symbols"
+      DESCRIPTION "Debug symbols for the OpenCL backend."
+      GROUP debug
+      DISABLED
+      INSTALL_TYPES Development)
+  
+    af_component(
+      COMPONENT afcuda_debug_symbols
+      DISPLAY_NAME "CUDA Debug Symbols"
+      DESCRIPTION "Debug symbols for CUDA backend backend."
+      GROUP debug
+      DISABLED
+      INSTALL_TYPES Development)
+  
+    af_component(
+      COMPONENT afcpu_debug_symbols
+      DISPLAY_NAME "CPU Debug Symbols"
+      DESCRIPTION "Debug symbols for CPU backend backend."
+      GROUP debug
+      DISABLED
+      INSTALL_TYPES Development)
+  
+    af_component(
+      COMPONENT af_debug_symbols
+      DISPLAY_NAME "Unified Debug Symbols"
+      DESCRIPTION "Debug symbols for the Unified backend."
+      GROUP debug
+      DISABLED
+      INSTALL_TYPES Development)
+  endif()
 endif()
 
 # if (AF_INSTALL_FORGE_DEV)
diff --git a/CMakeModules/FindAF_MKL.cmake b/CMakeModules/FindAF_MKL.cmake
index 123b6bee61..18037ca4fc 100644
--- a/CMakeModules/FindAF_MKL.cmake
+++ b/CMakeModules/FindAF_MKL.cmake
@@ -303,9 +303,15 @@ function(find_mkl_library)
       NAMES
         ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}
         ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}${md_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}.2${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}.5${CMAKE_SHARED_LIBRARY_SUFFIX}
+        ${CMAKE_SHARED_LIBRARY_PREFIX}${mkl_args_LIBRARY_NAME}12${CMAKE_SHARED_LIBRARY_SUFFIX}
         lib${mkl_args_LIBRARY_NAME}${md_suffix}${CMAKE_SHARED_LIBRARY_SUFFIX}
         $ENV{LIB}
         $ENV{LIBRARY_PATH}
+      PATHS
+        ${MKL_ROOT}/bin
+        ${TBB_ROOT}/bin
       PATH_SUFFIXES
         IntelSWTools/compilers_and_libraries/windows/redist/intel64/mkl
         IntelSWTools/compilers_and_libraries/windows/redist/intel64/compiler
diff --git a/CMakeModules/FindFreeImage.cmake b/CMakeModules/FindFreeImage.cmake
index b049ec06a3..3b2d3fca29 100644
--- a/CMakeModules/FindFreeImage.cmake
+++ b/CMakeModules/FindFreeImage.cmake
@@ -75,12 +75,14 @@ find_library(FreeImage_STATIC_LIBRARY
   DOC "The FreeImage static library")
 
 if (WIN32)
+  get_filename_component(FreeImage_LIB_PATH ${FreeImage_LINK_LIBRARY} DIRECTORY)
   find_file(FreeImage_DLL_LIBRARY
     NAMES
       ${CMAKE_SHARED_LIBRARY_PREFIX}FreeImage${CMAKE_SHARED_LIBRARY_SUFFIX}
       ${CMAKE_SHARED_LIBRARY_PREFIX}freeimage${CMAKE_SHARED_LIBRARY_SUFFIX}
     PATHS
       ${FreeImage_ROOT}
+      ${FreeImage_LIB_PATH}/../bin
     DOC "The FreeImage dll")
 	mark_as_advanced(FreeImage_DLL_LIBRARY)
 endif ()
diff --git a/CMakeModules/nsis/NSIS.InstallOptions.ini.in b/CMakeModules/nsis/NSIS.InstallOptions.ini.in
index d92d77959c..cc17d8268a 100644
--- a/CMakeModules/nsis/NSIS.InstallOptions.ini.in
+++ b/CMakeModules/nsis/NSIS.InstallOptions.ini.in
@@ -3,7 +3,7 @@ NumFields=5
 
 [Field 1]
 Type=label
-Text=By default @CPACK_PACKAGE_INSTALL_DIRECTORY@ does not add its directory to the system PATH.
+Text=By default @CPACK_PACKAGE_INSTALL_DIRECTORY@ will add its directory to the system PATH. This will make the dynamic libraries available to all users and software on the system.
 Left=0
 Right=-1
 Top=0
@@ -16,7 +16,7 @@ Left=0
 Right=-1
 Top=30
 Bottom=40
-State=1
+State=0
 
 [Field 3]
 Type=radiobutton
@@ -25,7 +25,7 @@ Left=0
 Right=-1
 Top=40
 Bottom=50
-State=0
+State=1
 
 [Field 4]
 Type=radiobutton
diff --git a/CMakeModules/nsis/NSIS.definitions.nsh.in b/CMakeModules/nsis/NSIS.definitions.nsh.in
index feedbd7c8d..1062271940 100644
--- a/CMakeModules/nsis/NSIS.definitions.nsh.in
+++ b/CMakeModules/nsis/NSIS.definitions.nsh.in
@@ -8,18 +8,18 @@ A few lines of code in ArrayFire can replace dozens of lines of parallel compute
 saving you valuable time and lowering development costs.\r\n\r\n\
 Follow these steps to install the ArrayFire libraries."
 
-!define MUI_ICON "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@.ico"
-!define MUI_UNICON "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@.ico"
+!define MUI_ICON "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@.ico"
+!define MUI_UNICON "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@.ico"
 
-!define MUI_WELCOMEFINISHPAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@_sym.bmp"
-!define MUI_UNWELCOMEFINISHPAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@_sym.bmp"
+!define MUI_WELCOMEFINISHPAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@_sym.bmp"
+!define MUI_UNWELCOMEFINISHPAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@_sym.bmp"
 !define MUI_WELCOMEFINISHPAGE_UNBITMAP_NOSTRETCH
 !define MUI_UNWELCOMEFINISHPAGE_BITMAP_NOSTRETCH
 
 !define MUI_HEADERIMAGE
 !define MUI_HEADERIMAGE_RIGHT
-!define MUI_HEADERIMAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@_logo.bmp"
-!define MUI_HEADERIMAGE_UNBITMAP "@CPACK_AF_ASSETS_DIR@@CPACK_PACKAGE_NAME@_logo.bmp"
+!define MUI_HEADERIMAGE_BITMAP "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@_logo.bmp"
+!define MUI_HEADERIMAGE_UNBITMAP "@CPACK_AF_ASSETS_DIR@@APP_LOW_NAME@_logo.bmp"
 !define MUI_HEADERIMAGE_BITMAP_NOSTRETCH
 !define MUI_HEADERIMAGE_UNBITMAP_NOSTRETCH
 !define MUI_ABORTWARNING
diff --git a/CMakeModules/nsis/NSIS.template.in b/CMakeModules/nsis/NSIS.template.in
index 971eea59bf..3eaad1c383 100644
--- a/CMakeModules/nsis/NSIS.template.in
+++ b/CMakeModules/nsis/NSIS.template.in
@@ -740,6 +740,11 @@ Section "-Core installation"
 
 SectionEnd
 
+Section "-Visual C++ installation"
+  ExecWait "$INSTDIR\lib\vc_redist.x64.exe /install /passive"
+  Delete "$INSTDIR\lib\vc_redist.x64.exe"
+SectionEnd
+
 Section "-Add to path"
   Push $INSTDIR\lib
   StrCmp "@CPACK_NSIS_MODIFY_PATH@" "ON" 0 doNotAddToPath
diff --git a/src/backend/cuda/CMakeLists.txt b/src/backend/cuda/CMakeLists.txt
index a4783b4936..5085c57717 100644
--- a/src/backend/cuda/CMakeLists.txt
+++ b/src/backend/cuda/CMakeLists.txt
@@ -50,7 +50,10 @@ set(CUDA_architecture_build_targets "Auto" CACHE
 
 find_cuda_helper_libs(nvrtc)
 find_cuda_helper_libs(nvrtc-builtins)
-list(APPEND nvrtc_libs ${CUDA_nvrtc_LIBRARY} ${CUDA_nvrtc-builtins_LIBRARY})
+list(APPEND nvrtc_libs ${CUDA_nvrtc_LIBRARY})
+if(UNIX)
+  list(APPEND nvrtc_libs ${CUDA_nvrtc-builtins_LIBRARY})
+endif()
 
 if(UNIX AND AF_WITH_STATIC_CUDA_NUMERIC_LIBS)
   # The libraries that may be staticly linked or may be loaded at runtime
@@ -789,7 +792,9 @@ function(afcu_collect_libs libname)
       NAMES
         "${PX}${libname}64_${lib_major}${SX}"
         "${PX}${libname}64_${lib_major}${lib_minor}${SX}"
+        "${PX}${libname}64_${lib_major}0_0${SX}"
         "${PX}${libname}64_${lib_major}${lib_minor}_0${SX}"
+        "${PX}${libname}_${lib_major}0_0${SX}"
       PATHS ${dlib_path_prefix}
     )
     mark_as_advanced(CUDA_${libname}_LIBRARY_DLL)
diff --git a/src/backend/oneapi/CMakeLists.txt b/src/backend/oneapi/CMakeLists.txt
index 210c8f59a9..a41d3fa3b7 100644
--- a/src/backend/oneapi/CMakeLists.txt
+++ b/src/backend/oneapi/CMakeLists.txt
@@ -271,6 +271,11 @@ function(set_sycl_language)
       PROPERTIES
         LINKER_LANGUAGE SYCL)
 
+    get_target_property(target_type ${target} TYPE)
+    if(NOT (${target_type} STREQUAL "INTERFACE_LIBRARY"))
+      target_compile_options(${target} PRIVATE ${MSVC_RUNTIME})
+    endif()
+
     get_target_property(TGT_SOURCES ${target} SOURCES)
     if(NOT TGT_SOURCES)
       get_target_property(TGT_SOURCES ${target} INTERFACE_SOURCES)
diff --git a/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt b/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
index 394d593d6e..08b1d35f73 100644
--- a/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
+++ b/src/backend/oneapi/kernel/sort_by_key/CMakeLists.txt
@@ -49,6 +49,7 @@ foreach(SBK_TYPE ${SBK_TYPES})
     PRIVATE
       $<$<COMPILE_LANGUAGE:SYCL>: -fno-sycl-id-queries-fit-in-int
                                   -sycl-std=2020
+                                  ${MSVC_RUNTIME}
                                   $<$<PLATFORM_ID:Linux>: -fno-sycl-rdc>>)
 
   target_include_directories(oneapi_sort_by_key_${SBK_TYPE}
diff --git a/vcpkg.json b/vcpkg.json
index 063e402a02..d811275a6f 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -71,6 +71,7 @@
             "description": "Build OpenCL backend",
             "dependencies": [
                 "boost-compute",
+                "boost-program-options",
                 "opencl"
             ]
         },

From 83feba1826ae3dcfdd7cfbafdb1abdf706fc6175 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Thu, 12 Jun 2025 23:19:05 -0400
Subject: [PATCH 452/473] Add support for CUDA 12.9 (#3657)

---
 src/backend/cuda/device_manager.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/backend/cuda/device_manager.cpp b/src/backend/cuda/device_manager.cpp
index 88cbe487a8..ee7ce76980 100644
--- a/src/backend/cuda/device_manager.cpp
+++ b/src/backend/cuda/device_manager.cpp
@@ -101,6 +101,7 @@ static const int jetsonComputeCapabilities[] = {
 
 // clang-format off
 static const cuNVRTCcompute Toolkit2MaxCompute[] = {
+    {12090, 9, 0, 0},
     {12080, 9, 0, 0},
     {12070, 9, 0, 0},
     {12060, 9, 0, 0},
@@ -146,6 +147,7 @@ struct ComputeCapabilityToStreamingProcessors {
 // clang-format off
 static const ToolkitDriverVersions
     CudaToDriverVersion[] = {
+        {12090, 525.60f, 528.33f},
         {12080, 525.60f, 528.33f},
         {12070, 525.60f, 528.33f},
         {12060, 525.60f, 528.33f},

From dd6b43d28a34e6b4cb554d897bd52acf181e952c Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Thu, 12 Jun 2025 23:19:52 -0400
Subject: [PATCH 453/473] Use correct offset for lookup on oneapi back end.
 (#3659)

This was fixed in the opencl back end in PR #3650 but the issue also existed in the oneapi back end and is fixed here.
---
 src/backend/oneapi/kernel/lookup.hpp | 2 +-
 src/backend/oneapi/lookup.cpp        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/backend/oneapi/kernel/lookup.hpp b/src/backend/oneapi/kernel/lookup.hpp
index f3e2fcdcde..6bceca3e97 100644
--- a/src/backend/oneapi/kernel/lookup.hpp
+++ b/src/backend/oneapi/kernel/lookup.hpp
@@ -64,7 +64,7 @@ class lookupNDCreateKernel {
         int gx = g.get_local_range(0) * (g.get_group_id(0) - gz * nBBS0_) + lx;
         int gy = g.get_local_range(1) * (g.get_group_id(1) - gw * nBBS1_) + ly;
 
-        const idx_t *idxPtr = indices_.get_pointer();
+        const idx_t *idxPtr = indices_.get_pointer() + idxInfo_.offset;
 
         int i = iInfo_.strides[0] *
                 (DIM_ == 0 ? trimIndex((int)idxPtr[gx], iInfo_.dims[0]) : gx);
diff --git a/src/backend/oneapi/lookup.cpp b/src/backend/oneapi/lookup.cpp
index de0a017c55..da658e12aa 100644
--- a/src/backend/oneapi/lookup.cpp
+++ b/src/backend/oneapi/lookup.cpp
@@ -25,8 +25,8 @@ Array<in_t> lookup(const Array<in_t> &input, const Array<idx_t> &indices,
     const dim4 &iDims = input.dims();
 
     dim4 oDims(1);
-    for (int d = 0; d < 4; ++d) {
-        oDims[d] = (d == int(dim) ? indices.elements() : iDims[d]);
+    for (dim_t d = 0; d < 4; ++d) {
+        oDims[d] = (d == dim ? indices.elements() : iDims[d]);
     }
 
     Array<in_t> out = createEmptyArray<in_t>(oDims);

From 9cac22e7ec596fa8e17bef44a2c28a514dababc5 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Fri, 13 Jun 2025 17:48:46 -0400
Subject: [PATCH 454/473] Fix JIT source for casting to signed char. (#3661)

An incorrect function name for casting to a signed char was used when generating the source for oneAPI JIT kernels resulting in a compilation error. This has been fixed with a template specialization of CastOp.
---
 src/backend/oneapi/cast.hpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/backend/oneapi/cast.hpp b/src/backend/oneapi/cast.hpp
index 7d4e2be76f..11b64c9631 100644
--- a/src/backend/oneapi/cast.hpp
+++ b/src/backend/oneapi/cast.hpp
@@ -34,11 +34,15 @@ struct CastOp {
 
 CAST_FN(int)
 CAST_FN(uint)
-CAST_FN(schar)
 CAST_FN(uchar)
 CAST_FN(float)
 CAST_FN(double)
 
+template<typename Ti>
+struct CastOp<schar, Ti> {
+    const char *name() { return "convert_char"; }
+};
+
 #define CAST_CFN(TYPE)                                    \
     template<typename Ti>                                 \
     struct CastOp<TYPE, Ti> {                             \

From d4e96e35c4c7cb5434315f0836b47b7d5d53c9e4 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Fri, 13 Jun 2025 17:53:02 -0400
Subject: [PATCH 455/473] Implement oneAPI half precision RNG (#3662)

The writeOut routines for the uniform and normal distribution RNG were not implemented for the half type on the oneAPI back end. This resulted in undefined behavior when using the randn and randu methods. They have now been implemented.
---
 .../oneapi/kernel/random_engine_write.hpp     | 303 +++++-------------
 1 file changed, 85 insertions(+), 218 deletions(-)

diff --git a/src/backend/oneapi/kernel/random_engine_write.hpp b/src/backend/oneapi/kernel/random_engine_write.hpp
index 3ebf0a113e..a96d7d07fe 100644
--- a/src/backend/oneapi/kernel/random_engine_write.hpp
+++ b/src/backend/oneapi/kernel/random_engine_write.hpp
@@ -8,71 +8,12 @@
  ********************************************************/
 #pragma once
 #include <sycl/sycl.hpp>
+#include <math.hpp>
 
 namespace arrayfire {
 namespace oneapi {
 namespace kernel {
 
-// TODO: !!!! half functions still need to be ported !!!!
-
-//// Conversion to half adapted from Random123
-//// #define HALF_FACTOR (1.0f) / (std::numeric_limits<ushort>::max() + (1.0f))
-//// #define HALF_HALF_FACTOR ((0.5f) * HALF_FACTOR)
-////
-//// NOTE: The following constants for half were calculated using the formulas
-//// above. This is done so that we can avoid unnecessary computations because
-/// the / __half datatype is not a constexprable type. This prevents the
-/// compiler from / peforming these operations at compile time.
-// #define HALF_FACTOR __ushort_as_half(0x100u)
-// #define HALF_HALF_FACTOR __ushort_as_half(0x80)
-//
-//// Conversion to half adapted from Random123
-////#define SIGNED_HALF_FACTOR                                \
-//    //((1.0f) / (std::numeric_limits<short>::max() + (1.0f)))
-////#define SIGNED_HALF_HALF_FACTOR ((0.5f) * SIGNED_HALF_FACTOR)
-////
-//// NOTE: The following constants for half were calculated using the formulas
-//// above. This is done so that we can avoid unnecessary computations because
-/// the / __half datatype is not a constexprable type. This prevents the
-/// compiler from / peforming these operations at compile time
-// #define SIGNED_HALF_FACTOR __ushort_as_half(0x200u)
-// #define SIGNED_HALF_HALF_FACTOR __ushort_as_half(0x100u)
-//
-///// This is the largest integer representable by fp16. We need to
-///// make sure that the value converted from ushort is smaller than this
-///// value to avoid generating infinity
-// constexpr ushort max_int_before_infinity = 65504;
-//
-//// Generates rationals in (0, 1]
-//__device__ static __half oneMinusGetHalf01(uint num) {
-//    // convert to ushort before the min operation
-//    ushort v = min(max_int_before_infinity, ushort(num));
-// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 530
-//    return (1.0f - __half2float(__hfma(__ushort2half_rn(v), HALF_FACTOR,
-//                                       HALF_HALF_FACTOR)));
-// #else
-//    __half out = __ushort_as_half(0x3c00u) /*1.0h*/ -
-//                 __hfma(__ushort2half_rn(v), HALF_FACTOR, HALF_HALF_FACTOR);
-//    if (__hisinf(out)) printf("val: %d ushort: %d\n", num, v);
-//    return out;
-// #endif
-//}
-//
-//// Generates rationals in (0, 1]
-//__device__ static __half getHalf01(uint num) {
-//    // convert to ushort before the min operation
-//    ushort v = min(max_int_before_infinity, ushort(num));
-//    return __hfma(__ushort2half_rn(v), HALF_FACTOR, HALF_HALF_FACTOR);
-//}
-//
-//// Generates rationals in (-1, 1]
-//__device__ static __half getHalfNegative11(uint num) {
-//    // convert to ushort before the min operation
-//    ushort v = min(max_int_before_infinity, ushort(num));
-//    return __hfma(__ushort2half_rn(v), SIGNED_HALF_FACTOR,
-//                  SIGNED_HALF_HALF_FACTOR);
-//}
-//
 // Generates rationals in (0, 1]
 static float getFloat01(uint num) {
     // Conversion to floats adapted from Random123
@@ -126,94 +67,43 @@ static double getDoubleNegative11(uint num1, uint num2) {
     return sycl::fma(static_cast<double>(num), signed_factor, half_factor);
 }
 
+/// This is the largest integer representable by fp16. We need to
+/// make sure that the value converted from ushort is smaller than this
+/// value to avoid generating infinity
+#define MAX_INT_BEFORE_INFINITY (ushort)65504u
+
+// Generates rationals in (0, 1]
+sycl::half getHalf01(uint num, uint index) {
+    sycl::half v = static_cast<sycl::half>(min(MAX_INT_BEFORE_INFINITY,
+                       static_cast<ushort>(num >> (16U * (index & 1U)) & 0x0000ffff)));
+
+    const sycl::half half_factor{1.526e-5}; // (1 / (USHRT_MAX + 1))
+    const sycl::half half_half_factor{7.6e-6}; // (0.5 * half_factor)
+    return sycl::fma(v, half_factor, half_half_factor);
+}
+
+sycl::half oneMinusGetHalf01(uint num, uint index) {
+    return static_cast<sycl::half>(1.) - getHalf01(num, index);
+}
+
+// Generates rationals in (-1, 1]
+sycl::half getHalfNegative11(uint num, uint index) {
+    sycl::half v = static_cast<sycl::half>(min(MAX_INT_BEFORE_INFINITY,
+                       static_cast<ushort>(num >> (16U * (index & 1U)) & 0x0000ffff)));
+
+    const sycl::half signed_half_factor{3.05e-5}; // (1 / (SHRT_MAX + 1))
+    const sycl::half signed_half_half_factor{1.526e-5}; // (0.5 * signed_half_factor)
+    return sycl::fma(v, signed_half_factor, signed_half_half_factor);
+}
+
 namespace {
-//
-// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
-// #define HALF_MATH_FUNC(OP, HALF_OP)    \
-//    template<>                         \
-//    __device__ __half OP(__half val) { \
-//        return ::HALF_OP(val);         \
-//    }
-// #else
-// #define HALF_MATH_FUNC(OP, HALF_OP)     \
-//    template<>                          \
-//    __device__ __half OP(__half val) {  \
-//        float fval = __half2float(val); \
-//        return __float2half(OP(fval));  \
-//    }
-// #endif
-//
-// #define MATH_FUNC(OP, DOUBLE_OP, FLOAT_OP, HALF_OP) \
-//    template<typename T>                            \
-//    __device__ T OP(T val);                         \
-//    template<>                                      \
-//    __device__ double OP(double val) {              \
-//        return ::DOUBLE_OP(val);                    \
-//    }                                               \
-//    template<>                                      \
-//    __device__ float OP(float val) {                \
-//        return ::FLOAT_OP(val);                     \
-//    }                                               \
-//    HALF_MATH_FUNC(OP, HALF_OP)
-//
-// MATH_FUNC(log, log, logf, hlog)
-// MATH_FUNC(sqrt, sqrt, sqrtf, hsqrt)
-// MATH_FUNC(sin, sin, sinf, hsin)
-// MATH_FUNC(cos, cos, cosf, hcos)
-//
-// template<typename T>
-//__device__ void sincos(T val, T *sptr, T *cptr);
-//
-// template<>
-//__device__ void sincos(double val, double *sptr, double *cptr) {
-//    ::sincos(val, sptr, cptr);
-//}
-//
-// template<>
-//__device__ void sincos(float val, float *sptr, float *cptr) {
-//    sincosf(val, sptr, cptr);
-//}
-//
-// template<>
-//__device__ void sincos(__half val, __half *sptr, __half *cptr) {
-// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
-//    *sptr = sin(val);
-//    *cptr = cos(val);
-// #else
-//    float s, c;
-//    float fval = __half2float(val);
-//    sincos(fval, &s, &c);
-//    *sptr = __float2half(s);
-//    *cptr = __float2half(c);
-// #endif
-//}
-//
 template<typename T>
 void sincospi(T val, T *sptr, T *cptr) {
     *sptr = sycl::sinpi(val);
     *cptr = sycl::cospi(val);
 }
-
-// template<>
-//__device__ void sincospi(__half val, __half *sptr, __half *cptr) {
-//    // CUDA cannot make __half into a constexpr as of CUDA 11 so we are
-//    // converting this offline
-// #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
-//    const __half pi_val = __ushort_as_half(0x4248);  // 0x4248 == 3.14062h
-//    val *= pi_val;
-//    *sptr = sin(val);
-//    *cptr = cos(val);
-// #else
-//    float fval = __half2float(val);
-//    float s, c;
-//    sincospi(fval, &s, &c);
-//    *sptr = __float2half(s);
-//    *cptr = __float2half(c);
-// #endif
-//}
-//
 }  // namespace
-//
+
 template<typename T>
 constexpr T neg_two() {
     return -2.0;
@@ -273,13 +163,6 @@ static void boxMullerTransform(Td *const out1, Td *const out2, const Tc &r1,
     *out1 = static_cast<Td>(r * s);
     *out2 = static_cast<Td>(r * c);
 }
-// template<>
-//__device__ void boxMullerTransform<arrayfire::common::half, __half>(
-//    arrayfire::common::half *const out1, arrayfire::common::half *const out2,
-//    const __half &r1, const __half &r2) { float o1, o2; float fr1 =
-//    __half2float(r1); float fr2 = __half2float(r2); boxMullerTransform(&o1,
-//    &o2, fr1, fr2); *out1 = o1; *out2 = o2;
-//}
 
 // Writes without boundary checking
 static void writeOut128Bytes(uchar *out, const uint &index, const uint groupSz,
@@ -413,14 +296,14 @@ static void writeOut128Bytes(cdouble *out, const uint &index,
 static void writeOut128Bytes(arrayfire::common::half *out, const uint &index,
                              const uint groupSz, const uint &r1, const uint &r2,
                              const uint &r3, const uint &r4) {
-    // out[index]               = oneMinusGetHalf01(r1);
-    // out[index + groupSz]     = oneMinusGetHalf01(r1 >> 16);
-    // out[index + 2 * groupSz] = oneMinusGetHalf01(r2);
-    // out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16);
-    // out[index + 4 * groupSz] = oneMinusGetHalf01(r3);
-    // out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16);
-    // out[index + 6 * groupSz] = oneMinusGetHalf01(r4);
-    // out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16);
+    out[index]               = oneMinusGetHalf01(r1, 0);
+    out[index + groupSz]     = oneMinusGetHalf01(r1, 1);
+    out[index + 2 * groupSz] = oneMinusGetHalf01(r2, 0);
+    out[index + 3 * groupSz] = oneMinusGetHalf01(r2, 1);
+    out[index + 4 * groupSz] = oneMinusGetHalf01(r3, 0);
+    out[index + 5 * groupSz] = oneMinusGetHalf01(r3, 1);
+    out[index + 6 * groupSz] = oneMinusGetHalf01(r4, 0);
+    out[index + 7 * groupSz] = oneMinusGetHalf01(r4, 1);
 }
 
 // Normalized writes without boundary checking
@@ -464,17 +347,14 @@ static void boxMullerWriteOut128Bytes(arrayfire::common::half *out,
                                       const uint &index, const uint groupSz,
                                       const uint &r1, const uint &r2,
                                       const uint &r3, const uint &r4) {
-    //   boxMullerTransform(&out[index], &out[index + groupSz],
-    //                      getHalfNegative11(r1), getHalf01(r1 >> 16));
-    //   boxMullerTransform(&out[index + 2 * groupSz],
-    //                      &out[index + 3 * groupSz], getHalfNegative11(r2),
-    //                      getHalf01(r2 >> 16));
-    //   boxMullerTransform(&out[index + 4 * groupSz],
-    //                      &out[index + 5 * groupSz], getHalfNegative11(r3),
-    //                      getHalf01(r3 >> 16));
-    //   boxMullerTransform(&out[index + 6 * groupSz],
-    //                      &out[index + 7 * groupSz], getHalfNegative11(r4),
-    //                      getHalf01(r4 >> 16));
+    boxMullerTransform(&out[index], &out[index + groupSz],
+                       getHalfNegative11(r1, 0), getHalf01(r1, 1));
+    boxMullerTransform(&out[index + 2 * groupSz], &out[index + 3 * groupSz],
+                       getHalfNegative11(r2, 0), getHalf01(r2, 1));
+    boxMullerTransform(&out[index + 4 * groupSz], &out[index + 5 * groupSz],
+                       getHalfNegative11(r3, 0), getHalf01(r3, 1));
+    boxMullerTransform(&out[index + 6 * groupSz], &out[index + 7 * groupSz],
+                       getHalfNegative11(r4, 0), getHalf01(r4, 1));
 }
 
 // Writes with boundary checking
@@ -727,28 +607,28 @@ static void partialWriteOut128Bytes(arrayfire::common::half *out,
                                     const uint &r1, const uint &r2,
                                     const uint &r3, const uint &r4,
                                     const uint &elements) {
-    //  if (index < elements) { out[index] = oneMinusGetHalf01(r1); }
-    //  if (index + groupSz < elements) {
-    //      out[index + groupSz] = oneMinusGetHalf01(r1 >> 16);
-    //  }
-    //  if (index + 2 * groupSz < elements) {
-    //      out[index + 2 * groupSz] = oneMinusGetHalf01(r2);
-    //  }
-    //  if (index + 3 * groupSz < elements) {
-    //      out[index + 3 * groupSz] = oneMinusGetHalf01(r2 >> 16);
-    //  }
-    //  if (index + 4 * groupSz < elements) {
-    //      out[index + 4 * groupSz] = oneMinusGetHalf01(r3);
-    //  }
-    //  if (index + 5 * groupSz < elements) {
-    //      out[index + 5 * groupSz] = oneMinusGetHalf01(r3 >> 16);
-    //  }
-    //  if (index + 6 * groupSz < elements) {
-    //      out[index + 6 * groupSz] = oneMinusGetHalf01(r4);
-    //  }
-    //  if (index + 7 * groupSz < elements) {
-    //      out[index + 7 * groupSz] = oneMinusGetHalf01(r4 >> 16);
-    //  }
+    if (index < elements) { out[index] = oneMinusGetHalf01(r1, 0); }
+    if (index + groupSz < elements) {
+        out[index + groupSz] = oneMinusGetHalf01(r1, 1);
+    }
+    if (index + 2 * groupSz < elements) {
+        out[index + 2 * groupSz] = oneMinusGetHalf01(r2, 0);
+    }
+    if (index + 3 * groupSz < elements) {
+        out[index + 3 * groupSz] = oneMinusGetHalf01(r2, 1);
+    }
+    if (index + 4 * groupSz < elements) {
+        out[index + 4 * groupSz] = oneMinusGetHalf01(r3, 0);
+    }
+    if (index + 5 * groupSz < elements) {
+        out[index + 5 * groupSz] = oneMinusGetHalf01(r3, 1);
+    }
+    if (index + 6 * groupSz < elements) {
+        out[index + 6 * groupSz] = oneMinusGetHalf01(r4, 0);
+    }
+    if (index + 7 * groupSz < elements) {
+        out[index + 7 * groupSz] = oneMinusGetHalf01(r4, 1);
+    }
 }
 
 // Normalized writes with boundary checking
@@ -758,35 +638,22 @@ static void partialBoxMullerWriteOut128Bytes(arrayfire::common::half *out,
                                              const uint &r2, const uint &r3,
                                              const uint &r4,
                                              const uint &elements) {
-    //    arrayfire::common::half n[8];
-    //    boxMullerTransform(n + 0, n + 1, getHalfNegative11(r1),
-    //                       getHalf01(r1 >> 16));
-    //    boxMullerTransform(n + 2, n + 3, getHalfNegative11(r2),
-    //                       getHalf01(r2 >> 16));
-    //    boxMullerTransform(n + 4, n + 5, getHalfNegative11(r3),
-    //                       getHalf01(r3 >> 16));
-    //    boxMullerTransform(n + 6, n + 7, getHalfNegative11(r4),
-    //                       getHalf01(r4 >> 16));
-    //    if (index < elements) { out[index] = n[0]; }
-    //    if (index + groupSz < elements) { out[index + groupSz] = n[1]; }
-    //    if (index + 2 * groupSz < elements) {
-    //        out[index + 2 * groupSz] = n[2];
-    //    }
-    //    if (index + 3 * groupSz < elements) {
-    //        out[index + 3 * groupSz] = n[3];
-    //    }
-    //    if (index + 4 * groupSz < elements) {
-    //        out[index + 4 * groupSz] = n[4];
-    //    }
-    //    if (index + 5 * groupSz < elements) {
-    //        out[index + 5 * groupSz] = n[5];
-    //    }
-    //    if (index + 6 * groupSz < elements) {
-    //        out[index + 6 * groupSz] = n[6];
-    //    }
-    //    if (index + 7 * groupSz < elements) {
-    //        out[index + 7 * groupSz] = n[7];
-    //    }
+    sycl::half n1, n2;
+    boxMullerTransform(&n1, &n2, getHalfNegative11(r1, 0), getHalf01(r1, 1));
+    if (index < elements) { out[index] = n1; }
+    if (index + groupSz < elements) { out[index + groupSz] = n2; }
+
+    boxMullerTransform(&n1, &n2, getHalfNegative11(r2, 0), getHalf01(r2, 1));
+    if (index + 2 * groupSz < elements) { out[index + 2 * groupSz] = n1; }
+    if (index + 3 * groupSz < elements) { out[index + 3 * groupSz] = n2; }
+
+    boxMullerTransform(&n1, &n2, getHalfNegative11(r3, 0), getHalf01(r3, 1));
+    if (index + 4 * groupSz < elements) { out[index + 4 * groupSz] = n1; }
+    if (index + 5 * groupSz < elements) { out[index + 5 * groupSz] = n2; }
+
+    boxMullerTransform(&n1, &n2, getHalfNegative11(r4, 0), getHalf01(r4, 1));
+    if (index + 6 * groupSz < elements) { out[index + 6 * groupSz] = n1; }
+    if (index + 7 * groupSz < elements) { out[index + 7 * groupSz] = n2; }
 }
 
 }  // namespace kernel

From 492f808f7781dd849099ddbbc20d6946e7f841a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Fri, 20 Jun 2025 12:15:07 -0700
Subject: [PATCH 456/473] Fixed span lite add_subdirectory command missing
 build directory (#3669)

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4ce33555e1..21bc48d39e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -316,7 +316,7 @@ if(NOT TARGET nonstd::span-lite)
     URI https://github.com/martinmoene/span-lite
     REF "ccf2351"
     )
-  add_subdirectory(${span-lite_SOURCE_DIR} EXCLUDE_FROM_ALL)
+  add_subdirectory(${span-lite_SOURCE_DIR} ${span-lite_BINARY_DIR} EXCLUDE_FROM_ALL)
   get_property(span_include_dir
     TARGET span-lite
     PROPERTY INTERFACE_INCLUDE_DIRECTORIES)

From 0e8a6900c338c72fa8b217eb13ed4c3b529475e3 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Wed, 25 Jun 2025 11:02:09 -0400
Subject: [PATCH 457/473] Fixes to the indexed reduce function for the cpu,
 opencl and cuda back ends (#3658)

* Fixes to the indexed reduce function for the cpu, opencl and cuda back ends.

These back ends were incorrectly assuming a linear array. In the case of the cuda and opencl back end this was just for the cpu-fallback methods which are used when the total number of elements in the array is less than or equal to 4096.

* Ensure array is evaluated before reducing

Added an eval() to the input array on the CPU back end for the ireduce method to ensure that the array has been evaluated before reducing.
---
 src/backend/cpu/ireduce.cpp           |   6 +-
 src/backend/cuda/kernel/ireduce.hpp   |  14 +--
 src/backend/opencl/kernel/ireduce.hpp |  13 +--
 test/ireduce.cpp                      | 133 ++++++++++++++++++++++++++
 4 files changed, 151 insertions(+), 15 deletions(-)

diff --git a/src/backend/cpu/ireduce.cpp b/src/backend/cpu/ireduce.cpp
index a20df27c1a..b87c12bc87 100644
--- a/src/backend/cpu/ireduce.cpp
+++ b/src/backend/cpu/ireduce.cpp
@@ -58,11 +58,13 @@ void rreduce(Array<T> &out, Array<uint> &loc, const Array<T> &in, const int dim,
 
 template<af_op_t op, typename T>
 T ireduce_all(unsigned *loc, const Array<T> &in) {
+    in.eval();
     getQueue().sync();
 
     af::dim4 dims    = in.dims();
     af::dim4 strides = in.strides();
     const T *inPtr   = in.get();
+    dim_t idx = 0;
 
     kernel::MinMaxOp<op, T> Op(inPtr[0], 0);
 
@@ -76,8 +78,8 @@ T ireduce_all(unsigned *loc, const Array<T> &in) {
                 dim_t off1 = j * strides[1];
 
                 for (dim_t i = 0; i < dims[0]; i++) {
-                    dim_t idx = i + off1 + off2 + off3;
-                    Op(inPtr[idx], idx);
+                    dim_t d_idx = i + off1 + off2 + off3;
+                    Op(inPtr[d_idx], idx++);
                 }
             }
         }
diff --git a/src/backend/cuda/kernel/ireduce.hpp b/src/backend/cuda/kernel/ireduce.hpp
index c394c01f83..992d0871c4 100644
--- a/src/backend/cuda/kernel/ireduce.hpp
+++ b/src/backend/cuda/kernel/ireduce.hpp
@@ -165,14 +165,14 @@ T ireduce_all(uint *idx, CParam<T> in) {
     using std::unique_ptr;
     int in_elements = in.dims[0] * in.dims[1] * in.dims[2] * in.dims[3];
 
-    // FIXME: Use better heuristics to get to the optimum number
-    if (in_elements > 4096) {
-        bool is_linear = (in.strides[0] == 1);
-        for (int k = 1; k < 4; k++) {
-            is_linear &=
-                (in.strides[k] == (in.strides[k - 1] * in.dims[k - 1]));
-        }
+    bool is_linear = (in.strides[0] == 1);
+    for (int k = 1; k < 4; k++) {
+        is_linear &=
+            (in.strides[k] == (in.strides[k - 1] * in.dims[k - 1]));
+    }
 
+    // FIXME: Use better heuristics to get to the optimum number
+    if (!is_linear || in_elements > 4096) {
         if (is_linear) {
             in.dims[0] = in_elements;
             for (int k = 1; k < 4; k++) {
diff --git a/src/backend/opencl/kernel/ireduce.hpp b/src/backend/opencl/kernel/ireduce.hpp
index 1bbcf08d2b..d056fb8fea 100644
--- a/src/backend/opencl/kernel/ireduce.hpp
+++ b/src/backend/opencl/kernel/ireduce.hpp
@@ -251,13 +251,14 @@ T ireduceAll(uint *loc, Param in) {
     int in_elements =
         in.info.dims[0] * in.info.dims[1] * in.info.dims[2] * in.info.dims[3];
 
+    bool is_linear = (in.info.strides[0] == 1);
+    for (int k = 1; k < 4; k++) {
+        is_linear &= (in.info.strides[k] ==
+                      (in.info.strides[k - 1] * in.info.dims[k - 1]));
+    }
+
     // FIXME: Use better heuristics to get to the optimum number
-    if (in_elements > 4096) {
-        bool is_linear = (in.info.strides[0] == 1);
-        for (int k = 1; k < 4; k++) {
-            is_linear &= (in.info.strides[k] ==
-                          (in.info.strides[k - 1] * in.info.dims[k - 1]));
-        }
+    if (!is_linear || in_elements > 4096) {
         if (is_linear) {
             in.info.dims[0] = in_elements;
             for (int k = 1; k < 4; k++) {
diff --git a/test/ireduce.cpp b/test/ireduce.cpp
index e93a8267b4..b155512e32 100644
--- a/test/ireduce.cpp
+++ b/test/ireduce.cpp
@@ -420,3 +420,136 @@ TEST(IndexedReduce, MaxCplxPreferSmallerIdxIfEqual) {
 
     ASSERT_EQ(h_max_idx[0], gold_max_idx);
 }
+
+#define SUBA_TEST_DATA                                              \
+    float test_data[25] = {0.0168, 0.0278, 0.0317, 0.0248, 0.0131,  \
+                           0.0197, 0.0321, 0.0362, 0.0279, 0.0141,  \
+                           0.0218, 0.0353, 0.0394, 0.0297, 0.0143,  \
+                           0.0224, 0.0363, 0.0104, 0.0302, 0.0142,  \
+                           0.0217, 0.0409, 0.0398, 0.0302, 0.0144}; \
+    array a(5, 5, test_data);                                       \
+    array a_sub = a(seq(1, 3), seq(2,4))
+
+TEST(IndexedReduce, max_subarray_all) {
+    SUBA_TEST_DATA;
+
+    float gold_max_val = 0.0409;
+    unsigned gold_max_idx   = 6;
+
+    float max_val;
+    unsigned max_idx;
+    max<float>(&max_val, &max_idx, a_sub);
+
+    ASSERT_FLOAT_EQ(max_val, gold_max_val);
+    ASSERT_EQ(max_idx, gold_max_idx);
+}
+
+TEST(IndexedReduce, min_subarray_all) {
+    SUBA_TEST_DATA;
+
+    float gold_min_val = 0.0104;
+    unsigned gold_min_idx   = 4;
+
+    float min_val;
+    unsigned min_idx;
+    min<float>(&min_val, &min_idx, a_sub);
+
+    ASSERT_FLOAT_EQ(min_val, gold_min_val);
+    ASSERT_EQ(min_idx, gold_min_idx);
+}
+
+TEST(IndexedReduce, max_subarray_0) {
+    SUBA_TEST_DATA;
+
+    float gold_val[3] = {0.0394, 0.0363, 0.0409};
+    unsigned gold_idx[3] = {1, 0, 0};
+
+    array val;
+    array idx;
+    float h_val[3];
+    unsigned h_idx[3];
+
+    max(val, idx, a_sub);
+    val.host(&h_val);
+    idx.host(&h_idx);
+
+    for(int i = 0; i < 3; ++i) {
+        ASSERT_FLOAT_EQ(h_val[i], gold_val[i]);
+        ASSERT_EQ(h_idx[i], gold_idx[i]);
+    }
+}
+
+TEST(IndexedReduce, min_subarray_0) {
+    SUBA_TEST_DATA;
+
+    float gold_val[3] = {0.0297, 0.0104, 0.0302};
+    unsigned gold_idx[3] = {2, 1, 2};
+
+    array val;
+    array idx;
+    float h_val[3];
+    unsigned h_idx[3];
+
+    min(val, idx, a_sub);
+    val.host(&h_val);
+    idx.host(&h_idx);
+
+    for(int i = 0; i < 3; ++i) {
+        ASSERT_FLOAT_EQ(h_val[i], gold_val[i]);
+        ASSERT_EQ(h_idx[i], gold_idx[i]);
+    }
+}
+
+TEST(IndexedReduce, max_subarray_1) {
+    SUBA_TEST_DATA;
+
+    float gold_val[3] = {0.0409, 0.0398, 0.0302};
+    unsigned gold_idx[3] = {2, 2, 1};
+
+    array val;
+    array idx;
+    float h_val[3];
+    unsigned h_idx[3];
+
+    max(val, idx, a_sub, 1);
+    val.host(&h_val);
+    idx.host(&h_idx);
+
+    for(int i = 0; i < 3; ++i) {
+        ASSERT_FLOAT_EQ(h_val[i], gold_val[i]);
+        ASSERT_EQ(h_idx[i], gold_idx[i]);
+    }
+}
+
+TEST(IndexedReduce, min_subarray_1) {
+    SUBA_TEST_DATA;
+
+    float gold_val[3] = {0.0353, 0.0104, 0.0297};
+    unsigned gold_idx[3] = {0, 1, 0};
+
+    array val;
+    array idx;
+    float h_val[3];
+    unsigned h_idx[3];
+
+    min(val, idx, a_sub, 1);
+    val.host(&h_val);
+    idx.host(&h_idx);
+
+    for(int i = 0; i < 3; ++i) {
+        ASSERT_FLOAT_EQ(h_val[i], gold_val[i]);
+        ASSERT_EQ(h_idx[i], gold_idx[i]);
+    }
+}
+
+//Ensure that array is evaluated before reducing
+TEST(IndexedReduce, reduce_jit_array) {
+    af::array jit(af::dim4(2),{1.0f, 2.0f});
+    jit += af::constant(1.0f, af::dim4(2));
+    float val; unsigned idx;
+    float gold_val = 2.0f;
+    unsigned gold_idx = 0;
+    af::min(&val, &idx, jit);
+    ASSERT_EQ(val, gold_val);
+    ASSERT_EQ(idx, gold_idx);
+}

From 700db10ccd5074d6ee4f32a5314fec6005e6d01f Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Wed, 25 Jun 2025 19:19:21 +0200
Subject: [PATCH 458/473] Join does not always respect the order of provided
 parameters (oneapi) (#3511)(#3513) (#3667)

* Adds test helpers for temporary array formats (JIT, SUB, ...)

* Join does not always respect the order of provided parameters (oneapi) (#3511)(#3513)
---
 src/backend/oneapi/jit.cpp | 115 ++++++++++++++----------
 test/arrayfire_test.cpp    | 178 ++++++++++++++++++++++++++++++++++---
 test/join.cpp              |  25 +++++-
 test/testHelpers.hpp       |  32 ++++++-
 4 files changed, 283 insertions(+), 67 deletions(-)

diff --git a/src/backend/oneapi/jit.cpp b/src/backend/oneapi/jit.cpp
index 2bd34a5dc4..bda9e43ccf 100644
--- a/src/backend/oneapi/jit.cpp
+++ b/src/backend/oneapi/jit.cpp
@@ -218,61 +218,75 @@ __kernel void )JIT";
     thread_local stringstream outOffsetStream;
     thread_local stringstream inOffsetsStream;
     thread_local stringstream opsStream;
+    thread_local stringstream kerStream;
 
-    int oid{0};
-    for (size_t i{0}; i < full_nodes.size(); i++) {
-        const auto& node{full_nodes[i]};
-        const auto& ids_curr{full_ids[i]};
-        // Generate input parameters, only needs current id
-        node->genParams(inParamStream, ids_curr.id, is_linear);
-        // Generate input offsets, only needs current id
-        node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
-        // Generate the core function body, needs children ids as well
-        node->genFuncs(opsStream, ids_curr);
-        for (auto outIt{begin(output_ids)}, endIt{end(output_ids)};
-             (outIt = find(outIt, endIt, ids_curr.id)) != endIt; ++outIt) {
-            // Generate also output parameters
-            outParamStream << "__global "
-                           << full_nodes[ids_curr.id]->getTypeStr() << " *out"
-                           << oid << ", int offset" << oid << ",\n";
-            // Apply output offset
-            outOffsetStream << "\nout" << oid << " += offset" << oid << ';';
-            // Generate code to write the output
-            opsStream << "out" << oid << "[idx] = val" << ids_curr.id << ";\n";
-            ++oid;
+    string ret;
+    try {
+        int oid{0};
+        for (size_t i{0}; i < full_nodes.size(); i++) {
+            const auto& node{full_nodes[i]};
+            const auto& ids_curr{full_ids[i]};
+            // Generate input parameters, only needs current id
+            node->genParams(inParamStream, ids_curr.id, is_linear);
+            // Generate input offsets, only needs current id
+            node->genOffsets(inOffsetsStream, ids_curr.id, is_linear);
+            // Generate the core function body, needs children ids as well
+            node->genFuncs(opsStream, ids_curr);
+            for (size_t output_idx{0}; output_idx < output_ids.size();
+                 ++output_idx) {
+                if (output_ids[output_idx] == ids_curr.id) {
+                    outParamStream
+                        << "__global " << full_nodes[ids_curr.id]->getTypeStr()
+                        << " *out" << oid << ", int offset" << oid << ",\n";
+                    // Apply output offset
+                    outOffsetStream << "\nout" << oid << " += offset" << oid
+                                    << ';';
+                    // Generate code to write the output
+                    opsStream << "out" << output_idx << "[idx] = val"
+                              << ids_curr.id << ";\n";
+                    ++oid;
+                }
+            }
         }
-    }
 
-    thread_local stringstream kerStream;
-    kerStream << DEFAULT_MACROS_STR << kernelVoid << funcName << "(\n"
-              << inParamStream.str() << outParamStream.str() << dimParams << ")"
-              << blockStart;
-    if (is_linear) {
-        kerStream << linearInit << inOffsetsStream.str()
-                  << outOffsetStream.str() << '\n';
-        if (loop0) kerStream << linearLoop0Start;
-        kerStream << "\n\n" << opsStream.str();
-        if (loop0) kerStream << linearLoop0End;
-        kerStream << linearEnd;
-    } else {
-        if (loop0) {
-            kerStream << stridedLoop0Init << outOffsetStream.str() << '\n'
-                      << stridedLoop0Start;
+        kerStream << DEFAULT_MACROS_STR << kernelVoid << funcName << "(\n"
+                  << inParamStream.str() << outParamStream.str() << dimParams
+                  << ")" << blockStart;
+        if (is_linear) {
+            kerStream << linearInit << inOffsetsStream.str()
+                      << outOffsetStream.str() << '\n';
+            if (loop0) kerStream << linearLoop0Start;
+            kerStream << "\n\n" << opsStream.str();
+            if (loop0) kerStream << linearLoop0End;
+            kerStream << linearEnd;
         } else {
-            kerStream << stridedLoopNInit << outOffsetStream.str() << '\n';
-            if (loop3) kerStream << stridedLoop3Init;
-            if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
-            if (loop3) kerStream << stridedLoop3Start;
+            if (loop0) {
+                kerStream << stridedLoop0Init << outOffsetStream.str() << '\n'
+                          << stridedLoop0Start;
+            } else {
+                kerStream << stridedLoopNInit << outOffsetStream.str() << '\n';
+                if (loop3) kerStream << stridedLoop3Init;
+                if (loop1) kerStream << stridedLoop1Init << stridedLoop1Start;
+                if (loop3) kerStream << stridedLoop3Start;
+            }
+            kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
+            if (loop3) kerStream << stridedLoop3End;
+            if (loop1) kerStream << stridedLoop1End;
+            if (loop0) kerStream << stridedLoop0End;
+            kerStream << stridedEnd;
         }
-        kerStream << "\n\n" << inOffsetsStream.str() << opsStream.str();
-        if (loop3) kerStream << stridedLoop3End;
-        if (loop1) kerStream << stridedLoop1End;
-        if (loop0) kerStream << stridedLoop0End;
-        kerStream << stridedEnd;
+        kerStream << blockEnd;
+        ret = kerStream.str();
+    } catch (...) {
+        // Prepare for next round, limit memory
+        inParamStream.str("");
+        outParamStream.str("");
+        inOffsetsStream.str("");
+        outOffsetStream.str("");
+        opsStream.str("");
+        kerStream.str("");
+        throw;
     }
-    kerStream << blockEnd;
-    const string ret{kerStream.str()};
-
     // Prepare for next round, limit memory
     inParamStream.str("");
     outParamStream.str("");
@@ -381,9 +395,11 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
 
     bool is_linear{true};
     dim_t numOutElems{1};
+    assert(outputs.size() == output_nodes.size());
     KParam& out_info{outputs[0].info};
     dim_t* outDims{out_info.dims};
     dim_t* outStrides{out_info.strides};
+    // unsigned nrInputs{0};
 
     dim_t ndims{outDims[3] > 1   ? 4
                 : outDims[2] > 1 ? 3
@@ -409,6 +425,7 @@ void evalNodes(vector<Param<T>>& outputs, const vector<Node*>& output_nodes) {
     for (const Node* node : full_nodes) {
         is_linear &= node->isLinear(outDims);
         moddimsFound |= (node->getOp() == af_moddims_t);
+        // if (node->isBuffer()) { ++nrInputs; }
     }
 
     bool emptyColumnsFound{false};
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index dedaedbf75..6803cc586d 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -105,17 +105,16 @@ std::string readNextNonEmptyLine(std::ifstream &file) {
 
 std::string getBackendName(bool lower) {
     af::Backend backend = af::getActiveBackend();
-    switch(backend) {
-    case AF_BACKEND_CPU:
-        return lower ? std::string("cpu") : std::string("CPU");
-    case AF_BACKEND_CUDA:
-        return lower ? std::string("cuda") : std::string("CUDA");
-    case AF_BACKEND_OPENCL:
-        return lower ? std::string("opencl") : std::string("OpenCL");
-    case AF_BACKEND_ONEAPI:
-        return lower ? std::string("oneapi") : std::string("oneAPI");
-    default:
-        return lower ? std::string("unknown") : std::string("Unknown");
+    switch (backend) {
+        case AF_BACKEND_CPU:
+            return lower ? std::string("cpu") : std::string("CPU");
+        case AF_BACKEND_CUDA:
+            return lower ? std::string("cuda") : std::string("CUDA");
+        case AF_BACKEND_OPENCL:
+            return lower ? std::string("opencl") : std::string("OpenCL");
+        case AF_BACKEND_ONEAPI:
+            return lower ? std::string("oneapi") : std::string("oneAPI");
+        default: return lower ? std::string("unknown") : std::string("Unknown");
     }
 }
 
@@ -2046,6 +2045,163 @@ INSTANTIATE(std::complex<float>);
 INSTANTIATE(std::complex<double>);
 #undef INSTANTIATE
 
+af::array toTempFormat(tempFormat form, const af::array &in) {
+    af::array ret;
+    const af::dim4 &dims = in.dims();
+    switch (form) {
+        case JIT_FORMAT:
+            switch (in.type()) {
+                case b8: ret = not(in); break;
+                default: ret = in * 2;
+            }
+            // Make sure that the base array is <> form original
+            ret.eval();
+            switch (in.type()) {
+                case b8: ret = not(ret); break;
+                default: ret /= 2;
+            }
+            break;
+        case SUB_FORMAT_dim0: {
+            af::dim4 pdims(dims);
+            pdims[0] += 2;
+            af::array parent = af::randu(pdims, in.type());
+            parent(af::seq(1, dims[0]), af::span, af::span, af::span) = in;
+            ret = parent(af::seq(1, dims[0]), af::span, af::span, af::span);
+        }; break;
+        case SUB_FORMAT_dim1: {
+            af::dim4 pdims(dims);
+            pdims[1] += 2;
+            af::array parent = af::randu(pdims, in.type());
+            parent(af::span, af::seq(1, dims[1]), af::span, af::span) = in;
+            ret = parent(af::span, af::seq(1, dims[1]), af::span, af::span);
+        }; break;
+        case SUB_FORMAT_dim2: {
+            af::dim4 pdims(dims);
+            pdims[2] += 2;
+            af::array parent = af::randu(pdims, in.type());
+            parent(af::span, af::span, af::seq(1, dims[2]), af::span) = in;
+            ret = parent(af::span, af::span, af::seq(1, dims[2]), af::span);
+        }; break;
+        case SUB_FORMAT_dim3: {
+            af::dim4 pdims(dims);
+            pdims[3] += 2;
+            af::array parent = af::randu(pdims, in.type());
+            parent(af::span, af::span, af::span, af::seq(1, dims[3])) = in;
+            ret = parent(af::span, af::span, af::span, af::seq(1, dims[3]));
+        }; break;
+        case REORDERED_FORMAT: {
+            const dim_t idxs[4] = {0, 3, 1, 2};
+            // idxs[0] has to be 0, to keep the same data in mem
+            dim_t rev_idxs[4];
+            for (dim_t i = 0; i < 4; ++i) { rev_idxs[idxs[i]] = i; };
+            ret = af::reorder(in, idxs[0], idxs[1], idxs[2], idxs[3]);
+            ret = ret.copy();  // make data linear
+            ret = af::reorder(ret, rev_idxs[0], rev_idxs[1], rev_idxs[2],
+                              rev_idxs[3]);
+            // ret has same content as in, although data is stored in
+            // different order
+        }; break;
+        case LINEAR_FORMAT:
+        default: ret = in.copy();
+    };
+    return ret;
+}
+
+void toTempFormat(tempFormat form, af_array *out, const af_array &in) {
+    dim_t dims[4];
+    af_get_dims(dims, dims + 1, dims + 2, dims + 3, in);
+    unsigned numdims;
+    af_get_numdims(&numdims, in);
+    af_dtype ty;
+    af_get_type(&ty, in);
+    switch (form) {
+        case JIT_FORMAT: {
+            // af_array one = nullptr, min_one = nullptr, res = nullptr;
+            af_array res = nullptr, two = nullptr;
+            ASSERT_SUCCESS(af_constant(&two, 2, numdims, dims, ty));
+            switch (ty) {
+                case b8: af_not(&res, in); break;
+                default:
+                    // ret = in + af::constant(1, dims, in.type());
+                    ASSERT_SUCCESS(af_mul(&res, in, two, false));
+            }
+            // Make sure that the base array is <> form original
+            ASSERT_SUCCESS(af_eval(res));
+            switch (ty) {
+                case b8: af_not(out, res); break;
+                default:
+                    ASSERT_SUCCESS(af_div(out, res, two, false));  // NO EVAL!!
+            }
+            ASSERT_SUCCESS(af_release_array(two));
+            two = nullptr;
+            ASSERT_SUCCESS(af_release_array(res));
+            res = nullptr;
+        }; break;
+        case SUB_FORMAT_dim0: {
+            const dim_t pdims[4] = {dims[0] + 2, dims[1], dims[2], dims[3]};
+            af_array parent      = nullptr;
+            ASSERT_SUCCESS(af_randu(&parent, std::max(1u, numdims), pdims, ty));
+            const af_seq idxs[4] = {af_make_seq(1, dims[0], 1), af_span,
+                                    af_span, af_span};
+
+            ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in));
+            ASSERT_SUCCESS(af_index(out, parent, numdims, idxs));
+            ASSERT_SUCCESS(af_release_array(parent));
+        }; break;
+        case SUB_FORMAT_dim1: {
+            const dim_t pdims[4] = {dims[0], dims[1] + 2, dims[2], dims[3]};
+            af_array parent      = nullptr;
+            ASSERT_SUCCESS(af_randu(&parent, std::max(2u, numdims), pdims, ty));
+            const af_seq idxs[4] = {af_span, af_make_seq(1, dims[1], 1),
+                                    af_span, af_span};
+            ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in));
+            ASSERT_SUCCESS(af_index(out, parent, numdims, idxs));
+            ASSERT_SUCCESS(af_release_array(parent));
+            parent = nullptr;
+        }; break;
+        case SUB_FORMAT_dim2: {
+            const dim_t pdims[4] = {dims[0], dims[1], dims[2] + 2, dims[3]};
+            af_array parent      = nullptr;
+            ASSERT_SUCCESS(af_randu(&parent, std::max(3u, numdims), pdims, ty));
+            const af_seq idxs[4] = {af_span, af_span,
+                                    af_make_seq(1, dims[2], 1), af_span};
+            ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in));
+            ASSERT_SUCCESS(af_index(out, parent, numdims, idxs));
+            ASSERT_SUCCESS(af_release_array(parent));
+            parent = nullptr;
+        }; break;
+        case SUB_FORMAT_dim3: {
+            const dim_t pdims[4] = {dims[0], dims[1], dims[2], dims[3] + 2};
+            af_array parent      = nullptr;
+            ASSERT_SUCCESS(af_randu(&parent, std::max(4u, numdims), pdims, ty));
+            const af_seq idxs[4] = {af_span, af_span, af_span,
+                                    af_make_seq(1, dims[3], 1)};
+            ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in));
+            ASSERT_SUCCESS(af_index(out, parent, numdims, idxs));
+            ASSERT_SUCCESS(af_release_array(parent));
+            parent = nullptr;
+        }; break;
+        case REORDERED_FORMAT: {
+            const unsigned idxs[4] = {0, 3, 1, 2};
+            // idxs[0] has to be 0, to keep the same data in mem
+            dim_t rev_idxs[4];
+            for (dim_t i = 0; i < 4; ++i) { rev_idxs[idxs[i]] = i; };
+            af_array rev = nullptr;
+            ASSERT_SUCCESS(
+                af_reorder(&rev, in, idxs[0], idxs[1], idxs[2], idxs[3]));
+            ASSERT_SUCCESS(af_copy_array(out, rev));
+            ASSERT_SUCCESS(af_reorder(out, rev, rev_idxs[0], rev_idxs[1],
+                                      rev_idxs[2], rev_idxs[3]));
+            // ret has same content as in, although data is stored in
+            // different order
+            ASSERT_SUCCESS(af_release_array(rev));
+            rev = nullptr;
+        }; break;
+        case LINEAR_FORMAT:
+        default: af_copy_array(out, in);
+    };
+}
+
 int main(int argc, char **argv) {
     ::testing::InitGoogleTest(&argc, argv);
     return RUN_ALL_TESTS();
diff --git a/test/join.cpp b/test/join.cpp
index aef578bcf2..5cd470780f 100644
--- a/test/join.cpp
+++ b/test/join.cpp
@@ -280,9 +280,9 @@ TEST(Join, respect_parameters_order_ISSUE3511) {
     const af::array jit2{buf2 + 2.0};
     const std::array<af::array, 8> cases{jit1,  -jit1,       jit1 + 1.0, jit2,
                                          -jit2, jit1 + jit2, buf1,       buf2};
-    const std::array<char*, 8> cases_name{"JIT1", "-JIT1", "JIT1+1.0",
-                                          "JIT2", "-JIT2", "JIT1+JIT2",
-                                          "BUF1", "BUF2"};
+    const std::array<const char*, 8> cases_name{"JIT1", "-JIT1", "JIT1+1.0",
+                                                "JIT2", "-JIT2", "JIT1+JIT2",
+                                                "BUF1", "BUF2"};
     assert(cases.size() == cases_name.size());
     for (size_t cl0{0}; cl0 < cases.size(); ++cl0) {
         for (size_t cl1{0}; cl1 < cases.size(); ++cl1) {
@@ -312,3 +312,22 @@ TEST(Join, respect_parameters_order_ISSUE3511) {
         }
     }
 }
+
+#define TEST_TEMP_FORMAT(form, d)                                           \
+    TEST(TEMP_FORMAT, form##_dim##d) {                                      \
+        const dim4 dims(2, 2, 2, 2);                                        \
+        const array a(randu(dims));                                         \
+        const array b(randu(dims));                                         \
+                                                                            \
+        array out  = join(d, toTempFormat(form, a), toTempFormat(form, b)); \
+        array gold = join(d, a, b);                                         \
+        EXPECT_ARRAYS_EQ(gold, out);                                        \
+    }
+
+#define TEST_TEMP_FORMATS(form) \
+    TEST_TEMP_FORMAT(form, 0)   \
+    TEST_TEMP_FORMAT(form, 1)   \
+    TEST_TEMP_FORMAT(form, 2)   \
+    TEST_TEMP_FORMAT(form, 3)
+
+FOREACH_TEMP_FORMAT(TEST_TEMP_FORMATS)
diff --git a/test/testHelpers.hpp b/test/testHelpers.hpp
index 5f6b02b5a4..405f23309d 100644
--- a/test/testHelpers.hpp
+++ b/test/testHelpers.hpp
@@ -244,10 +244,10 @@ bool noHalfTests(af::dtype ty);
     GTEST_SKIP() << "Device doesn't support Half"
 
 #ifdef SKIP_UNSUPPORTED_TESTS
-#define UNSUPPORTED_BACKEND(backend)                        \
-    if(backend == af::getActiveBackend())                   \
-        GTEST_SKIP() << "Skipping unsupported function on " \
-                        + getBackendName() + " backend"
+#define UNSUPPORTED_BACKEND(backend)                                         \
+    if (backend == af::getActiveBackend())                                   \
+    GTEST_SKIP() << "Skipping unsupported function on " + getBackendName() + \
+                        " backend"
 #else
 #define UNSUPPORTED_BACKEND(backend)
 #endif
@@ -653,6 +653,30 @@ ::testing::AssertionResult assertArrayEq(std::string aName, std::string bName,
                                          const af_array a, const af_array b,
                                          TestOutputArrayInfo *metadata);
 
+enum tempFormat {
+    LINEAR_FORMAT,    // Linear array (= default)
+    JIT_FORMAT,       // Array which has JIT operations outstanding
+    SUB_FORMAT_dim0,  // Array where only a subset is allocated for dim0
+    SUB_FORMAT_dim1,  // Array where only a subset is allocated for dim1
+    SUB_FORMAT_dim2,  // Array where only a subset is allocated for dim2
+    SUB_FORMAT_dim3,  // Array where only a subset is allocated for dim3
+    REORDERED_FORMAT  // Array where the dimensions are reordered
+};
+// Calls the function fn for all available formats
+#define FOREACH_TEMP_FORMAT(TESTS) \
+    TESTS(LINEAR_FORMAT)           \
+    TESTS(JIT_FORMAT)              \
+    TESTS(SUB_FORMAT_dim0)         \
+    TESTS(SUB_FORMAT_dim1)         \
+    TESTS(SUB_FORMAT_dim2)         \
+    TESTS(SUB_FORMAT_dim3)         \
+    TESTS(REORDERED_FORMAT)
+
+// formats the "in" array according to provided format.  The content remains
+// unchanged.
+af::array toTempFormat(tempFormat form, const af::array &in);
+void toTempFormat(tempFormat form, af_array *out, const af_array &in);
+
 #ifdef __GNUC__
 #pragma GCC diagnostic pop
 #endif

From 6034d5fc0e2212914caae2a2c692386f8571cf2f Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Wed, 25 Jun 2025 21:56:30 +0200
Subject: [PATCH 459/473] Fixes sub-array support for scan (oneapi) (#3663)

* Adds test helpers for temporary array formats (JIT, SUB, ...)

* Fixes sub-array support for scan (oneapi)
---
 src/backend/oneapi/kernel/scan_dim.hpp   |  2 +-
 src/backend/oneapi/kernel/scan_first.hpp |  2 +-
 test/scan.cpp                            | 19 +++++++++++++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/backend/oneapi/kernel/scan_dim.hpp b/src/backend/oneapi/kernel/scan_dim.hpp
index eea34ffff7..52450f5c98 100644
--- a/src/backend/oneapi/kernel/scan_dim.hpp
+++ b/src/backend/oneapi/kernel/scan_dim.hpp
@@ -82,7 +82,7 @@ class scanDimKernel {
         optr += ids[3] * oInfo_.strides[3] + ids[2] * oInfo_.strides[2] +
                 ids[1] * oInfo_.strides[1] + ids[0];
         iptr += ids[3] * iInfo_.strides[3] + ids[2] * iInfo_.strides[2] +
-                ids[1] * iInfo_.strides[1] + ids[0];
+                ids[1] * iInfo_.strides[1] + ids[0] + iInfo_.offset;
         int id_dim        = ids[dim];
         const int out_dim = oInfo_.dims[dim];
 
diff --git a/src/backend/oneapi/kernel/scan_first.hpp b/src/backend/oneapi/kernel/scan_first.hpp
index dd483f069b..4aa7fc502e 100644
--- a/src/backend/oneapi/kernel/scan_first.hpp
+++ b/src/backend/oneapi/kernel/scan_first.hpp
@@ -71,7 +71,7 @@ class scanFirstKernel {
         To *tptr       = tmp_acc_.get_pointer();
 
         iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] +
-                yid * iInfo_.strides[1];
+                yid * iInfo_.strides[1] + iInfo_.offset;
         optr += wid * oInfo_.strides[3] + zid * oInfo_.strides[2] +
                 yid * oInfo_.strides[1];
         tptr += wid * tInfo_.strides[3] + zid * tInfo_.strides[2] +
diff --git a/test/scan.cpp b/test/scan.cpp
index 8bfbe0dd20..afb488278d 100644
--- a/test/scan.cpp
+++ b/test/scan.cpp
@@ -346,3 +346,22 @@ TEST(Scan, ExclusiveSum2D_Dim3) {
 
     ASSERT_ARRAYS_EQ(gold, out);
 }
+
+#define TEST_TEMP_FORMAT(form, dim)                                      \
+    TEST(TEMP_FORMAT, form##_Dim##dim) {                                 \
+        const dim4 dims(2, 2, 2, 2);                                     \
+        const array in(af::moddims(range(dim4(dims.elements())), dims)); \
+        in.eval();                                                       \
+        const array gold = scan(in, dim);                                \
+                                                                         \
+        array out = scan(toTempFormat(form, in), dim);                   \
+        ASSERT_ARRAYS_EQ(gold, out);                                     \
+    }
+
+#define TEST_TEMP_FORMATS(form) \
+    TEST_TEMP_FORMAT(form, 0)   \
+    TEST_TEMP_FORMAT(form, 1)   \
+    TEST_TEMP_FORMAT(form, 2)   \
+    TEST_TEMP_FORMAT(form, 3)
+
+FOREACH_TEMP_FORMAT(TEST_TEMP_FORMATS)
\ No newline at end of file

From 0e49da28d3948ff763bab004bac9ad270ffb4fd6 Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Thu, 26 Jun 2025 00:34:30 +0200
Subject: [PATCH 460/473] Fixes sub-array (oneapi) support for where (#3666)

* Adds test helpers for temporary array formats (JIT, SUB, ...)

* Fixes sub-array (oneapi) support for where
---
 src/backend/oneapi/kernel/where.hpp |  2 +-
 test/where.cpp                      | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/backend/oneapi/kernel/where.hpp b/src/backend/oneapi/kernel/where.hpp
index b65e0d9333..69f2f7719a 100644
--- a/src/backend/oneapi/kernel/where.hpp
+++ b/src/backend/oneapi/kernel/where.hpp
@@ -73,7 +73,7 @@ class whereKernel {
         otptr += wid * otInfo_.strides[3] + zid * otInfo_.strides[2] +
                  yid * otInfo_.strides[1];
         iptr += wid * iInfo_.strides[3] + zid * iInfo_.strides[2] +
-                yid * iInfo_.strides[1];
+                yid * iInfo_.strides[1] + iInfo_.offset;
 
         size_t odims0 = otInfo_.dims[0];
         size_t odims1 = otInfo_.dims[1];
diff --git a/test/where.cpp b/test/where.cpp
index 265c0d4d7b..a6c8dcde46 100644
--- a/test/where.cpp
+++ b/test/where.cpp
@@ -136,3 +136,22 @@ TEST(Where, ISSUE_1259) {
     array indices = where(a > 2);
     ASSERT_EQ(indices.elements(), 0);
 }
+
+#define TEST_TEMP_FORMAT(form, dim)                                      \
+    TEST(TEMP_FORMAT, form##_Dim##dim) {                                 \
+        const dim4 dims(2, 3, 4, 5);                                     \
+        const array in(af::moddims(range(dim4(dims.elements())), dims)); \
+        in.eval();                                                       \
+        const array gold = where(in > 3.0);                              \
+                                                                         \
+        array out = where(toTempFormat(form, in) > 3.0);                 \
+        ASSERT_ARRAYS_EQ(gold, out);                                     \
+    }
+
+#define TEST_TEMP_FORMATS(form) \
+    TEST_TEMP_FORMAT(form, 0)   \
+    TEST_TEMP_FORMAT(form, 1)   \
+    TEST_TEMP_FORMAT(form, 2)   \
+    TEST_TEMP_FORMAT(form, 3)
+
+FOREACH_TEMP_FORMAT(TEST_TEMP_FORMATS)
\ No newline at end of file

From 7185202b800afae802026731e9a68112224bdbb0 Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Thu, 26 Jun 2025 01:33:17 +0200
Subject: [PATCH 461/473] Fixes sub-array support for scan by key (opencl)
 (#3664)

* Adds test helpers for temporary array formats (JIT, SUB, ...)

* Fixes sub-array support for scanByKey (opencl)
---
 src/backend/opencl/kernel/scan_dim_by_key.cl  | 12 ++++------
 .../opencl/kernel/scan_first_by_key.cl        | 14 +++++------
 test/scan_by_key.cpp                          | 23 +++++++++++++++++++
 3 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/src/backend/opencl/kernel/scan_dim_by_key.cl b/src/backend/opencl/kernel/scan_dim_by_key.cl
index 5446b28e29..eacd7f9283 100644
--- a/src/backend/opencl/kernel/scan_dim_by_key.cl
+++ b/src/backend/opencl/kernel/scan_dim_by_key.cl
@@ -34,7 +34,7 @@ kernel void scanDimByKeyNonfinal(
     // Hence increment ids[kDim] just after offseting out and before offsetting
     // in
     tData += ids[3] * tInfo.strides[3] + ids[2] * tInfo.strides[2] +
-             ids[1] * tInfo.strides[1] + ids[0];
+             ids[1] * tInfo.strides[1] + ids[0] ;
     tfData += ids[3] * tfInfo.strides[3] + ids[2] * tfInfo.strides[2] +
               ids[1] * tfInfo.strides[1] + ids[0];
     tiData += ids[3] * tiInfo.strides[3] + ids[2] * tiInfo.strides[2] +
@@ -45,10 +45,9 @@ kernel void scanDimByKeyNonfinal(
     oData += ids[3] * oInfo.strides[3] + ids[2] * oInfo.strides[2] +
              ids[1] * oInfo.strides[1] + ids[0];
     iData += ids[3] * iInfo.strides[3] + ids[2] * iInfo.strides[2] +
-             ids[1] * iInfo.strides[1] + ids[0];
+             ids[1] * iInfo.strides[1] + ids[0] + iInfo.offset;
     kData += ids[3] * kInfo.strides[3] + ids[2] * kInfo.strides[2] +
-             ids[1] * kInfo.strides[1] + ids[0];
-    iData += iInfo.offset;
+             ids[1] * kInfo.strides[1] + ids[0] + kInfo.offset;
 
     int id_dim        = ids[kDim];
     const int out_dim = oInfo.dims[kDim];
@@ -192,10 +191,9 @@ kernel void scanDimByKeyFinal(global To *oData, KParam oInfo,
     oData += ids[3] * oInfo.strides[3] + ids[2] * oInfo.strides[2] +
              ids[1] * oInfo.strides[1] + ids[0];
     iData += ids[3] * iInfo.strides[3] + ids[2] * iInfo.strides[2] +
-             ids[1] * iInfo.strides[1] + ids[0];
+             ids[1] * iInfo.strides[1] + ids[0] + iInfo.offset;
     kData += ids[3] * kInfo.strides[3] + ids[2] * kInfo.strides[2] +
-             ids[1] * kInfo.strides[1] + ids[0];
-    iData += iInfo.offset;
+             ids[1] * kInfo.strides[1] + ids[0] + kInfo.offset;
 
     int id_dim        = ids[kDim];
     const int out_dim = oInfo.dims[kDim];
diff --git a/src/backend/opencl/kernel/scan_first_by_key.cl b/src/backend/opencl/kernel/scan_first_by_key.cl
index 54d572d965..1793f0b293 100644
--- a/src/backend/opencl/kernel/scan_first_by_key.cl
+++ b/src/backend/opencl/kernel/scan_first_by_key.cl
@@ -39,13 +39,13 @@ kernel void scanFirstByKeyNonfinal(global To *oData, KParam oInfo,
              yid * kInfo.strides[1] + kInfo.offset;
 
     tData += wid * tInfo.strides[3] + zid * tInfo.strides[2] +
-             yid * tInfo.strides[1] + tInfo.offset;
+             yid * tInfo.strides[1];
 
     tfData += wid * tfInfo.strides[3] + zid * tfInfo.strides[2] +
-              yid * tfInfo.strides[1] + tfInfo.offset;
+              yid * tfInfo.strides[1];
 
     tiData += wid * tiInfo.strides[3] + zid * tiInfo.strides[2] +
-              yid * tiInfo.strides[1] + tiInfo.offset;
+              yid * tiInfo.strides[1];
 
     oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
              yid * oInfo.strides[1] + oInfo.offset;
@@ -179,7 +179,7 @@ kernel void scanFirstByKeyFinal(global To *oData, KParam oInfo,
              yid * kInfo.strides[1] + kInfo.offset;
 
     oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
-             yid * oInfo.strides[1] + oInfo.offset;
+             yid * oInfo.strides[1];
 
     local To l_val0[SHARED_MEM_SIZE];
     local To l_val1[SHARED_MEM_SIZE];
@@ -283,13 +283,13 @@ kernel void bcastFirstByKey(global To *oData, KParam oInfo,
 
         if (cond) {
             tiData += wid * tiInfo.strides[3] + zid * tiInfo.strides[2] +
-                      yid * tiInfo.strides[1] + tiInfo.offset;
+                      yid * tiInfo.strides[1];
 
             tData += wid * tInfo.strides[3] + zid * tInfo.strides[2] +
-                     yid * tInfo.strides[1] + tInfo.offset;
+                     yid * tInfo.strides[1];
 
             oData += wid * oInfo.strides[3] + zid * oInfo.strides[2] +
-                     yid * oInfo.strides[1] + oInfo.offset;
+                     yid * oInfo.strides[1];
 
             int boundary = tiData[groupId_x];
             To accum     = tData[groupId_x - 1];
diff --git a/test/scan_by_key.cpp b/test/scan_by_key.cpp
index 0ea1dd8ecb..08928b5fdc 100644
--- a/test/scan_by_key.cpp
+++ b/test/scan_by_key.cpp
@@ -240,3 +240,26 @@ TEST(ScanByKey, FixOverflowWrite) {
 
     ASSERT_EQ(prior, valsAF(0).scalar<float>());
 }
+
+#define TEST_TEMP_FORMAT(form, dim)                                           \
+    TEST(TEMP_FORMAT, form##_Dim##dim) {                                      \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                               \
+        const dim4 dims(2, 2, 2, 2);                                          \
+        const array in(af::moddims(range(dim4(dims.elements())), dims));      \
+        in.eval();                                                            \
+        const array keys(af::constant(0, dims, u32));                         \
+        keys.eval();                                                          \
+        const array gold = scanByKey(keys, in, dim);                          \
+                                                                              \
+        array out =                                                           \
+            scanByKey(toTempFormat(form, keys), toTempFormat(form, in), dim); \
+        ASSERT_ARRAYS_EQ(gold, out);                                          \
+    }
+
+#define TEST_TEMP_FORMATS(form) \
+    TEST_TEMP_FORMAT(form, 0)   \
+    TEST_TEMP_FORMAT(form, 1)   \
+    TEST_TEMP_FORMAT(form, 2)   \
+    TEST_TEMP_FORMAT(form, 3)
+
+FOREACH_TEMP_FORMAT(TEST_TEMP_FORMATS)
\ No newline at end of file

From eaa49caced87c6eb21d612b1d546ab3061d30a73 Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Fri, 27 Jun 2025 19:50:36 +0200
Subject: [PATCH 462/473] Fixes sub-array (opencl, oneapi) support for reduce
 by key (#3665)

* Adds test helpers for temporary array formats (JIT, SUB, ...)

* Fixes sub-array (opencl, oneapi) support  for reduce by key

* Update reduce.cpp

Revert line breaks. While these lines are quite long, introducing more breaks here makes it more difficult to read.

---------

Co-authored-by: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
---
 src/backend/oneapi/kernel/reduce_by_key.hpp   |  31 ++---
 src/backend/oneapi/kernel/reduce_dim.hpp      |   3 +-
 .../opencl/kernel/reduce_blocks_by_key_dim.cl |   4 +-
 .../kernel/reduce_blocks_by_key_first.cl      |   4 +-
 .../opencl/kernel/reduce_by_key_compact.cl    |   4 +-
 .../kernel/reduce_by_key_compact_dim.cl       |   4 +-
 .../kernel/reduce_by_key_needs_reduction.cl   |   4 +-
 test/reduce.cpp                               | 113 ++++++++++++++++--
 8 files changed, 128 insertions(+), 39 deletions(-)

diff --git a/src/backend/oneapi/kernel/reduce_by_key.hpp b/src/backend/oneapi/kernel/reduce_by_key.hpp
index 3b5058a6bf..329fd33109 100644
--- a/src/backend/oneapi/kernel/reduce_by_key.hpp
+++ b/src/backend/oneapi/kernel/reduce_by_key.hpp
@@ -51,8 +51,8 @@ class finalBoundaryReduceKernel {
         common::Binary<compute_t<To>, op> binOp;
         if (gid == ((bid + 1) * it.get_local_range(0)) - 1 &&
             bid < g.get_group_range(0) - 1) {
-            Tk k0 = iKeys_[gid];
-            Tk k1 = iKeys_[gid + 1];
+            Tk k0 = iKeys_[gid + iKInfo_.offset];
+            Tk k1 = iKeys_[gid + 1 + iKInfo_.offset];
 
             if (k0 == k1) {
                 compute_t<To> v0          = compute_t<To>(oVals_[gid]);
@@ -104,8 +104,8 @@ class finalBoundaryReduceDimKernel {
         common::Binary<compute_t<To>, op> binOp;
         if (gid == ((bid + 1) * it.get_local_range(0)) - 1 &&
             bid < g.get_group_range(0) - 1) {
-            Tk k0 = iKeys_[gid];
-            Tk k1 = iKeys_[gid + 1];
+            Tk k0 = iKeys_[gid + iKInfo_.offset];
+            Tk k1 = iKeys_[gid + 1 + iKInfo_.offset];
 
             if (k0 == k1) {
                 compute_t<To> v0          = compute_t<To>(oVals_[gid]);
@@ -163,7 +163,7 @@ class testNeedsReductionKernel {
         const uint bid = g.get_group_id(0);
 
         Tk k = scalar<Tk>(0);
-        if (gid < n_) { k = iKeys_[gid]; }
+        if (gid < n_) { k = iKeys_[gid + iKInfo_.offset]; }
 
         l_keys_[lid] = k;
         it.barrier();
@@ -181,8 +181,8 @@ class testNeedsReductionKernel {
         // reduction
         if (gid == ((bid + 1) * DIMX_) - 1 &&
             bid < (g.get_group_range(0) - 1)) {
-            int k0 = iKeys_[gid];
-            int k1 = iKeys_[gid + 1];
+            int k0 = iKeys_[gid + iKInfo_.offset];
+            int k1 = iKeys_[gid + 1 + iKInfo_.offset];
             if (k0 == k1) {
                 global_atomic_ref<int>(needs_block_boundary_reduced_[0]) |= 1;
             }
@@ -240,8 +240,8 @@ class compactKernel {
                 : (reduced_block_sizes_[bid] - reduced_block_sizes_[bid - 1]);
         int writeloc = (bid == 0) ? 0 : reduced_block_sizes_[bid - 1];
 
-        Tk k = iKeys_[gid];
-        To v = iVals_[bOffset + gid];
+        Tk k = iKeys_[gid + iKInfo_.offset];
+        To v = iVals_[bOffset + gid + iVInfo_.offset];
 
         if (lid < nwrite) {
             oKeys_[writeloc + lid]           = k;
@@ -316,8 +316,8 @@ class compactDimKernel {
                         bidz * iVInfo_.strides[dims_ordering[2]] +
                         bidy * iVInfo_.strides[dims_ordering[1]] +
                         gidx * iVInfo_.strides[DIM_];
-        k = iKeys_[gidx];
-        v = iVals_[tid];
+        k = iKeys_[gidx + iKInfo_.offset];
+        v = iVals_[tid + iVInfo_.offset];
 
         if (lid < nwrite) {
             oKeys_[writeloc + lid] = k;
@@ -403,11 +403,11 @@ class reduceBlocksByKeyKernel {
         Tk k            = scalar<Tk>(0);
         compute_t<To> v = init_val;
         if (gid < n_) {
-            k                 = iKeys_[gid];
+            k                 = iKeys_[gid + iKInfo_.offset];
             const int bOffset = bidw * iVInfo_.strides[3] +
                                 bidz * iVInfo_.strides[2] +
                                 bidy * iVInfo_.strides[1];
-            v = transform(iVals_[bOffset + gid]);
+            v = transform(iVals_[bOffset + gid + iVInfo_.offset]);
             if (change_nan_) v = IS_NAN(v) ? nanval_ : v;
         }
 
@@ -579,11 +579,12 @@ class reduceBlocksByKeyDimKernel {
         Tk k            = scalar<Tk>(0);
         compute_t<To> v = init_val;
         if (gid < n_) {
-            k                 = iKeys_[gid];
+            k                 = iKeys_[gid + iKInfo_.offset];
             const int bOffset = bidw * iVInfo_.strides[dims_ordering[3]] +
                                 bidz * iVInfo_.strides[dims_ordering[2]] +
                                 bidy * iVInfo_.strides[dims_ordering[1]];
-            v = transform(iVals_[bOffset + gid * iVInfo_.strides[DIM_]]);
+            v = transform(
+                iVals_[bOffset + gid * iVInfo_.strides[DIM_] + iVInfo_.offset]);
             if (change_nan_) v = IS_NAN(v) ? nanval_ : v;
         }
 
diff --git a/src/backend/oneapi/kernel/reduce_dim.hpp b/src/backend/oneapi/kernel/reduce_dim.hpp
index b1d3d81648..0cc7055f14 100644
--- a/src/backend/oneapi/kernel/reduce_dim.hpp
+++ b/src/backend/oneapi/kernel/reduce_dim.hpp
@@ -74,7 +74,8 @@ class reduceDimKernelSMEM {
 
         const data_t<Ti> *iptr =
             in_.get_pointer() + ids[3] * iInfo_.strides[3] +
-            ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + ids[0];
+            ids[2] * iInfo_.strides[2] + ids[1] * iInfo_.strides[1] + ids[0] +
+            iInfo_.offset;
 
         const uint id_dim_in   = ids[dim];
         const uint istride_dim = iInfo_.strides[dim];
diff --git a/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl b/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl
index 66bbb3e6d2..76941ebbd7 100644
--- a/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl
+++ b/src/backend/opencl/kernel/reduce_blocks_by_key_dim.cl
@@ -82,12 +82,12 @@ kernel void reduce_blocks_by_key_dim(global int *reduced_block_sizes,
     Tk k;
     To v;
     if (gidx < n) {
-        k             = iKeys[gidx];
+        k             = iKeys[gidx + iKInfo.offset];
         const int gid = bidw * iVInfo.strides[dims_ordering[3]] +
                         bidz * iVInfo.strides[dims_ordering[2]] +
                         bidy * iVInfo.strides[dims_ordering[1]] +
                         gidx * iVInfo.strides[DIM];
-        v = transform(iVals[gid]);
+        v = transform(iVals[gid + iVInfo.offset]);
         if (change_nan) v = IS_NAN(v) ? nanval : v;
     } else {
         v = init_val;
diff --git a/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl b/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
index f184e94818..c01d3c250d 100644
--- a/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
+++ b/src/backend/opencl/kernel/reduce_blocks_by_key_first.cl
@@ -72,10 +72,10 @@ kernel void reduce_blocks_by_key_first(global int *reduced_block_sizes,
     Tk k;
     To v;
     if (gid < n) {
-        k                 = iKeys[gid];
+        k                 = iKeys[gid + iKInfo.offset];
         const int bOffset = bidw * iVInfo.strides[3] +
                             bidz * iVInfo.strides[2] + bidy * iVInfo.strides[1];
-        v = transform(iVals[bOffset + gid]);
+        v = transform(iVals[bOffset + gid + iVInfo.offset]);
         if (change_nan) v = IS_NAN(v) ? nanval : v;
     } else {
         v = init_val;
diff --git a/src/backend/opencl/kernel/reduce_by_key_compact.cl b/src/backend/opencl/kernel/reduce_by_key_compact.cl
index c8081e45e9..58b78cd894 100644
--- a/src/backend/opencl/kernel/reduce_by_key_compact.cl
+++ b/src/backend/opencl/kernel/reduce_by_key_compact.cl
@@ -31,8 +31,8 @@ kernel void compact(global int *reduced_block_sizes, global Tk *oKeys,
                    : (reduced_block_sizes[bid] - reduced_block_sizes[bid - 1]);
     int writeloc = (bid == 0) ? 0 : reduced_block_sizes[bid - 1];
 
-    k = iKeys[gid];
-    v = iVals[bOffset + gid];
+    k = iKeys[gid + iKInfo.offset];
+    v = iVals[bOffset + gid + iVInfo.offset];
 
     if (lid < nwrite) {
         oKeys[writeloc + lid]           = k;
diff --git a/src/backend/opencl/kernel/reduce_by_key_compact_dim.cl b/src/backend/opencl/kernel/reduce_by_key_compact_dim.cl
index 285d4cc20c..3d07a63eb7 100644
--- a/src/backend/opencl/kernel/reduce_by_key_compact_dim.cl
+++ b/src/backend/opencl/kernel/reduce_by_key_compact_dim.cl
@@ -43,8 +43,8 @@ kernel void compact_dim(global int *reduced_block_sizes, global Tk *oKeys,
                     bidz * iVInfo.strides[dim_ordering[2]] +
                     bidy * iVInfo.strides[dim_ordering[1]] +
                     gidx * iVInfo.strides[DIM];
-    k = iKeys[gidx];
-    v = iVals[tid];
+    k = iKeys[gidx + iKInfo.offset];
+    v = iVals[tid + iVInfo.offset];
 
     if (lid < nwrite) {
         oKeys[writeloc + lid] = k;
diff --git a/src/backend/opencl/kernel/reduce_by_key_needs_reduction.cl b/src/backend/opencl/kernel/reduce_by_key_needs_reduction.cl
index 4b12830aaf..c505689bff 100644
--- a/src/backend/opencl/kernel/reduce_by_key_needs_reduction.cl
+++ b/src/backend/opencl/kernel/reduce_by_key_needs_reduction.cl
@@ -32,8 +32,8 @@ kernel void test_needs_reduction(global int *needs_another_reduction,
     // last thread in each block checks if any inter-block keys need further
     // reduction
     if (gid == ((bid + 1) * DIMX) - 1 && bid < get_num_groups(0) - 1) {
-        int k0 = iKeys[gid];
-        int k1 = iKeys[gid + 1];
+        int k0 = iKeys[gid + iKInfo.offset];
+        int k1 = iKeys[gid + 1 + iKInfo.offset];
         if (k0 == k1) { atomic_or(needs_block_boundary_reduced, 1); }
     }
 }
diff --git a/test/reduce.cpp b/test/reduce.cpp
index 0d4ab59225..c50f95d924 100644
--- a/test/reduce.cpp
+++ b/test/reduce.cpp
@@ -454,7 +454,7 @@ template<typename T>
 struct generateConsq {
     T vals;
 
-    generateConsq(T v_i = 0) : vals(v_i){};
+    generateConsq(T v_i = 0) : vals(v_i) {};
 
     T operator()() { return vals++; }
 };
@@ -463,7 +463,7 @@ template<typename T>
 struct generateConst {
     T vals;
 
-    generateConst(T v_i) : vals(v_i){};
+    generateConst(T v_i) : vals(v_i) {};
 
     T operator()() { return vals; }
 };
@@ -626,12 +626,12 @@ TYPED_TEST(ReduceByKey, MultiBlockReduceSingleval) {
     SUPPORTED_TYPE_CHECK(TypeParam);
     array keys = constant(0, 1024 * 1024, s32);
     array vals = constant(1, 1024 * 1024,
-            (af_dtype)af::dtype_traits<TypeParam>::af_type);
+                          (af_dtype)af::dtype_traits<TypeParam>::af_type);
 
     array keyResGold      = constant(0, 1);
-    using promoted_t = typename promote_type<TypeParam, af_sum>::type;
-    array valsReducedGold = constant(1024 * 1024, 1,
-            (af_dtype)af::dtype_traits<promoted_t>::af_type);
+    using promoted_t      = typename promote_type<TypeParam, af_sum>::type;
+    array valsReducedGold = constant(
+        1024 * 1024, 1, (af_dtype)af::dtype_traits<promoted_t>::af_type);
 
     array keyRes, valsReduced;
     sumByKey(keyRes, valsReduced, keys, vals);
@@ -842,9 +842,9 @@ TYPED_TEST(ReduceByKey, ReduceByKeyNans) {
 
     SKIP_IF_FAST_MATH_ENABLED();
     SUPPORTED_TYPE_CHECK(TypeParam);
-    const static int testSz          = 8;
-    const int testKeys[testSz]       = {0, 2, 2, 9, 5, 5, 5, 8};
-    const TypeParam nan = std::numeric_limits<TypeParam>::quiet_NaN();
+    const static int testSz    = 8;
+    const int testKeys[testSz] = {0, 2, 2, 9, 5, 5, 5, 8};
+    const TypeParam nan        = std::numeric_limits<TypeParam>::quiet_NaN();
     const TypeParam testVals[testSz] = {0, 7, nan, 6, 2, 5, 3, 4};
 
     array keys(testSz, testKeys);
@@ -906,7 +906,7 @@ TYPED_TEST(ReduceByKey, nDim1ReduceByKey) {
     const double nanval = 0.0;
     sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval);
 
-    const int goldSz                = 5;
+    const int goldSz = 5;
     using promoted_t = typename promote_type<TypeParam, af_sum>::type;
     const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4};
     vector<promoted_t> hreduce(reduced_vals.elements());
@@ -935,7 +935,7 @@ TYPED_TEST(ReduceByKey, nDim2ReduceByKey) {
     const double nanval = 0.0;
     sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval);
 
-    const int goldSz                = 5;
+    const int goldSz = 5;
     using promoted_t = typename promote_type<TypeParam, af_sum>::type;
     const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4};
     vector<promoted_t> h_a(reduced_vals.elements());
@@ -964,7 +964,7 @@ TYPED_TEST(ReduceByKey, nDim3ReduceByKey) {
     const double nanval = 0.0;
     sumByKey(reduced_keys, reduced_vals, keys, vals, dim, nanval);
 
-    const int goldSz                = 5;
+    const int goldSz = 5;
     using promoted_t = typename promote_type<TypeParam, af_sum>::type;
     const promoted_t gold_reduce[goldSz] = {0, 8, 6, 10, 4};
     vector<promoted_t> h_a(reduced_vals.elements());
@@ -2420,7 +2420,7 @@ TEST(Reduce, SNIPPET_algorithm_func_sum) {
                                           //      1, 3, 5]
 
     // Create b by summing across the first dimension
-    array b = sum(a);        // sum across the first dimension, same as sum(a, 0)
+    array b = sum(a);        // sum across the first dimension, same as sum(a,0)
 
     // Create c by summing across the second dimension
     array c = sum(a, 1);     // sum across the second dimension
@@ -2448,3 +2448,90 @@ TEST(Reduce, SNIPPET_algorithm_func_sum) {
     ASSERT_VEC_ARRAY_EQ(gold_a, d.dims(), d);
     ASSERT_VEC_ARRAY_EQ(gold_a, e.dims(), e);
 }
+
+#define TEMP_FORMAT_TESTS_reduce(form, op)                    \
+    TEST(TEMP_FORMAT, form##_##op##_array) {                  \
+        const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f});    \
+        const array gold = op(in, 3);                         \
+        array out        = op(toTempFormat(form, in), 3);     \
+        EXPECT_ARRAYS_EQ(out, gold);                          \
+    }                                                         \
+    TEST(TEMP_FORMAT, form##_##op##_value) {                  \
+        const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f});    \
+        const float gold = op<float>(in);                     \
+        float out        = op<float>(toTempFormat(form, in)); \
+        EXPECT_EQ(out, gold);                                 \
+    }
+
+#define TEMP_FORMAT_TESTS_ragged(form, op)                                     \
+    TEST(TEMP_FORMAT, form##_##op##_ragged) {                                  \
+        const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f});                     \
+        const array ragged_len(dim4(1), {(unsigned)in.elements()});            \
+        array gold_vals, gold_idxs;                                            \
+        op(gold_vals, gold_idxs, in, ragged_len, 3);                           \
+        array vals, idxs;                                                      \
+        op(vals, idxs, toTempFormat(form, in), toTempFormat(form, ragged_len), \
+           3);                                                                 \
+        EXPECT_ARRAYS_EQ(vals, gold_vals);                                     \
+        EXPECT_ARRAYS_EQ(idxs, gold_idxs);                                     \
+    }
+
+#define TEMP_FORMAT_TESTS_ByKey(form, op)                      \
+    TEST(TEMP_FORMAT, form##_##op) {                           \
+        const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f});     \
+        const array keys(constant(0, in.dims().dims[3], u32)); \
+        keys.eval();                                           \
+        array gold_keys, gold_vals;                            \
+        op(gold_keys, gold_vals, keys, in, 3);                 \
+        array out_keys, out_vals;                              \
+        op(out_keys, out_vals, toTempFormat(form, keys),       \
+           toTempFormat(form, in), 3);                         \
+        EXPECT_ARRAYS_EQ(gold_vals, out_vals);                 \
+        EXPECT_ARRAYS_EQ(gold_keys, out_keys);                 \
+    }
+
+#define TEMP_FORMAT_TESTS_allTest(form, op)                         \
+    TEST(TEMP_FORMAT, form##_##op##_array) {                        \
+        const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f});          \
+        const array gold = op(in > 2.0, 3);                         \
+        array out        = op(toTempFormat(form, in) > 2.0, 3);     \
+        EXPECT_ARRAYS_EQ(gold, out);                                \
+    }                                                               \
+    TEST(TEMP_FORMAT, form##_##op##_value) {                        \
+        const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f});          \
+        const float gold = op<float>(in > 2.0);                     \
+        float out        = op<float>(toTempFormat(form, in) > 2.0); \
+        EXPECT_EQ(gold, out);                                       \
+    }
+
+#define TEMP_FORMAT_TESTS_allTestByKey(form, op)               \
+    TEST(TEMP_FORMAT, form##_##op) {                           \
+        const array in(dim4(1, 1, 1, 3), {1.f, 2.f, 3.f});     \
+        const array keys(constant(0, in.dims().dims[3], u32)); \
+        array gold_vals, gold_keys;                            \
+        op(gold_keys, gold_vals, keys, in > 2.0, 3);           \
+        array out_vals, out_keys;                              \
+        op(out_keys, out_vals, toTempFormat(form, keys),       \
+           toTempFormat(form, in) > 2.0, 3);                   \
+        EXPECT_ARRAYS_EQ(gold_vals, out_vals);                 \
+        EXPECT_ARRAYS_EQ(gold_keys, out_keys);                 \
+    }
+
+#define TEMP_FORMATS_TESTS(form)                        \
+    TEMP_FORMAT_TESTS_reduce(form, min);                \
+    TEMP_FORMAT_TESTS_reduce(form, max);                \
+    TEMP_FORMAT_TESTS_reduce(form, sum);                \
+    TEMP_FORMAT_TESTS_reduce(form, product);            \
+    TEMP_FORMAT_TESTS_reduce(form, count);              \
+    TEMP_FORMAT_TESTS_ragged(form, max);                \
+    TEMP_FORMAT_TESTS_ByKey(form, minByKey);            \
+    TEMP_FORMAT_TESTS_ByKey(form, maxByKey);            \
+    TEMP_FORMAT_TESTS_ByKey(form, sumByKey);            \
+    TEMP_FORMAT_TESTS_ByKey(form, productByKey);        \
+    TEMP_FORMAT_TESTS_ByKey(form, countByKey);          \
+    TEMP_FORMAT_TESTS_allTest(form, allTrue);           \
+    TEMP_FORMAT_TESTS_allTest(form, anyTrue);           \
+    TEMP_FORMAT_TESTS_allTestByKey(form, allTrueByKey); \
+    TEMP_FORMAT_TESTS_allTestByKey(form, anyTrueByKey);
+
+FOREACH_TEMP_FORMAT(TEMP_FORMATS_TESTS)

From f01e6fe9dcddd058c3bcc217b1e707d35500320a Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Mon, 30 Jun 2025 08:38:03 +0200
Subject: [PATCH 463/473] Fixes sub array (opencl) support for confidenceCC
 (#3668)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Adds test helpers for temporary array formats (JIT, SUB, ...)

* Fixes sub-array (opencl) support for confidenceCC

* Update flood_fill.cpp

Removed unnecessary #include

* Added TODO comment to change this lines after subarrays fix

---------

Co-authored-by: Edwin Lester Solís Fuentes <68087165+edwinsolisf@users.noreply.github.com>
---
 src/api/c/confidence_connected.cpp      | 11 ++++-
 src/backend/opencl/kernel/flood_fill.cl |  9 ++--
 test/confidence_connected.cpp           | 59 ++++++++++++++++++++++---
 3 files changed, 67 insertions(+), 12 deletions(-)

diff --git a/src/api/c/confidence_connected.cpp b/src/api/c/confidence_connected.cpp
index ceb8ca7b75..903c06f87b 100644
--- a/src/api/c/confidence_connected.cpp
+++ b/src/api/c/confidence_connected.cpp
@@ -45,8 +45,15 @@ using std::swap;
 template<typename T>
 Array<T> pointList(const Array<T>& in, const Array<uint>& x,
                    const Array<uint>& y) {
-    af_array xcoords                          = getHandle<uint>(x);
-    af_array ycoords                          = getHandle<uint>(y);
+
+    // TODO: Temporary Fix, must fix handling subarrays upstream
+    // Array<T> has to be a basic array, to be accepted as af_index
+    Array<uint> x_ = (x.getOffset() == 0 && x.isLinear()) ? x : copyArray(x);
+    Array<uint> y_ = (y.getOffset() == 0 && y.isLinear()) ? y : copyArray(y);
+
+    af_array xcoords = getHandle<uint>(x_);
+    af_array ycoords = getHandle<uint>(y_);
+
     std::array<af_index_t, AF_MAX_DIMS> idxrs = {{{{xcoords}, false, false},
                                                   {{ycoords}, false, false},
                                                   createSpanIndex(),
diff --git a/src/backend/opencl/kernel/flood_fill.cl b/src/backend/opencl/kernel/flood_fill.cl
index ba8f8e109a..58d03b52e8 100644
--- a/src/backend/opencl/kernel/flood_fill.cl
+++ b/src/backend/opencl/kernel/flood_fill.cl
@@ -23,8 +23,8 @@ kernel void init_seeds(global T *out, KParam oInfo, global const uint *seedsx,
                        KParam syInfo) {
     uint tid = get_global_id(0);
     if (tid < sxInfo.dims[0]) {
-        uint x                                             = seedsx[tid];
-        uint y                                             = seedsy[tid];
+        uint x                                             = seedsx[tid + sxInfo.offset];
+        uint y                                             = seedsy[tid + syInfo.offset];
         out[(x * oInfo.strides[0] + y * oInfo.strides[1])] = VALID;
     }
 }
@@ -76,14 +76,15 @@ kernel void flood_step(global T *out, KParam oInfo, global const T *img,
 
     T tImgVal =
         img[(clamp(gx, 0, (int)(iInfo.dims[0] - 1)) * iInfo.strides[0] +
-             clamp(gy, 0, (int)(iInfo.dims[1] - 1)) * iInfo.strides[1])];
+             clamp(gy, 0, (int)(iInfo.dims[1] - 1)) * iInfo.strides[1])+ 
+             iInfo.offset];
     const int isPxBtwnThresholds =
         (tImgVal >= lowValue && tImgVal <= highValue);
 
     int tid = lx + get_local_size(0) * ly;
 
     barrier(CLK_LOCAL_MEM_FENCE);
-
+    
     T origOutVal     = lmem[j][i];
     bool isBorderPxl = (lx == 0 || ly == 0 || lx == (get_local_size(0) - 1) ||
                         ly == (get_local_size(1) - 1));
diff --git a/test/confidence_connected.cpp b/test/confidence_connected.cpp
index 22254e5532..39c0f8f0ff 100644
--- a/test/confidence_connected.cpp
+++ b/test/confidence_connected.cpp
@@ -92,9 +92,9 @@ void testImage(const std::string pTestFile, const size_t numSeeds,
         params.iterations = iter;
         params.replace    = 255.0;
 
-        ASSERT_SUCCESS(af_confidence_cc(&outArray, inArray, seedxArr, seedyArr, params.radius,
-                                        params.multiplier, params.iterations,
-                                        params.replace));
+        ASSERT_SUCCESS(af_confidence_cc(&outArray, inArray, seedxArr, seedyArr,
+                                        params.radius, params.multiplier,
+                                        params.iterations, params.replace));
         int device = 0;
         ASSERT_SUCCESS(af_get_device(&device));
         ASSERT_SUCCESS(af_sync(device));
@@ -141,9 +141,9 @@ void testData(CCCTestParams params) {
                                    (af_dtype)af::dtype_traits<T>::af_type));
 
     af_array outArray = 0;
-    ASSERT_SUCCESS(af_confidence_cc(&outArray, inArray, seedxArr, seedyArr, params.radius,
-                                    params.multiplier, params.iterations,
-                                    params.replace));
+    ASSERT_SUCCESS(af_confidence_cc(&outArray, inArray, seedxArr, seedyArr,
+                                    params.radius, params.multiplier,
+                                    params.iterations, params.replace));
     int device = 0;
     ASSERT_SUCCESS(af_get_device(&device));
     ASSERT_SUCCESS(af_sync(device));
@@ -201,3 +201,50 @@ INSTANTIATE_TEST_SUITE_P(
            << info.param.iterations << "_replace_" << info.param.replace;
         return ss.str();
     });
+
+#define TEST_FORMATS(form)                                                     \
+    TEST(TEMP_FORMAT, form##_2Dseed) {                                         \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                                \
+        const string filename(string(TEST_DIR) + "/confidence_cc/donut.png");  \
+        const af::array image(af::loadImage(filename.c_str()));                \
+        const af::array seed(dim4(1, 2), {10u, 8u});                           \
+                                                                               \
+        const af::array out =                                                  \
+            af::confidenceCC(toTempFormat(form, image),                        \
+                             toTempFormat(form, seed), 3, 3, 25, 255.0);       \
+        const af::array gold = af::confidenceCC(image, seed, 3, 3, 25, 255.0); \
+                                                                               \
+        EXPECT_ARRAYS_EQ(out, gold);                                           \
+    }                                                                          \
+                                                                               \
+    TEST(TEMP_FORMAT, form##_2xSeed) {                                         \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                                \
+        const string filename(string(TEST_DIR) + "/confidence_cc/donut.png");  \
+        const af::array image(af::loadImage(filename.c_str()));                \
+        const af::array seedx({10u});                                          \
+        const af::array seedy({8u});                                           \
+                                                                               \
+        const af::array out = af::confidenceCC(                                \
+            toTempFormat(form, image), toTempFormat(form, seedx),              \
+            toTempFormat(form, seedy), 3, 3, 25, 255.0);                       \
+        const af::array gold =                                                 \
+            af::confidenceCC(image, seedx, seedy, 3, 3, 25, 255.0);            \
+                                                                               \
+        EXPECT_ARRAYS_EQ(out, gold);                                           \
+    }                                                                          \
+    TEST(TEMP_FORMAT, form##_vectSeed) {                                       \
+        UNSUPPORTED_BACKEND(AF_BACKEND_ONEAPI);                                \
+        const string filename(string(TEST_DIR) + "/confidence_cc/donut.png");  \
+        const af::array image(af::loadImage(filename.c_str()));                \
+        const unsigned seedx[1] = {10u};                                       \
+        const unsigned seedy[1] = {8u};                                        \
+                                                                               \
+        const af::array out = af::confidenceCC(toTempFormat(form, image), 1,   \
+                                               seedx, seedy, 3, 3, 25, 255.0); \
+        const af::array gold =                                                 \
+            af::confidenceCC(image, 1, seedx, seedy, 3, 3, 25, 255.0);         \
+                                                                               \
+        EXPECT_ARRAYS_EQ(out, gold);                                           \
+    }
+
+FOREACH_TEMP_FORMAT(TEST_FORMATS)

From 8da6800e048152e233be3ba180a2c87b871acf06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Sun, 29 Jun 2025 23:47:34 -0700
Subject: [PATCH 464/473] Fix half precision pow function for openCL backend
 (#3676)

---
 src/backend/opencl/binary.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/backend/opencl/binary.hpp b/src/backend/opencl/binary.hpp
index 39f340942a..546c5bc085 100644
--- a/src/backend/opencl/binary.hpp
+++ b/src/backend/opencl/binary.hpp
@@ -9,6 +9,9 @@
 
 #pragma once
 #include <optypes.hpp>
+#include <common/half.hpp>
+
+using arrayfire::common::half;
 
 namespace arrayfire {
 namespace opencl {
@@ -98,6 +101,7 @@ struct BinOp<To, Ti, af_pow_t> {
 
 POW_BINARY_OP(double, "pow")
 POW_BINARY_OP(float, "pow")
+POW_BINARY_OP(half, "pow")
 POW_BINARY_OP(intl, "__powll")
 POW_BINARY_OP(uintl, "__powul")
 POW_BINARY_OP(uint, "__powui")

From 95fc0994346d93a4296d91d7d64bced740d5493b Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Mon, 30 Jun 2025 19:51:47 +0100
Subject: [PATCH 465/473] Fix half precision pow function for oneAPI back end
 (#3672)

* Fix half precision pow function for oneAPI back end

An incorrect power function was being used for half precision variables when building the JIT kernel for the oneAPI back end which lead to a fallback to an integer power function causing the result to be rounded. This fixes that.

* Apply half precision pow function fix to the OpenCL back end as well.

* Revert "Apply half precision pow function fix to the OpenCL back end as well."

This reverts commit e3357218911e5b592684c4f13e8a62f39c95b441.
---
 src/backend/oneapi/binary.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/backend/oneapi/binary.hpp b/src/backend/oneapi/binary.hpp
index a9bc4900e8..8bd36aff7e 100644
--- a/src/backend/oneapi/binary.hpp
+++ b/src/backend/oneapi/binary.hpp
@@ -9,6 +9,9 @@
 
 #pragma once
 #include <optypes.hpp>
+#include <common/half.hpp>
+
+using arrayfire::common::half;
 
 namespace arrayfire {
 namespace oneapi {
@@ -93,6 +96,7 @@ struct BinOp<To, Ti, af_pow_t> {
 
 POW_BINARY_OP(double, "pow")
 POW_BINARY_OP(float, "pow")
+POW_BINARY_OP(half, "pow")
 POW_BINARY_OP(intl, "__powll")
 POW_BINARY_OP(uintl, "__powul")
 POW_BINARY_OP(uint, "__powui")

From e58f6df8894537e3742451a0e645fa2406a3f249 Mon Sep 17 00:00:00 2001
From: willy born <70607676+willyborn@users.noreply.github.com>
Date: Tue, 8 Jul 2025 22:12:43 +0200
Subject: [PATCH 466/473] Fixes sub array (cpu, cuda, oneapi, opencl) for
 transform (#3679)

* Increased difficulty of sub-array testing

* Fixes sub-array (cpu, cuda, oneapi, opencl) for transform

* Added TODO comments for linear checks. This will be fixed at a higher level later.

---------

Co-authored-by: Christophe Murphy <christophe.murphy@outlook.com>
---
 src/backend/cpu/transform.cpp           |  17 ++-
 src/backend/cuda/transform.cpp          |   7 +-
 src/backend/oneapi/kernel/transform.hpp |   3 +-
 src/backend/oneapi/transform.cpp        |  14 ++-
 src/backend/opencl/kernel/transform.cl  |   2 +-
 src/backend/opencl/transform.cpp        |  14 ++-
 test/arrayfire_test.cpp                 | 145 ++++++++++++++----------
 test/transform.cpp                      |  40 +++++++
 test/transform_coordinates.cpp          |  25 +++-
 9 files changed, 188 insertions(+), 79 deletions(-)

diff --git a/src/backend/cpu/transform.cpp b/src/backend/cpu/transform.cpp
index bbcf689f25..0fbe10ea5c 100644
--- a/src/backend/cpu/transform.cpp
+++ b/src/backend/cpu/transform.cpp
@@ -8,6 +8,7 @@
  ********************************************************/
 
 #include <Array.hpp>
+#include <copy.hpp>
 #include <kernel/transform.hpp>
 #include <math.hpp>
 #include <platform.hpp>
@@ -22,23 +23,27 @@ void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const bool perspective) {
     out.eval();
     in.eval();
+
+    // TODO: Temporary Fix, must fix handling subarrays upstream
+    // tf has to be linear, although offset is allowed
+    const Array<float> tf_Lin = tf.isLinear() ? tf : copyArray(tf);
     tf.eval();
 
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            getQueue().enqueue(kernel::transform<T, 1>, out, in, tf, inverse,
-                               perspective, method);
+            getQueue().enqueue(kernel::transform<T, 1>, out, in, tf_Lin,
+                               inverse, perspective, method);
             break;
         case AF_INTERP_BILINEAR:
         case AF_INTERP_BILINEAR_COSINE:
-            getQueue().enqueue(kernel::transform<T, 2>, out, in, tf, inverse,
-                               perspective, method);
+            getQueue().enqueue(kernel::transform<T, 2>, out, in, tf_Lin,
+                               inverse, perspective, method);
             break;
         case AF_INTERP_BICUBIC:
         case AF_INTERP_BICUBIC_SPLINE:
-            getQueue().enqueue(kernel::transform<T, 3>, out, in, tf, inverse,
-                               perspective, method);
+            getQueue().enqueue(kernel::transform<T, 3>, out, in, tf_Lin,
+                               inverse, perspective, method);
             break;
         default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG); break;
     }
diff --git a/src/backend/cuda/transform.cpp b/src/backend/cuda/transform.cpp
index e0d0509c8d..af8b561191 100644
--- a/src/backend/cuda/transform.cpp
+++ b/src/backend/cuda/transform.cpp
@@ -9,6 +9,7 @@
 
 #include <transform.hpp>
 
+#include <copy.hpp>
 #include <kernel/transform.hpp>
 #include <utility.hpp>
 
@@ -19,7 +20,11 @@ template<typename T>
 void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const af::interpType method, const bool inverse,
                const bool perspective) {
-    kernel::transform<T>(out, in, tf, inverse, perspective, method,
+    // TODO: Temporary Fix, must fix handling subarrays upstream
+    // tf has to be linear, although offset is allowed.
+    const Array<float> tf_Lin = tf.isLinear() ? tf : copyArray(tf);
+
+    kernel::transform<T>(out, in, tf_Lin, inverse, perspective, method,
                          interpOrder(method));
 }
 
diff --git a/src/backend/oneapi/kernel/transform.hpp b/src/backend/oneapi/kernel/transform.hpp
index 07f70a3a62..874e9638c7 100644
--- a/src/backend/oneapi/kernel/transform.hpp
+++ b/src/backend/oneapi/kernel/transform.hpp
@@ -178,7 +178,8 @@ class transformCreateKernel {
         using TMatTy =
             typename std::conditional<PERSPECTIVE, float[9], float[6]>::type;
         TMatTy tmat;
-        const float *tmat_ptr = c_tmat_.get_pointer() + t_idx * transf_len;
+        const float *tmat_ptr =
+            c_tmat_.get_pointer() + tf_.offset + t_idx * transf_len;
 
         // We expect a inverse transform matrix by default
         // If it is an forward transform, then we need its inverse
diff --git a/src/backend/oneapi/transform.cpp b/src/backend/oneapi/transform.cpp
index a277df9661..00edc15817 100644
--- a/src/backend/oneapi/transform.cpp
+++ b/src/backend/oneapi/transform.cpp
@@ -9,6 +9,7 @@
 
 #include <transform.hpp>
 
+#include <copy.hpp>
 #include <err_oneapi.hpp>
 #include <kernel/transform.hpp>
 
@@ -19,18 +20,25 @@ template<typename T>
 void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const af_interp_type method, const bool inverse,
                const bool perspective) {
+    // TODO: Temporary Fix, must fix handling subarrays upstream
+    // tf has to be linear, although offset is allowed.
+    const Array<float> tf_Lin = tf.isLinear() ? tf : copyArray(tf);
+
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            kernel::transform<T>(out, in, tf, inverse, perspective, method, 1);
+            kernel::transform<T>(out, in, tf_Lin, inverse, perspective, method,
+                                 1);
             break;
         case AF_INTERP_BILINEAR:
         case AF_INTERP_BILINEAR_COSINE:
-            kernel::transform<T>(out, in, tf, inverse, perspective, method, 2);
+            kernel::transform<T>(out, in, tf_Lin, inverse, perspective, method,
+                                 2);
             break;
         case AF_INTERP_BICUBIC:
         case AF_INTERP_BICUBIC_SPLINE:
-            kernel::transform<T>(out, in, tf, inverse, perspective, method, 3);
+            kernel::transform<T>(out, in, tf_Lin, inverse, perspective, method,
+                                 3);
             break;
         default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG);
     }
diff --git a/src/backend/opencl/kernel/transform.cl b/src/backend/opencl/kernel/transform.cl
index 85c6a293ab..4fae1c05f8 100644
--- a/src/backend/opencl/kernel/transform.cl
+++ b/src/backend/opencl/kernel/transform.cl
@@ -133,7 +133,7 @@ kernel void transformKernel(global T *d_out, const KParam out,
     const int transf_len = 6;
     float tmat[6];
 #endif
-    global const float *tmat_ptr = c_tmat + t_idx * transf_len;
+    global const float *tmat_ptr = c_tmat + tf.offset + t_idx * transf_len;
 
     // We expect a inverse transform matrix by default
     // If it is an forward transform, then we need its inverse
diff --git a/src/backend/opencl/transform.cpp b/src/backend/opencl/transform.cpp
index 78428ed3a7..de99f48a60 100644
--- a/src/backend/opencl/transform.cpp
+++ b/src/backend/opencl/transform.cpp
@@ -9,6 +9,7 @@
 
 #include <transform.hpp>
 
+#include <copy.hpp>
 #include <kernel/transform.hpp>
 
 namespace arrayfire {
@@ -18,18 +19,25 @@ template<typename T>
 void transform(Array<T> &out, const Array<T> &in, const Array<float> &tf,
                const af_interp_type method, const bool inverse,
                const bool perspective) {
+    // TODO: Temporary Fix, must fix handling subarrays upstream
+    // tf has to be linear, although offset is allowed.
+    const Array<float> tf_Lin = tf.isLinear() ? tf : copyArray(tf);
+
     switch (method) {
         case AF_INTERP_NEAREST:
         case AF_INTERP_LOWER:
-            kernel::transform<T>(out, in, tf, inverse, perspective, method, 1);
+            kernel::transform<T>(out, in, tf_Lin, inverse, perspective, method,
+                                 1);
             break;
         case AF_INTERP_BILINEAR:
         case AF_INTERP_BILINEAR_COSINE:
-            kernel::transform<T>(out, in, tf, inverse, perspective, method, 2);
+            kernel::transform<T>(out, in, tf_Lin, inverse, perspective, method,
+                                 2);
             break;
         case AF_INTERP_BICUBIC:
         case AF_INTERP_BICUBIC_SPLINE:
-            kernel::transform<T>(out, in, tf, inverse, perspective, method, 3);
+            kernel::transform<T>(out, in, tf_Lin, inverse, perspective, method,
+                                 3);
             break;
         default: AF_ERROR("Unsupported interpolation type", AF_ERR_ARG);
     }
diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index 6803cc586d..eab07f5b41 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -229,9 +229,9 @@ ::testing::AssertionResult imageEq(std::string aName, std::string bName,
         af::saveImage(result_path.c_str(), b.as(f32));
         af::saveImage(diff_path.c_str(), abs(a.as(f32) - b.as(f32)));
 
-        std::cout
-            << "<DartMeasurementFile type=\"image/png\" name=\"ValidImage\">"
-            << valid_path << "</DartMeasurementFile>\n";
+        std::cout << "<DartMeasurementFile type=\"image/png\" "
+                     "name=\"ValidImage\">"
+                  << valid_path << "</DartMeasurementFile>\n";
         std::cout
             << "<DartMeasurementFile type=\"image/png\" name=\"TestImage\">"
             << result_path << "</DartMeasurementFile>\n";
@@ -526,7 +526,8 @@ dim_t ravelIdx(af::dim4 coords, af::dim4 strides) {
                               0LL);
 }
 
-// Calculate a linearized index's multi-dimensonal coordinates in an af::array,
+// Calculate a linearized index's multi-dimensonal coordinates in an
+// af::array,
 //  given its dimension sizes and strides
 af::dim4 unravelIdx(dim_t idx, af::dim4 dims, af::dim4 strides) {
     af::dim4 coords;
@@ -567,8 +568,9 @@ std::string minimalDim4(af::dim4 coords, af::dim4 dims) {
     return os.str();
 }
 
-// Generates a random array. testWriteToOutputArray expects that it will receive
-// the same af_array that this generates after the af_* function is called
+// Generates a random array. testWriteToOutputArray expects that it will
+// receive the same af_array that this generates after the af_* function is
+// called
 void genRegularArray(TestOutputArrayInfo *metadata, const unsigned ndims,
                      const dim_t *const dims, const af_dtype ty) {
     metadata->init(ndims, dims, ty);
@@ -581,9 +583,9 @@ void genRegularArray(TestOutputArrayInfo *metadata, double val,
 }
 
 // Generates a large, random array, and extracts a subarray for the af_*
-// function to use. testWriteToOutputArray expects that the large array that it
-// receives is equal to the same large array with the gold array injected on the
-// same subarray location
+// function to use. testWriteToOutputArray expects that the large array that
+// it receives is equal to the same large array with the gold array injected
+// on the same subarray location
 void genSubArray(TestOutputArrayInfo *metadata, const unsigned ndims,
                  const dim_t *const dims, const af_dtype ty) {
     const dim_t pad_size = 2;
@@ -596,8 +598,9 @@ void genSubArray(TestOutputArrayInfo *metadata, const unsigned ndims,
     }
 
     // Calculate index of sub-array. These will be used also by
-    // testWriteToOutputArray so that the gold sub array will be placed in the
-    // same location. Currently, this location is the center of the large array
+    // testWriteToOutputArray so that the gold sub array will be placed in
+    // the same location. Currently, this location is the center of the
+    // large array
     af_seq subarr_idxs[4] = {af_span, af_span, af_span, af_span};
     for (uint i = 0; i < ndims; ++i) {
         af_seq idx     = {pad_size, pad_size + dims[i] - 1.0, 1.0};
@@ -620,8 +623,9 @@ void genSubArray(TestOutputArrayInfo *metadata, double val,
     }
 
     // Calculate index of sub-array. These will be used also by
-    // testWriteToOutputArray so that the gold sub array will be placed in the
-    // same location. Currently, this location is the center of the large array
+    // testWriteToOutputArray so that the gold sub array will be placed in
+    // the same location. Currently, this location is the center of the
+    // large array
     af_seq subarr_idxs[4] = {af_span, af_span, af_span, af_span};
     for (uint i = 0; i < ndims; ++i) {
         af_seq idx     = {pad_size, pad_size + dims[i] - 1.0, 1.0};
@@ -631,13 +635,14 @@ void genSubArray(TestOutputArrayInfo *metadata, double val,
     metadata->init(val, ndims, full_arr_dims, ty, &subarr_idxs[0]);
 }
 
-// Generates a reordered array. testWriteToOutputArray expects that this array
-// will still have the correct output values from the af_* function, even though
-// the array was initially reordered.
+// Generates a reordered array. testWriteToOutputArray expects that this
+// array will still have the correct output values from the af_* function,
+// even though the array was initially reordered.
 void genReorderedArray(TestOutputArrayInfo *metadata, const unsigned ndims,
                        const dim_t *const dims, const af_dtype ty) {
-    // The rest of this function assumes that dims has 4 elements. Just in case
-    // dims has < 4 elements, use another dims array that is filled with 1s
+    // The rest of this function assumes that dims has 4 elements. Just in
+    // case dims has < 4 elements, use another dims array that is filled
+    // with 1s
     dim_t all_dims[4] = {1, 1, 1, 1};
     for (uint i = 0; i < ndims; ++i) { all_dims[i] = dims[i]; }
 
@@ -648,7 +653,8 @@ void genReorderedArray(TestOutputArrayInfo *metadata, const unsigned ndims,
     uint reorder_idxs[4] = {0, 2, 1, 3};
 
     // Shape the output array such that the reordered output array will have
-    // the correct dimensions that the test asks for (i.e. must match dims arg)
+    // the correct dimensions that the test asks for (i.e. must match dims
+    // arg)
     dim_t init_dims[4] = {all_dims[0], all_dims[1], all_dims[2], all_dims[3]};
     for (uint i = 0; i < 4; ++i) { init_dims[i] = all_dims[reorder_idxs[i]]; }
     metadata->init(4, init_dims, ty);
@@ -663,8 +669,9 @@ void genReorderedArray(TestOutputArrayInfo *metadata, const unsigned ndims,
 void genReorderedArray(TestOutputArrayInfo *metadata, double val,
                        const unsigned ndims, const dim_t *const dims,
                        const af_dtype ty) {
-    // The rest of this function assumes that dims has 4 elements. Just in case
-    // dims has < 4 elements, use another dims array that is filled with 1s
+    // The rest of this function assumes that dims has 4 elements. Just in
+    // case dims has < 4 elements, use another dims array that is filled
+    // with 1s
     dim_t all_dims[4] = {1, 1, 1, 1};
     for (uint i = 0; i < ndims; ++i) { all_dims[i] = dims[i]; }
 
@@ -675,7 +682,8 @@ void genReorderedArray(TestOutputArrayInfo *metadata, double val,
     uint reorder_idxs[4] = {0, 2, 1, 3};
 
     // Shape the output array such that the reordered output array will have
-    // the correct dimensions that the test asks for (i.e. must match dims arg)
+    // the correct dimensions that the test asks for (i.e. must match dims
+    // arg)
     dim_t init_dims[4] = {all_dims[0], all_dims[1], all_dims[2], all_dims[3]};
     for (uint i = 0; i < 4; ++i) { init_dims[i] = all_dims[reorder_idxs[i]]; }
     metadata->init(val, 4, init_dims, ty);
@@ -745,8 +753,8 @@ ::testing::AssertionResult testWriteToOutputArray(
 
     if (metadata->getOutputArrayType() == SUB_ARRAY) {
         // There are two full arrays. One will be injected with the gold
-        // subarray, the other should have already been injected with the af_*
-        // function's output. Then we compare the two full arrays
+        // subarray, the other should have already been injected with the
+        // af_* function's output. Then we compare the two full arrays
         af_array gold_full_array = metadata->getFullOutputCopy();
         af_assign_seq(&gold_full_array, gold_full_array,
                       metadata->getSubArrayNumDims(),
@@ -1293,9 +1301,11 @@ ::testing::AssertionResult mtxReadSparseMatrix(af::array &out,
                 return ::testing::AssertionFailure()
                        << "\nEnd of file reached, expected more data, "
                        << "following are some reasons this happens.\n"
-                       << "\t - use of template type that doesn't match data "
+                       << "\t - use of template type that doesn't match "
+                          "data "
                           "type\n"
-                       << "\t - the mtx file itself doesn't have enough data\n";
+                       << "\t - the mtx file itself doesn't have enough "
+                          "data\n";
             }
             I[i] = r - 1;
             J[i] = c - 1;
@@ -1319,9 +1329,11 @@ ::testing::AssertionResult mtxReadSparseMatrix(af::array &out,
                 return ::testing::AssertionFailure()
                        << "\nEnd of file reached, expected more data, "
                        << "following are some reasons this happens.\n"
-                       << "\t - use of template type that doesn't match data "
+                       << "\t - use of template type that doesn't match "
+                          "data "
                           "type\n"
-                       << "\t - the mtx file itself doesn't have enough data\n";
+                       << "\t - the mtx file itself doesn't have enough "
+                          "data\n";
             }
             I[i] = r - 1;
             J[i] = c - 1;
@@ -1531,8 +1543,8 @@ vector<sparseCooValue<T>> toCooVector(const af::array &arr) {
         }
     }
 
-    // Remove zero elements from result to ensure that only non-zero elements
-    // are compared
+    // Remove zero elements from result to ensure that only non-zero
+    // elements are compared
     out.erase(std::remove_if(out.begin(), out.end(), isZero<T>), out.end());
     std::sort(begin(out), end(out));
     return out;
@@ -1584,8 +1596,8 @@ std::string printContext(const std::vector<T> &hGold, std::string goldName,
 
     // Get dim0 positions and out/reference values for the context window
     //
-    // Also get the max string length between the position and out/ref values
-    // per item so that it can be used later as the field width for
+    // Also get the max string length between the position and out/ref
+    // values per item so that it can be used later as the field width for
     // displaying each item in the context window
     for (dim_t i = 0; i < ctxElems; ++i) {
         std::ostringstream tmpOs;
@@ -2063,31 +2075,35 @@ af::array toTempFormat(tempFormat form, const af::array &in) {
             break;
         case SUB_FORMAT_dim0: {
             af::dim4 pdims(dims);
-            pdims[0] += 2;
-            af::array parent = af::randu(pdims, in.type());
-            parent(af::seq(1, dims[0]), af::span, af::span, af::span) = in;
-            ret = parent(af::seq(1, dims[0]), af::span, af::span, af::span);
+            pdims[0] *= 2;
+            af::array parent  = af::randu(pdims, in.type());
+            const af::seq dim = af::seq(dims[0]) + static_cast<double>(dims[0]);
+            parent(dim, af::span, af::span, af::span) = in;
+            ret = parent(dim, af::span, af::span, af::span);
         }; break;
         case SUB_FORMAT_dim1: {
             af::dim4 pdims(dims);
-            pdims[1] += 2;
-            af::array parent = af::randu(pdims, in.type());
-            parent(af::span, af::seq(1, dims[1]), af::span, af::span) = in;
-            ret = parent(af::span, af::seq(1, dims[1]), af::span, af::span);
+            pdims[1] *= 2;
+            const af::seq dim = af::seq(dims[1]) + static_cast<double>(dims[1]);
+            af::array parent  = af::randu(pdims, in.type());
+            parent(af::span, dim, af::span, af::span) = in;
+            ret = parent(af::span, dim, af::span, af::span);
         }; break;
         case SUB_FORMAT_dim2: {
             af::dim4 pdims(dims);
-            pdims[2] += 2;
-            af::array parent = af::randu(pdims, in.type());
-            parent(af::span, af::span, af::seq(1, dims[2]), af::span) = in;
-            ret = parent(af::span, af::span, af::seq(1, dims[2]), af::span);
+            pdims[2] *= 2;
+            const af::seq dim = af::seq(dims[2]) + static_cast<double>(dims[2]);
+            af::array parent  = af::randu(pdims, in.type());
+            parent(af::span, af::span, dim, af::span) = in;
+            ret = parent(af::span, af::span, dim, af::span);
         }; break;
         case SUB_FORMAT_dim3: {
             af::dim4 pdims(dims);
-            pdims[3] += 2;
-            af::array parent = af::randu(pdims, in.type());
-            parent(af::span, af::span, af::span, af::seq(1, dims[3])) = in;
-            ret = parent(af::span, af::span, af::span, af::seq(1, dims[3]));
+            pdims[3] *= 2;
+            const af::seq dim = af::seq(dims[3]) + static_cast<double>(dims[3]);
+            af::array parent  = af::randu(pdims, in.type());
+            parent(af::span, af::span, af::span, dim) = in;
+            ret = parent(af::span, af::span, af::span, dim);
         }; break;
         case REORDERED_FORMAT: {
             const dim_t idxs[4] = {0, 3, 1, 2};
@@ -2138,21 +2154,22 @@ void toTempFormat(tempFormat form, af_array *out, const af_array &in) {
             res = nullptr;
         }; break;
         case SUB_FORMAT_dim0: {
-            const dim_t pdims[4] = {dims[0] + 2, dims[1], dims[2], dims[3]};
+            const dim_t pdims[4] = {dims[0] * 2, dims[1], dims[2], dims[3]};
             af_array parent      = nullptr;
-            ASSERT_SUCCESS(af_randu(&parent, std::max(1u, numdims), pdims, ty));
-            const af_seq idxs[4] = {af_make_seq(1, dims[0], 1), af_span,
-                                    af_span, af_span};
-
+            ASSERT_SUCCESS(af_randu(&parent, 4, pdims, ty));
+            const af_seq idxs[4] = {af_make_seq(dims[0], 2. * dims[0] - 1., 1.),
+                                    af_span, af_span, af_span};
             ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in));
             ASSERT_SUCCESS(af_index(out, parent, numdims, idxs));
             ASSERT_SUCCESS(af_release_array(parent));
+            parent = nullptr;
         }; break;
         case SUB_FORMAT_dim1: {
-            const dim_t pdims[4] = {dims[0], dims[1] + 2, dims[2], dims[3]};
+            const dim_t pdims[4] = {dims[0], dims[1] * 2, dims[2], dims[3]};
             af_array parent      = nullptr;
-            ASSERT_SUCCESS(af_randu(&parent, std::max(2u, numdims), pdims, ty));
-            const af_seq idxs[4] = {af_span, af_make_seq(1, dims[1], 1),
+            ASSERT_SUCCESS(af_randu(&parent, 4, pdims, ty));
+            const af_seq idxs[4] = {af_span,
+                                    af_make_seq(dims[1], 2. * dims[1] - 1., 1.),
                                     af_span, af_span};
             ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in));
             ASSERT_SUCCESS(af_index(out, parent, numdims, idxs));
@@ -2160,22 +2177,24 @@ void toTempFormat(tempFormat form, af_array *out, const af_array &in) {
             parent = nullptr;
         }; break;
         case SUB_FORMAT_dim2: {
-            const dim_t pdims[4] = {dims[0], dims[1], dims[2] + 2, dims[3]};
+            const dim_t pdims[4] = {dims[0], dims[1], dims[2] * 2, dims[3]};
             af_array parent      = nullptr;
-            ASSERT_SUCCESS(af_randu(&parent, std::max(3u, numdims), pdims, ty));
+            ASSERT_SUCCESS(af_randu(&parent, 4, pdims, ty));
             const af_seq idxs[4] = {af_span, af_span,
-                                    af_make_seq(1, dims[2], 1), af_span};
+                                    af_make_seq(dims[2], 2. * dims[2] - 1., 1.),
+                                    af_span};
             ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in));
             ASSERT_SUCCESS(af_index(out, parent, numdims, idxs));
             ASSERT_SUCCESS(af_release_array(parent));
             parent = nullptr;
         }; break;
         case SUB_FORMAT_dim3: {
-            const dim_t pdims[4] = {dims[0], dims[1], dims[2], dims[3] + 2};
+            const dim_t pdims[4] = {dims[0], dims[1], dims[2], dims[3] * 2};
             af_array parent      = nullptr;
-            ASSERT_SUCCESS(af_randu(&parent, std::max(4u, numdims), pdims, ty));
-            const af_seq idxs[4] = {af_span, af_span, af_span,
-                                    af_make_seq(1, dims[3], 1)};
+            ASSERT_SUCCESS(af_randu(&parent, 4, pdims, ty));
+            const af_seq idxs[4] = {
+                af_span, af_span, af_span,
+                af_make_seq(dims[3], 2. * dims[3] - 1., 1.)};
             ASSERT_SUCCESS(af_assign_seq(out, parent, numdims, idxs, in));
             ASSERT_SUCCESS(af_index(out, parent, numdims, idxs));
             ASSERT_SUCCESS(af_release_array(parent));
diff --git a/test/transform.cpp b/test/transform.cpp
index ef3b0dd4f9..e6026576ba 100644
--- a/test/transform.cpp
+++ b/test/transform.cpp
@@ -620,3 +620,43 @@ TEST(TransformBatching, CPP) {
         }
     }
 }
+
+#define TEST_TEMP_FORMAT(form, interp)                                         \
+    TEST(TEMP_FORMAT, form##_##interp) {                                       \
+        IMAGEIO_ENABLED_CHECK();                                               \
+                                                                               \
+        vector<dim4> inDims;                                                   \
+        vector<string> inFiles;                                                \
+        vector<dim_t> goldDim;                                                 \
+        vector<string> goldFiles;                                              \
+                                                                               \
+        vector<dim4> HDims;                                                    \
+        vector<vector<float>> HIn;                                             \
+        vector<vector<float>> HTests;                                          \
+        readTests<float, float, float>(TEST_DIR "/transform/tux_tmat.test",    \
+                                       HDims, HIn, HTests);                    \
+                                                                               \
+        readImageTests(string(TEST_DIR "/transform/tux_nearest.test"), inDims, \
+                       inFiles, goldDim, goldFiles);                           \
+        inFiles[1].insert(0, string(TEST_DIR "/transform/"));                  \
+        const array IH = array(HDims[0][0], HDims[0][1], &(HIn[0].front()));   \
+        const array scene_img = loadImage(inFiles[1].c_str(), false);          \
+                                                                               \
+        const array out =                                                      \
+            transform(toTempFormat(form, scene_img), toTempFormat(form, IH),   \
+                      inDims[0][0], inDims[0][1], interp, false);              \
+        const array gold = transform(scene_img, IH, inDims[0][0],              \
+                                     inDims[0][1], interp, false);             \
+                                                                               \
+        EXPECT_ARRAYS_EQ(out, gold);                                           \
+    }
+
+#define TESTS_TEMP_FORMAT(form)                       \
+    TEST_TEMP_FORMAT(form, AF_INTERP_NEAREST)         \
+    TEST_TEMP_FORMAT(form, AF_INTERP_BILINEAR)        \
+    TEST_TEMP_FORMAT(form, AF_INTERP_BILINEAR_COSINE) \
+    TEST_TEMP_FORMAT(form, AF_INTERP_BICUBIC)         \
+    TEST_TEMP_FORMAT(form, AF_INTERP_BICUBIC_SPLINE)  \
+    TEST_TEMP_FORMAT(form, AF_INTERP_LOWER)
+
+FOREACH_TEMP_FORMAT(TESTS_TEMP_FORMAT)
\ No newline at end of file
diff --git a/test/transform_coordinates.cpp b/test/transform_coordinates.cpp
index 2875f18c1a..bc5dbed4e9 100644
--- a/test/transform_coordinates.cpp
+++ b/test/transform_coordinates.cpp
@@ -61,7 +61,7 @@ void transformCoordinatesTest(string pTestFile) {
         dim_t outEl = 0;
         ASSERT_SUCCESS(af_get_elements(&outEl, outArray));
         vector<T> outData(outEl);
-        ASSERT_SUCCESS(af_get_data_ptr((void*)&outData.front(), outArray));
+        ASSERT_SUCCESS(af_get_data_ptr((void *)&outData.front(), outArray));
 
         ASSERT_SUCCESS(af_release_array(outArray));
         const float thr = 1.f;
@@ -114,3 +114,26 @@ TEST(TransformCoordinates, CPP) {
             << "at: " << elIter << endl;
     }
 }
+
+#define TESTS_TEMP_FORMAT(form)                                                \
+    TEST(TEMP_FORMAT, form) {                                                  \
+        vector<dim4> inDims;                                                   \
+        vector<vector<float>> in;                                              \
+        vector<vector<float>> gold;                                            \
+                                                                               \
+        readTests<float, float, float>(TEST_DIR                                \
+                                       "/transformCoordinates/3d_matrix.test", \
+                                       inDims, in, gold);                      \
+                                                                               \
+        const array tf(inDims[0][0], inDims[0][1], &(in[0].front()));          \
+        const float d0 = in[1][0];                                             \
+        const float d1 = in[1][1];                                             \
+                                                                               \
+        const array out =                                                      \
+            transformCoordinates(toTempFormat(form, tf), d0, d1);              \
+        const array gout = transformCoordinates(tf, d0, d1);                   \
+                                                                               \
+        EXPECT_ARRAYS_EQ(out, gout);                                           \
+    }
+
+FOREACH_TEMP_FORMAT(TESTS_TEMP_FORMAT)
\ No newline at end of file

From 82ca3b39d20e74962fa1163c57466edd9eb31295 Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Tue, 8 Jul 2025 22:51:49 +0200
Subject: [PATCH 467/473] Revert clBlast version to original reference. Newer
 versions are causing some test failures in the Cholesky decomposition.
 (#3678)

---
 CMakeModules/build_CLBlast.cmake | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CMakeModules/build_CLBlast.cmake b/CMakeModules/build_CLBlast.cmake
index a0d9fab435..7ea0b43256 100644
--- a/CMakeModules/build_CLBlast.cmake
+++ b/CMakeModules/build_CLBlast.cmake
@@ -24,9 +24,10 @@ if(TARGET clblast OR AF_WITH_EXTERNAL_PACKAGES_ONLY)
     message(ERROR "CLBlast now found")
   endif()
 else()
+  # This specific reference passes tests
   af_dep_check_and_populate(${clblast_prefix}
     URI https://github.com/cnugteren/CLBlast.git
-    REF 1.6.3
+    REF 4500a03440e2cc54998c0edab366babf5e504d67
   )
 
   include(ExternalProject)

From a699cb9eb2d7f7bfbeca9e00a2b53b26d90b2efb Mon Sep 17 00:00:00 2001
From: Edwin Solis <edwinsolisf12@gmail.com>
Date: Mon, 14 Jul 2025 00:15:53 -0700
Subject: [PATCH 468/473] Fixed topk for half, marked sort_index with half
 unsupported

---
 src/backend/opencl/sort_index.cpp |  6 +++++
 src/backend/opencl/topk.cpp       | 44 ++++++++++++++++++++++++++-----
 2 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/src/backend/opencl/sort_index.cpp b/src/backend/opencl/sort_index.cpp
index 4840c24277..afd8bf8413 100644
--- a/src/backend/opencl/sort_index.cpp
+++ b/src/backend/opencl/sort_index.cpp
@@ -25,6 +25,12 @@ namespace opencl {
 template<typename T>
 void sort_index(Array<T> &okey, Array<uint> &oval, const Array<T> &in,
                 const uint dim, bool isAscending) {
+    
+    // TODO: fix half implementation of sort0bykey to support this
+    if (std::is_same_v<T, half>) {
+        OPENCL_NOT_SUPPORTED("sort_index with half");
+    }
+
     try {
         // okey contains values, oval contains indices
         okey = copyArray<T>(in);
diff --git a/src/backend/opencl/topk.cpp b/src/backend/opencl/topk.cpp
index 18e03d2f0d..201ec06197 100644
--- a/src/backend/opencl/topk.cpp
+++ b/src/backend/opencl/topk.cpp
@@ -8,12 +8,17 @@
  ********************************************************/
 
 #include <Array.hpp>
+#include <common/cast.hpp>
 #include <common/half.hpp>
+#include <common/moddims.hpp>
 #include <err_opencl.hpp>
 #include <index.hpp>
 #include <sort.hpp>
 #include <sort_index.hpp>
 #include <types.hpp>
+#include <handle.hpp>
+#include <arith.hpp>
+#include <range.hpp>
 
 #include <algorithm>
 #include <cmath>
@@ -157,12 +162,39 @@ void topk(Array<T>& vals, Array<unsigned>& idxs, const Array<T>& in,
         vals = values;
         idxs = indices;
     } else {
-        auto values  = createEmptyArray<T>(in.dims());
-        auto indices = createEmptyArray<unsigned>(in.dims());
-        sort_index(values, indices, in, dim, order & AF_TOPK_MIN);
-        auto indVec = indexForTopK(k);
-        vals        = index<T>(values, indVec.data());
-        idxs        = index<unsigned>(indices, indVec.data());
+        
+        if (!std::is_same_v<T, half>) {
+            auto values  = createEmptyArray<T>(in.dims());
+            auto indices = createEmptyArray<unsigned>(in.dims());
+            sort_index(values, indices, in, dim, order & AF_TOPK_MIN);
+            auto indVec = indexForTopK(k);
+            idxs        = index<unsigned>(indices, indVec.data());
+            vals        = index<T>(values, indVec.data());
+        } else {
+            // Temporary implementation for topk due half not being supported in sort_index
+            // TODO: Fix sort_index and remove this
+
+            auto values  = createEmptyArray<float>(in.dims());
+            auto indices = createEmptyArray<unsigned>(in.dims());
+            sort_index(values, indices, common::cast<float>(in), dim, order & AF_TOPK_MIN);
+
+            auto indVec = indexForTopK(k);
+            idxs        = index<unsigned>(indices, indVec.data());
+
+            // Index values from original array by using the indices from the previous resuult
+            auto len = in.elements() / in.dims()[dim];
+            auto index_dims = dim4(k, len);
+            auto new_indices = common::flat(arithOp<unsigned, af_add_t>(arithOp<unsigned, af_mul_t>(range<unsigned>(index_dims, 1), createValueArray<unsigned>(index_dims, in.dims()[dim]), index_dims), idxs, index_dims));
+            auto indVecVals = indexForTopK(k);
+            indVecVals[0].idx.arr = getHandle(new_indices);
+            indVecVals[0].isSeq = false;
+            indVecVals[0].isBatch = false;
+            
+            vals = common::modDims(index<T>(common::flat(in), indVecVals.data()), idxs.dims());
+            vals.eval();
+
+            releaseHandle<unsigned>(indVecVals[0].idx.arr);
+        }
     }
 }
 

From 3ae9f0460e1d3dfeb17e10099c3c08b67c46bd8c Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Fri, 18 Jul 2025 01:09:23 +0200
Subject: [PATCH 469/473] Fix for evaluation containing array and its inverse.
 (#3671)

A bug was found in the jit calculation tree where under certain circumstances the incorrect result would be found when both an array and a transpose of the same array are used in the same evaluation. When the transpose method is used on an array the treatment depends on the structure of the jit tree of that array. In the particular case that the array is linear (e.g. not a sub-array) and is not simply a buffer (i.e. a combination of buffers and/or scalars with some operators) a moddim node will be added to the root of the tree with the new dimensions. When the tree is evaluated the dimensions of the child buffer(s) of a moddim node are changed and the moddim node is deleted. If an evaluation happens to include both an array and the transpose of that same array then when the moddim node is applied it changes the dimensions of the children for the remaining lifetime of the evaluation including any subsequent use of these nodes. As a result if the transpose of an array is used in an expression followed by the array itself the second instance will also be transposed.

In this fix instead of creating a moddims node a deep copy of the calculation tree is made at that point with the transpose applied to the child buffer nodes. These buffer node copies will have the transposed dimensions and strides but will still point to the same memory location for the data so no copy of the underlying data needs to be made.
---
 src/backend/common/jit/BufferNodeBase.hpp | 12 +++++++
 src/backend/common/jit/Node.hpp           |  5 +++
 src/backend/common/moddims.cpp            | 29 +++++++++-------
 src/backend/cpu/jit/BufferNode.hpp        | 13 ++++++++
 src/backend/cuda/jit/BufferNode.hpp       | 11 ++++++-
 src/backend/oneapi/jit/BufferNode.hpp     | 11 ++++++-
 src/backend/opencl/jit/BufferNode.hpp     | 11 ++++++-
 test/jit.cpp                              | 40 +++++++++++++++++++++++
 8 files changed, 118 insertions(+), 14 deletions(-)

diff --git a/src/backend/common/jit/BufferNodeBase.hpp b/src/backend/common/jit/BufferNodeBase.hpp
index fd63e89932..85576304ad 100644
--- a/src/backend/common/jit/BufferNodeBase.hpp
+++ b/src/backend/common/jit/BufferNodeBase.hpp
@@ -119,6 +119,18 @@ class BufferNodeBase : public common::Node {
         }
         return false;
     }
+
+    virtual void modDims(const af::dim4 &newDim) override {
+        af::dim4 strides(1, 1, 1, 1);
+        for(dim_t i = 1; i < 4; ++i) {
+            strides[i] = strides[i - 1] * newDim[i - 1];
+        }
+
+        for(dim_t i = 0; i < 4; ++i) {
+            m_param.dims[i] = newDim[i];
+            m_param.strides[i] = strides[i];
+        }
+    }
 };
 
 }  // namespace common
diff --git a/src/backend/common/jit/Node.hpp b/src/backend/common/jit/Node.hpp
index 4641ff182c..794c10c14c 100644
--- a/src/backend/common/jit/Node.hpp
+++ b/src/backend/common/jit/Node.hpp
@@ -13,6 +13,7 @@
 #include <optypes.hpp>
 #include <types.hpp>
 #include <af/defines.h>
+#include <af/dim4.hpp>
 
 #include <nonstd/span.hpp>
 #include <algorithm>
@@ -311,6 +312,10 @@ class Node {
     }
     virtual std::unique_ptr<Node> clone() = 0;
 
+    virtual void modDims(const af::dim4 &newDim) {
+        UNUSED(newDim);
+    }
+
 #ifdef AF_CPU
     template<typename U>
     friend void arrayfire::cpu::kernel::evalMultiple(
diff --git a/src/backend/common/moddims.cpp b/src/backend/common/moddims.cpp
index cf9d8d6bb9..25edfa5b0a 100644
--- a/src/backend/common/moddims.cpp
+++ b/src/backend/common/moddims.cpp
@@ -20,21 +20,27 @@ using detail::createNodeArray;
 
 using std::make_shared;
 using std::shared_ptr;
+using std::array;
+using arrayfire::common::Node;
+using arrayfire::common::Node_ptr;
 using std::vector;
 
 namespace arrayfire {
 namespace common {
+
+Node_ptr copyModdims(const Node_ptr &in, const af::dim4 &newDim) {
+
+    Node_ptr out = in->clone();
+    for(int i = 0; i < in->kMaxChildren && in->m_children[i] != nullptr; ++i) {
+        out->m_children[i] = copyModdims(in->m_children[i], newDim);
+    }
+    if(out->isBuffer()) out->modDims(newDim);
+
+    return out;
+}
+
 template<typename T>
 Array<T> moddimOp(const Array<T> &in, af::dim4 outDim) {
-    using arrayfire::common::Node;
-    using arrayfire::common::Node_ptr;
-    using std::array;
-
-    auto createModdim = [outDim](array<Node_ptr, 1> &operands) {
-        return make_shared<ModdimNode>(
-            outDim, static_cast<af::dtype>(af::dtype_traits<T>::af_type),
-            operands[0]);
-    };
 
     const auto &node = in.getNode();
 
@@ -49,8 +55,9 @@ Array<T> moddimOp(const Array<T> &in, af::dim4 outDim) {
     }
     if (all_linear == false) in.eval();
 
-    Node_ptr out = createNaryNode<T, 1>(outDim, createModdim, {&in});
-    return createNodeArray<T>(outDim, out);
+    Array<T> out = createNodeArray<T>(outDim, copyModdims(in.getNode(), outDim));
+
+    return out;
 }
 
 template<typename T>
diff --git a/src/backend/cpu/jit/BufferNode.hpp b/src/backend/cpu/jit/BufferNode.hpp
index 32a94b2a74..ca3cfe7bb5 100644
--- a/src/backend/cpu/jit/BufferNode.hpp
+++ b/src/backend/cpu/jit/BufferNode.hpp
@@ -175,6 +175,19 @@ class BufferNode : public TNode<T> {
         }
         return false;
     }
+
+    virtual void modDims(const af::dim4 &newDim) override {
+        af::dim4 strides(1, 1, 1, 1);
+        for(dim_t i = 1; i < 4; ++i) {
+            strides[i] = strides[i - 1] * newDim[i - 1];
+        }
+
+        for(dim_t i = 0; i < 4; ++i) {
+            m_dims[i] = newDim[i];
+            m_strides[i] = strides[i];
+        }
+    }
+
 };
 
 }  // namespace jit
diff --git a/src/backend/cuda/jit/BufferNode.hpp b/src/backend/cuda/jit/BufferNode.hpp
index 195353fdd8..8692b72515 100644
--- a/src/backend/cuda/jit/BufferNode.hpp
+++ b/src/backend/cuda/jit/BufferNode.hpp
@@ -27,7 +27,16 @@ bool BufferNodeBase<DataType, ParamType>::operator==(
     // clang-format off
     return m_data.get() == other.m_data.get() &&
            m_bytes == other.m_bytes &&
-           m_param.ptr == other.m_param.ptr;
+           m_param.ptr == other.m_param.ptr &&
+           m_linear_buffer == other.m_linear_buffer &&
+           m_param.dims[0] == other.m_param.dims[0] &&
+           m_param.dims[1] == other.m_param.dims[1] &&
+           m_param.dims[2] == other.m_param.dims[2] &&
+           m_param.dims[3] == other.m_param.dims[3] &&
+           m_param.strides[0] == other.m_param.strides[0] &&
+           m_param.strides[1] == other.m_param.strides[1] &&
+           m_param.strides[2] == other.m_param.strides[2] &&
+           m_param.strides[3] == other.m_param.strides[3];
     // clang-format on
 }
 
diff --git a/src/backend/oneapi/jit/BufferNode.hpp b/src/backend/oneapi/jit/BufferNode.hpp
index 94655f23e7..d10ca24cc3 100644
--- a/src/backend/oneapi/jit/BufferNode.hpp
+++ b/src/backend/oneapi/jit/BufferNode.hpp
@@ -31,7 +31,16 @@ bool BufferNodeBase<DataType, ParamType>::operator==(
     // clang-format off
     return m_data.get() == other.m_data.get() &&
            m_bytes == other.m_bytes &&
-           m_param.offset == other.m_param.offset;
+           m_param.offset == other.m_param.offset &&
+           m_linear_buffer == other.m_linear_buffer &&
+           m_param.dims[0] == other.m_param.dims[0] &&
+           m_param.dims[1] == other.m_param.dims[1] &&
+           m_param.dims[2] == other.m_param.dims[2] &&
+           m_param.dims[3] == other.m_param.dims[3] &&
+           m_param.strides[0] == other.m_param.strides[0] &&
+           m_param.strides[1] == other.m_param.strides[1] &&
+           m_param.strides[2] == other.m_param.strides[2] &&
+           m_param.strides[3] == other.m_param.strides[3];
     // clang-format on
 }
 
diff --git a/src/backend/opencl/jit/BufferNode.hpp b/src/backend/opencl/jit/BufferNode.hpp
index e188fb429f..14521030f7 100644
--- a/src/backend/opencl/jit/BufferNode.hpp
+++ b/src/backend/opencl/jit/BufferNode.hpp
@@ -28,7 +28,16 @@ bool BufferNodeBase<DataType, ParamType>::operator==(
     // clang-format off
     return m_data.get() == other.m_data.get() &&
            m_bytes == other.m_bytes &&
-           m_param.offset == other.m_param.offset;
+           m_param.offset == other.m_param.offset &&
+           m_linear_buffer == other.m_linear_buffer &&
+           m_param.dims[0] == other.m_param.dims[0] &&
+           m_param.dims[1] == other.m_param.dims[1] &&
+           m_param.dims[2] == other.m_param.dims[2] &&
+           m_param.dims[3] == other.m_param.dims[3] &&
+           m_param.strides[0] == other.m_param.strides[0] &&
+           m_param.strides[1] == other.m_param.strides[1] &&
+           m_param.strides[2] == other.m_param.strides[2] &&
+           m_param.strides[3] == other.m_param.strides[3];
     // clang-format on
 }
 
diff --git a/test/jit.cpp b/test/jit.cpp
index 3848a22242..487fdcb6e2 100644
--- a/test/jit.cpp
+++ b/test/jit.cpp
@@ -814,3 +814,43 @@ TEST(JIT, setKernelCacheDirectory) {
   // Reset to the old path
   ASSERT_SUCCESS(af_set_kernel_cache_directory(old_path.c_str(), false));
 }
+
+// Ensure that a correct result is obtained when evaluating an expression
+// that contains both an array and its transpose - see ISSUE 3660
+TEST(JIT, evaluateBothArrayAndItsTranspose) {
+  float X2_ptr[25] = { -1.,  -1.,  -1.,  -1.,  -1.,
+                      -0.5, -0.5, -0.5, -0.5, -0.5,
+                        0.,   0.,   0.,   0.,   0.,
+                       0.5,  0.5,  0.5,  0.5,  0.5,
+                        1.,   1.,   1.,   1.,   1. };
+  array X2_gold(5, 5, X2_ptr);
+
+  float Y2_ptr[25] = { -1., -0.5,   0.,  0.5,   1.,
+                       -1., -0.5,   0.,  0.5,   1.,
+                       -1., -0.5,   0.,  0.5,   1.,
+                       -1., -0.5,   0.,  0.5,   1.,
+                       -1., -0.5,   0.,  0.5,   1. };
+  array Y2_gold(5, 5, Y2_ptr);
+
+  float X2Y2_ptr[25] = {  -2., -1.5,  -1., -0.5,  0.,
+                         -1.5,  -1., -0.5,   0., 0.5,
+                          -1., -0.5,   0.,  0.5,  1.,
+                         -0.5,   0.,  0.5,   1., 1.5,
+                           0.,  0.5,   1.,  1.5,  2. };
+  array X2Y2_gold(5, 5, X2Y2_ptr);
+
+  int n = 5;
+  int half = (n - 1) / 2;
+  double delta = 1.0 / half;
+
+  array coord = delta * (af::range(n) - half);
+
+  array X2 = tile(coord.T(), n, 1);
+  array Y2 = tile(coord, 1, n);
+
+  array X2Y2 = X2 + Y2;
+
+  ASSERT_ARRAYS_EQ(X2_gold, X2);
+  ASSERT_ARRAYS_EQ(Y2_gold, Y2);
+  ASSERT_ARRAYS_EQ(X2Y2_gold, X2Y2);
+}

From 3994d1ec8bccd8a01e2a68e1fc9864fa13479307 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Thu, 17 Jul 2025 16:19:26 -0700
Subject: [PATCH 470/473] Added release notes for v3.10 (#3681)

* Added release notes for v3.10

* Update ArrayFire version in vckpg json file.

---------

Co-authored-by: Christophe Murphy <christophe.murphy@outlook.com>
---
 docs/pages/release_notes.md | 47 ++++++++++++++++++++++++++++++++++++-
 vcpkg.json                  |  2 +-
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 1b55fea448..525542246f 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -1,6 +1,51 @@
 Release Notes {#releasenotes}
 ==============
 
+v3.10.0
+======
+
+## Improvements
+- Added signed int8 support \PR{3661} \PR{3508} \PR{3507} \PR{3503}
+- Increased support for half (fp16) \PR{3680} \PR{3258} \PR{3561} \PR{3627} \PR{3561} \PR{3627} \PR{3559}
+- Updated oneAPI to use Intel oneAPI (R) 2025.1 \PR{3643} \PR{3573}
+- Updated cl2hpp dependency \PR{3651} \pr{3562}
+- Add support for CUDA 12.3, 12.4, 12.5, 12.6, 12.8, and 12.9 \PR{3657} \PR{3645} \PR{3641} \PR{3636} \PR{3588} \PR{3552} \PR{3586} \PR{3541} 
+- Added minimum driver version check for CUDA GPUs \PR{3648}
+- Add more examples \PR{3530} \PR{3455} \PR{3375} \PR{3612} \PR{3584} \PR{3577}
+- Updated documentation \PR{3496} \PR{3613}
+- Improved performance of matrix multiplication of sparse matrices on the OpenCL backend \PR{3608}
+- Improved cmake configure \PR{3581} \PR{3569} \PR{3567} \PR{3564} \PR{3554}
+- Loosen indexing assertions for assignments \PR{3514}
+
+## Fixes
+- Fix jit tree when doing operations containing moddims and original array \PR{3671} 
+- Fix incorrect behavior of sub-arrays with multiple functions \PR{3679} \PR{3668} \PR{3666} \PR{3665} \PR{3664} \PR{3663} \PR{3658} \PR{3659} \PR{3650} \PR{3611} \PR{3633} \PR{3602}
+- Fix half precision operations in multiple backends \PR{3676} \PR{3662}
+- Fix for join not always respecting the order of parameters \PR{3667} \PR{3513}
+- Fix for cmake building as an external project (needed by arrayfire python wheels) \PR{3669}
+- Fix for cmake build in Windows (including with vcpkg) \PR{3655} \PR{3646} \PR{3644} \PR{3512} \PR{3626} \PR{3566} \PR{3557} \pr{3591} \PR{3592}
+- Fix race condition in OpenCL flood fill \PR{3535}
+- Fix indexing array using sequences `af_seq` that have non-unit steps \PR{3587}
+- Fix padding issue convolve2GradientNN \PR{3519}
+- Fix incorrect axis values for histogram \PR{3590}
+- Fix unified exceptions errors \PR{3617}
+- Fix OpenCL memory migration on devices with different contexts \PR{3510}
+- Fix conversion of COO Sparse to Dense matrix \PR{3589} \PR{3579}
+- Fix `AF_JIT_KERNEL_TRACE` on Windows \PR{3517}
+- Fix cmake build with CUDNN \PR{3521}
+- Fix cmake build with `AF_DISABLE_CPU_ASYNC` \PR{3551}
+
+
+## Contributions
+
+Special thanks to our contributors:
+[Willy Born](https://github.com/willyborn)
+[verstatx](https://github.com/verstatx)
+[Filip Matzner](https://github.com/FloopCZ)
+[Fraser Cormack](https://github.com/frasercrmck)
+[errata-c](https://github.com/errata-c)
+[Tyler Hilbert](https://github.com/Tyler-Hilbert)
+
 v3.9.0
 ======
 
@@ -24,7 +69,7 @@ v3.9.0
 
 ## Fixes
 - Improve Errors when creating OpenCL contexts from devices \PR{3257}
-- Improvements to vcpkg builds \PR{3376 \PR{3476}
+- Improvements to vcpkg builds \PR{3376} \PR{3476}
 - Fix reduce by key when nan's are present \PR{3261}
 - Fix error in convolve where the ndims parameter was forced to be equal to 2
   \PR{3277}
diff --git a/vcpkg.json b/vcpkg.json
index d811275a6f..7b8d9bca2f 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -1,6 +1,6 @@
 {
     "name": "arrayfire",
-    "version": "3.9.0",
+    "version": "3.10.0",
     "homepage": "https://github.com/arrayfire/arrayfire",
     "description": "ArrayFire is a HPC general-purpose library targeting parallel and massively-parallel architectures such as CPUs, GPUs, etc.",
     "supports": "x64",

From d12e298f8feaecd45ecb90135045951b0612992e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Mon, 28 Jul 2025 08:40:51 -0700
Subject: [PATCH 471/473] not keyword not recognized in msvc (#3684)

---
 test/arrayfire_test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/arrayfire_test.cpp b/test/arrayfire_test.cpp
index eab07f5b41..687de09aab 100644
--- a/test/arrayfire_test.cpp
+++ b/test/arrayfire_test.cpp
@@ -2063,13 +2063,13 @@ af::array toTempFormat(tempFormat form, const af::array &in) {
     switch (form) {
         case JIT_FORMAT:
             switch (in.type()) {
-                case b8: ret = not(in); break;
+                case b8: ret = !(in); break;
                 default: ret = in * 2;
             }
             // Make sure that the base array is <> form original
             ret.eval();
             switch (in.type()) {
-                case b8: ret = not(ret); break;
+                case b8: ret = !(ret); break;
                 default: ret /= 2;
             }
             break;

From 3d50c357cbb386d3be255c8ef48d6c5656687792 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Edwin=20Lester=20Sol=C3=ADs=20Fuentes?=
 <68087165+edwinsolisf@users.noreply.github.com>
Date: Mon, 28 Jul 2025 08:43:32 -0700
Subject: [PATCH 472/473] Fixed dlls not found due to missing search paths
 (#3683)

---
 CMakeModules/FindAF_MKL.cmake | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/CMakeModules/FindAF_MKL.cmake b/CMakeModules/FindAF_MKL.cmake
index 18037ca4fc..2da1ed4584 100644
--- a/CMakeModules/FindAF_MKL.cmake
+++ b/CMakeModules/FindAF_MKL.cmake
@@ -310,8 +310,9 @@ function(find_mkl_library)
         $ENV{LIB}
         $ENV{LIBRARY_PATH}
       PATHS
-        ${MKL_ROOT}/bin
-        ${TBB_ROOT}/bin
+        $ENV{MKLROOT}/bin
+        $ENV{TBBROOT}/bin
+        $ENV{ONEAPI_ROOT}/compiler/latest/bin
       PATH_SUFFIXES
         IntelSWTools/compilers_and_libraries/windows/redist/intel64/mkl
         IntelSWTools/compilers_and_libraries/windows/redist/intel64/compiler

From 492718b5a256d4a9d5198fdce89d8fd21772bfda Mon Sep 17 00:00:00 2001
From: Christophe Murphy <72265703+christophe-murphy@users.noreply.github.com>
Date: Mon, 28 Jul 2025 08:44:18 -0700
Subject: [PATCH 473/473] Don't restart automatically after installing VC
 redistributable (#3685)

Add flag to VC redistributable installer to prevent it from restarting the computer automatically without prompting the user.

Co-authored-by: Abc <xyz@abc.com>
---
 CMakeModules/nsis/NSIS.template.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeModules/nsis/NSIS.template.in b/CMakeModules/nsis/NSIS.template.in
index 3eaad1c383..c46274518c 100644
--- a/CMakeModules/nsis/NSIS.template.in
+++ b/CMakeModules/nsis/NSIS.template.in
@@ -741,7 +741,7 @@ Section "-Core installation"
 SectionEnd
 
 Section "-Visual C++ installation"
-  ExecWait "$INSTDIR\lib\vc_redist.x64.exe /install /passive"
+  ExecWait "$INSTDIR\lib\vc_redist.x64.exe /install /passive /norestart"
   Delete "$INSTDIR\lib\vc_redist.x64.exe"
 SectionEnd